Fix/update geo layer export
This commit is contained in:
phil
parent
53c2e359da
commit
ccf6710225
5 changed files with 232 additions and 11 deletions
|
|
@ -1 +1 @@
|
|||
__version__: str = '2023.4.dev68+gd08e40d.d20240420'
|
||||
__version__: str = '0.1.dev70+g53c2e35.d20240422'
|
||||
|
|
@ -16,6 +16,7 @@ from gisaf.security import (
|
|||
from gisaf.models.authentication import (User, UserRead, Role, RoleRead)
|
||||
from gisaf.registry import registry, NotInRegistry
|
||||
from gisaf.plugins import DownloadResponse, manager as plugin_manager
|
||||
from gisaf.exporters import export_with_fiona, export_with_pyshp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -80,6 +81,87 @@ async def download_csv(
|
|||
return response
|
||||
|
||||
|
||||
@api.get('/geodata/{stores}')
|
||||
async def download_geodata(
|
||||
user: Annotated[UserRead, Depends(get_current_active_user)],
|
||||
stores: str,
|
||||
format: str = 'gpkg',
|
||||
reproject: bool = False,
|
||||
) -> Response:
|
||||
## Check permissions
|
||||
try:
|
||||
layers_registry = registry.stores.loc[stores.split(',')]
|
||||
except KeyError:
|
||||
## Empty dataframe
|
||||
layers_registry = registry.stores[0:0]
|
||||
|
||||
live_stores = [
|
||||
layer for layer in stores.split(',')
|
||||
if layer.startswith('live:')
|
||||
]
|
||||
|
||||
def _check_permission(model):
|
||||
if not hasattr(model, 'downloadable_role'):
|
||||
return True
|
||||
if user is None:
|
||||
return False
|
||||
return user.has_role(model.downloadable_role)
|
||||
|
||||
stores_allowed: list[str] = [
|
||||
c[0] for c in layers_registry.model.items()
|
||||
if _check_permission(c[1])
|
||||
] # type: ignore
|
||||
store_names = ','.join(stores_allowed + live_stores)
|
||||
|
||||
if len(store_names) == 0:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
formats = {
|
||||
'gpkg': (
|
||||
export_with_fiona, {
|
||||
'extension': 'gpkg',
|
||||
'driver': 'GPKG',
|
||||
'mimetype': 'application/geopackage+vnd.sqlite3',
|
||||
}
|
||||
),
|
||||
'dxf': (
|
||||
export_with_fiona, {
|
||||
'extension': 'dxf',
|
||||
'driver': 'DXF',
|
||||
'mimetype': 'application/x-dxf',
|
||||
'filter_columns': ['geom'],
|
||||
}
|
||||
),
|
||||
'shapefile': (
|
||||
## Fiona/ogr has several limitations for writing shapefiles (eg. dbf, shx not supported): use export_with_pyshp
|
||||
#export_with_fiona, {
|
||||
# 'extension': 'shp',
|
||||
# 'driver': 'ESRI Shapefile',
|
||||
# 'mimetype': 'application/octet-stream',
|
||||
#}
|
||||
export_with_pyshp, {}
|
||||
),
|
||||
}
|
||||
|
||||
if format not in formats:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"{format} not in known formats: {', '.join(formats.keys())}")
|
||||
|
||||
fn, kwargs = formats[format]
|
||||
if reproject and reproject not in ['false', '0', 'no']:
|
||||
kwargs['reproject'] = True
|
||||
body, filename, content_type = await fn(store_names=store_names, **kwargs)
|
||||
|
||||
headers = {
|
||||
'Content-Disposition': f'attachment; filename="{filename}"'
|
||||
}
|
||||
return Response(
|
||||
body,
|
||||
headers=headers,
|
||||
media_type=content_type,
|
||||
)
|
||||
|
||||
|
||||
@api.get('/plugin/{name}/{store}/{id}')
|
||||
async def execute_action(
|
||||
name: str,
|
||||
|
|
|
|||
108
src/gisaf/exporters.py
Normal file
108
src/gisaf/exporters.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
from os import remove
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from zipfile import ZipFile
|
||||
import logging
|
||||
import tempfile
|
||||
|
||||
from geopandas.io.file import infer_schema
|
||||
|
||||
import fiona
|
||||
|
||||
from gisaf.registry import registry
|
||||
from gisaf.redis_tools import store as redis_store, RedisError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
async def export_with_fiona(store_names, driver, mimetype, extension, filter_columns=None, reproject=False):
|
||||
"""
|
||||
Use fiona to export geo data.
|
||||
registry: gisaf.registry.ModelRegistry
|
||||
store_names: comma separated string of store (aka. layer) names
|
||||
driver: fiona driver (one of fiona.supported_drivers)
|
||||
extension: extension of the file name
|
||||
filter_columns: list of column names to filter out
|
||||
reproject: if true-ish, the geometries are reprojected to the srid specified in conf.srid_for_proj
|
||||
"""
|
||||
layers_features = {}
|
||||
for store_name in store_names.split(','):
|
||||
try:
|
||||
if store_name in registry.geom:
|
||||
layers_features[store_name] = await registry.geom[store_name].get_geo_df(reproject=reproject)
|
||||
else:
|
||||
## Live
|
||||
## TODO: make live check more explicit
|
||||
layers_features[store_name] = await redis_store.get_gdf(store_name, reproject=reproject)
|
||||
except RedisError as err:
|
||||
logger.warn(f'Cannot get store {store_name}: {err}')
|
||||
except Exception as err:
|
||||
logger.warn(f'Cannot get store {store_name}, see below')
|
||||
logger.exception(err)
|
||||
|
||||
## FIXME: only 1 layer gets exported with BytesIO, so use a real file
|
||||
|
||||
#filename = '{}_{:%Y-%m-%d_%H:%M:%S}.{}'.format(layers, datetime.now(), extension)
|
||||
filename = 'Gisaf export {:%Y-%m-%d_%H:%M:%S}_{}.{}'.format(
|
||||
datetime.now(), next(tempfile._get_candidate_names()), extension)
|
||||
## XXX: fails in case of a lot of layers
|
||||
data_file_name = Path(tempfile._get_default_tempdir()) / filename
|
||||
#data_file_name = Path(tempfile._get_default_tempdir()) / next(tempfile._get_candidate_names())
|
||||
|
||||
## XXX: MemoryFile doesn't support multiple layers (I opened https://github.com/Toblerity/Fiona/issues/830)
|
||||
#with fiona.io.MemoryFile(filename='selected_layers.gpkg') as mem_file:
|
||||
# for layer_name, gdf in layers_features.items():
|
||||
# if filter_columns:
|
||||
# gdf = gdf.filter(filter_columns)
|
||||
# schema = infer_schema(gdf)
|
||||
# with mem_file.open(layer=layer_name, driver=driver, crs=gdf.crs, schema=schema) as mem_sink:
|
||||
# mem_sink.writerecords(gdf.iterfeatures())
|
||||
#return mem_file, filename, mimetype
|
||||
|
||||
with fiona.Env():
|
||||
for layer_name, gdf in layers_features.items():
|
||||
## XXX: geopandas doesn't accept BytesIO: using fiona directly
|
||||
#gdf.to_file(data, driver=driver, mode='a')
|
||||
_gdf = gdf.reset_index()
|
||||
_gdf['fid'] = _gdf['id']
|
||||
if filter_columns:
|
||||
_gdf = _gdf.filter(filter_columns)
|
||||
schema = infer_schema(_gdf)
|
||||
with fiona.Env(OSR_WKT_FORMAT="WKT2_2018"), fiona.open(
|
||||
data_file_name, 'w',
|
||||
driver=driver,
|
||||
crs=_gdf.crs.to_string(),
|
||||
layer=layer_name,
|
||||
schema=schema) as colxn:
|
||||
colxn.writerecords(_gdf.iterfeatures())
|
||||
#data.seek(0)
|
||||
with open(data_file_name, 'rb') as data_file:
|
||||
data = data_file.read()
|
||||
remove(data_file_name)
|
||||
return data, filename, mimetype
|
||||
|
||||
|
||||
async def export_with_pyshp(store_names, reproject=False):
|
||||
"""
|
||||
Zip and return data using "old style", ie. with pyshp 1.2
|
||||
"""
|
||||
## TODO: migrate to fiona, see below
|
||||
zip_file = BytesIO()
|
||||
with ZipFile(zip_file, 'w') as zip:
|
||||
for layer_name in store_names.split(','):
|
||||
model = registry.geom[layer_name]
|
||||
dbf, shp, shx, qml, proj_str = await model.get_shapefile_files()
|
||||
zip.writestr('{}.dbf'.format(layer_name), dbf.getvalue())
|
||||
zip.writestr('{}.shp'.format(layer_name), shp.getvalue())
|
||||
zip.writestr('{}.shx'.format(layer_name), shx.getvalue())
|
||||
if qml:
|
||||
zip.writestr('{}.qml'.format(layer_name), qml)
|
||||
if proj_str:
|
||||
zip.writestr('{}.prj'.format(layer_name), proj_str)
|
||||
|
||||
zip_file.seek(0)
|
||||
filename = '{}_{:%Y-%m-%d_%H:%M}.zip'.format(store_names, datetime.now(), )
|
||||
content_type = 'application/zip'
|
||||
|
||||
return zip_file.read(), filename, content_type
|
||||
|
||||
|
|
@ -721,9 +721,7 @@ class GeoModelNoStatus(Model):
|
|||
:param reproject: should reproject to conf.srid_for_proj
|
||||
:return:
|
||||
"""
|
||||
return await cls.get_gdf(where=where, **kwargs)
|
||||
|
||||
# df = await cls.get_df(where=where, **kwargs)
|
||||
gdf = await cls.get_gdf(where=where, **kwargs)
|
||||
# df.dropna(subset=['geom'], inplace=True)
|
||||
# df.set_index('id', inplace=True)
|
||||
# df.sort_index(inplace=True)
|
||||
|
|
@ -754,18 +752,18 @@ class GeoModelNoStatus(Model):
|
|||
# float(cls.simplify) / conf.geo.simplify_geom_factor,
|
||||
# preserve_topology=conf.geo.simplify_preserve_topology)
|
||||
|
||||
# if reproject:
|
||||
# gdf.to_crs(crs=conf.crs.for_proj, inplace=True)
|
||||
if reproject:
|
||||
gdf.to_crs(crs=conf.crs.for_proj, inplace=True)
|
||||
|
||||
# ## Filter out columns
|
||||
# if filter_columns:
|
||||
# gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
|
||||
# inplace=True)
|
||||
if filter_columns:
|
||||
gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
|
||||
inplace=True)
|
||||
|
||||
# if with_popup:
|
||||
# gdf['popup'] = await cls.get_popup(gdf)
|
||||
|
||||
# return gdf
|
||||
return gdf
|
||||
|
||||
@classmethod
|
||||
def get_attachment_dir(cls):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue