diff --git a/pdm.lock b/pdm.lock index daa384b..acf09e4 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "mqtt"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:581ae31055bb26abe5b7bf7cab172337258913d8960892dbd206e00421b309b7" +content_hash = "sha256:413d13a6c7a8e7bf7f53340357f32ff8c62ad28ea0fab5df9f65e705a06f585f" [[package]] name = "aiofile" @@ -982,6 +982,39 @@ files = [ {file = "psycopg-3.1.18.tar.gz", hash = "sha256:31144d3fb4c17d78094d9e579826f047d4af1da6a10427d91dfcfb6ecdf6f12b"}, ] +[[package]] +name = "psycopg2-binary" +version = "2.9.9" +requires_python = ">=3.7" +summary = "psycopg2 - Python-PostgreSQL Database Adapter" +files = [ + {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, +] + [[package]] name = "ptyprocess" version = "0.7.0" diff --git a/src/gisaf/_version.py b/src/gisaf/_version.py index 55c3ec9..a9c16b1 100644 --- a/src/gisaf/_version.py +++ b/src/gisaf/_version.py @@ -1 +1 @@ -__version__: str = '2023.4.dev68+gd08e40d.d20240420' \ No newline at end of file +__version__: str = '0.1.dev70+g53c2e35.d20240422' \ No newline at end of file diff --git a/src/gisaf/api/download.py b/src/gisaf/api/download.py index 1077dd0..e6e2e08 100644 --- a/src/gisaf/api/download.py +++ b/src/gisaf/api/download.py @@ -16,6 +16,7 @@ from gisaf.security import ( from gisaf.models.authentication import (User, UserRead, Role, RoleRead) from gisaf.registry import registry, NotInRegistry from gisaf.plugins import DownloadResponse, manager as plugin_manager +from gisaf.exporters import export_with_fiona, export_with_pyshp logger = logging.getLogger(__name__) @@ -80,6 +81,87 @@ async def download_csv( return response +@api.get('/geodata/{stores}') +async def download_geodata( + user: Annotated[UserRead, Depends(get_current_active_user)], + stores: str, + format: str = 'gpkg', + reproject: bool = False, +) -> Response: + ## Check permissions + try: + layers_registry = registry.stores.loc[stores.split(',')] + except KeyError: + ## Empty dataframe + layers_registry = registry.stores[0:0] + + live_stores = [ + layer for layer in stores.split(',') + if layer.startswith('live:') + ] + + def _check_permission(model): + if not hasattr(model, 'downloadable_role'): + return True + if user is None: + return False + return user.has_role(model.downloadable_role) + + stores_allowed: list[str] = [ + c[0] for c in layers_registry.model.items() + if _check_permission(c[1]) + ] # type: ignore + store_names = ','.join(stores_allowed + live_stores) + + if len(store_names) == 0: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + + formats = { + 'gpkg': ( + export_with_fiona, { + 'extension': 'gpkg', + 'driver': 'GPKG', + 'mimetype': 'application/geopackage+vnd.sqlite3', + } + ), + 'dxf': ( + export_with_fiona, { + 'extension': 'dxf', + 'driver': 'DXF', + 'mimetype': 'application/x-dxf', + 'filter_columns': ['geom'], + } + ), + 'shapefile': ( + ## Fiona/ogr has several limitations for writing shapefiles (eg. dbf, shx not supported): use export_with_pyshp + #export_with_fiona, { + # 'extension': 'shp', + # 'driver': 'ESRI Shapefile', + # 'mimetype': 'application/octet-stream', + #} + export_with_pyshp, {} + ), + } + + if format not in formats: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, + detail=f"{format} not in known formats: {', '.join(formats.keys())}") + + fn, kwargs = formats[format] + if reproject and reproject not in ['false', '0', 'no']: + kwargs['reproject'] = True + body, filename, content_type = await fn(store_names=store_names, **kwargs) + + headers = { + 'Content-Disposition': f'attachment; filename="{filename}"' + } + return Response( + body, + headers=headers, + media_type=content_type, + ) + + @api.get('/plugin/{name}/{store}/{id}') async def execute_action( name: str, diff --git a/src/gisaf/exporters.py b/src/gisaf/exporters.py new file mode 100644 index 0000000..5832d81 --- /dev/null +++ b/src/gisaf/exporters.py @@ -0,0 +1,108 @@ +from os import remove +from pathlib import Path +from datetime import datetime +from io import BytesIO +from zipfile import ZipFile +import logging +import tempfile + +from geopandas.io.file import infer_schema + +import fiona + +from gisaf.registry import registry +from gisaf.redis_tools import store as redis_store, RedisError + +logger = logging.getLogger(__name__) + +async def export_with_fiona(store_names, driver, mimetype, extension, filter_columns=None, reproject=False): + """ + Use fiona to export geo data. + registry: gisaf.registry.ModelRegistry + store_names: comma separated string of store (aka. layer) names + driver: fiona driver (one of fiona.supported_drivers) + extension: extension of the file name + filter_columns: list of column names to filter out + reproject: if true-ish, the geometries are reprojected to the srid specified in conf.srid_for_proj + """ + layers_features = {} + for store_name in store_names.split(','): + try: + if store_name in registry.geom: + layers_features[store_name] = await registry.geom[store_name].get_geo_df(reproject=reproject) + else: + ## Live + ## TODO: make live check more explicit + layers_features[store_name] = await redis_store.get_gdf(store_name, reproject=reproject) + except RedisError as err: + logger.warn(f'Cannot get store {store_name}: {err}') + except Exception as err: + logger.warn(f'Cannot get store {store_name}, see below') + logger.exception(err) + + ## FIXME: only 1 layer gets exported with BytesIO, so use a real file + + #filename = '{}_{:%Y-%m-%d_%H:%M:%S}.{}'.format(layers, datetime.now(), extension) + filename = 'Gisaf export {:%Y-%m-%d_%H:%M:%S}_{}.{}'.format( + datetime.now(), next(tempfile._get_candidate_names()), extension) + ## XXX: fails in case of a lot of layers + data_file_name = Path(tempfile._get_default_tempdir()) / filename + #data_file_name = Path(tempfile._get_default_tempdir()) / next(tempfile._get_candidate_names()) + + ## XXX: MemoryFile doesn't support multiple layers (I opened https://github.com/Toblerity/Fiona/issues/830) + #with fiona.io.MemoryFile(filename='selected_layers.gpkg') as mem_file: + # for layer_name, gdf in layers_features.items(): + # if filter_columns: + # gdf = gdf.filter(filter_columns) + # schema = infer_schema(gdf) + # with mem_file.open(layer=layer_name, driver=driver, crs=gdf.crs, schema=schema) as mem_sink: + # mem_sink.writerecords(gdf.iterfeatures()) + #return mem_file, filename, mimetype + + with fiona.Env(): + for layer_name, gdf in layers_features.items(): + ## XXX: geopandas doesn't accept BytesIO: using fiona directly + #gdf.to_file(data, driver=driver, mode='a') + _gdf = gdf.reset_index() + _gdf['fid'] = _gdf['id'] + if filter_columns: + _gdf = _gdf.filter(filter_columns) + schema = infer_schema(_gdf) + with fiona.Env(OSR_WKT_FORMAT="WKT2_2018"), fiona.open( + data_file_name, 'w', + driver=driver, + crs=_gdf.crs.to_string(), + layer=layer_name, + schema=schema) as colxn: + colxn.writerecords(_gdf.iterfeatures()) + #data.seek(0) + with open(data_file_name, 'rb') as data_file: + data = data_file.read() + remove(data_file_name) + return data, filename, mimetype + + +async def export_with_pyshp(store_names, reproject=False): + """ + Zip and return data using "old style", ie. with pyshp 1.2 + """ + ## TODO: migrate to fiona, see below + zip_file = BytesIO() + with ZipFile(zip_file, 'w') as zip: + for layer_name in store_names.split(','): + model = registry.geom[layer_name] + dbf, shp, shx, qml, proj_str = await model.get_shapefile_files() + zip.writestr('{}.dbf'.format(layer_name), dbf.getvalue()) + zip.writestr('{}.shp'.format(layer_name), shp.getvalue()) + zip.writestr('{}.shx'.format(layer_name), shx.getvalue()) + if qml: + zip.writestr('{}.qml'.format(layer_name), qml) + if proj_str: + zip.writestr('{}.prj'.format(layer_name), proj_str) + + zip_file.seek(0) + filename = '{}_{:%Y-%m-%d_%H:%M}.zip'.format(store_names, datetime.now(), ) + content_type = 'application/zip' + + return zip_file.read(), filename, content_type + diff --git a/src/gisaf/models/geo_models_base.py b/src/gisaf/models/geo_models_base.py index 40444e1..a09f295 100644 --- a/src/gisaf/models/geo_models_base.py +++ b/src/gisaf/models/geo_models_base.py @@ -721,9 +721,7 @@ class GeoModelNoStatus(Model): :param reproject: should reproject to conf.srid_for_proj :return: """ - return await cls.get_gdf(where=where, **kwargs) - - # df = await cls.get_df(where=where, **kwargs) + gdf = await cls.get_gdf(where=where, **kwargs) # df.dropna(subset=['geom'], inplace=True) # df.set_index('id', inplace=True) # df.sort_index(inplace=True) @@ -754,18 +752,18 @@ class GeoModelNoStatus(Model): # float(cls.simplify) / conf.geo.simplify_geom_factor, # preserve_topology=conf.geo.simplify_preserve_topology) - # if reproject: - # gdf.to_crs(crs=conf.crs.for_proj, inplace=True) + if reproject: + gdf.to_crs(crs=conf.crs.for_proj, inplace=True) # ## Filter out columns - # if filter_columns: - # gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map), - # inplace=True) + if filter_columns: + gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map), + inplace=True) # if with_popup: # gdf['popup'] = await cls.get_popup(gdf) - # return gdf + return gdf @classmethod def get_attachment_dir(cls):