Fix/update geo layer export

pdm.lock

@@ -5,7 +5,7 @@
 groups = ["default", "dev", "mqtt"]
 strategy = ["cross_platform"]
 lock_version = "4.4.1"
-content_hash = "sha256:581ae31055bb26abe5b7bf7cab172337258913d8960892dbd206e00421b309b7"
+content_hash = "sha256:413d13a6c7a8e7bf7f53340357f32ff8c62ad28ea0fab5df9f65e705a06f585f"
 
 [[package]]
 name = "aiofile"
@@ -982,6 +982,39 @@ files = [
     {file = "psycopg-3.1.18.tar.gz", hash = "sha256:31144d3fb4c17d78094d9e579826f047d4af1da6a10427d91dfcfb6ecdf6f12b"},
 ]
 
+[[package]]
+name = "psycopg2-binary"
+version = "2.9.9"
+requires_python = ">=3.7"
+summary = "psycopg2 - Python-PostgreSQL Database Adapter"
+files = [
+    {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"},
+]
+
 [[package]]
 name = "ptyprocess"
 version = "0.7.0"

src/gisaf/_version.py

@@ -1 +1 @@
-__version__: str = '2023.4.dev68+gd08e40d.d20240420'
+__version__: str = '0.1.dev70+g53c2e35.d20240422'

src/gisaf/api/download.py

@@ -16,6 +16,7 @@ from gisaf.security import (
 from gisaf.models.authentication import (User, UserRead, Role, RoleRead)
 from gisaf.registry import registry, NotInRegistry
 from gisaf.plugins import DownloadResponse, manager as plugin_manager
+from gisaf.exporters import export_with_fiona, export_with_pyshp
 
 logger = logging.getLogger(__name__)
 
@@ -80,6 +81,87 @@ async def download_csv(
     return response
 
 
+@api.get('/geodata/{stores}')
+async def download_geodata(
+    user: Annotated[UserRead, Depends(get_current_active_user)],
+    stores: str,
+    format: str = 'gpkg',
+    reproject: bool = False,
+) -> Response:
+    ## Check permissions
+    try:
+        layers_registry = registry.stores.loc[stores.split(',')]
+    except KeyError:
+        ## Empty dataframe
+        layers_registry = registry.stores[0:0]
+
+    live_stores = [
+        layer for layer in stores.split(',')
+        if layer.startswith('live:')
+    ]
+
+    def _check_permission(model):
+        if not hasattr(model, 'downloadable_role'):
+            return True
+        if user is None:
+            return False
+        return user.has_role(model.downloadable_role)
+
+    stores_allowed: list[str] = [
+        c[0] for c in layers_registry.model.items()
+        if _check_permission(c[1])
+    ] # type: ignore
+    store_names = ','.join(stores_allowed + live_stores)
+
+    if len(store_names) == 0:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+    formats = {
+        'gpkg': (
+            export_with_fiona, {
+                'extension': 'gpkg',
+                'driver': 'GPKG',
+                'mimetype': 'application/geopackage+vnd.sqlite3',
+            }
+        ),
+        'dxf': (
+            export_with_fiona, {
+                'extension': 'dxf',
+                'driver': 'DXF',
+                'mimetype': 'application/x-dxf',
+                'filter_columns': ['geom'],
+            }
+        ),
+        'shapefile': (
+            ## Fiona/ogr has several limitations for writing shapefiles (eg. dbf, shx not supported): use export_with_pyshp
+            #export_with_fiona, {
+            #    'extension': 'shp',
+            #    'driver': 'ESRI Shapefile',
+            #    'mimetype': 'application/octet-stream',
+            #}
+            export_with_pyshp, {}
+        ),
+    }
+
+    if format not in formats:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
+                            detail=f"{format} not in known formats: {', '.join(formats.keys())}")
+
+    fn, kwargs = formats[format]
+    if reproject and reproject not in ['false', '0', 'no']:
+        kwargs['reproject'] = True
+    body, filename, content_type = await fn(store_names=store_names, **kwargs)
+
+    headers = {
+        'Content-Disposition': f'attachment; filename="{filename}"'
+    }
+    return Response(
+        body,
+        headers=headers,
+        media_type=content_type,
+    )
+
+
 @api.get('/plugin/{name}/{store}/{id}')
 async def execute_action(
         name: str,

src/gisaf/exporters.py

@@ -0,0 +1,108 @@
+from os import remove
+from pathlib import Path
+from datetime import datetime
+from io import BytesIO
+from zipfile import ZipFile
+import logging
+import tempfile
+
+from geopandas.io.file import infer_schema
+
+import fiona
+
+from gisaf.registry import registry
+from gisaf.redis_tools import store as redis_store, RedisError
+
+logger = logging.getLogger(__name__)
+
+async def export_with_fiona(store_names, driver, mimetype, extension, filter_columns=None, reproject=False):
+    """
+    Use fiona to export geo data.
+    registry: gisaf.registry.ModelRegistry
+    store_names: comma separated string of store (aka. layer) names
+    driver: fiona driver (one of fiona.supported_drivers)
+    extension: extension of the file name
+    filter_columns: list of column names to filter out
+    reproject: if true-ish, the geometries are reprojected to the srid specified in conf.srid_for_proj
+    """
+    layers_features = {}
+    for store_name in store_names.split(','):
+        try:
+            if store_name in registry.geom:
+                layers_features[store_name] = await registry.geom[store_name].get_geo_df(reproject=reproject)
+            else:
+                ## Live
+                ## TODO: make live check more explicit
+                layers_features[store_name] = await redis_store.get_gdf(store_name, reproject=reproject)
+        except RedisError as err:
+            logger.warn(f'Cannot get store {store_name}: {err}')
+        except Exception as err:
+            logger.warn(f'Cannot get store {store_name}, see below')
+            logger.exception(err)
+
+    ## FIXME: only 1 layer gets exported with BytesIO, so use a real file
+
+    #filename = '{}_{:%Y-%m-%d_%H:%M:%S}.{}'.format(layers, datetime.now(), extension)
+    filename = 'Gisaf export {:%Y-%m-%d_%H:%M:%S}_{}.{}'.format(
+        datetime.now(), next(tempfile._get_candidate_names()), extension)
+    ## XXX: fails in case of a lot of layers
+    data_file_name = Path(tempfile._get_default_tempdir()) / filename
+    #data_file_name = Path(tempfile._get_default_tempdir()) / next(tempfile._get_candidate_names())
+
+    ## XXX: MemoryFile doesn't support multiple layers (I opened https://github.com/Toblerity/Fiona/issues/830)
+    #with fiona.io.MemoryFile(filename='selected_layers.gpkg') as mem_file:
+    #    for layer_name, gdf in layers_features.items():
+    #        if filter_columns:
+    #            gdf = gdf.filter(filter_columns)
+    #        schema = infer_schema(gdf)
+    #        with mem_file.open(layer=layer_name, driver=driver, crs=gdf.crs, schema=schema) as mem_sink:
+    #            mem_sink.writerecords(gdf.iterfeatures())
+    #return mem_file, filename, mimetype
+
+    with fiona.Env():
+        for layer_name, gdf in layers_features.items():
+            ## XXX: geopandas doesn't accept BytesIO: using fiona directly
+            #gdf.to_file(data, driver=driver, mode='a')
+            _gdf = gdf.reset_index()
+            _gdf['fid'] = _gdf['id']
+            if filter_columns:
+                _gdf = _gdf.filter(filter_columns)
+            schema = infer_schema(_gdf)
+            with fiona.Env(OSR_WKT_FORMAT="WKT2_2018"), fiona.open(
+                data_file_name, 'w',
+                driver=driver,
+                crs=_gdf.crs.to_string(),
+                layer=layer_name,
+                schema=schema) as colxn:
+                colxn.writerecords(_gdf.iterfeatures())
+        #data.seek(0)
+        with open(data_file_name, 'rb') as data_file:
+            data = data_file.read()
+        remove(data_file_name)
+        return data, filename, mimetype
+
+
+async def export_with_pyshp(store_names, reproject=False):
+    """
+    Zip and return data using "old style", ie. with pyshp 1.2
+    """
+    ## TODO: migrate to fiona, see below
+    zip_file = BytesIO()
+    with ZipFile(zip_file, 'w') as zip:
+        for layer_name in store_names.split(','):
+            model = registry.geom[layer_name]
+            dbf, shp, shx, qml, proj_str = await model.get_shapefile_files()
+            zip.writestr('{}.dbf'.format(layer_name), dbf.getvalue())
+            zip.writestr('{}.shp'.format(layer_name), shp.getvalue())
+            zip.writestr('{}.shx'.format(layer_name), shx.getvalue())
+            if qml:
+                zip.writestr('{}.qml'.format(layer_name), qml)
+            if proj_str:
+                zip.writestr('{}.prj'.format(layer_name), proj_str)
+
+    zip_file.seek(0)
+    filename = '{}_{:%Y-%m-%d_%H:%M}.zip'.format(store_names, datetime.now(), )
+    content_type = 'application/zip'
+
+    return zip_file.read(), filename, content_type
+

src/gisaf/models/geo_models_base.py

@@ -721,9 +721,7 @@ class GeoModelNoStatus(Model):
         :param reproject: should reproject to conf.srid_for_proj
         :return:
         """
-        return await cls.get_gdf(where=where, **kwargs)
+        gdf = await cls.get_gdf(where=where, **kwargs)
-
-        # df = await cls.get_df(where=where, **kwargs)
         # df.dropna(subset=['geom'], inplace=True)
         # df.set_index('id', inplace=True)
         # df.sort_index(inplace=True)
@@ -754,18 +752,18 @@ class GeoModelNoStatus(Model):
         #         float(cls.simplify) / conf.geo.simplify_geom_factor,
         #         preserve_topology=conf.geo.simplify_preserve_topology)
 
-        # if reproject:
+        if reproject:
-        #     gdf.to_crs(crs=conf.crs.for_proj, inplace=True)
+            gdf.to_crs(crs=conf.crs.for_proj, inplace=True)
 
         # ## Filter out columns
-        # if filter_columns:
+        if filter_columns:
-        #     gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
+            gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
-        #              inplace=True)
+                     inplace=True)
 
         # if with_popup:
         #     gdf['popup'] = await cls.get_popup(gdf)
 
-        # return gdf
+        return gdf
 
     @classmethod
     def get_attachment_dir(cls):