Fix/update geo layer export
This commit is contained in:
phil
parent
53c2e359da
commit
ccf6710225
5 changed files with 232 additions and 11 deletions
35
pdm.lock
generated
35
pdm.lock
generated
|
|
@ -5,7 +5,7 @@
|
|||
groups = ["default", "dev", "mqtt"]
|
||||
strategy = ["cross_platform"]
|
||||
lock_version = "4.4.1"
|
||||
content_hash = "sha256:581ae31055bb26abe5b7bf7cab172337258913d8960892dbd206e00421b309b7"
|
||||
content_hash = "sha256:413d13a6c7a8e7bf7f53340357f32ff8c62ad28ea0fab5df9f65e705a06f585f"
|
||||
|
||||
[[package]]
|
||||
name = "aiofile"
|
||||
|
|
@ -982,6 +982,39 @@ files = [
|
|||
{file = "psycopg-3.1.18.tar.gz", hash = "sha256:31144d3fb4c17d78094d9e579826f047d4af1da6a10427d91dfcfb6ecdf6f12b"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg2-binary"
|
||||
version = "2.9.9"
|
||||
requires_python = ">=3.7"
|
||||
summary = "psycopg2 - Python-PostgreSQL Database Adapter"
|
||||
files = [
|
||||
{file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"},
|
||||
{file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"},
|
||||
{file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ptyprocess"
|
||||
version = "0.7.0"
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
__version__: str = '2023.4.dev68+gd08e40d.d20240420'
|
||||
__version__: str = '0.1.dev70+g53c2e35.d20240422'
|
||||
|
|
@ -16,6 +16,7 @@ from gisaf.security import (
|
|||
from gisaf.models.authentication import (User, UserRead, Role, RoleRead)
|
||||
from gisaf.registry import registry, NotInRegistry
|
||||
from gisaf.plugins import DownloadResponse, manager as plugin_manager
|
||||
from gisaf.exporters import export_with_fiona, export_with_pyshp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -80,6 +81,87 @@ async def download_csv(
|
|||
return response
|
||||
|
||||
|
||||
@api.get('/geodata/{stores}')
|
||||
async def download_geodata(
|
||||
user: Annotated[UserRead, Depends(get_current_active_user)],
|
||||
stores: str,
|
||||
format: str = 'gpkg',
|
||||
reproject: bool = False,
|
||||
) -> Response:
|
||||
## Check permissions
|
||||
try:
|
||||
layers_registry = registry.stores.loc[stores.split(',')]
|
||||
except KeyError:
|
||||
## Empty dataframe
|
||||
layers_registry = registry.stores[0:0]
|
||||
|
||||
live_stores = [
|
||||
layer for layer in stores.split(',')
|
||||
if layer.startswith('live:')
|
||||
]
|
||||
|
||||
def _check_permission(model):
|
||||
if not hasattr(model, 'downloadable_role'):
|
||||
return True
|
||||
if user is None:
|
||||
return False
|
||||
return user.has_role(model.downloadable_role)
|
||||
|
||||
stores_allowed: list[str] = [
|
||||
c[0] for c in layers_registry.model.items()
|
||||
if _check_permission(c[1])
|
||||
] # type: ignore
|
||||
store_names = ','.join(stores_allowed + live_stores)
|
||||
|
||||
if len(store_names) == 0:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
formats = {
|
||||
'gpkg': (
|
||||
export_with_fiona, {
|
||||
'extension': 'gpkg',
|
||||
'driver': 'GPKG',
|
||||
'mimetype': 'application/geopackage+vnd.sqlite3',
|
||||
}
|
||||
),
|
||||
'dxf': (
|
||||
export_with_fiona, {
|
||||
'extension': 'dxf',
|
||||
'driver': 'DXF',
|
||||
'mimetype': 'application/x-dxf',
|
||||
'filter_columns': ['geom'],
|
||||
}
|
||||
),
|
||||
'shapefile': (
|
||||
## Fiona/ogr has several limitations for writing shapefiles (eg. dbf, shx not supported): use export_with_pyshp
|
||||
#export_with_fiona, {
|
||||
# 'extension': 'shp',
|
||||
# 'driver': 'ESRI Shapefile',
|
||||
# 'mimetype': 'application/octet-stream',
|
||||
#}
|
||||
export_with_pyshp, {}
|
||||
),
|
||||
}
|
||||
|
||||
if format not in formats:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"{format} not in known formats: {', '.join(formats.keys())}")
|
||||
|
||||
fn, kwargs = formats[format]
|
||||
if reproject and reproject not in ['false', '0', 'no']:
|
||||
kwargs['reproject'] = True
|
||||
body, filename, content_type = await fn(store_names=store_names, **kwargs)
|
||||
|
||||
headers = {
|
||||
'Content-Disposition': f'attachment; filename="{filename}"'
|
||||
}
|
||||
return Response(
|
||||
body,
|
||||
headers=headers,
|
||||
media_type=content_type,
|
||||
)
|
||||
|
||||
|
||||
@api.get('/plugin/{name}/{store}/{id}')
|
||||
async def execute_action(
|
||||
name: str,
|
||||
|
|
|
|||
108
src/gisaf/exporters.py
Normal file
108
src/gisaf/exporters.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
from os import remove
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from zipfile import ZipFile
|
||||
import logging
|
||||
import tempfile
|
||||
|
||||
from geopandas.io.file import infer_schema
|
||||
|
||||
import fiona
|
||||
|
||||
from gisaf.registry import registry
|
||||
from gisaf.redis_tools import store as redis_store, RedisError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
async def export_with_fiona(store_names, driver, mimetype, extension, filter_columns=None, reproject=False):
|
||||
"""
|
||||
Use fiona to export geo data.
|
||||
registry: gisaf.registry.ModelRegistry
|
||||
store_names: comma separated string of store (aka. layer) names
|
||||
driver: fiona driver (one of fiona.supported_drivers)
|
||||
extension: extension of the file name
|
||||
filter_columns: list of column names to filter out
|
||||
reproject: if true-ish, the geometries are reprojected to the srid specified in conf.srid_for_proj
|
||||
"""
|
||||
layers_features = {}
|
||||
for store_name in store_names.split(','):
|
||||
try:
|
||||
if store_name in registry.geom:
|
||||
layers_features[store_name] = await registry.geom[store_name].get_geo_df(reproject=reproject)
|
||||
else:
|
||||
## Live
|
||||
## TODO: make live check more explicit
|
||||
layers_features[store_name] = await redis_store.get_gdf(store_name, reproject=reproject)
|
||||
except RedisError as err:
|
||||
logger.warn(f'Cannot get store {store_name}: {err}')
|
||||
except Exception as err:
|
||||
logger.warn(f'Cannot get store {store_name}, see below')
|
||||
logger.exception(err)
|
||||
|
||||
## FIXME: only 1 layer gets exported with BytesIO, so use a real file
|
||||
|
||||
#filename = '{}_{:%Y-%m-%d_%H:%M:%S}.{}'.format(layers, datetime.now(), extension)
|
||||
filename = 'Gisaf export {:%Y-%m-%d_%H:%M:%S}_{}.{}'.format(
|
||||
datetime.now(), next(tempfile._get_candidate_names()), extension)
|
||||
## XXX: fails in case of a lot of layers
|
||||
data_file_name = Path(tempfile._get_default_tempdir()) / filename
|
||||
#data_file_name = Path(tempfile._get_default_tempdir()) / next(tempfile._get_candidate_names())
|
||||
|
||||
## XXX: MemoryFile doesn't support multiple layers (I opened https://github.com/Toblerity/Fiona/issues/830)
|
||||
#with fiona.io.MemoryFile(filename='selected_layers.gpkg') as mem_file:
|
||||
# for layer_name, gdf in layers_features.items():
|
||||
# if filter_columns:
|
||||
# gdf = gdf.filter(filter_columns)
|
||||
# schema = infer_schema(gdf)
|
||||
# with mem_file.open(layer=layer_name, driver=driver, crs=gdf.crs, schema=schema) as mem_sink:
|
||||
# mem_sink.writerecords(gdf.iterfeatures())
|
||||
#return mem_file, filename, mimetype
|
||||
|
||||
with fiona.Env():
|
||||
for layer_name, gdf in layers_features.items():
|
||||
## XXX: geopandas doesn't accept BytesIO: using fiona directly
|
||||
#gdf.to_file(data, driver=driver, mode='a')
|
||||
_gdf = gdf.reset_index()
|
||||
_gdf['fid'] = _gdf['id']
|
||||
if filter_columns:
|
||||
_gdf = _gdf.filter(filter_columns)
|
||||
schema = infer_schema(_gdf)
|
||||
with fiona.Env(OSR_WKT_FORMAT="WKT2_2018"), fiona.open(
|
||||
data_file_name, 'w',
|
||||
driver=driver,
|
||||
crs=_gdf.crs.to_string(),
|
||||
layer=layer_name,
|
||||
schema=schema) as colxn:
|
||||
colxn.writerecords(_gdf.iterfeatures())
|
||||
#data.seek(0)
|
||||
with open(data_file_name, 'rb') as data_file:
|
||||
data = data_file.read()
|
||||
remove(data_file_name)
|
||||
return data, filename, mimetype
|
||||
|
||||
|
||||
async def export_with_pyshp(store_names, reproject=False):
|
||||
"""
|
||||
Zip and return data using "old style", ie. with pyshp 1.2
|
||||
"""
|
||||
## TODO: migrate to fiona, see below
|
||||
zip_file = BytesIO()
|
||||
with ZipFile(zip_file, 'w') as zip:
|
||||
for layer_name in store_names.split(','):
|
||||
model = registry.geom[layer_name]
|
||||
dbf, shp, shx, qml, proj_str = await model.get_shapefile_files()
|
||||
zip.writestr('{}.dbf'.format(layer_name), dbf.getvalue())
|
||||
zip.writestr('{}.shp'.format(layer_name), shp.getvalue())
|
||||
zip.writestr('{}.shx'.format(layer_name), shx.getvalue())
|
||||
if qml:
|
||||
zip.writestr('{}.qml'.format(layer_name), qml)
|
||||
if proj_str:
|
||||
zip.writestr('{}.prj'.format(layer_name), proj_str)
|
||||
|
||||
zip_file.seek(0)
|
||||
filename = '{}_{:%Y-%m-%d_%H:%M}.zip'.format(store_names, datetime.now(), )
|
||||
content_type = 'application/zip'
|
||||
|
||||
return zip_file.read(), filename, content_type
|
||||
|
||||
|
|
@ -721,9 +721,7 @@ class GeoModelNoStatus(Model):
|
|||
:param reproject: should reproject to conf.srid_for_proj
|
||||
:return:
|
||||
"""
|
||||
return await cls.get_gdf(where=where, **kwargs)
|
||||
|
||||
# df = await cls.get_df(where=where, **kwargs)
|
||||
gdf = await cls.get_gdf(where=where, **kwargs)
|
||||
# df.dropna(subset=['geom'], inplace=True)
|
||||
# df.set_index('id', inplace=True)
|
||||
# df.sort_index(inplace=True)
|
||||
|
|
@ -754,18 +752,18 @@ class GeoModelNoStatus(Model):
|
|||
# float(cls.simplify) / conf.geo.simplify_geom_factor,
|
||||
# preserve_topology=conf.geo.simplify_preserve_topology)
|
||||
|
||||
# if reproject:
|
||||
# gdf.to_crs(crs=conf.crs.for_proj, inplace=True)
|
||||
if reproject:
|
||||
gdf.to_crs(crs=conf.crs.for_proj, inplace=True)
|
||||
|
||||
# ## Filter out columns
|
||||
# if filter_columns:
|
||||
# gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
|
||||
# inplace=True)
|
||||
if filter_columns:
|
||||
gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
|
||||
inplace=True)
|
||||
|
||||
# if with_popup:
|
||||
# gdf['popup'] = await cls.get_popup(gdf)
|
||||
|
||||
# return gdf
|
||||
return gdf
|
||||
|
||||
@classmethod
|
||||
def get_attachment_dir(cls):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue