From 1b7db43ee75f516c14f6f7a0431f5ddd96121a44 Mon Sep 17 00:00:00 2001 From: phil Date: Sun, 17 Dec 2023 12:20:07 +0530 Subject: [PATCH] Return geoapi store geojson --- src/gisaf/database.py | 4 ++-- src/gisaf/geoapi.py | 31 ++++++++++++++----------- src/gisaf/models/geo_models_base.py | 36 +++++++++++++++++++---------- src/gisaf/registry.py | 13 ----------- 4 files changed, 44 insertions(+), 40 deletions(-) diff --git a/src/gisaf/database.py b/src/gisaf/database.py index 8e62e40..375db90 100644 --- a/src/gisaf/database.py +++ b/src/gisaf/database.py @@ -1,5 +1,5 @@ from contextlib import asynccontextmanager -from typing import Annotated +from typing import Annotated, AsyncContextManager from sqlalchemy.ext.asyncio import create_async_engine from sqlmodel.ext.asyncio.session import AsyncSession @@ -23,7 +23,7 @@ async def get_db_session() -> AsyncSession: yield session @asynccontextmanager -async def db_session() -> AsyncSession: +async def db_session() -> AsyncContextManager[AsyncSession]: async with AsyncSession(engine) as session: yield session diff --git a/src/gisaf/geoapi.py b/src/gisaf/geoapi.py index fc161e1..9cfb81b 100644 --- a/src/gisaf/geoapi.py +++ b/src/gisaf/geoapi.py @@ -3,9 +3,11 @@ Geographical json stores, served under /gj Used for displaying features on maps """ import logging +from typing import Annotated from asyncio import CancelledError -from fastapi import FastAPI, HTTPException, status, responses +from fastapi import FastAPI, HTTPException, Response, status, responses, Header + from .redis_tools import store as redis_store # from gisaf.live import live_server from .registry import registry @@ -41,7 +43,10 @@ async def live_layer(store: str): return ws @api.get('/{store_name}') -async def get_geojson(store_name): +async def get_geojson(store_name, + If_None_Match: Annotated[str | None, Header()] = None, + simplify: Annotated[float | None, Header()] = 50.0, + ): """ Some REST stores coded manually (route prefixed with "gj": geojson). :param store_name: name of the model @@ -59,22 +64,22 @@ async def get_geojson(store_name): if await redis_store.has_channel(store_name): ## Live layers data = await redis_store.get_layer_as_json(store_name) - return web.Response(text=data.decode(), content_type='application/json') + return data.decode() # elif not model: # raise HTTPException(status.HTTP_404_NOT_FOUND) if model.cache_enabled: ttag = await redis_store.get_ttag(store_name) - if ttag and request.headers.get('If-None-Match') == ttag: - return web.HTTPNotModified() + if ttag and If_None_Match == ttag: + return status.HTTP_304_NOT_MODIFIED if hasattr(model, 'get_geojson'): - geojson = await model.get_geojson(simplify_tolerance=float(request.headers.get('simplify', 50.0))) + geojson = await model.get_geojson(simplify_tolerance=simplify, registry=registry) ## Store to redis for caching if use_cache: await redis_store.store_json(model, geojson) - resp = web.Response(text=geojson, content_type='application/json') + resp = geojson elif model.can_get_features_as_df: ## Get the GeoDataframe (gdf) with GeoPandas @@ -86,7 +91,7 @@ async def get_geojson(store_name): raise err except Exception as err: logger.exception(err) - raise web.HTTPInternalServerError() + raise status.HTTP_500_INTERNAL_SERVER_ERROR ## The query of category defined models gets the status (not sure how and this could be skipped) ## Other models do not have: just add it manually from the model itself if 'status' not in gdf.columns: @@ -106,19 +111,19 @@ async def get_geojson(store_name): else: logger.warn(f"{model} doesn't allow using dataframe for generating json!") - attrs, features_kwargs = await model.get_features_attrs( - float(request.headers.get('simplify', 50.0))) + attrs, features_kwargs = await model.get_features_attrs(simplify) ## Using gino: allows OO model (get_info, etc) try: attrs['features'] = await model.get_features_in_bulk_gino(**features_kwargs) except Exception as err: logger.exception(err) - raise web.HTTPInternalServerError() + raise status.HTTP_500_INTERNAL_SERVER_ERROR resp = attrs + headers = {} if model.cache_enabled and ttag: - resp.headers.add('ETag', ttag) - return resp + headers['ETag'] = ttag + return Response(content=resp, media_type="application/json", headers=headers) @api.get('/gj/{store_name}/popup/{id}') diff --git a/src/gisaf/models/geo_models_base.py b/src/gisaf/models/geo_models_base.py index ff3d226..ddf2a4d 100644 --- a/src/gisaf/models/geo_models_base.py +++ b/src/gisaf/models/geo_models_base.py @@ -16,11 +16,12 @@ import shapely import pyproj from sqlmodel import SQLModel, Field +from sqlmodel.ext.asyncio.session import AsyncSession from pydantic import BaseModel from geoalchemy2.shape import from_shape from sqlalchemy.dialects.postgresql import BIGINT -from sqlalchemy import BigInteger, Column, String, func, and_ +from sqlalchemy import BigInteger, Column, MetaData, String, func, and_, text from sqlalchemy.sql import sqltypes from psycopg2.extensions import adapt @@ -36,13 +37,16 @@ from shapefile import (Writer as ShapeFileWriter, POLYGON, POLYGONZ, ) + +from ..database import db_session from ..config import conf from .models_base import Model +from ..models.metadata import survey, raw_survey +from ..utils import upsert_df from .survey import Equipment, Surveyor, Accuracy from .misc import Qml from .category import Category from .project import Project -from ..utils import upsert_df LOCALE_DATE_FORMAT = locale.nl_langinfo(locale.D_FMT) @@ -138,6 +142,7 @@ class SurveyModel(BaseSurveyModel): """ Base mixin class for defining final (reprojected) survey data, with a status """ + metadata: ClassVar[MetaData] = survey status: str = Field(sa_type=String(1)) get_gdf_with_related: ClassVar[bool] = False @@ -198,7 +203,9 @@ class SurveyModel(BaseSurveyModel): '] #' + df.index.astype('U') @classmethod - async def get_geojson(cls, simplify_tolerance=0): + async def get_geojson(cls, registry=None, simplify_tolerance=0): + if registry is None: + from ..registry import registry ## Fastest, but the id is in properties and needs the front end (eg Gisaf for mapbox) ## to move it at the feature level @@ -211,7 +218,7 @@ class SurveyModel(BaseSurveyModel): FROM ( SELECT f.geom, f.id::varchar, - {description} || ' [' || '{model.category.group}' || '-' || '{model.category.minor_group_1}' || '] #' || f.id as popup, + {description} || ' [' || '{category.group}' || '-' || '{category.minor_group_1}' || '] #' || f.id as popup, f.status FROM "{schema}"."{table}" as f WHERE f.geom is not null @@ -232,7 +239,7 @@ class SurveyModel(BaseSurveyModel): 'geometry', ST_AsGeoJSON(geom)::jsonb, 'properties', jsonb_build_object( 'popup', - {description} || ' [' || '{model.category.group}' || '-' || '{model.category.minor_group_1}' || '] #' || inputs.id::varchar, + {description} || ' [' || '{category.group}' || '-' || '{category.minor_group_1}' || '] #' || inputs.id::varchar, 'status', status ) ) AS feature @@ -241,16 +248,20 @@ class SurveyModel(BaseSurveyModel): """ sql_query_for_geojson = sql_query_for_geojson_fast - - async with db.acquire(reuse=False) as conn: + category = registry.categories.loc[cls.category_name] + session: AsyncSession + async with db_session() as session: query = sql_query_for_geojson.format( model=cls, - schema=cls.__table_args__['schema'], - table=cls.__tablename__, - description=adapt(cls.category.description), + category=category, + schema=cls.metadata.schema, + #table=cls.__tablename__, + # FIXME: should be __tablename__, but see SQLModel.__tablename__ which use lower(__name__) + table=cls.__name__, + description=adapt(category.description), ) - result = await conn.scalar(query) - return result + result = await session.exec(text(query)) + return result.first()[0] def to_row(self): @@ -1048,6 +1059,7 @@ class RawSurveyBaseModel(BaseSurveyModel, GeoPointMModel): Abstract base class for category based raw survey point models """ #__abstract__ = True + metadata: ClassVar[MetaData] = raw_survey geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINTZ', dimension=3, srid=conf.geo.raw_survey.srid)) status: str = Field(sa_type=String(1)) diff --git a/src/gisaf/registry.py b/src/gisaf/registry.py index 67a0e0d..9778a97 100644 --- a/src/gisaf/registry.py +++ b/src/gisaf/registry.py @@ -145,14 +145,7 @@ class ModelRegistry: __model_name=category.raw_survey_table_name, __cls_kwargs__={ 'table': True, - 'metadata': raw_survey, '__tablename__': category.raw_survey_table_name, - # ## FIXME: RawSurveyBaseModel.category should be a Category, not category.name - # 'category_name': category.name, - # ## FIXME: Same for RawSurveyBaseModel.group - # 'group_name': category.category_group.name, - # 'viewable_role': category.viewable_role, - # 'store_name': raw_store_name, }, **raw_survey_field_definitions ) @@ -180,13 +173,7 @@ class ModelRegistry: __model_name=category.table_name, __cls_kwargs__={ 'table': True, - 'metadata': survey, '__tablename__': category.table_name, - # 'category_name': category.name, - # 'group_name': category.category_group.name, - # 'raw_store_name': raw_store_name, - # 'viewable_role': category.viewable_role, - # 'symbol': category.symbol, }, **survey_field_definitions, )