From ec71b6ed15ca4fdcdd74d5c939a2a08f84cd64c1 Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 4 Jan 2024 18:50:23 +0530 Subject: [PATCH] Remove custom sqlalchemy metadata, manage with __table_args__ Allow sqlmodel queries, with relations Remode join_with mechanisms coming from gino Handlew ith_only_columns in get_df and get_gdf Implement feature-info --- src/gisaf/api.py | 123 +++++++++++++++++++++++++++- src/gisaf/database.py | 22 +++-- src/gisaf/models/admin.py | 6 +- src/gisaf/models/authentication.py | 19 +++-- src/gisaf/models/category.py | 13 +-- src/gisaf/models/geo_models_base.py | 93 ++++++++++----------- src/gisaf/models/map_bases.py | 8 +- src/gisaf/models/metadata.py | 14 ++-- src/gisaf/models/misc.py | 4 +- src/gisaf/models/models_base.py | 2 +- src/gisaf/models/project.py | 4 +- src/gisaf/models/raw_survey.py | 6 +- src/gisaf/models/reconcile.py | 8 +- src/gisaf/models/survey.py | 16 ++-- src/gisaf/models/tags.py | 6 +- src/gisaf/models/to_migrate.py | 63 +++++++++++++- src/gisaf/registry.py | 35 +++++--- src/gisaf/utils.py | 52 ++++++------ 18 files changed, 353 insertions(+), 141 deletions(-) diff --git a/src/gisaf/api.py b/src/gisaf/api.py index abd0cbe..2c65c2b 100644 --- a/src/gisaf/api.py +++ b/src/gisaf/api.py @@ -22,7 +22,12 @@ from gisaf.security import ( Token, authenticate_user, get_current_user, create_access_token, ) -from gisaf.registry import registry +from gisaf.registry import registry, NotInRegistry +from gisaf.redis_tools import store as redis_store +from gisaf.custom_store_base import BaseStore +from gisaf.models.to_migrate import ( + FeatureInfo, InfoItem, Attachment, InfoCategory +) logger = logging.getLogger(__name__) @@ -43,7 +48,6 @@ async def bootstrap( @api.post("/token") async def login_for_access_token( - db_session: db_session, form_data: OAuth2PasswordRequestForm = Depends() ) -> Token: user = await authenticate_user(form_data.username, form_data.password) @@ -117,6 +121,121 @@ async def get_projects( df = await db_session.run_sync(pandas_query, query) return df.to_dict(orient="records") +@api.get("/feature-info/{store}/{id}") +async def get_feature_info( + store: str, id: str, + ) -> FeatureInfo: + store_record = registry.stores.loc[store] + model = store_record.model + + if store_record.is_live: + item = await redis_store.get_feature_info(store, id) + geom = item.geometry + ## Reproject to default coordinate system (WGS84), + ## XXX: only for shapely geometries + if not isinstance(geom, pygeos.Geometry): + geom_reprojected = transform(reproject_func, geom) + geoInfoItems = OrderedDict() + if isinstance(geom, Point): + geoInfoItems['longitude'] = f'{geom.x:.6f}' + geoInfoItems['latitude'] = f'{geom.y:.6f}' + if geom.has_z: + geoInfoItems['elevation (m)'] = f'{geom.z:.6f}' + elif isinstance(geom, (LineString, MultiLineString)): + bounds = geom.bounds + geoInfoItems['longitude'] = f'{bounds[0]:.6f} - {bounds[2]:.6f}' + geoInfoItems['latitude'] = f'{bounds[1]:.6f} - {bounds[3]:.6f}' + geoInfoItems['length (m)'] = f'{geom_reprojected.length:.2f}' + ## TODO: elevation for MultiLineString + if geom.has_z and not isinstance(geom, MultiLineString): + elevations = [cc[2] for cc in geom.coords] + elev_min = min(elevations) + elev_max = max(elevations) + if elev_min == elev_max: + geoInfoItems['elevation (m)'] = f'{elev_min:.2f}' + else: + geoInfoItems['elevation (m)'] = f'{elev_min:.2f} - {elev_max:.2f}' + elif isinstance(geom, (Polygon, MultiPolygon)): + area = geom_reprojected.area + bounds = geom.bounds + geoInfoItems['longitude'] = f'{bounds[0]:.6f} - {bounds[2]:.6f}' + geoInfoItems['latitude'] = f'{bounds[1]:.6f} - {bounds[3]:.6f}' + geoInfoItems['area (sq. m)'] = f'{area:.1f} sq. m' + geoInfoItems['area (ha)'] = f'{area / 10000:.1f} ha' + geoInfoItems['area (acre)'] = f'{area / 4046.85643005078874:.1f} acres' + ## TODO: elevation for MultiPolygon + if geom.has_z and not isinstance(geom, MultiPolygon): + if hasattr(geom, 'exterior'): + coords = geom.exterior.coords + else: + coords = geom.coords + elevations = [coord[2] for coord in coords] + elev_min = min(elevations) + elev_max = max(elevations) + if elev_min == elev_max: + geoInfoItems['elevation (m)'] = f'{elev_min:.2f}' + else: + geoInfoItems['elevation (m)'] = f'{elev_min:.2f} - {elev_max:.2f}' + + feature_info_dict = { + 'itemName': item.get('popup', f'Live: {store} #{id}'), + 'geoInfoItems': geoInfoItems, + 'surveyInfoItems': { + 'Note': 'Live layers do not have survey info', + }, + 'infoItems': dict(item.drop(set(item.keys()).intersection(('geometry', 'popup')))), + 'tags': {}, + } + elif issubclass(model, BaseStore): + feature_info_dict = await model.get_item_params(id) + else: + ## Not a live layer + try: + feature_info_dict = await registry.get_model_id_params(model, int(id)) + except NotInRegistry: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + + feature_info = FeatureInfo( + id=id, + itemName=feature_info_dict['itemName'], + geoInfoItems=[InfoItem(key=k, value=v) + for k, v in feature_info_dict['geoInfoItems'].items()], + surveyInfoItems=[InfoItem(key=k, value=v) + for k, v in feature_info_dict['surveyInfoItems'].items()], + infoItems=[InfoItem(key=k, value=v) + for k, v in feature_info_dict['infoItems'].items()], + tags=[InfoItem(key=k, value=v) + for k, v in feature_info_dict['tags'].items()], + graph=feature_info_dict.get('graph'), + ) + if 'files' in feature_info_dict and feature_info_dict['files'] is not None: + feature_info.files = [ + Attachment(name=k, path=v) + for k, v in feature_info_dict['files'].items() + ] + else: + feature_info.files = [] + if 'images' in feature_info_dict and feature_info_dict['images'] is not None: + feature_info.images = [ + Attachment(name=k, path=v) + for k, v in feature_info_dict['images'].items() + ] + else: + feature_info.images = [] + if 'categorized_info_items' in feature_info_dict and feature_info_dict['categorized_info_items'] != None: + feature_info.categorizedInfoItems = [ + InfoCategory( + name=name, + infoItems=[ + InfoItem(key=k, value=v) + for k, v in info_items.items() + ] + ) + for name, info_items in feature_info_dict['categorized_info_items'].items() + ] + feature_info.externalRecordUrl = feature_info_dict.get('externalRecordUrl') + return feature_info + # @api.get("/user-role") # async def get_user_role_relation( # *, db_session: AsyncSession = Depends(get_db_session) diff --git a/src/gisaf/database.py b/src/gisaf/database.py index c0e7b36..10ef829 100644 --- a/src/gisaf/database.py +++ b/src/gisaf/database.py @@ -40,17 +40,29 @@ class BaseModel(SQLModel): return [] @classmethod - async def get_df(cls, where=None, with_related=True, **kwargs) -> pd.DataFrame: + async def get_df(cls, *, + where=None, with_related=True, **kwargs + ) -> pd.DataFrame: return await cls._get_df(pandas_query, where=None, with_related=True, **kwargs) @classmethod - async def get_gdf(cls, *, where=None, with_related=True, **kwargs) -> gpd.GeoDataFrame: - return await cls._get_df(geopandas_query, where=None, with_related=True, **kwargs) + async def get_gdf(cls, *, + where=None, with_related=True, **kwargs + ) -> gpd.GeoDataFrame: + return await cls._get_df(geopandas_query, + where=None, with_related=True, **kwargs) @classmethod - async def _get_df(cls, method, *, where=None, with_related=True, **kwargs) -> pd.DataFrame | gpd.GeoDataFrame: + async def _get_df(cls, method, *, + where=None, with_related=True, with_only_columns=[], **kwargs + ) -> pd.DataFrame | gpd.GeoDataFrame: async with db_session() as session: - query = select(cls) + if len(with_only_columns) == 0: + query = select(cls) + else: + columns = set(with_only_columns) + columns.add(*(col.name for col in cls.__table__.primary_key.columns)) + query = select(*(getattr(cls, col) for col in columns)) if where is not None: query.append_whereclause(where) ## Get the joined tables diff --git a/src/gisaf/models/admin.py b/src/gisaf/models/admin.py index e82f1f0..6c3705f 100644 --- a/src/gisaf/models/admin.py +++ b/src/gisaf/models/admin.py @@ -9,7 +9,7 @@ import pandas as pd from gisaf.models.models_base import Model from gisaf.models.survey import Surveyor, Equipment from gisaf.models.project import Project -from gisaf.models.metadata import gisaf_admin +from gisaf.models.metadata import gisaf_admin_table_args re_file_import_record_date_expr = '^(\S+)-(\d\d\d\d)-(\d\d)-(\d\d).*$' @@ -45,7 +45,7 @@ class FileImport(Model): Give either url or path. """ __tablename__ = 'file_import' - metadata = gisaf_admin + __table_args__ = gisaf_admin_table_args id: int | None = Field(default=None, primary_key=True) url: str @@ -117,7 +117,7 @@ class FeatureImportData(Model): Keep track of imported data, typically from shapefiles """ __tablename__ = 'feature_import_data' - metadata = gisaf_admin + __table_args__ = gisaf_admin_table_args id: int | None = Field(default=None, primary_key=True) store: str = Field(index=True) diff --git a/src/gisaf/models/authentication.py b/src/gisaf/models/authentication.py index 7138dea..efd41ce 100644 --- a/src/gisaf/models/authentication.py +++ b/src/gisaf/models/authentication.py @@ -1,15 +1,20 @@ -from sqlmodel import Field, SQLModel, MetaData, Relationship +from sqlmodel import Field, SQLModel, Relationship + +from gisaf.models.metadata import gisaf_admin_table_args -from gisaf.models.metadata import gisaf_admin class UserRoleLink(SQLModel, table=True): - metadata = gisaf_admin __tablename__ = 'roles_users' + __table_args__ = gisaf_admin_table_args user_id: int | None = Field( - default=None, foreign_key="user.id", primary_key=True + default=None, + foreign_key=gisaf_admin_table_args['schema'] + '.user.id', + primary_key=True ) role_id: int | None = Field( - default=None, foreign_key="role.id", primary_key=True + default=None, + foreign_key=gisaf_admin_table_args['schema'] + '.role.id', + primary_key=True ) @@ -20,7 +25,7 @@ class UserBase(SQLModel): class User(UserBase, table=True): - metadata = gisaf_admin + __table_args__ = gisaf_admin_table_args id: int | None = Field(default=None, primary_key=True) roles: list["Role"] = Relationship(back_populates="users", link_model=UserRoleLink) @@ -40,7 +45,7 @@ class RoleWithDescription(RoleBase): description: str | None class Role(RoleWithDescription, table=True): - metadata = gisaf_admin + __table_args__ = gisaf_admin_table_args id: int | None = Field(default=None, primary_key=True) users: list[User] = Relationship(back_populates="roles", link_model=UserRoleLink) diff --git a/src/gisaf/models/category.py b/src/gisaf/models/category.py index 4d1b092..f8c1b42 100644 --- a/src/gisaf/models/category.py +++ b/src/gisaf/models/category.py @@ -4,8 +4,8 @@ from sqlalchemy import String from pydantic import computed_field, ConfigDict from sqlmodel import Field, Relationship, JSON, TEXT -from gisaf.models.metadata import gisaf_survey from gisaf.database import BaseModel +from gisaf.models.metadata import gisaf_survey_table_args mapbox_type_mapping = { 'Point': 'symbol', @@ -15,8 +15,8 @@ mapbox_type_mapping = { class CategoryGroup(BaseModel, table=True): - metadata = gisaf_survey __tablename__ = 'category_group' + __table_args__ = gisaf_survey_table_args name: str | None = Field(sa_type=String(4), default=None, primary_key=True) major: str long_name: str @@ -28,8 +28,8 @@ class CategoryGroup(BaseModel, table=True): class CategoryModelType(BaseModel, table=True): - metadata = gisaf_survey __tablename__ = 'category_model_type' + __table_args__ = gisaf_survey_table_args name: str | None = Field(default=None, primary_key=True) class Admin: @@ -47,14 +47,15 @@ class CategoryBase(BaseModel): domain: ClassVar[str] = 'V' description: str | None group: str = Field(sa_type=String(4), - foreign_key="category_group.name", index=True) + foreign_key=gisaf_survey_table_args['schema'] + ".category_group.name", + index=True) minor_group_1: str = Field(sa_type=String(4), default='----') minor_group_2: str = Field(sa_type=String(4), default='----') status: str = Field(sa_type=String(1)) custom: bool | None auto_import: bool = True gis_type: str = Field(sa_type=String(50), - foreign_key='category_model_type.name', + foreign_key=gisaf_survey_table_args['schema'] + '.category_model_type.name', default='Point') long_name: str | None = Field(sa_type=String(50)) style: str | None = Field(sa_type=TEXT) @@ -105,7 +106,7 @@ class CategoryBase(BaseModel): class Category(CategoryBase, table=True): - metadata = gisaf_survey + __table_args__ = gisaf_survey_table_args name: str | None = Field(default=None, primary_key=True) category_group: CategoryGroup = Relationship(back_populates="categories") diff --git a/src/gisaf/models/geo_models_base.py b/src/gisaf/models/geo_models_base.py index f5fa8d5..ea88a66 100644 --- a/src/gisaf/models/geo_models_base.py +++ b/src/gisaf/models/geo_models_base.py @@ -4,6 +4,7 @@ from datetime import date, datetime from collections import OrderedDict from io import BytesIO from zipfile import ZipFile +from functools import cached_property import locale import logging @@ -12,19 +13,16 @@ import geopandas as gpd # type: ignore import shapely # type: ignore import pyproj -from sqlmodel import SQLModel, Field, Relationship -from sqlmodel.ext.asyncio.session import AsyncSession from pydantic import BaseModel - -from geoalchemy2.shape import from_shape -from sqlalchemy.dialects.postgresql import BIGINT -from sqlalchemy import BigInteger, Column, MetaData, String, func, and_, text +from sqlmodel import select, Field, Relationship +from sqlmodel.ext.asyncio.session import AsyncSession +from sqlalchemy import BigInteger, MetaData, String, func, and_, text from sqlalchemy.sql import sqltypes +from sqlalchemy.orm import declared_attr from psycopg2.extensions import adapt - +from geoalchemy2.shape import from_shape from geoalchemy2.types import Geometry, WKBElement - -from shapely import wkb +from shapely import wkb, from_wkb from shapely.geometry import mapping from shapely.ops import transform # type: ignore @@ -35,15 +33,15 @@ from shapefile import ( POLYGON, POLYGONZ, ) - from gisaf.database import db_session from gisaf.config import conf from gisaf.models.models_base import Model -from gisaf.models.metadata import survey, raw_survey -from gisaf.models.survey import Equipment, Surveyor, Accuracy +from gisaf.models.metadata import ( + gisaf_survey_table_args, gisaf_admin_table_args, survey_table_args) from gisaf.models.misc import Qml from gisaf.models.category import Category -from gisaf.models.project import Project +# from gisaf.models.survey import Equipment, Surveyor, Accuracy +# from gisaf.models.project import Project LOCALE_DATE_FORMAT = locale.nl_langinfo(locale.D_FMT) @@ -82,14 +80,14 @@ class BaseSurveyModel(BaseModel): - projected ('V_*') """ id: int | None = Field(sa_type=BigInteger, primary_key=True, default=None) - equip_id: int = Field(foreign_key='equipment.id') - # equipment: Equipment = Relationship() - srvyr_id: int = Field('surveyor.id') - # surveyor: Surveyor = Relationship() - accur_id: int = Field('accuracy.id') - # accuracy: Accuracy = Relationship() - project_id: int = Field('project.id') - # project: Project = Relationship() + equip_id: int = Field( + foreign_key=gisaf_survey_table_args['schema'] + '.equipment.id') + srvyr_id: int = Field( + foreign_key=gisaf_survey_table_args['schema'] + '.surveyor.id') + accur_id: int = Field( + foreign_key=gisaf_survey_table_args['schema'] + '.accuracy.id') + project_id: int = Field( + foreign_key=gisaf_admin_table_args['schema'] + '.project.id') orig_id: str date: date @@ -151,7 +149,7 @@ class SurveyModel(BaseSurveyModel): """ Base mixin class for defining final (reprojected) survey data, with a status """ - metadata: ClassVar[MetaData] = survey + __table_args__ = survey_table_args # status: str = Field(sa_type=String(1)) get_gdf_with_related: ClassVar[bool] = False @@ -182,6 +180,10 @@ class SurveyModel(BaseSurveyModel): 'gisaf_admin_project_skip_columns', ] + @declared_attr + def __tablename__(cls) -> str: + return cls.__name__ # type: nocheck + async def get_survey_info(self): info = await super(SurveyModel, self).get_survey_info() if self.srvyr_id: @@ -203,7 +205,7 @@ class SurveyModel(BaseSurveyModel): @property def caption(self): - return '{self.category.description} [{self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self) + return '{self.category.description} - {self.category.name} [{self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self) @classmethod async def get_popup(cls, df): @@ -425,30 +427,28 @@ class GeoModelNoStatus(Model): async def get_tags(self): from gisaf.models.tags import Tags - tags = await Tags.get_df( - where=and_( - Tags.store == self.__class__.get_store_name(), - Tags.ref_id == self.id - ), - with_only_columns=['tags'] - ) - if len(tags) > 0: - return tags.loc[0, 'tags'] - else: - return {} + async with db_session() as session: + query = select(Tags.tags).where(Tags.store == self.get_store_name(), + Tags.ref_id == self.id) + tags = await session.exec(query) + return tags.one_or_none() or {} - @property + @cached_property def shapely_geom(self): - if not hasattr(self, '_shapely_geom'): - if isinstance(self.geom, WKBElement): - bytes = self.geom.data - if bytes: - self._shapely_geom = wkb.loads(bytes) - else: - self._shapely_geom = None - else: - self._shapely_geom = None - return self._shapely_geom + return from_wkb(self.geom.data) + + # @property + # def shapely_geom(self): + # if not hasattr(self, '_shapely_geom'): + # if isinstance(self.geom, WKBElement): + # bytes = self.geom.data + # if bytes: + # self._shapely_geom = wkb.loads(bytes) + # else: + # self._shapely_geom = None + # else: + # self._shapely_geom = None + # return self._shapely_geom def get_bgColor(self): """ @@ -760,6 +760,7 @@ class GeoModelNoStatus(Model): def get_attachment_base_dir(cls): return Path(conf.attachments.base_dir) / cls.get_attachment_dir() + class GeoModel(GeoModelNoStatus): status: ClassVar[str] = 'E' """ @@ -1061,7 +1062,7 @@ class RawSurveyBaseModel(BaseSurveyModel, GeoPointModelNoStatus): """ Abstract base class for category based raw survey point models """ - metadata: ClassVar[MetaData] = raw_survey + # metadata: ClassVar[MetaData] = raw_survey geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINTZ', dimension=3, srid=conf.geo.raw_survey.srid)) status: str = Field(sa_type=String(1)) diff --git a/src/gisaf/models/map_bases.py b/src/gisaf/models/map_bases.py index 574c5a0..04116eb 100644 --- a/src/gisaf/models/map_bases.py +++ b/src/gisaf/models/map_bases.py @@ -3,11 +3,11 @@ from typing import Any from sqlmodel import Field, String, JSON, Relationship from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_map +from gisaf.models.metadata import gisaf_map_table_args class BaseStyle(Model): - metadata = gisaf_map + __table_args__ = gisaf_map_table_args __tablename__ = 'map_base_style' class Admin: @@ -26,7 +26,7 @@ class BaseStyle(Model): class BaseMap(Model): - metadata = gisaf_map + __table_args__ = gisaf_map_table_args __tablename__ = 'base_map' class Admin: @@ -43,7 +43,7 @@ class BaseMap(Model): class BaseMapLayer(Model): - metadata = gisaf_map + __table_args__ = gisaf_map_table_args __tablename__ = 'base_map_layer' class Admin: diff --git a/src/gisaf/models/metadata.py b/src/gisaf/models/metadata.py index a87ac1f..8a67182 100644 --- a/src/gisaf/models/metadata.py +++ b/src/gisaf/models/metadata.py @@ -1,10 +1,8 @@ -from sqlmodel import MetaData - from gisaf.config import conf -gisaf = MetaData(schema='gisaf') -gisaf_survey = MetaData(schema='gisaf_survey') -gisaf_admin = MetaData(schema='gisaf_admin') -gisaf_map = MetaData(schema='gisaf_map') -raw_survey = MetaData(schema=conf.survey.db_schema_raw) -survey = MetaData(schema=conf.survey.db_schema) \ No newline at end of file +gisaf_table_args = dict(schema= 'gisaf') +gisaf_survey_table_args = dict(schema='gisaf_survey') +gisaf_admin_table_args = dict(schema='gisaf_admin') +gisaf_map_table_args = dict(schema='gisaf_map') +raw_survey_table_args = dict(schema=conf.survey.db_schema_raw) +survey_table_args = dict(schema=conf.survey.db_schema) \ No newline at end of file diff --git a/src/gisaf/models/misc.py b/src/gisaf/models/misc.py index b4161d9..3d588a6 100644 --- a/src/gisaf/models/misc.py +++ b/src/gisaf/models/misc.py @@ -5,7 +5,7 @@ from pydantic import ConfigDict from sqlmodel import Field, JSON, Column from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_map +from gisaf.models.metadata import gisaf_map_table_args logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ class Qml(Model): Model for storing qml (QGis style) """ model_config = ConfigDict(protected_namespaces=()) - metadata = gisaf_map + __table_args__ = gisaf_map_table_args class Admin: menu = 'Other' diff --git a/src/gisaf/models/models_base.py b/src/gisaf/models/models_base.py index f438f30..24b1f04 100644 --- a/src/gisaf/models/models_base.py +++ b/src/gisaf/models/models_base.py @@ -38,7 +38,7 @@ class Model(BaseModel): if hasattr(cls, '__table__'): return cls.__table__.fullname elif hasattr(cls, '__table_args__') and 'schema' in cls.__table_args__: - return f"{cls.__table_args__.schema}.{cls.__tablename__}" + return f"{cls.__table_args__['schema']}.{cls.__tablename__}" else: return f'{cls.metadata.schema}.{cls.__tablename__}' diff --git a/src/gisaf/models/project.py b/src/gisaf/models/project.py index 6130d11..811809b 100644 --- a/src/gisaf/models/project.py +++ b/src/gisaf/models/project.py @@ -9,10 +9,10 @@ from shapely.geometry import Point from gisaf.config import conf from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_admin +from gisaf.models.metadata import gisaf_admin_table_args class Project(Model, table=True): - metadata = gisaf_admin + __table_args__ = gisaf_admin_table_args class Admin: menu = 'Other' diff --git a/src/gisaf/models/raw_survey.py b/src/gisaf/models/raw_survey.py index 8295c40..318b09a 100644 --- a/src/gisaf/models/raw_survey.py +++ b/src/gisaf/models/raw_survey.py @@ -5,10 +5,10 @@ from gisaf.models.models_base import Model from gisaf.models.geo_models_base import GeoPointMModel, BaseSurveyModel from gisaf.models.project import Project from gisaf.models.category import Category -from gisaf.models.metadata import gisaf_survey +from gisaf.models.metadata import gisaf_survey_table_args class RawSurveyModel(BaseSurveyModel, GeoPointMModel): - metadata = gisaf_survey + __table_args__ = gisaf_survey_table_args __tablename__ = 'raw_survey' hidden: ClassVar[bool] = True @@ -94,7 +94,7 @@ class OriginRawPoint(Model): for each line and polygon shape Filled when importing shapefiles """ - metadata = gisaf_survey + __table_args__ = gisaf_survey_table_args __tablename__ = 'origin_raw_point' id: int | None = Field(default=None, primary_key=True) diff --git a/src/gisaf/models/reconcile.py b/src/gisaf/models/reconcile.py index 08db7b8..4ce2403 100644 --- a/src/gisaf/models/reconcile.py +++ b/src/gisaf/models/reconcile.py @@ -4,11 +4,11 @@ from sqlalchemy import BigInteger, String from sqlmodel import Field from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_admin +from gisaf.models.metadata import gisaf_admin_table_args class Reconciliation(Model): - metadata = gisaf_admin + __table_args__ = gisaf_admin_table_args class Admin: menu = 'Other' @@ -21,7 +21,7 @@ class Reconciliation(Model): class StatusChange(Model): - metadata = gisaf_admin + __table_args__ = gisaf_admin_table_args __tablename__ = 'status_change' id: int = Field(primary_key=True, sa_type=BigInteger, @@ -34,7 +34,7 @@ class StatusChange(Model): class FeatureDeletion(Model): - metadata = gisaf_admin + __table_args__ = gisaf_admin_table_args __tablename__ = 'feature_deletion' id: int = Field(BigInteger, primary_key=True, diff --git a/src/gisaf/models/survey.py b/src/gisaf/models/survey.py index 065a8f7..7a3e023 100644 --- a/src/gisaf/models/survey.py +++ b/src/gisaf/models/survey.py @@ -3,11 +3,11 @@ from enum import Enum from sqlmodel import Field, Relationship from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_survey +from gisaf.models.metadata import gisaf_survey_table_args class Accuracy(Model, table=True): - metadata = gisaf_survey + __table_args__ = gisaf_survey_table_args class Admin: menu = 'Other' @@ -25,7 +25,7 @@ class Accuracy(Model, table=True): class Surveyor(Model, table=True): - metadata = gisaf_survey + __table_args__ = gisaf_survey_table_args class Admin: menu = 'Other' @@ -42,7 +42,7 @@ class Surveyor(Model, table=True): class Equipment(Model, table=True): - metadata = gisaf_survey + __table_args__ = gisaf_survey_table_args class Admin: menu = 'Other' @@ -62,17 +62,17 @@ class GeometryType(str, Enum): line_work = 'Line_work' class AccuracyEquimentSurveyorMapping(Model, table=True): - metadata = gisaf_survey + __table_args__ = gisaf_survey_table_args __tablename__ = 'accuracy_equiment_surveyor_mapping' class Admin: menu = 'Other' id: int | None= Field(default=None, primary_key=True) - surveyor_id: int = Field(foreign_key='surveyor.id', index=True) - equipment_id: int = Field(foreign_key='equipment.id', index=True) + surveyor_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.surveyor.id', index=True) + equipment_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.equipment.id', index=True) geometry_type: GeometryType = Field(default='Point', index=True) - accuracy_id: int = Field(foreign_key='accuracy.id') + accuracy_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.accuracy.id') surveyor: Surveyor = Relationship() accuracy: Accuracy = Relationship() equipment: Equipment = Relationship() diff --git a/src/gisaf/models/tags.py b/src/gisaf/models/tags.py index 5ee87c1..8f3b586 100644 --- a/src/gisaf/models/tags.py +++ b/src/gisaf/models/tags.py @@ -5,11 +5,11 @@ from sqlalchemy.dialects.postgresql import HSTORE from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column from pydantic import computed_field -from gisaf.models.metadata import gisaf +from gisaf.models.metadata import gisaf_table_args from gisaf.models.geo_models_base import GeoPointModel class Tags(GeoPointModel, table=True): - metadata = gisaf + __table_args__ = gisaf_table_args hidden: ClassVar[bool] = True class Admin: @@ -29,7 +29,7 @@ class Tags(GeoPointModel, table=True): class TagKey(SQLModel, table=True): - metadata = gisaf + __table_args__ = gisaf_table_args ## CREATE TABLE gisaf.tagkey (key VARCHAR(255) primary key); class Admin: diff --git a/src/gisaf/models/to_migrate.py b/src/gisaf/models/to_migrate.py index aa92c5b..e6f3fe2 100644 --- a/src/gisaf/models/to_migrate.py +++ b/src/gisaf/models/to_migrate.py @@ -3,20 +3,81 @@ from pydantic import BaseModel class ActionResult(BaseModel): message: str + class ActionResults(BaseModel): name: str message: str actionResults: list[ActionResult] + class FormField(BaseModel): name: str type: str + class ModelAction(BaseModel): name: str icon: str formFields: list[FormField] + class DataProvider(BaseModel): name: str - values: list[str] \ No newline at end of file + values: list[str] + + +class InfoItem(BaseModel): + key: str + value: str + + +class InfoCategory(BaseModel): + name: str + infoItems: list[InfoItem] + + +class PlotBgShape(BaseModel): + name: str + valueTop: float + valueBottom: float + color: str + + +class PlotBaseLine(BaseModel): + name: str + value: float + color: str + + +class PlotParams(BaseModel): + baseLines: list[PlotBaseLine] + bgShape: list[PlotBgShape] + barBase: float + + +class Attachment(BaseModel): + name: str + path: str + + +class FeatureInfo(BaseModel): + id: str + itemName: str + geoInfoItems: list[InfoItem] = [] + surveyInfoItems: list[InfoItem] = [] + infoItems: list[InfoItem] = [] + categorizedInfoItems: list[InfoCategory] = [] + tags: list[InfoItem] = [] + graph: str | None = None + plotParams: PlotParams | None = None + files: list[Attachment] = [] + images: list[Attachment] = [] + externalRecordUrl: str | None = None + + +class MapboxPaint(BaseModel): + ... + + +class MapboxLayout(BaseModel): + ... \ No newline at end of file diff --git a/src/gisaf/registry.py b/src/gisaf/registry.py index 83e9ce4..0e8ff9c 100644 --- a/src/gisaf/registry.py +++ b/src/gisaf/registry.py @@ -9,10 +9,10 @@ from importlib.metadata import entry_points from typing import Any, ClassVar, Literal from pydantic import create_model -from pydantic_core import PydanticUndefined from sqlalchemy import text -from sqlalchemy.orm import selectinload -from sqlmodel import SQLModel, select, inspect +from sqlalchemy.orm import selectinload, joinedload +from sqlalchemy.exc import NoResultFound +from sqlmodel import SQLModel, select, inspect, Relationship import pandas as pd import numpy as np @@ -30,11 +30,14 @@ from gisaf.models.geo_models_base import ( GeoLineSurveyModel, GeoPolygonSurveyModel, ) +from gisaf.models.survey import Equipment, Surveyor, Accuracy +from gisaf.models.project import Project from gisaf.utils import ToMigrate from gisaf.models.category import Category, CategoryGroup from gisaf.database import db_session from gisaf import models -from gisaf.models.metadata import survey, raw_survey +from gisaf.models.metadata import ( + gisaf_survey_table_args, raw_survey_table_args, survey_table_args) logger = logging.getLogger(__name__) @@ -139,8 +142,8 @@ class ModelRegistry: ## Use pydantic create_model, supported by SQLModel ## See https://github.com/tiangolo/sqlmodel/issues/377 - store_name = f'{survey.schema}.{category.table_name}' - raw_store_name = f'{raw_survey.schema}.RAW_{category.table_name}' + store_name = f"{survey_table_args['schema']}.{category.table_name}" + raw_store_name = f"{raw_survey_table_args['schema']}.RAW_{category.table_name}" raw_survey_field_definitions = { ## FIXME: RawSurveyBaseModel.category should be a Category, not category.name 'category_name': (ClassVar[str], category.name), @@ -157,7 +160,6 @@ class ModelRegistry: __model_name=category.raw_survey_table_name, __cls_kwargs__={ 'table': True, - '__tablename__': category.raw_survey_table_name, }, **raw_survey_field_definitions ) @@ -173,10 +175,15 @@ class ModelRegistry: if model_class: survey_field_definitions = { 'category_name': (ClassVar[str], category.name), + 'category': (ClassVar[Category], category), 'group_name': (ClassVar[str], category.category_group.name), 'raw_store_name': (ClassVar[str], raw_store_name), 'viewable_role': (ClassVar[str], category.viewable_role), 'symbol': (ClassVar[str], category.symbol), + 'equipment': (Equipment, Relationship()), + 'surveyor': (Surveyor, Relationship()), + 'accuracy': (Accuracy, Relationship()), + 'project': (Project, Relationship()), #'raw_model': (str, self.raw_survey_models.get(raw_store_name)), # 'icon': (str, f'{survey.schema}-{category.table_name}'), } @@ -185,7 +192,6 @@ class ModelRegistry: __model_name=category.table_name, __cls_kwargs__={ 'table': True, - '__tablename__': category.table_name, }, **survey_field_definitions, ) @@ -335,7 +341,16 @@ class ModelRegistry: """ if not model: return {} - item = await model.load(**model.get_join_with()).query.where(model.id==id).gino.first() + async with db_session() as session: + query = select(model).where(model.id == id).options( + *(joinedload(jt) for jt in model.selectinload())) + result = await session.exec(query) + try: + item = result.one() + except NoResultFound: + raise NotInRegistry + + # item = await model.load(**model.get_join_with()).query.where(model.id==id).gino.first() if not item: return {} resp = {} @@ -386,7 +401,7 @@ class ModelRegistry: if category.minor_group_2 != '----': fragments.append(category.minor_group_2) return '.'.join([ - survey.schema, + survey_table_args['schema'], '_'.join(fragments) ]) diff --git a/src/gisaf/utils.py b/src/gisaf/utils.py index edf65ec..9fc43d5 100644 --- a/src/gisaf/utils.py +++ b/src/gisaf/utils.py @@ -126,32 +126,32 @@ class NumpyEncoder(JSONEncoder): # headers=headers, content_type=content_type, **kwargs) -def get_join_with(cls, recursive=True): - """ - Helper function for loading related tables with a Gino loader (left outer join) - Should work recursively... - Eg: - cls.load(**get_join_with(cls)).query.gino.all() - :param cls: - :return: - """ - if hasattr(cls, 'dyn_join_with'): - joins = cls.dyn_join_with() - else: - joins = {} - if hasattr(cls, '_join_with'): - joins.update(cls._join_with) - if not recursive: - return joins - recursive_joins = {} - for name, join in joins.items(): - more_joins = get_join_with(join) - if more_joins: - aliased = {name: join.alias() for name, join in more_joins.items()} - recursive_joins[name] = join.load(**aliased) - else: - recursive_joins[name] = join - return recursive_joins +# def get_join_with(cls, recursive=True): +# """ +# Helper function for loading related tables with a Gino loader (left outer join) +# Should work recursively... +# Eg: +# cls.load(**get_join_with(cls)).query.gino.all() +# :param cls: +# :return: +# """ +# if hasattr(cls, 'dyn_join_with'): +# joins = cls.dyn_join_with() +# else: +# joins = {} +# if hasattr(cls, '_join_with'): +# joins.update(cls._join_with) +# if not recursive: +# return joins +# recursive_joins = {} +# for name, join in joins.items(): +# more_joins = get_join_with(join) +# if more_joins: +# aliased = {name: join.alias() for name, join in more_joins.items()} +# recursive_joins[name] = join.load(**aliased) +# else: +# recursive_joins[name] = join +# return recursive_joins def get_joined_query(cls): """