philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Remove custom sqlalchemy metadata, manage with __table_args__

Browse source 
Allow sqlmodel queries, with relations
Remode join_with mechanisms coming from gino
Handlew ith_only_columns in get_df and get_gdf
Implement feature-info
This commit is contained in:
phil 2024-01-04 18:50:23 +05:30
parent 1e3678fb69
commit ec71b6ed15
18 changed files with 353 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

123
src/gisaf/api.py
View file

@ -22,7 +22,12 @@ from gisaf.security import (
Token,
authenticate_user, get_current_user, create_access_token,
)
from gisaf.registry import registry
from gisaf.registry import registry, NotInRegistry
from gisaf.redis_tools import store as redis_store
from gisaf.custom_store_base import BaseStore
from gisaf.models.to_migrate import (
FeatureInfo, InfoItem, Attachment, InfoCategory
)
logger = logging.getLogger(__name__)
@ -43,7 +48,6 @@ async def bootstrap(
@api.post("/token")
async def login_for_access_token(
db_session: db_session,
form_data: OAuth2PasswordRequestForm = Depends()
) -> Token:
user = await authenticate_user(form_data.username, form_data.password)
@ -117,6 +121,121 @@ async def get_projects(
df = await db_session.run_sync(pandas_query, query)
return df.to_dict(orient="records")
@api.get("/feature-info/{store}/{id}")
async def get_feature_info(
store: str, id: str,
) -> FeatureInfo:
store_record = registry.stores.loc[store]
model = store_record.model
if store_record.is_live:
item = await redis_store.get_feature_info(store, id)
geom = item.geometry
## Reproject to default coordinate system (WGS84),
## XXX: only for shapely geometries
if not isinstance(geom, pygeos.Geometry):
geom_reprojected = transform(reproject_func, geom)
geoInfoItems = OrderedDict()
if isinstance(geom, Point):
geoInfoItems['longitude'] = f'{geom.x:.6f}'
geoInfoItems['latitude'] = f'{geom.y:.6f}'
if geom.has_z:
geoInfoItems['elevation (m)'] = f'{geom.z:.6f}'
elif isinstance(geom, (LineString, MultiLineString)):
bounds = geom.bounds
geoInfoItems['longitude'] = f'{bounds[0]:.6f} - {bounds[2]:.6f}'
geoInfoItems['latitude'] = f'{bounds[1]:.6f} - {bounds[3]:.6f}'
geoInfoItems['length (m)'] = f'{geom_reprojected.length:.2f}'
## TODO: elevation for MultiLineString
if geom.has_z and not isinstance(geom, MultiLineString):
elevations = [cc[2] for cc in geom.coords]
elev_min = min(elevations)
elev_max = max(elevations)
if elev_min == elev_max:
geoInfoItems['elevation (m)'] = f'{elev_min:.2f}'
else:
geoInfoItems['elevation (m)'] = f'{elev_min:.2f} - {elev_max:.2f}'
elif isinstance(geom, (Polygon, MultiPolygon)):
area = geom_reprojected.area
bounds = geom.bounds
geoInfoItems['longitude'] = f'{bounds[0]:.6f} - {bounds[2]:.6f}'
geoInfoItems['latitude'] = f'{bounds[1]:.6f} - {bounds[3]:.6f}'
geoInfoItems['area (sq. m)'] = f'{area:.1f} sq. m'
geoInfoItems['area (ha)'] = f'{area / 10000:.1f} ha'
geoInfoItems['area (acre)'] = f'{area / 4046.85643005078874:.1f} acres'
## TODO: elevation for MultiPolygon
if geom.has_z and not isinstance(geom, MultiPolygon):
if hasattr(geom, 'exterior'):
coords = geom.exterior.coords
else:
coords = geom.coords
elevations = [coord[2] for coord in coords]
elev_min = min(elevations)
elev_max = max(elevations)
if elev_min == elev_max:
geoInfoItems['elevation (m)'] = f'{elev_min:.2f}'
else:
geoInfoItems['elevation (m)'] = f'{elev_min:.2f} - {elev_max:.2f}'
feature_info_dict = {
'itemName': item.get('popup', f'Live: {store} #{id}'),
'geoInfoItems': geoInfoItems,
'surveyInfoItems': {
'Note': 'Live layers do not have survey info',
},
'infoItems': dict(item.drop(set(item.keys()).intersection(('geometry', 'popup')))),
'tags': {},
}
elif issubclass(model, BaseStore):
feature_info_dict = await model.get_item_params(id)
else:
## Not a live layer
try:
feature_info_dict = await registry.get_model_id_params(model, int(id))
except NotInRegistry:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
feature_info = FeatureInfo(
id=id,
itemName=feature_info_dict['itemName'],
geoInfoItems=[InfoItem(key=k, value=v)
for k, v in feature_info_dict['geoInfoItems'].items()],
surveyInfoItems=[InfoItem(key=k, value=v)
for k, v in feature_info_dict['surveyInfoItems'].items()],
infoItems=[InfoItem(key=k, value=v)
for k, v in feature_info_dict['infoItems'].items()],
tags=[InfoItem(key=k, value=v)
for k, v in feature_info_dict['tags'].items()],
graph=feature_info_dict.get('graph'),
)
if 'files' in feature_info_dict and feature_info_dict['files'] is not None:
feature_info.files = [
Attachment(name=k, path=v)
for k, v in feature_info_dict['files'].items()
]
else:
feature_info.files = []
if 'images' in feature_info_dict and feature_info_dict['images'] is not None:
feature_info.images = [
Attachment(name=k, path=v)
for k, v in feature_info_dict['images'].items()
]
else:
feature_info.images = []
if 'categorized_info_items' in feature_info_dict and feature_info_dict['categorized_info_items'] != None:
feature_info.categorizedInfoItems = [
InfoCategory(
name=name,
infoItems=[
InfoItem(key=k, value=v)
for k, v in info_items.items()
]
)
for name, info_items in feature_info_dict['categorized_info_items'].items()
]
feature_info.externalRecordUrl = feature_info_dict.get('externalRecordUrl')
return feature_info
# @api.get("/user-role")
# async def get_user_role_relation(
# *, db_session: AsyncSession = Depends(get_db_session)

22
src/gisaf/database.py
View file

@ -40,17 +40,29 @@ class BaseModel(SQLModel):
return []
@classmethod
async def get_df(cls, where=None, with_related=True, **kwargs) -> pd.DataFrame:
async def get_df(cls, *,
where=None, with_related=True, **kwargs
) -> pd.DataFrame:
return await cls._get_df(pandas_query, where=None, with_related=True, **kwargs)
@classmethod
async def get_gdf(cls, *, where=None, with_related=True, **kwargs) -> gpd.GeoDataFrame:
return await cls._get_df(geopandas_query, where=None, with_related=True, **kwargs)
async def get_gdf(cls, *,
where=None, with_related=True, **kwargs
) -> gpd.GeoDataFrame:
return await cls._get_df(geopandas_query,
where=None, with_related=True, **kwargs)
@classmethod
async def _get_df(cls, method, *, where=None, with_related=True, **kwargs) -> pd.DataFrame | gpd.GeoDataFrame:
async def _get_df(cls, method, *,
where=None, with_related=True, with_only_columns=[], **kwargs
) -> pd.DataFrame | gpd.GeoDataFrame:
async with db_session() as session:
query = select(cls)
if len(with_only_columns) == 0:
query = select(cls)
else:
columns = set(with_only_columns)
columns.add(*(col.name for col in cls.__table__.primary_key.columns))
query = select(*(getattr(cls, col) for col in columns))
if where is not None:
query.append_whereclause(where)
## Get the joined tables

6
src/gisaf/models/admin.py
View file

@ -9,7 +9,7 @@ import pandas as pd
from gisaf.models.models_base import Model
from gisaf.models.survey import Surveyor, Equipment
from gisaf.models.project import Project
from gisaf.models.metadata import gisaf_admin
from gisaf.models.metadata import gisaf_admin_table_args
re_file_import_record_date_expr = '^(\S+)-(\d\d\d\d)-(\d\d)-(\d\d).*$'
@ -45,7 +45,7 @@ class FileImport(Model):
Give either url or path.
"""
__tablename__ = 'file_import'
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
id: int | None = Field(default=None, primary_key=True)
url: str
@ -117,7 +117,7 @@ class FeatureImportData(Model):
Keep track of imported data, typically from shapefiles
"""
__tablename__ = 'feature_import_data'
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
id: int | None = Field(default=None, primary_key=True)
store: str = Field(index=True)

19
src/gisaf/models/authentication.py
View file

@ -1,15 +1,20 @@
from sqlmodel import Field, SQLModel, MetaData, Relationship
from sqlmodel import Field, SQLModel, Relationship
from gisaf.models.metadata import gisaf_admin_table_args
from gisaf.models.metadata import gisaf_admin
class UserRoleLink(SQLModel, table=True):
metadata = gisaf_admin
__tablename__ = 'roles_users'
__table_args__ = gisaf_admin_table_args
user_id: int | None = Field(
default=None, foreign_key="user.id", primary_key=True
default=None,
foreign_key=gisaf_admin_table_args['schema'] + '.user.id',
primary_key=True
)
role_id: int | None = Field(
default=None, foreign_key="role.id", primary_key=True
default=None,
foreign_key=gisaf_admin_table_args['schema'] + '.role.id',
primary_key=True
)
@ -20,7 +25,7 @@ class UserBase(SQLModel):
class User(UserBase, table=True):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
id: int | None = Field(default=None, primary_key=True)
roles: list["Role"] = Relationship(back_populates="users",
link_model=UserRoleLink)
@ -40,7 +45,7 @@ class RoleWithDescription(RoleBase):
description: str | None
class Role(RoleWithDescription, table=True):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
id: int | None = Field(default=None, primary_key=True)
users: list[User] = Relationship(back_populates="roles",
link_model=UserRoleLink)

13
src/gisaf/models/category.py
View file

@ -4,8 +4,8 @@ from sqlalchemy import String
from pydantic import computed_field, ConfigDict
from sqlmodel import Field, Relationship, JSON, TEXT
from gisaf.models.metadata import gisaf_survey
from gisaf.database import BaseModel
from gisaf.models.metadata import gisaf_survey_table_args
mapbox_type_mapping = {
'Point': 'symbol',
@ -15,8 +15,8 @@ mapbox_type_mapping = {
class CategoryGroup(BaseModel, table=True):
metadata = gisaf_survey
__tablename__ = 'category_group'
__table_args__ = gisaf_survey_table_args
name: str | None = Field(sa_type=String(4), default=None, primary_key=True)
major: str
long_name: str
@ -28,8 +28,8 @@ class CategoryGroup(BaseModel, table=True):
class CategoryModelType(BaseModel, table=True):
metadata = gisaf_survey
__tablename__ = 'category_model_type'
__table_args__ = gisaf_survey_table_args
name: str | None = Field(default=None, primary_key=True)
class Admin:
@ -47,14 +47,15 @@ class CategoryBase(BaseModel):
domain: ClassVar[str] = 'V'
description: str | None
group: str = Field(sa_type=String(4),
foreign_key="category_group.name", index=True)
foreign_key=gisaf_survey_table_args['schema'] + ".category_group.name",
index=True)
minor_group_1: str = Field(sa_type=String(4), default='----')
minor_group_2: str = Field(sa_type=String(4), default='----')
status: str = Field(sa_type=String(1))
custom: bool | None
auto_import: bool = True
gis_type: str = Field(sa_type=String(50),
foreign_key='category_model_type.name',
foreign_key=gisaf_survey_table_args['schema'] + '.category_model_type.name',
default='Point')
long_name: str | None = Field(sa_type=String(50))
style: str | None = Field(sa_type=TEXT)
@ -105,7 +106,7 @@ class CategoryBase(BaseModel):
class Category(CategoryBase, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
name: str | None = Field(default=None, primary_key=True)
category_group: CategoryGroup = Relationship(back_populates="categories")

93
src/gisaf/models/geo_models_base.py
View file

@ -4,6 +4,7 @@ from datetime import date, datetime
from collections import OrderedDict
from io import BytesIO
from zipfile import ZipFile
from functools import cached_property
import locale
import logging
@ -12,19 +13,16 @@ import geopandas as gpd # type: ignore
import shapely # type: ignore
import pyproj
from sqlmodel import SQLModel, Field, Relationship
from sqlmodel.ext.asyncio.session import AsyncSession
from pydantic import BaseModel
from geoalchemy2.shape import from_shape
from sqlalchemy.dialects.postgresql import BIGINT
from sqlalchemy import BigInteger, Column, MetaData, String, func, and_, text
from sqlmodel import select, Field, Relationship
from sqlmodel.ext.asyncio.session import AsyncSession
from sqlalchemy import BigInteger, MetaData, String, func, and_, text
from sqlalchemy.sql import sqltypes
from sqlalchemy.orm import declared_attr
from psycopg2.extensions import adapt
from geoalchemy2.shape import from_shape
from geoalchemy2.types import Geometry, WKBElement
from shapely import wkb
from shapely import wkb, from_wkb
from shapely.geometry import mapping
from shapely.ops import transform # type: ignore
@ -35,15 +33,15 @@ from shapefile import (
POLYGON, POLYGONZ,
)
from gisaf.database import db_session
from gisaf.config import conf
from gisaf.models.models_base import Model
from gisaf.models.metadata import survey, raw_survey
from gisaf.models.survey import Equipment, Surveyor, Accuracy
from gisaf.models.metadata import (
gisaf_survey_table_args, gisaf_admin_table_args, survey_table_args)
from gisaf.models.misc import Qml
from gisaf.models.category import Category
from gisaf.models.project import Project
# from gisaf.models.survey import Equipment, Surveyor, Accuracy
# from gisaf.models.project import Project
LOCALE_DATE_FORMAT = locale.nl_langinfo(locale.D_FMT)
@ -82,14 +80,14 @@ class BaseSurveyModel(BaseModel):
- projected ('V_*')
"""
id: int | None = Field(sa_type=BigInteger, primary_key=True, default=None)
equip_id: int = Field(foreign_key='equipment.id')
# equipment: Equipment = Relationship()
srvyr_id: int = Field('surveyor.id')
# surveyor: Surveyor = Relationship()
accur_id: int = Field('accuracy.id')
# accuracy: Accuracy = Relationship()
project_id: int = Field('project.id')
# project: Project = Relationship()
equip_id: int = Field(
foreign_key=gisaf_survey_table_args['schema'] + '.equipment.id')
srvyr_id: int = Field(
foreign_key=gisaf_survey_table_args['schema'] + '.surveyor.id')
accur_id: int = Field(
foreign_key=gisaf_survey_table_args['schema'] + '.accuracy.id')
project_id: int = Field(
foreign_key=gisaf_admin_table_args['schema'] + '.project.id')
orig_id: str
date: date
@ -151,7 +149,7 @@ class SurveyModel(BaseSurveyModel):
"""
Base mixin class for defining final (reprojected) survey data, with a status
"""
metadata: ClassVar[MetaData] = survey
__table_args__ = survey_table_args
# status: str = Field(sa_type=String(1))
get_gdf_with_related: ClassVar[bool] = False
@ -182,6 +180,10 @@ class SurveyModel(BaseSurveyModel):
'gisaf_admin_project_skip_columns',
]
@declared_attr
def __tablename__(cls) -> str:
return cls.__name__ # type: nocheck
async def get_survey_info(self):
info = await super(SurveyModel, self).get_survey_info()
if self.srvyr_id:
@ -203,7 +205,7 @@ class SurveyModel(BaseSurveyModel):
@property
def caption(self):
return '{self.category.description} [{self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self)
return '{self.category.description} - {self.category.name} [{self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self)
@classmethod
async def get_popup(cls, df):
@ -425,30 +427,28 @@ class GeoModelNoStatus(Model):
async def get_tags(self):
from gisaf.models.tags import Tags
tags = await Tags.get_df(
where=and_(
Tags.store == self.__class__.get_store_name(),
Tags.ref_id == self.id
),
with_only_columns=['tags']
)
if len(tags) > 0:
return tags.loc[0, 'tags']
else:
return {}
async with db_session() as session:
query = select(Tags.tags).where(Tags.store == self.get_store_name(),
Tags.ref_id == self.id)
tags = await session.exec(query)
return tags.one_or_none() or {}
@property
@cached_property
def shapely_geom(self):
if not hasattr(self, '_shapely_geom'):
if isinstance(self.geom, WKBElement):
bytes = self.geom.data
if bytes:
self._shapely_geom = wkb.loads(bytes)
else:
self._shapely_geom = None
else:
self._shapely_geom = None
return self._shapely_geom
return from_wkb(self.geom.data)
# @property
# def shapely_geom(self):
# if not hasattr(self, '_shapely_geom'):
# if isinstance(self.geom, WKBElement):
# bytes = self.geom.data
# if bytes:
# self._shapely_geom = wkb.loads(bytes)
# else:
# self._shapely_geom = None
# else:
# self._shapely_geom = None
# return self._shapely_geom
def get_bgColor(self):
"""
@ -760,6 +760,7 @@ class GeoModelNoStatus(Model):
def get_attachment_base_dir(cls):
return Path(conf.attachments.base_dir) / cls.get_attachment_dir()
class GeoModel(GeoModelNoStatus):
status: ClassVar[str] = 'E'
"""
@ -1061,7 +1062,7 @@ class RawSurveyBaseModel(BaseSurveyModel, GeoPointModelNoStatus):
"""
Abstract base class for category based raw survey point models
"""
metadata: ClassVar[MetaData] = raw_survey
# metadata: ClassVar[MetaData] = raw_survey
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINTZ', dimension=3,
srid=conf.geo.raw_survey.srid))
status: str = Field(sa_type=String(1))

8
src/gisaf/models/map_bases.py
View file

@ -3,11 +3,11 @@ from typing import Any
from sqlmodel import Field, String, JSON, Relationship
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_map
from gisaf.models.metadata import gisaf_map_table_args
class BaseStyle(Model):
metadata = gisaf_map
__table_args__ = gisaf_map_table_args
__tablename__ = 'map_base_style'
class Admin:
@ -26,7 +26,7 @@ class BaseStyle(Model):
class BaseMap(Model):
metadata = gisaf_map
__table_args__ = gisaf_map_table_args
__tablename__ = 'base_map'
class Admin:
@ -43,7 +43,7 @@ class BaseMap(Model):
class BaseMapLayer(Model):
metadata = gisaf_map
__table_args__ = gisaf_map_table_args
__tablename__ = 'base_map_layer'
class Admin:

14
src/gisaf/models/metadata.py
View file

@ -1,10 +1,8 @@
from sqlmodel import MetaData
from gisaf.config import conf
gisaf = MetaData(schema='gisaf')
gisaf_survey = MetaData(schema='gisaf_survey')
gisaf_admin = MetaData(schema='gisaf_admin')
gisaf_map = MetaData(schema='gisaf_map')
raw_survey = MetaData(schema=conf.survey.db_schema_raw)
survey = MetaData(schema=conf.survey.db_schema)
gisaf_table_args = dict(schema= 'gisaf')
gisaf_survey_table_args = dict(schema='gisaf_survey')
gisaf_admin_table_args = dict(schema='gisaf_admin')
gisaf_map_table_args = dict(schema='gisaf_map')
raw_survey_table_args = dict(schema=conf.survey.db_schema_raw)
survey_table_args = dict(schema=conf.survey.db_schema)

4
src/gisaf/models/misc.py
View file

@ -5,7 +5,7 @@ from pydantic import ConfigDict
from sqlmodel import Field, JSON, Column
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_map
from gisaf.models.metadata import gisaf_map_table_args
logger = logging.getLogger(__name__)
@ -19,7 +19,7 @@ class Qml(Model):
Model for storing qml (QGis style)
"""
model_config = ConfigDict(protected_namespaces=())
metadata = gisaf_map
__table_args__ = gisaf_map_table_args
class Admin:
menu = 'Other'

2
src/gisaf/models/models_base.py
View file

@ -38,7 +38,7 @@ class Model(BaseModel):
if hasattr(cls, '__table__'):
return cls.__table__.fullname
elif hasattr(cls, '__table_args__') and 'schema' in cls.__table_args__:
return f"{cls.__table_args__.schema}.{cls.__tablename__}"
return f"{cls.__table_args__['schema']}.{cls.__tablename__}"
else:
return f'{cls.metadata.schema}.{cls.__tablename__}'

4
src/gisaf/models/project.py
View file

@ -9,10 +9,10 @@ from shapely.geometry import Point
from gisaf.config import conf
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_admin
from gisaf.models.metadata import gisaf_admin_table_args
class Project(Model, table=True):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
class Admin:
menu = 'Other'

6
src/gisaf/models/raw_survey.py
View file

@ -5,10 +5,10 @@ from gisaf.models.models_base import Model
from gisaf.models.geo_models_base import GeoPointMModel, BaseSurveyModel
from gisaf.models.project import Project
from gisaf.models.category import Category
from gisaf.models.metadata import gisaf_survey
from gisaf.models.metadata import gisaf_survey_table_args
class RawSurveyModel(BaseSurveyModel, GeoPointMModel):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
__tablename__ = 'raw_survey'
hidden: ClassVar[bool] = True
@ -94,7 +94,7 @@ class OriginRawPoint(Model):
for each line and polygon shape
Filled when importing shapefiles
"""
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
__tablename__ = 'origin_raw_point'
id: int | None = Field(default=None, primary_key=True)

8
src/gisaf/models/reconcile.py
View file

@ -4,11 +4,11 @@ from sqlalchemy import BigInteger, String
from sqlmodel import Field
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_admin
from gisaf.models.metadata import gisaf_admin_table_args
class Reconciliation(Model):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
class Admin:
menu = 'Other'
@ -21,7 +21,7 @@ class Reconciliation(Model):
class StatusChange(Model):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
__tablename__ = 'status_change'
id: int = Field(primary_key=True, sa_type=BigInteger,
@ -34,7 +34,7 @@ class StatusChange(Model):
class FeatureDeletion(Model):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
__tablename__ = 'feature_deletion'
id: int = Field(BigInteger, primary_key=True,

16
src/gisaf/models/survey.py
View file

@ -3,11 +3,11 @@ from enum import Enum
from sqlmodel import Field, Relationship
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_survey
from gisaf.models.metadata import gisaf_survey_table_args
class Accuracy(Model, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
class Admin:
menu = 'Other'
@ -25,7 +25,7 @@ class Accuracy(Model, table=True):
class Surveyor(Model, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
class Admin:
menu = 'Other'
@ -42,7 +42,7 @@ class Surveyor(Model, table=True):
class Equipment(Model, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
class Admin:
menu = 'Other'
@ -62,17 +62,17 @@ class GeometryType(str, Enum):
line_work = 'Line_work'
class AccuracyEquimentSurveyorMapping(Model, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
__tablename__ = 'accuracy_equiment_surveyor_mapping'
class Admin:
menu = 'Other'
id: int | None= Field(default=None, primary_key=True)
surveyor_id: int = Field(foreign_key='surveyor.id', index=True)
equipment_id: int = Field(foreign_key='equipment.id', index=True)
surveyor_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.surveyor.id', index=True)
equipment_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.equipment.id', index=True)
geometry_type: GeometryType = Field(default='Point', index=True)
accuracy_id: int = Field(foreign_key='accuracy.id')
accuracy_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.accuracy.id')
surveyor: Surveyor = Relationship()
accuracy: Accuracy = Relationship()
equipment: Equipment = Relationship()

6
src/gisaf/models/tags.py
View file

@ -5,11 +5,11 @@ from sqlalchemy.dialects.postgresql import HSTORE
from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
from pydantic import computed_field
from gisaf.models.metadata import gisaf
from gisaf.models.metadata import gisaf_table_args
from gisaf.models.geo_models_base import GeoPointModel
class Tags(GeoPointModel, table=True):
metadata = gisaf
__table_args__ = gisaf_table_args
hidden: ClassVar[bool] = True
class Admin:
@ -29,7 +29,7 @@ class Tags(GeoPointModel, table=True):
class TagKey(SQLModel, table=True):
metadata = gisaf
__table_args__ = gisaf_table_args
## CREATE TABLE gisaf.tagkey (key VARCHAR(255) primary key);
class Admin:

61
src/gisaf/models/to_migrate.py
View file

@ -3,20 +3,81 @@ from pydantic import BaseModel
class ActionResult(BaseModel):
message: str
class ActionResults(BaseModel):
name: str
message: str
actionResults: list[ActionResult]
class FormField(BaseModel):
name: str
type: str
class ModelAction(BaseModel):
name: str
icon: str
formFields: list[FormField]
class DataProvider(BaseModel):
name: str
values: list[str]
class InfoItem(BaseModel):
key: str
value: str
class InfoCategory(BaseModel):
name: str
infoItems: list[InfoItem]
class PlotBgShape(BaseModel):
name: str
valueTop: float
valueBottom: float
color: str
class PlotBaseLine(BaseModel):
name: str
value: float
color: str
class PlotParams(BaseModel):
baseLines: list[PlotBaseLine]
bgShape: list[PlotBgShape]
barBase: float
class Attachment(BaseModel):
name: str
path: str
class FeatureInfo(BaseModel):
id: str
itemName: str
geoInfoItems: list[InfoItem] = []
surveyInfoItems: list[InfoItem] = []
infoItems: list[InfoItem] = []
categorizedInfoItems: list[InfoCategory] = []
tags: list[InfoItem] = []
graph: str | None = None
plotParams: PlotParams | None = None
files: list[Attachment] = []
images: list[Attachment] = []
externalRecordUrl: str | None = None
class MapboxPaint(BaseModel):
...
class MapboxLayout(BaseModel):
...

35
src/gisaf/registry.py
View file

@ -9,10 +9,10 @@ from importlib.metadata import entry_points
from typing import Any, ClassVar, Literal
from pydantic import create_model
from pydantic_core import PydanticUndefined
from sqlalchemy import text
from sqlalchemy.orm import selectinload
from sqlmodel import SQLModel, select, inspect
from sqlalchemy.orm import selectinload, joinedload
from sqlalchemy.exc import NoResultFound
from sqlmodel import SQLModel, select, inspect, Relationship
import pandas as pd
import numpy as np
@ -30,11 +30,14 @@ from gisaf.models.geo_models_base import (
GeoLineSurveyModel,
GeoPolygonSurveyModel,
)
from gisaf.models.survey import Equipment, Surveyor, Accuracy
from gisaf.models.project import Project
from gisaf.utils import ToMigrate
from gisaf.models.category import Category, CategoryGroup
from gisaf.database import db_session
from gisaf import models
from gisaf.models.metadata import survey, raw_survey
from gisaf.models.metadata import (
gisaf_survey_table_args, raw_survey_table_args, survey_table_args)
logger = logging.getLogger(__name__)
@ -139,8 +142,8 @@ class ModelRegistry:
## Use pydantic create_model, supported by SQLModel
## See https://github.com/tiangolo/sqlmodel/issues/377
store_name = f'{survey.schema}.{category.table_name}'
raw_store_name = f'{raw_survey.schema}.RAW_{category.table_name}'
store_name = f"{survey_table_args['schema']}.{category.table_name}"
raw_store_name = f"{raw_survey_table_args['schema']}.RAW_{category.table_name}"
raw_survey_field_definitions = {
## FIXME: RawSurveyBaseModel.category should be a Category, not category.name
'category_name': (ClassVar[str], category.name),
@ -157,7 +160,6 @@ class ModelRegistry:
__model_name=category.raw_survey_table_name,
__cls_kwargs__={
'table': True,
'__tablename__': category.raw_survey_table_name,
},
**raw_survey_field_definitions
)
@ -173,10 +175,15 @@ class ModelRegistry:
if model_class:
survey_field_definitions = {
'category_name': (ClassVar[str], category.name),
'category': (ClassVar[Category], category),
'group_name': (ClassVar[str], category.category_group.name),
'raw_store_name': (ClassVar[str], raw_store_name),
'viewable_role': (ClassVar[str], category.viewable_role),
'symbol': (ClassVar[str], category.symbol),
'equipment': (Equipment, Relationship()),
'surveyor': (Surveyor, Relationship()),
'accuracy': (Accuracy, Relationship()),
'project': (Project, Relationship()),
#'raw_model': (str, self.raw_survey_models.get(raw_store_name)),
# 'icon': (str, f'{survey.schema}-{category.table_name}'),
}
@ -185,7 +192,6 @@ class ModelRegistry:
__model_name=category.table_name,
__cls_kwargs__={
'table': True,
'__tablename__': category.table_name,
},
**survey_field_definitions,
)
@ -335,7 +341,16 @@ class ModelRegistry:
"""
if not model:
return {}
item = await model.load(**model.get_join_with()).query.where(model.id==id).gino.first()
async with db_session() as session:
query = select(model).where(model.id == id).options(
*(joinedload(jt) for jt in model.selectinload()))
result = await session.exec(query)
try:
item = result.one()
except NoResultFound:
raise NotInRegistry
# item = await model.load(**model.get_join_with()).query.where(model.id==id).gino.first()
if not item:
return {}
resp = {}
@ -386,7 +401,7 @@ class ModelRegistry:
if category.minor_group_2 != '----':
fragments.append(category.minor_group_2)
return '.'.join([
survey.schema,
survey_table_args['schema'],
'_'.join(fragments)
])

52
src/gisaf/utils.py
View file

@ -126,32 +126,32 @@ class NumpyEncoder(JSONEncoder):
# headers=headers, content_type=content_type, **kwargs)
def get_join_with(cls, recursive=True):
"""
Helper function for loading related tables with a Gino loader (left outer join)
Should work recursively...
Eg:
cls.load(**get_join_with(cls)).query.gino.all()
:param cls:
:return:
"""
if hasattr(cls, 'dyn_join_with'):
joins = cls.dyn_join_with()
else:
joins = {}
if hasattr(cls, '_join_with'):
joins.update(cls._join_with)
if not recursive:
return joins
recursive_joins = {}
for name, join in joins.items():
more_joins = get_join_with(join)
if more_joins:
aliased = {name: join.alias() for name, join in more_joins.items()}
recursive_joins[name] = join.load(**aliased)
else:
recursive_joins[name] = join
return recursive_joins
# def get_join_with(cls, recursive=True):
# """
# Helper function for loading related tables with a Gino loader (left outer join)
# Should work recursively...
# Eg:
# cls.load(**get_join_with(cls)).query.gino.all()
# :param cls:
# :return:
# """
# if hasattr(cls, 'dyn_join_with'):
# joins = cls.dyn_join_with()
# else:
# joins = {}
# if hasattr(cls, '_join_with'):
# joins.update(cls._join_with)
# if not recursive:
# return joins
# recursive_joins = {}
# for name, join in joins.items():
# more_joins = get_join_with(join)
# if more_joins:
# aliased = {name: join.alias() for name, join in more_joins.items()}
# recursive_joins[name] = join.load(**aliased)
# else:
# recursive_joins[name] = join
# return recursive_joins
def get_joined_query(cls):
"""