philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Remove custom sqlalchemy metadata, manage with __table_args__

Browse source 
Allow sqlmodel queries, with relations
Remode join_with mechanisms coming from gino
Handlew ith_only_columns in get_df and get_gdf
Implement feature-info
This commit is contained in:
phil 2024-01-04 18:50:23 +05:30
parent 1e3678fb69
commit ec71b6ed15
18 changed files with 353 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/gisaf/models/admin.py
View file

@ -9,7 +9,7 @@ import pandas as pd
from gisaf.models.models_base import Model
from gisaf.models.survey import Surveyor, Equipment
from gisaf.models.project import Project
from gisaf.models.metadata import gisaf_admin
from gisaf.models.metadata import gisaf_admin_table_args
re_file_import_record_date_expr = '^(\S+)-(\d\d\d\d)-(\d\d)-(\d\d).*$'
@ -45,7 +45,7 @@ class FileImport(Model):
Give either url or path.
"""
__tablename__ = 'file_import'
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
id: int | None = Field(default=None, primary_key=True)
url: str
@ -117,7 +117,7 @@ class FeatureImportData(Model):
Keep track of imported data, typically from shapefiles
"""
__tablename__ = 'feature_import_data'
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
id: int | None = Field(default=None, primary_key=True)
store: str = Field(index=True)

19
src/gisaf/models/authentication.py
View file

@ -1,15 +1,20 @@
from sqlmodel import Field, SQLModel, MetaData, Relationship
from sqlmodel import Field, SQLModel, Relationship
from gisaf.models.metadata import gisaf_admin_table_args
from gisaf.models.metadata import gisaf_admin
class UserRoleLink(SQLModel, table=True):
metadata = gisaf_admin
__tablename__ = 'roles_users'
__table_args__ = gisaf_admin_table_args
user_id: int | None = Field(
default=None, foreign_key="user.id", primary_key=True
default=None,
foreign_key=gisaf_admin_table_args['schema'] + '.user.id',
primary_key=True
)
role_id: int | None = Field(
default=None, foreign_key="role.id", primary_key=True
default=None,
foreign_key=gisaf_admin_table_args['schema'] + '.role.id',
primary_key=True
)
@ -20,7 +25,7 @@ class UserBase(SQLModel):
class User(UserBase, table=True):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
id: int | None = Field(default=None, primary_key=True)
roles: list["Role"] = Relationship(back_populates="users",
link_model=UserRoleLink)
@ -40,7 +45,7 @@ class RoleWithDescription(RoleBase):
description: str | None
class Role(RoleWithDescription, table=True):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
id: int | None = Field(default=None, primary_key=True)
users: list[User] = Relationship(back_populates="roles",
link_model=UserRoleLink)

13
src/gisaf/models/category.py
View file

@ -4,8 +4,8 @@ from sqlalchemy import String
from pydantic import computed_field, ConfigDict
from sqlmodel import Field, Relationship, JSON, TEXT
from gisaf.models.metadata import gisaf_survey
from gisaf.database import BaseModel
from gisaf.models.metadata import gisaf_survey_table_args
mapbox_type_mapping = {
'Point': 'symbol',
@ -15,8 +15,8 @@ mapbox_type_mapping = {
class CategoryGroup(BaseModel, table=True):
metadata = gisaf_survey
__tablename__ = 'category_group'
__table_args__ = gisaf_survey_table_args
name: str | None = Field(sa_type=String(4), default=None, primary_key=True)
major: str
long_name: str
@ -28,8 +28,8 @@ class CategoryGroup(BaseModel, table=True):
class CategoryModelType(BaseModel, table=True):
metadata = gisaf_survey
__tablename__ = 'category_model_type'
__table_args__ = gisaf_survey_table_args
name: str | None = Field(default=None, primary_key=True)
class Admin:
@ -47,14 +47,15 @@ class CategoryBase(BaseModel):
domain: ClassVar[str] = 'V'
description: str | None
group: str = Field(sa_type=String(4),
foreign_key="category_group.name", index=True)
foreign_key=gisaf_survey_table_args['schema'] + ".category_group.name",
index=True)
minor_group_1: str = Field(sa_type=String(4), default='----')
minor_group_2: str = Field(sa_type=String(4), default='----')
status: str = Field(sa_type=String(1))
custom: bool | None
auto_import: bool = True
gis_type: str = Field(sa_type=String(50),
foreign_key='category_model_type.name',
foreign_key=gisaf_survey_table_args['schema'] + '.category_model_type.name',
default='Point')
long_name: str | None = Field(sa_type=String(50))
style: str | None = Field(sa_type=TEXT)
@ -105,7 +106,7 @@ class CategoryBase(BaseModel):
class Category(CategoryBase, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
name: str | None = Field(default=None, primary_key=True)
category_group: CategoryGroup = Relationship(back_populates="categories")

93
src/gisaf/models/geo_models_base.py
View file

@ -4,6 +4,7 @@ from datetime import date, datetime
from collections import OrderedDict
from io import BytesIO
from zipfile import ZipFile
from functools import cached_property
import locale
import logging
@ -12,19 +13,16 @@ import geopandas as gpd # type: ignore
import shapely # type: ignore
import pyproj
from sqlmodel import SQLModel, Field, Relationship
from sqlmodel.ext.asyncio.session import AsyncSession
from pydantic import BaseModel
from geoalchemy2.shape import from_shape
from sqlalchemy.dialects.postgresql import BIGINT
from sqlalchemy import BigInteger, Column, MetaData, String, func, and_, text
from sqlmodel import select, Field, Relationship
from sqlmodel.ext.asyncio.session import AsyncSession
from sqlalchemy import BigInteger, MetaData, String, func, and_, text
from sqlalchemy.sql import sqltypes
from sqlalchemy.orm import declared_attr
from psycopg2.extensions import adapt
from geoalchemy2.shape import from_shape
from geoalchemy2.types import Geometry, WKBElement
from shapely import wkb
from shapely import wkb, from_wkb
from shapely.geometry import mapping
from shapely.ops import transform # type: ignore
@ -35,15 +33,15 @@ from shapefile import (
POLYGON, POLYGONZ,
)
from gisaf.database import db_session
from gisaf.config import conf
from gisaf.models.models_base import Model
from gisaf.models.metadata import survey, raw_survey
from gisaf.models.survey import Equipment, Surveyor, Accuracy
from gisaf.models.metadata import (
gisaf_survey_table_args, gisaf_admin_table_args, survey_table_args)
from gisaf.models.misc import Qml
from gisaf.models.category import Category
from gisaf.models.project import Project
# from gisaf.models.survey import Equipment, Surveyor, Accuracy
# from gisaf.models.project import Project
LOCALE_DATE_FORMAT = locale.nl_langinfo(locale.D_FMT)
@ -82,14 +80,14 @@ class BaseSurveyModel(BaseModel):
- projected ('V_*')
"""
id: int | None = Field(sa_type=BigInteger, primary_key=True, default=None)
equip_id: int = Field(foreign_key='equipment.id')
# equipment: Equipment = Relationship()
srvyr_id: int = Field('surveyor.id')
# surveyor: Surveyor = Relationship()
accur_id: int = Field('accuracy.id')
# accuracy: Accuracy = Relationship()
project_id: int = Field('project.id')
# project: Project = Relationship()
equip_id: int = Field(
foreign_key=gisaf_survey_table_args['schema'] + '.equipment.id')
srvyr_id: int = Field(
foreign_key=gisaf_survey_table_args['schema'] + '.surveyor.id')
accur_id: int = Field(
foreign_key=gisaf_survey_table_args['schema'] + '.accuracy.id')
project_id: int = Field(
foreign_key=gisaf_admin_table_args['schema'] + '.project.id')
orig_id: str
date: date
@ -151,7 +149,7 @@ class SurveyModel(BaseSurveyModel):
"""
Base mixin class for defining final (reprojected) survey data, with a status
"""
metadata: ClassVar[MetaData] = survey
__table_args__ = survey_table_args
# status: str = Field(sa_type=String(1))
get_gdf_with_related: ClassVar[bool] = False
@ -182,6 +180,10 @@ class SurveyModel(BaseSurveyModel):
'gisaf_admin_project_skip_columns',
]
@declared_attr
def __tablename__(cls) -> str:
return cls.__name__ # type: nocheck
async def get_survey_info(self):
info = await super(SurveyModel, self).get_survey_info()
if self.srvyr_id:
@ -203,7 +205,7 @@ class SurveyModel(BaseSurveyModel):
@property
def caption(self):
return '{self.category.description} [{self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self)
return '{self.category.description} - {self.category.name} [{self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self)
@classmethod
async def get_popup(cls, df):
@ -425,30 +427,28 @@ class GeoModelNoStatus(Model):
async def get_tags(self):
from gisaf.models.tags import Tags
tags = await Tags.get_df(
where=and_(
Tags.store == self.__class__.get_store_name(),
Tags.ref_id == self.id
),
with_only_columns=['tags']
)
if len(tags) > 0:
return tags.loc[0, 'tags']
else:
return {}
async with db_session() as session:
query = select(Tags.tags).where(Tags.store == self.get_store_name(),
Tags.ref_id == self.id)
tags = await session.exec(query)
return tags.one_or_none() or {}
@property
@cached_property
def shapely_geom(self):
if not hasattr(self, '_shapely_geom'):
if isinstance(self.geom, WKBElement):
bytes = self.geom.data
if bytes:
self._shapely_geom = wkb.loads(bytes)
else:
self._shapely_geom = None
else:
self._shapely_geom = None
return self._shapely_geom
return from_wkb(self.geom.data)
# @property
# def shapely_geom(self):
# if not hasattr(self, '_shapely_geom'):
# if isinstance(self.geom, WKBElement):
# bytes = self.geom.data
# if bytes:
# self._shapely_geom = wkb.loads(bytes)
# else:
# self._shapely_geom = None
# else:
# self._shapely_geom = None
# return self._shapely_geom
def get_bgColor(self):
"""
@ -760,6 +760,7 @@ class GeoModelNoStatus(Model):
def get_attachment_base_dir(cls):
return Path(conf.attachments.base_dir) / cls.get_attachment_dir()
class GeoModel(GeoModelNoStatus):
status: ClassVar[str] = 'E'
"""
@ -1061,7 +1062,7 @@ class RawSurveyBaseModel(BaseSurveyModel, GeoPointModelNoStatus):
"""
Abstract base class for category based raw survey point models
"""
metadata: ClassVar[MetaData] = raw_survey
# metadata: ClassVar[MetaData] = raw_survey
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINTZ', dimension=3,
srid=conf.geo.raw_survey.srid))
status: str = Field(sa_type=String(1))

8
src/gisaf/models/map_bases.py
View file

@ -3,11 +3,11 @@ from typing import Any
from sqlmodel import Field, String, JSON, Relationship
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_map
from gisaf.models.metadata import gisaf_map_table_args
class BaseStyle(Model):
metadata = gisaf_map
__table_args__ = gisaf_map_table_args
__tablename__ = 'map_base_style'
class Admin:
@ -26,7 +26,7 @@ class BaseStyle(Model):
class BaseMap(Model):
metadata = gisaf_map
__table_args__ = gisaf_map_table_args
__tablename__ = 'base_map'
class Admin:
@ -43,7 +43,7 @@ class BaseMap(Model):
class BaseMapLayer(Model):
metadata = gisaf_map
__table_args__ = gisaf_map_table_args
__tablename__ = 'base_map_layer'
class Admin:

14
src/gisaf/models/metadata.py
View file

@ -1,10 +1,8 @@
from sqlmodel import MetaData
from gisaf.config import conf
gisaf = MetaData(schema='gisaf')
gisaf_survey = MetaData(schema='gisaf_survey')
gisaf_admin = MetaData(schema='gisaf_admin')
gisaf_map = MetaData(schema='gisaf_map')
raw_survey = MetaData(schema=conf.survey.db_schema_raw)
survey = MetaData(schema=conf.survey.db_schema)
gisaf_table_args = dict(schema= 'gisaf')
gisaf_survey_table_args = dict(schema='gisaf_survey')
gisaf_admin_table_args = dict(schema='gisaf_admin')
gisaf_map_table_args = dict(schema='gisaf_map')
raw_survey_table_args = dict(schema=conf.survey.db_schema_raw)
survey_table_args = dict(schema=conf.survey.db_schema)

4
src/gisaf/models/misc.py
View file

@ -5,7 +5,7 @@ from pydantic import ConfigDict
from sqlmodel import Field, JSON, Column
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_map
from gisaf.models.metadata import gisaf_map_table_args
logger = logging.getLogger(__name__)
@ -19,7 +19,7 @@ class Qml(Model):
Model for storing qml (QGis style)
"""
model_config = ConfigDict(protected_namespaces=())
metadata = gisaf_map
__table_args__ = gisaf_map_table_args
class Admin:
menu = 'Other'

2
src/gisaf/models/models_base.py
View file

@ -38,7 +38,7 @@ class Model(BaseModel):
if hasattr(cls, '__table__'):
return cls.__table__.fullname
elif hasattr(cls, '__table_args__') and 'schema' in cls.__table_args__:
return f"{cls.__table_args__.schema}.{cls.__tablename__}"
return f"{cls.__table_args__['schema']}.{cls.__tablename__}"
else:
return f'{cls.metadata.schema}.{cls.__tablename__}'

4
src/gisaf/models/project.py
View file

@ -9,10 +9,10 @@ from shapely.geometry import Point
from gisaf.config import conf
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_admin
from gisaf.models.metadata import gisaf_admin_table_args
class Project(Model, table=True):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
class Admin:
menu = 'Other'

6
src/gisaf/models/raw_survey.py
View file

@ -5,10 +5,10 @@ from gisaf.models.models_base import Model
from gisaf.models.geo_models_base import GeoPointMModel, BaseSurveyModel
from gisaf.models.project import Project
from gisaf.models.category import Category
from gisaf.models.metadata import gisaf_survey
from gisaf.models.metadata import gisaf_survey_table_args
class RawSurveyModel(BaseSurveyModel, GeoPointMModel):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
__tablename__ = 'raw_survey'
hidden: ClassVar[bool] = True
@ -94,7 +94,7 @@ class OriginRawPoint(Model):
for each line and polygon shape
Filled when importing shapefiles
"""
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
__tablename__ = 'origin_raw_point'
id: int | None = Field(default=None, primary_key=True)

8
src/gisaf/models/reconcile.py
View file

@ -4,11 +4,11 @@ from sqlalchemy import BigInteger, String
from sqlmodel import Field
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_admin
from gisaf.models.metadata import gisaf_admin_table_args
class Reconciliation(Model):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
class Admin:
menu = 'Other'
@ -21,7 +21,7 @@ class Reconciliation(Model):
class StatusChange(Model):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
__tablename__ = 'status_change'
id: int = Field(primary_key=True, sa_type=BigInteger,
@ -34,7 +34,7 @@ class StatusChange(Model):
class FeatureDeletion(Model):
metadata = gisaf_admin
__table_args__ = gisaf_admin_table_args
__tablename__ = 'feature_deletion'
id: int = Field(BigInteger, primary_key=True,

16
src/gisaf/models/survey.py
View file

@ -3,11 +3,11 @@ from enum import Enum
from sqlmodel import Field, Relationship
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_survey
from gisaf.models.metadata import gisaf_survey_table_args
class Accuracy(Model, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
class Admin:
menu = 'Other'
@ -25,7 +25,7 @@ class Accuracy(Model, table=True):
class Surveyor(Model, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
class Admin:
menu = 'Other'
@ -42,7 +42,7 @@ class Surveyor(Model, table=True):
class Equipment(Model, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
class Admin:
menu = 'Other'
@ -62,17 +62,17 @@ class GeometryType(str, Enum):
line_work = 'Line_work'
class AccuracyEquimentSurveyorMapping(Model, table=True):
metadata = gisaf_survey
__table_args__ = gisaf_survey_table_args
__tablename__ = 'accuracy_equiment_surveyor_mapping'
class Admin:
menu = 'Other'
id: int | None= Field(default=None, primary_key=True)
surveyor_id: int = Field(foreign_key='surveyor.id', index=True)
equipment_id: int = Field(foreign_key='equipment.id', index=True)
surveyor_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.surveyor.id', index=True)
equipment_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.equipment.id', index=True)
geometry_type: GeometryType = Field(default='Point', index=True)
accuracy_id: int = Field(foreign_key='accuracy.id')
accuracy_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.accuracy.id')
surveyor: Surveyor = Relationship()
accuracy: Accuracy = Relationship()
equipment: Equipment = Relationship()

6
src/gisaf/models/tags.py
View file

@ -5,11 +5,11 @@ from sqlalchemy.dialects.postgresql import HSTORE
from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
from pydantic import computed_field
from gisaf.models.metadata import gisaf
from gisaf.models.metadata import gisaf_table_args
from gisaf.models.geo_models_base import GeoPointModel
class Tags(GeoPointModel, table=True):
metadata = gisaf
__table_args__ = gisaf_table_args
hidden: ClassVar[bool] = True
class Admin:
@ -29,7 +29,7 @@ class Tags(GeoPointModel, table=True):
class TagKey(SQLModel, table=True):
metadata = gisaf
__table_args__ = gisaf_table_args
## CREATE TABLE gisaf.tagkey (key VARCHAR(255) primary key);
class Admin:

63
src/gisaf/models/to_migrate.py
View file

@ -3,20 +3,81 @@ from pydantic import BaseModel
class ActionResult(BaseModel):
message: str
class ActionResults(BaseModel):
name: str
message: str
actionResults: list[ActionResult]
class FormField(BaseModel):
name: str
type: str
class ModelAction(BaseModel):
name: str
icon: str
formFields: list[FormField]
class DataProvider(BaseModel):
name: str
values: list[str]
values: list[str]
class InfoItem(BaseModel):
key: str
value: str
class InfoCategory(BaseModel):
name: str
infoItems: list[InfoItem]
class PlotBgShape(BaseModel):
name: str
valueTop: float
valueBottom: float
color: str
class PlotBaseLine(BaseModel):
name: str
value: float
color: str
class PlotParams(BaseModel):
baseLines: list[PlotBaseLine]
bgShape: list[PlotBgShape]
barBase: float
class Attachment(BaseModel):
name: str
path: str
class FeatureInfo(BaseModel):
id: str
itemName: str
geoInfoItems: list[InfoItem] = []
surveyInfoItems: list[InfoItem] = []
infoItems: list[InfoItem] = []
categorizedInfoItems: list[InfoCategory] = []
tags: list[InfoItem] = []
graph: str | None = None
plotParams: PlotParams | None = None
files: list[Attachment] = []
images: list[Attachment] = []
externalRecordUrl: str | None = None
class MapboxPaint(BaseModel):
...
class MapboxLayout(BaseModel):
...