From 71cb4916175eb776bb7b51db4995ba577d3cf795 Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 5 Jan 2024 01:50:32 +0530 Subject: [PATCH] Helper class for db schema --- src/gisaf/models/admin.py | 6 +++--- src/gisaf/models/authentication.py | 12 ++++++------ src/gisaf/models/category.py | 12 ++++++------ src/gisaf/models/geo_models_base.py | 19 +++++++------------ src/gisaf/models/map_bases.py | 8 ++++---- src/gisaf/models/metadata.py | 24 ++++++++++++++++++------ src/gisaf/models/misc.py | 4 ++-- src/gisaf/models/project.py | 4 ++-- src/gisaf/models/raw_survey.py | 6 +++--- src/gisaf/models/reconcile.py | 8 ++++---- src/gisaf/models/survey.py | 16 ++++++++-------- src/gisaf/models/tags.py | 6 +++--- src/gisaf/registry.py | 12 ++++-------- 13 files changed, 70 insertions(+), 67 deletions(-) diff --git a/src/gisaf/models/admin.py b/src/gisaf/models/admin.py index 6c3705f..3780730 100644 --- a/src/gisaf/models/admin.py +++ b/src/gisaf/models/admin.py @@ -9,7 +9,7 @@ import pandas as pd from gisaf.models.models_base import Model from gisaf.models.survey import Surveyor, Equipment from gisaf.models.project import Project -from gisaf.models.metadata import gisaf_admin_table_args +from gisaf.models.metadata import gisaf_admin re_file_import_record_date_expr = '^(\S+)-(\d\d\d\d)-(\d\d)-(\d\d).*$' @@ -45,7 +45,7 @@ class FileImport(Model): Give either url or path. """ __tablename__ = 'file_import' - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args id: int | None = Field(default=None, primary_key=True) url: str @@ -117,7 +117,7 @@ class FeatureImportData(Model): Keep track of imported data, typically from shapefiles """ __tablename__ = 'feature_import_data' - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args id: int | None = Field(default=None, primary_key=True) store: str = Field(index=True) diff --git a/src/gisaf/models/authentication.py b/src/gisaf/models/authentication.py index efd41ce..7d4cbea 100644 --- a/src/gisaf/models/authentication.py +++ b/src/gisaf/models/authentication.py @@ -1,19 +1,19 @@ from sqlmodel import Field, SQLModel, Relationship -from gisaf.models.metadata import gisaf_admin_table_args +from gisaf.models.metadata import gisaf_admin class UserRoleLink(SQLModel, table=True): __tablename__ = 'roles_users' - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args user_id: int | None = Field( default=None, - foreign_key=gisaf_admin_table_args['schema'] + '.user.id', + foreign_key=gisaf_admin.table('user.id'), primary_key=True ) role_id: int | None = Field( default=None, - foreign_key=gisaf_admin_table_args['schema'] + '.role.id', + foreign_key=gisaf_admin.table('role.id'), primary_key=True ) @@ -25,7 +25,7 @@ class UserBase(SQLModel): class User(UserBase, table=True): - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args id: int | None = Field(default=None, primary_key=True) roles: list["Role"] = Relationship(back_populates="users", link_model=UserRoleLink) @@ -45,7 +45,7 @@ class RoleWithDescription(RoleBase): description: str | None class Role(RoleWithDescription, table=True): - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args id: int | None = Field(default=None, primary_key=True) users: list[User] = Relationship(back_populates="roles", link_model=UserRoleLink) diff --git a/src/gisaf/models/category.py b/src/gisaf/models/category.py index f8c1b42..5fa7a76 100644 --- a/src/gisaf/models/category.py +++ b/src/gisaf/models/category.py @@ -5,7 +5,7 @@ from pydantic import computed_field, ConfigDict from sqlmodel import Field, Relationship, JSON, TEXT from gisaf.database import BaseModel -from gisaf.models.metadata import gisaf_survey_table_args +from gisaf.models.metadata import gisaf_survey mapbox_type_mapping = { 'Point': 'symbol', @@ -16,7 +16,7 @@ mapbox_type_mapping = { class CategoryGroup(BaseModel, table=True): __tablename__ = 'category_group' - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args name: str | None = Field(sa_type=String(4), default=None, primary_key=True) major: str long_name: str @@ -29,7 +29,7 @@ class CategoryGroup(BaseModel, table=True): class CategoryModelType(BaseModel, table=True): __tablename__ = 'category_model_type' - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args name: str | None = Field(default=None, primary_key=True) class Admin: @@ -47,7 +47,7 @@ class CategoryBase(BaseModel): domain: ClassVar[str] = 'V' description: str | None group: str = Field(sa_type=String(4), - foreign_key=gisaf_survey_table_args['schema'] + ".category_group.name", + foreign_key=gisaf_survey.table('category_group.name'), index=True) minor_group_1: str = Field(sa_type=String(4), default='----') minor_group_2: str = Field(sa_type=String(4), default='----') @@ -55,7 +55,7 @@ class CategoryBase(BaseModel): custom: bool | None auto_import: bool = True gis_type: str = Field(sa_type=String(50), - foreign_key=gisaf_survey_table_args['schema'] + '.category_model_type.name', + foreign_key=gisaf_survey.table('category_model_type.name'), default='Point') long_name: str | None = Field(sa_type=String(50)) style: str | None = Field(sa_type=TEXT) @@ -106,7 +106,7 @@ class CategoryBase(BaseModel): class Category(CategoryBase, table=True): - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args name: str | None = Field(default=None, primary_key=True) category_group: CategoryGroup = Relationship(back_populates="categories") diff --git a/src/gisaf/models/geo_models_base.py b/src/gisaf/models/geo_models_base.py index ea88a66..f7e5529 100644 --- a/src/gisaf/models/geo_models_base.py +++ b/src/gisaf/models/geo_models_base.py @@ -36,8 +36,7 @@ from shapefile import ( from gisaf.database import db_session from gisaf.config import conf from gisaf.models.models_base import Model -from gisaf.models.metadata import ( - gisaf_survey_table_args, gisaf_admin_table_args, survey_table_args) +from gisaf.models.metadata import gisaf_survey, gisaf_admin, survey from gisaf.models.misc import Qml from gisaf.models.category import Category # from gisaf.models.survey import Equipment, Surveyor, Accuracy @@ -80,14 +79,10 @@ class BaseSurveyModel(BaseModel): - projected ('V_*') """ id: int | None = Field(sa_type=BigInteger, primary_key=True, default=None) - equip_id: int = Field( - foreign_key=gisaf_survey_table_args['schema'] + '.equipment.id') - srvyr_id: int = Field( - foreign_key=gisaf_survey_table_args['schema'] + '.surveyor.id') - accur_id: int = Field( - foreign_key=gisaf_survey_table_args['schema'] + '.accuracy.id') - project_id: int = Field( - foreign_key=gisaf_admin_table_args['schema'] + '.project.id') + equip_id: int = Field(foreign_key=gisaf_survey.table('equipment.id')) + srvyr_id: int = Field(foreign_key=gisaf_survey.table('surveyor.id')) + accur_id: int = Field(foreign_key=gisaf_survey.table('accuracy.id')) + project_id: int = Field(foreign_key=gisaf_admin.table('project.id')) orig_id: str date: date @@ -149,7 +144,7 @@ class SurveyModel(BaseSurveyModel): """ Base mixin class for defining final (reprojected) survey data, with a status """ - __table_args__ = survey_table_args + __table_args__ = survey.table_args # status: str = Field(sa_type=String(1)) get_gdf_with_related: ClassVar[bool] = False @@ -205,7 +200,7 @@ class SurveyModel(BaseSurveyModel): @property def caption(self): - return '{self.category.description} - {self.category.name} [{self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self) + return '{self.category.description} [{self.category.name}: {self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self) @classmethod async def get_popup(cls, df): diff --git a/src/gisaf/models/map_bases.py b/src/gisaf/models/map_bases.py index 04116eb..cee33c6 100644 --- a/src/gisaf/models/map_bases.py +++ b/src/gisaf/models/map_bases.py @@ -3,11 +3,11 @@ from typing import Any from sqlmodel import Field, String, JSON, Relationship from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_map_table_args +from gisaf.models.metadata import gisaf_map class BaseStyle(Model): - __table_args__ = gisaf_map_table_args + __table_args__ = gisaf_map.table_args __tablename__ = 'map_base_style' class Admin: @@ -26,7 +26,7 @@ class BaseStyle(Model): class BaseMap(Model): - __table_args__ = gisaf_map_table_args + __table_args__ = gisaf_map.table_args __tablename__ = 'base_map' class Admin: @@ -43,7 +43,7 @@ class BaseMap(Model): class BaseMapLayer(Model): - __table_args__ = gisaf_map_table_args + __table_args__ = gisaf_map.table_args __tablename__ = 'base_map_layer' class Admin: diff --git a/src/gisaf/models/metadata.py b/src/gisaf/models/metadata.py index 8a67182..4c7efa5 100644 --- a/src/gisaf/models/metadata.py +++ b/src/gisaf/models/metadata.py @@ -1,8 +1,20 @@ from gisaf.config import conf -gisaf_table_args = dict(schema= 'gisaf') -gisaf_survey_table_args = dict(schema='gisaf_survey') -gisaf_admin_table_args = dict(schema='gisaf_admin') -gisaf_map_table_args = dict(schema='gisaf_map') -raw_survey_table_args = dict(schema=conf.survey.db_schema_raw) -survey_table_args = dict(schema=conf.survey.db_schema) \ No newline at end of file +class Schema: + name: str + table_args: dict[str, str] + + def __init__(self, name: str): + self.name = name + self.table_args = dict(schema=name) + + def table(self, name: str) -> str: + return f'{self.name}.{name}' + + +gisaf = Schema(name='gisaf') +gisaf_survey = Schema(name='gisaf_survey') +gisaf_admin = Schema(name='gisaf_admin') +gisaf_map = Schema(name='gisaf_map') +raw_survey = Schema(name=conf.survey.db_schema_raw) +survey = Schema(name=conf.survey.db_schema) \ No newline at end of file diff --git a/src/gisaf/models/misc.py b/src/gisaf/models/misc.py index 3d588a6..7a8b4c0 100644 --- a/src/gisaf/models/misc.py +++ b/src/gisaf/models/misc.py @@ -5,7 +5,7 @@ from pydantic import ConfigDict from sqlmodel import Field, JSON, Column from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_map_table_args +from gisaf.models.metadata import gisaf_map logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ class Qml(Model): Model for storing qml (QGis style) """ model_config = ConfigDict(protected_namespaces=()) - __table_args__ = gisaf_map_table_args + __table_args__ = gisaf_map.table_args class Admin: menu = 'Other' diff --git a/src/gisaf/models/project.py b/src/gisaf/models/project.py index 811809b..a2673af 100644 --- a/src/gisaf/models/project.py +++ b/src/gisaf/models/project.py @@ -9,10 +9,10 @@ from shapely.geometry import Point from gisaf.config import conf from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_admin_table_args +from gisaf.models.metadata import gisaf_admin class Project(Model, table=True): - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args class Admin: menu = 'Other' diff --git a/src/gisaf/models/raw_survey.py b/src/gisaf/models/raw_survey.py index 318b09a..9e167f0 100644 --- a/src/gisaf/models/raw_survey.py +++ b/src/gisaf/models/raw_survey.py @@ -5,10 +5,10 @@ from gisaf.models.models_base import Model from gisaf.models.geo_models_base import GeoPointMModel, BaseSurveyModel from gisaf.models.project import Project from gisaf.models.category import Category -from gisaf.models.metadata import gisaf_survey_table_args +from gisaf.models.metadata import gisaf_survey class RawSurveyModel(BaseSurveyModel, GeoPointMModel): - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args __tablename__ = 'raw_survey' hidden: ClassVar[bool] = True @@ -94,7 +94,7 @@ class OriginRawPoint(Model): for each line and polygon shape Filled when importing shapefiles """ - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args __tablename__ = 'origin_raw_point' id: int | None = Field(default=None, primary_key=True) diff --git a/src/gisaf/models/reconcile.py b/src/gisaf/models/reconcile.py index 4ce2403..b15eba7 100644 --- a/src/gisaf/models/reconcile.py +++ b/src/gisaf/models/reconcile.py @@ -4,11 +4,11 @@ from sqlalchemy import BigInteger, String from sqlmodel import Field from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_admin_table_args +from gisaf.models.metadata import gisaf_admin class Reconciliation(Model): - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args class Admin: menu = 'Other' @@ -21,7 +21,7 @@ class Reconciliation(Model): class StatusChange(Model): - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args __tablename__ = 'status_change' id: int = Field(primary_key=True, sa_type=BigInteger, @@ -34,7 +34,7 @@ class StatusChange(Model): class FeatureDeletion(Model): - __table_args__ = gisaf_admin_table_args + __table_args__ = gisaf_admin.table_args __tablename__ = 'feature_deletion' id: int = Field(BigInteger, primary_key=True, diff --git a/src/gisaf/models/survey.py b/src/gisaf/models/survey.py index 7a3e023..7e98c0e 100644 --- a/src/gisaf/models/survey.py +++ b/src/gisaf/models/survey.py @@ -3,11 +3,11 @@ from enum import Enum from sqlmodel import Field, Relationship from gisaf.models.models_base import Model -from gisaf.models.metadata import gisaf_survey_table_args +from gisaf.models.metadata import gisaf_survey class Accuracy(Model, table=True): - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args class Admin: menu = 'Other' @@ -25,7 +25,7 @@ class Accuracy(Model, table=True): class Surveyor(Model, table=True): - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args class Admin: menu = 'Other' @@ -42,7 +42,7 @@ class Surveyor(Model, table=True): class Equipment(Model, table=True): - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args class Admin: menu = 'Other' @@ -62,17 +62,17 @@ class GeometryType(str, Enum): line_work = 'Line_work' class AccuracyEquimentSurveyorMapping(Model, table=True): - __table_args__ = gisaf_survey_table_args + __table_args__ = gisaf_survey.table_args __tablename__ = 'accuracy_equiment_surveyor_mapping' class Admin: menu = 'Other' id: int | None= Field(default=None, primary_key=True) - surveyor_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.surveyor.id', index=True) - equipment_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.equipment.id', index=True) + surveyor_id: int = Field(foreign_key=gisaf_survey.table('surveyor.id'), index=True) + equipment_id: int = Field(foreign_key=gisaf_survey.table('equipment.id'), index=True) geometry_type: GeometryType = Field(default='Point', index=True) - accuracy_id: int = Field(foreign_key=gisaf_survey_table_args['schema'] + '.accuracy.id') + accuracy_id: int = Field(foreign_key=gisaf_survey.table('accuracy.id')) surveyor: Surveyor = Relationship() accuracy: Accuracy = Relationship() equipment: Equipment = Relationship() diff --git a/src/gisaf/models/tags.py b/src/gisaf/models/tags.py index 8f3b586..5d0ef5a 100644 --- a/src/gisaf/models/tags.py +++ b/src/gisaf/models/tags.py @@ -5,11 +5,11 @@ from sqlalchemy.dialects.postgresql import HSTORE from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column from pydantic import computed_field -from gisaf.models.metadata import gisaf_table_args +from gisaf.models.metadata import gisaf from gisaf.models.geo_models_base import GeoPointModel class Tags(GeoPointModel, table=True): - __table_args__ = gisaf_table_args + __table_args__ = gisaf.table_args hidden: ClassVar[bool] = True class Admin: @@ -29,7 +29,7 @@ class Tags(GeoPointModel, table=True): class TagKey(SQLModel, table=True): - __table_args__ = gisaf_table_args + __table_args__ = gisaf.table_args ## CREATE TABLE gisaf.tagkey (key VARCHAR(255) primary key); class Admin: diff --git a/src/gisaf/registry.py b/src/gisaf/registry.py index 0e8ff9c..e6d3b68 100644 --- a/src/gisaf/registry.py +++ b/src/gisaf/registry.py @@ -36,8 +36,7 @@ from gisaf.utils import ToMigrate from gisaf.models.category import Category, CategoryGroup from gisaf.database import db_session from gisaf import models -from gisaf.models.metadata import ( - gisaf_survey_table_args, raw_survey_table_args, survey_table_args) +from gisaf.models.metadata import gisaf_survey, raw_survey, survey logger = logging.getLogger(__name__) @@ -142,8 +141,8 @@ class ModelRegistry: ## Use pydantic create_model, supported by SQLModel ## See https://github.com/tiangolo/sqlmodel/issues/377 - store_name = f"{survey_table_args['schema']}.{category.table_name}" - raw_store_name = f"{raw_survey_table_args['schema']}.RAW_{category.table_name}" + store_name = survey.table(category.table_name) + raw_store_name = raw_survey.table(f'RAW_{category.table_name}') raw_survey_field_definitions = { ## FIXME: RawSurveyBaseModel.category should be a Category, not category.name 'category_name': (ClassVar[str], category.name), @@ -400,10 +399,7 @@ class ModelRegistry: fragments = ['V', category.group, category.minor_group_1] if category.minor_group_2 != '----': fragments.append(category.minor_group_2) - return '.'.join([ - survey_table_args['schema'], - '_'.join(fragments) - ]) + return '.'.join([survey.name, '_'.join(fragments)]) self.categories = await Category.get_df() self.categories['title'] = self.categories.long_name.fillna(self.categories.description)