philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Remove relative imports

Browse source 
Fix primary keys (optional)
Add baskets, importers, plugins, reactor
Add fake replacement fro graphql defs (to_migrate)
Add typing marker (py.typed)
This commit is contained in:
phil 2023-12-25 15:50:45 +05:30
parent a974eea3d3
commit 741050db89
35 changed files with 2097 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

129
src/gisaf/models/admin.py Normal file
View file

@ -0,0 +1,129 @@
import re
from datetime import datetime
from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
import pandas as pd
# from graphene import ObjectType, Int, String, DateTime, List
from gisaf.models.models_base import Model
from gisaf.models.survey import Surveyor, Equipment
from gisaf.models.project import Project
re_file_import_record_date_expr = '^(\S+)-(\d\d\d\d)-(\d\d)-(\d\d).*$'
re_file_import_record_date_combined_expr = '^\S+-(\d\d\d\d-\d\d-\d\d).*$'
re_file_import_record_date = re.compile(re_file_import_record_date_expr)
class BadSurveyFileName(Exception):
pass
def get_file_import_date(record):
"""
Utility function that returns the date of survey from the file name,
if it matches the convention for CSV survey files.
Return exception otherwise.
"""
fname_search = re_file_import_record_date.search(record['name'])
if not fname_search:
raise BadSurveyFileName(
'The file name is not OK '
'(format should be: "PPP-DESCRIPTION-YYYY-MM-DD", '
'PPP being the project name, DESCRITION is optional and discarded)'
)
return datetime.date(day=int(fname_search.group(4)),
month=int(fname_search.group(3)),
year=int(fname_search.group(2)))
class FileImport(Model):
"""
Files to import or imported in the DB.
Give either url or path.
"""
__tablename__ = 'file_import'
__table_args__ = {'schema' : 'gisaf_admin'}
id: int | None = Field(default=None, primary_key=True)
url: str
## TODO: Deprecate FileImport.path, in favour of dir + name
path: str
dir: str
name: str
md5: str
time: datetime
comment: str
status: str
store: str
basket: str
project_id: int = Field(foreign_key='project.id')
# ALTER TABLE gisaf_admin.file_import add column project_id INT REFERENCES gisaf_admin.project;
surveyor_id: int = Field(foreign_key='surveyor.id')
# ALTER TABLE gisaf_admin.file_import add column surveyor_id INT REFERENCES gisaf_survey.surveyor;
equipment_id: int = Field(foreign_key='equipment.id')
# ALTER TABLE gisaf_admin.file_import add column equipment_id INT REFERENCES gisaf_survey.equipment;
def __str__(self):
return f'{self.path:s} for project id {self.project_id}'
def __repr__(self):
return f'<gisaf.misc.FileImport (gisaf_admin.file_import) {self.path}>'
@classmethod
def dyn_join_with(cls):
return {
'project': Project,
'surveyor': Surveyor,
'equipment': Equipment,
}
def set_import_time(self):
self.time = datetime.now()
db.session.commit()
@classmethod
async def get_df(cls, *args, **kwargs):
"""
Add a column 'date' based on the file name
"""
df = await super().get_df(*args, **kwargs)
dates = df['name'].str.extract(re_file_import_record_date_combined_expr)
df['date'] = pd.to_datetime(dates[0], format='%Y-%m-%d')
return df
#def get_parent_dir(self):
# split_path = self.path.split(os_path.sep)
# if len(split_path) == 1:
# return None
# else:
# return split_path[-2]
#def get_absolute_path(self):
# return os_path.join(conf.shapefiles_import_path, self.path)
#def exists_in_file_system(self):
# """
# Check if the file exists
# :return:
# """
# return os_path.exists(self.get_absolute_path())
class FeatureImportData(Model):
"""
Keep track of imported data, typically from shapefiles
"""
__tablename__ = 'feature_import_data'
__table_args__ = {'schema' : 'gisaf_admin'}
id: int | None = Field(default=None, primary_key=True)
store: str = Field(index=True)
feature_id: int = Field(index=True)
orig_id: int
time: datetime
origin: str
file_path: str
file_md5: str

4
src/gisaf/models/authentication.py
View file

@ -1,10 +1,10 @@
from sqlmodel import Field, SQLModel, MetaData, Relationship
from .metadata import gisaf_admin
from gisaf.models.metadata import gisaf_admin
class UserRoleLink(SQLModel, table=True):
metadata = gisaf_admin
__tablename__: str = 'roles_users'
__tablename__ = 'roles_users'
user_id: int | None = Field(
default=None, foreign_key="user.id", primary_key=True
)

5
src/gisaf/models/bootstrap.py
View file

@ -1,6 +1,7 @@
from pydantic import BaseModel
from ..config import conf, Map, Measures, Geo
from .authentication import UserRead
from gisaf.config import conf, Map, Measures, Geo
from gisaf.models.authentication import UserRead
class Proj(BaseModel):
srid: str

16
src/gisaf/models/category.py
View file

@ -1,10 +1,11 @@
from typing import Any, ClassVar
from sqlalchemy import String
from pydantic import computed_field, ConfigDict
from sqlmodel import Field, Relationship, SQLModel, JSON, TEXT, Column, select
from sqlmodel import Field, Relationship, SQLModel, JSON, TEXT, select
from .metadata import gisaf_survey
from ..database import db_session, pandas_query
from gisaf.models.metadata import gisaf_survey
from gisaf.database import db_session, pandas_query
mapbox_type_mapping = {
'Point': 'symbol',
@ -23,8 +24,7 @@ class BaseModel(SQLModel):
class CategoryGroup(BaseModel, table=True):
metadata = gisaf_survey
__tablename__ = 'category_group'
name: str | None = Field(min_length=4, max_length=4,
default=None, primary_key=True)
name: str | None = Field(sa_type=String(4), default=None, primary_key=True)
major: str
long_name: str
categories: list['Category'] = Relationship(back_populates='category_group')
@ -37,7 +37,7 @@ class CategoryGroup(BaseModel, table=True):
class CategoryModelType(BaseModel, table=True):
metadata = gisaf_survey
__tablename__ = 'category_model_type'
name: str = Field(default=None, primary_key=True)
name: str | None = Field(default=None, primary_key=True)
class Admin:
menu = 'Other'
@ -45,7 +45,7 @@ class CategoryModelType(BaseModel, table=True):
class CategoryBase(BaseModel):
model_config = ConfigDict(protected_namespaces=())
model_config = ConfigDict(protected_namespaces=()) # type: ignore
class Admin:
menu = 'Other'
flask_admin_model_view = 'CategoryModelView'
@ -113,7 +113,7 @@ class CategoryBase(BaseModel):
class Category(CategoryBase, table=True):
metadata = gisaf_survey
name: str = Field(default=None, primary_key=True)
name: str | None = Field(default=None, primary_key=True)
category_group: CategoryGroup = Relationship(back_populates="categories")

43
src/gisaf/models/geo_models_base.py
View file

@ -6,13 +6,10 @@ from io import BytesIO
from zipfile import ZipFile
import locale
import logging
from json import dumps
import numpy as np
import pandas as pd
import geopandas as gpd
import geopandas as gpd # type: ignore
import shapely
import shapely # type: ignore
import pyproj
from sqlmodel import SQLModel, Field
@ -29,24 +26,24 @@ from geoalchemy2.types import Geometry, WKBElement
from shapely import wkb
from shapely.geometry import mapping
from shapely.ops import transform
from shapely.ops import transform # type: ignore
from shapefile import (Writer as ShapeFileWriter,
POINT, POINTZ,
POLYLINE, POLYLINEZ,
POLYGON, POLYGONZ,
)
from shapefile import (
Writer as ShapeFileWriter, # type: ignore
POINT, POINTZ,
POLYLINE, POLYLINEZ,
POLYGON, POLYGONZ,
)
from ..database import db_session
from ..config import conf
from .models_base import Model
from ..models.metadata import survey, raw_survey
from ..utils import upsert_df
from .survey import Equipment, Surveyor, Accuracy
from .misc import Qml
from .category import Category
from .project import Project
from gisaf.database import db_session
from gisaf.config import conf
from gisaf.models.models_base import Model
from gisaf.models.metadata import survey, raw_survey
from gisaf.models.survey import Equipment, Surveyor, Accuracy
from gisaf.models.misc import Qml
from gisaf.models.category import Category
from gisaf.models.project import Project
LOCALE_DATE_FORMAT = locale.nl_langinfo(locale.D_FMT)
@ -84,7 +81,7 @@ class BaseSurveyModel(BaseModel):
- raw survey (RAW_V_*')
- projected ('V_*')
"""
id: int = Field(sa_type=BigInteger, primary_key=True, default=None)
id: int | None = Field(sa_type=BigInteger, primary_key=True, default=None)
equip_id: int = Field(foreign_key='equipment.id')
srvyr_id: int = Field('surveyor.id')
accur_id: int = Field('accuracy.id')
@ -368,7 +365,7 @@ class GeoModel(Model):
"""
return OrderedDict()
async def get_info(self):
async def get_info(self) -> dict[str, str]:
"""
Model specific info
"""
@ -1088,7 +1085,7 @@ class PlottableModel(Model):
#__abstract__ = True
float_format: ClassVar[str] = '%.1f'
values: dict[Any, Any] = {}
values: ClassVar[list[dict[str, str]]] = []
@classmethod
async def get_as_dataframe(cls, model_id=None, where=None, **kwargs):

10
src/gisaf/models/map_bases.py
View file

@ -2,8 +2,8 @@ from typing import Any
from sqlmodel import Field, String, JSON, Column
from .models_base import Model
from .metadata import gisaf_map
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_map
class BaseStyle(Model):
@ -14,7 +14,7 @@ class BaseStyle(Model):
menu = 'Other'
flask_admin_model_view = 'MapBaseStyleModelView'
id: int = Field(primary_key=True)
id: int | None = Field(primary_key=True, default=None)
name: str
style: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True))
mbtiles: str = Field(sa_type=String(50))
@ -32,7 +32,7 @@ class BaseMap(Model):
class Admin:
menu = 'Other'
id: int = Field(primary_key=True)
id: int | None = Field(primary_key=True, default=None)
name: str
def __repr__(self):
@ -49,7 +49,7 @@ class BaseMapLayer(Model):
class Admin:
menu = 'Other'
id: int = Field(primary_key=True)
id: int | None = Field(primary_key=True, default=None)
base_map_id: int = Field(foreign_key='base_map.id', index=True)
store: str = Field(sa_type=String(100))

2
src/gisaf/models/metadata.py
View file

@ -1,6 +1,6 @@
from sqlmodel import MetaData
from ..config import conf
from gisaf.config import conf
gisaf = MetaData(schema='gisaf')
gisaf_survey = MetaData(schema='gisaf_survey')

6
src/gisaf/models/misc.py
View file

@ -4,8 +4,8 @@ from pydantic import ConfigDict
from sqlmodel import Field, JSON, Column
from .models_base import Model
from .metadata import gisaf_map
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_map
logger = logging.getLogger(__name__)
@ -25,7 +25,7 @@ class Qml(Model):
menu = 'Other'
flask_admin_model_view = 'QmlModelView'
model_name: str = Field(default=None, primary_key=True)
model_name: str | None = Field(default=None, primary_key=True)
qml: str
attr: str
style: str

8
src/gisaf/models/project.py
View file

@ -7,9 +7,9 @@ from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
import pyproj
from shapely.geometry import Point
from ..config import conf
from .models_base import Model
from .metadata import gisaf_admin
from gisaf.config import conf
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_admin
class Project(Model):
metadata = gisaf_admin
@ -18,7 +18,7 @@ class Project(Model):
menu = 'Other'
flask_admin_model_view = 'ProjectModelView'
id: int = Field(default=None, primary_key=True)
id: int | None = Field(default=None, primary_key=True)
name: str
contact_person: str
site: str

14
src/gisaf/models/raw_survey.py
View file

@ -1,18 +1,18 @@
from typing import ClassVar
from sqlmodel import Field, BigInteger
from .models_base import Model
from .geo_models_base import GeoPointMModel, BaseSurveyModel
from .project import Project
from .category import Category
from .metadata import gisaf_survey
from gisaf.models.models_base import Model
from gisaf.models.geo_models_base import GeoPointMModel, BaseSurveyModel
from gisaf.models.project import Project
from gisaf.models.category import Category
from gisaf.models.metadata import gisaf_survey
class RawSurveyModel(BaseSurveyModel, GeoPointMModel):
metadata = gisaf_survey
__tablename__ = 'raw_survey'
hidden: ClassVar[bool] = True
id: int = Field(default=None, primary_key=True)
id: int | None = Field(default=None, primary_key=True)
project_id: int | None = Field(foreign_key='project.id')
category: str = Field(foreign_key='category.name')
in_menu: bool = False
@ -87,7 +87,7 @@ class OriginRawPoint(Model):
metadata = gisaf_survey
__tablename__ = 'origin_raw_point'
id: int = Field(default=None, primary_key=True)
id: int | None = Field(default=None, primary_key=True)
shape_table: str = Field(index=True)
shape_id: int = Field(index=True)
raw_point_id: int = Field(sa_type=BigInteger())

9
src/gisaf/models/reconcile.py
View file

@ -1,9 +1,10 @@
from datetime import datetime
from sqlalchemy import BigInteger
from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column, String
from .models_base import Model
from .metadata import gisaf_admin
from sqlalchemy import BigInteger, String
from sqlmodel import Field
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_admin
class Reconciliation(Model):

2
src/gisaf/models/store.py
View file

@ -1,6 +1,6 @@
from typing import Any
from pydantic import BaseModel
from .geo_models_base import GeoModel, RawSurveyBaseModel, GeoPointSurveyModel
from gisaf.models.geo_models_base import GeoModel, RawSurveyBaseModel, GeoPointSurveyModel
class MapLibreStyle(BaseModel):

14
src/gisaf/models/survey.py
View file

@ -1,9 +1,9 @@
from enum import Enum
from sqlmodel import Field, SQLModel
from sqlmodel import Field
from .models_base import Model
from .metadata import gisaf_survey
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_survey
class Accuracy(Model):
@ -13,7 +13,7 @@ class Accuracy(Model):
menu = 'Other'
flask_admin_model_view = 'MyModelViewWithPrimaryKey'
id: int = Field(default=None, primary_key=True)
id: int | None = Field(default=None, primary_key=True)
name: str
accuracy: float
@ -31,7 +31,7 @@ class Surveyor(Model):
menu = 'Other'
flask_admin_model_view = 'MyModelViewWithPrimaryKey'
id: int = Field(default=None, primary_key=True)
id: int | None = Field(default=None, primary_key=True)
name: str
def __str__(self):
@ -48,7 +48,7 @@ class Equipment(Model):
menu = 'Other'
flask_admin_model_view = 'MyModelViewWithPrimaryKey'
id: int = Field(default=None, primary_key=True)
id: int | None = Field(default=None, primary_key=True)
name: str
def __str__(self):
@ -68,7 +68,7 @@ class AccuracyEquimentSurveyorMapping(Model):
class Admin:
menu = 'Other'
id: int = Field(default=None, primary_key=True)
id: int | None= Field(default=None, primary_key=True)
surveyor_id: int = Field(foreign_key='surveyor.id', index=True)
equipment_id: int = Field(foreign_key='equipment.id', index=True)
geometry_type: GeometryType = Field(default='Point', index=True)

8
src/gisaf/models/tags.py
View file

@ -5,8 +5,8 @@ from sqlalchemy.dialects.postgresql import HSTORE
from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
from pydantic import computed_field
from .metadata import gisaf
from .geo_models_base import GeoPointModel
from gisaf.models.metadata import gisaf
from gisaf.models.geo_models_base import GeoPointModel
class Tags(GeoPointModel, table=True):
metadata = gisaf
@ -16,7 +16,7 @@ class Tags(GeoPointModel, table=True):
menu = 'Other'
flask_admin_model_view = 'TagModelView'
id: int | None = Field(primary_key=True)
id: int | None = Field(primary_key=True, default=None)
store: str = Field(index=True)
ref_id: int = Field(index=True, sa_type=BigInteger)
tags: dict = Field(sa_type=MutableDict.as_mutable(HSTORE))
@ -36,7 +36,7 @@ class TagKey(SQLModel, table=True):
menu = 'Other'
flask_admin_model_view = 'TagKeyModelView'
id: str | None = Field(primary_key=True)
id: str | None = Field(default=None, primary_key=True)
def __str__(self):
return self.key

18
src/gisaf/models/to_migrate.py Normal file
View file

@ -0,0 +1,18 @@
from pydantic import BaseModel
class ActionResult(BaseModel):
message: str
class ActionResults(BaseModel):
name: str
message: str
actionResults: list[ActionResult]
class FormField(BaseModel):
name: str
type: str
class ModelAction(BaseModel):
name: str
icon: str
formFields: list[FormField]