Migrate joins to sqlalchemy's query options
Use native pandas read_sql_query and geopandas from_postgis Fix definiiton of status in models Fix table names Fix category fields
This commit is contained in:
parent
956147aea8
commit
75bedb3e91
8 changed files with 236 additions and 190 deletions
src/gisaf/models
|
|
@ -12,7 +12,7 @@ import geopandas as gpd # type: ignore
|
|||
import shapely # type: ignore
|
||||
import pyproj
|
||||
|
||||
from sqlmodel import SQLModel, Field
|
||||
from sqlmodel import SQLModel, Field, Relationship
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
|
@ -83,9 +83,13 @@ class BaseSurveyModel(BaseModel):
|
|||
"""
|
||||
id: int | None = Field(sa_type=BigInteger, primary_key=True, default=None)
|
||||
equip_id: int = Field(foreign_key='equipment.id')
|
||||
# equipment: Equipment = Relationship()
|
||||
srvyr_id: int = Field('surveyor.id')
|
||||
# surveyor: Surveyor = Relationship()
|
||||
accur_id: int = Field('accuracy.id')
|
||||
# accuracy: Accuracy = Relationship()
|
||||
project_id: int = Field('project.id')
|
||||
# project: Project = Relationship()
|
||||
|
||||
orig_id: str
|
||||
date: date
|
||||
|
|
@ -140,7 +144,7 @@ class SurveyModel(BaseSurveyModel):
|
|||
Base mixin class for defining final (reprojected) survey data, with a status
|
||||
"""
|
||||
metadata: ClassVar[MetaData] = survey
|
||||
status: ClassVar[str] = Field(sa_type=String(1))
|
||||
# status: str = Field(sa_type=String(1))
|
||||
|
||||
get_gdf_with_related: ClassVar[bool] = False
|
||||
|
||||
|
|
@ -251,7 +255,7 @@ class SurveyModel(BaseSurveyModel):
|
|||
query = sql_query_for_geojson.format(
|
||||
model=cls,
|
||||
category=category,
|
||||
schema=cls.metadata.schema,
|
||||
schema=cls.__table__.schema,
|
||||
#table=cls.__tablename__,
|
||||
# FIXME: should be __tablename__, but see SQLModel.__tablename__ which use lower(__name__)
|
||||
table=cls.__name__,
|
||||
|
|
@ -282,12 +286,10 @@ class SurveyModel(BaseSurveyModel):
|
|||
]
|
||||
|
||||
|
||||
class GeoModel(Model):
|
||||
class GeoModelNoStatus(Model):
|
||||
"""
|
||||
Base class for all geo models
|
||||
"""
|
||||
#__abstract__ = True
|
||||
|
||||
id: int | None = Field(default=None, primary_key=True)
|
||||
|
||||
description: ClassVar[str] = ''
|
||||
|
|
@ -337,11 +339,6 @@ class GeoModel(Model):
|
|||
Style for the model, used in the map, etc
|
||||
"""
|
||||
|
||||
# status: ClassVar[str] = 'E'
|
||||
# """
|
||||
# Status (ISO layers definition) of the layer. E -> Existing.
|
||||
# """
|
||||
|
||||
_join_with: ClassVar[dict[str, Any]] = {
|
||||
}
|
||||
"""
|
||||
|
|
@ -474,7 +471,7 @@ class GeoModel(Model):
|
|||
shapely_geom = self.shapely_geom
|
||||
|
||||
if simplify_tolerance:
|
||||
shapely_geom = shapely_geom.simplify(simplify_tolerance / conf.geo['simplify_geom_factor'],
|
||||
shapely_geom = shapely_geom.simplify(simplify_tolerance / conf.geo.simplify_geom_factor,
|
||||
preserve_topology=False)
|
||||
if shapely_geom.is_empty:
|
||||
raise NoPoint
|
||||
|
|
@ -682,7 +679,7 @@ class GeoModel(Model):
|
|||
if not gpd.options.use_pygeos:
|
||||
logger.warn(f'Using get_geos_df for {cls} but gpd.options.use_pygeos not set')
|
||||
if not crs:
|
||||
crs = conf.crs['geojson']
|
||||
crs = conf.crs.geojson
|
||||
df = await cls.get_df(where=where, **kwargs)
|
||||
df.set_index('id', inplace=True)
|
||||
df.rename(columns={'geom': 'wkb'}, inplace=True)
|
||||
|
|
@ -694,78 +691,84 @@ class GeoModel(Model):
|
|||
|
||||
@classmethod
|
||||
async def get_geo_df(cls, where=None, crs=None, reproject=False,
|
||||
filter_columns=False, with_popup=False, **kwargs):
|
||||
filter_columns=False, with_popup=False, **kwargs) -> gpd.GeoDataFrame:
|
||||
"""
|
||||
Return a Pandas dataframe of all records
|
||||
Return a GeoPandas GeoDataFrame of all records
|
||||
:param where: where clause for the query (eg. Model.attr=='foo')
|
||||
:param crs: coordinate system (eg. 'epsg:4326') (priority over the reproject parameter)
|
||||
:param reproject: should reproject to conf.srid_for_proj
|
||||
:return:
|
||||
"""
|
||||
df = await cls.get_df(where=where, **kwargs)
|
||||
df.dropna(subset=['geom'], inplace=True)
|
||||
df.set_index('id', inplace=True)
|
||||
df.sort_index(inplace=True)
|
||||
df_clean = df[df.geom != None]
|
||||
## Drop coordinates
|
||||
df_clean.drop(columns=set(df_clean.columns).intersection(['ST_X_1', 'ST_Y_1', 'ST_Z_1']),
|
||||
inplace=True)
|
||||
if not crs:
|
||||
crs = conf.crs['geojson']
|
||||
return await cls.get_gdf(where=where, **kwargs)
|
||||
|
||||
## XXX: is it right? Waiting for application.py to remove pygeos support to know more.
|
||||
if getattr(gpd.options, 'use_pygeos', False):
|
||||
geometry = shapely.from_wkb(df_clean.geom)
|
||||
else:
|
||||
geometry = [wkb.loads(geom) for geom in df_clean.geom]
|
||||
# df = await cls.get_df(where=where, **kwargs)
|
||||
# df.dropna(subset=['geom'], inplace=True)
|
||||
# df.set_index('id', inplace=True)
|
||||
# df.sort_index(inplace=True)
|
||||
# df_clean = df[~df.geom.isna()]
|
||||
# ## Drop coordinates
|
||||
# df_clean.drop(columns=set(df_clean.columns).intersection(['ST_X_1', 'ST_Y_1', 'ST_Z_1']),
|
||||
# inplace=True)
|
||||
# if not crs:
|
||||
# crs = conf.crs.geojson
|
||||
|
||||
gdf = gpd.GeoDataFrame(
|
||||
df_clean.drop('geom', axis=1),
|
||||
crs=crs,
|
||||
geometry=geometry
|
||||
)
|
||||
# ## XXX: is it right? Waiting for application.py to remove pygeos support to know more.
|
||||
# # if getattr(gpd.options, 'use_pygeos', False):
|
||||
# # geometry = shapely.from_wkb(df_clean.geom)
|
||||
# # else:
|
||||
# # geometry = [wkb.loads(geom) for geom in df_clean.geom]
|
||||
|
||||
if hasattr(cls, 'simplify') and cls.simplify:
|
||||
#shapely_geom = shapely_geom.simplify(simplify_tolerance / conf.geo['simplify_geom_factor'],
|
||||
#preserve_topology=False)
|
||||
gdf['geometry'] = gdf['geometry'].simplify(
|
||||
float(cls.simplify) / conf.geo['simplify_geom_factor'],
|
||||
preserve_topology=False)
|
||||
# gdf = gpd.GeoDataFrame(
|
||||
# df_clean.drop('geom', axis=1),
|
||||
# crs=crs,
|
||||
# geometry=geometry
|
||||
# )
|
||||
|
||||
if reproject:
|
||||
gdf.to_crs(crs=conf.crs['for_proj'], inplace=True)
|
||||
# if hasattr(cls, 'simplify') and cls.simplify:
|
||||
# #shapely_geom = shapely_geom.simplify(simplify_tolerance / conf.geo.simplify_geom_factor,
|
||||
# #preserve_topology=False)
|
||||
# gdf['geometry'] = gdf['geometry'].simplify(
|
||||
# float(cls.simplify) / conf.geo.simplify_geom_factor,
|
||||
# preserve_topology=False)
|
||||
|
||||
## Filter out columns
|
||||
if filter_columns:
|
||||
gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
|
||||
inplace=True)
|
||||
# if reproject:
|
||||
# gdf.to_crs(crs=conf.crs.for_proj, inplace=True)
|
||||
|
||||
if with_popup:
|
||||
gdf['popup'] = await cls.get_popup(gdf)
|
||||
# ## Filter out columns
|
||||
# if filter_columns:
|
||||
# gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
|
||||
# inplace=True)
|
||||
|
||||
return gdf
|
||||
# if with_popup:
|
||||
# gdf['popup'] = await cls.get_popup(gdf)
|
||||
|
||||
# return gdf
|
||||
|
||||
@classmethod
|
||||
def get_attachment_dir(cls):
|
||||
return f'{cls.__table__.schema}.{cls.__table__.name}'
|
||||
return cls.__table__.fullname
|
||||
|
||||
@classmethod
|
||||
def get_attachment_base_dir(cls):
|
||||
return Path(conf.attachments['base_dir'])/cls.get_attachment_dir()
|
||||
return Path(conf.attachments.base_dir) / cls.get_attachment_dir()
|
||||
|
||||
class GeoModel(GeoModelNoStatus):
|
||||
status: ClassVar[str] = 'E'
|
||||
"""
|
||||
Status (ISO layers definition) of the layer. E -> Existing.
|
||||
"""
|
||||
|
||||
class LiveGeoModel(GeoModel):
|
||||
status: ClassVar[str] = 'E'
|
||||
store: ClassVar[str]
|
||||
group: ClassVar[str] ='Live'
|
||||
custom: ClassVar[bool] = True
|
||||
is_live: ClassVar[bool] = True
|
||||
is_db: ClassVar[bool] = False
|
||||
|
||||
class Geom(str):
|
||||
pass
|
||||
# class Geom(str):
|
||||
# pass
|
||||
|
||||
class GeoPointModel(GeoModel):
|
||||
#__abstract__ = True
|
||||
class GeoPointModelNoStatus(GeoModelNoStatus):
|
||||
shapefile_model: ClassVar[int] = POINT
|
||||
## geometry typing, see https://stackoverflow.com/questions/77333100/geoalchemy2-geometry-schema-for-pydantic-fastapi
|
||||
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINT', srid=conf.geo.srid))
|
||||
|
|
@ -827,9 +830,11 @@ class GeoPointModel(GeoModel):
|
|||
info['latitude'] = '{:.6f}'.format(self.shapely_geom.y)
|
||||
return info
|
||||
|
||||
class GeoPointModel(GeoPointModelNoStatus, GeoModel):
|
||||
...
|
||||
|
||||
|
||||
class GeoPointZModel(GeoPointModel):
|
||||
#__abstract__ = True
|
||||
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINTZ', dimension=3, srid=conf.geo.srid))
|
||||
shapefile_model: ClassVar[int] = POINTZ
|
||||
|
||||
|
|
@ -843,13 +848,11 @@ class GeoPointZModel(GeoPointModel):
|
|||
|
||||
|
||||
class GeoPointMModel(GeoPointZModel):
|
||||
#__abstract__ = True
|
||||
shapefile_model: ClassVar[int] = POINTZ
|
||||
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINTZ', dimension=3, srid=conf.geo.srid))
|
||||
|
||||
|
||||
class GeoLineModel(GeoModel):
|
||||
#__abstract__ = True
|
||||
shapefile_model: ClassVar[int] = POLYLINE
|
||||
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('LINESTRING', srid=conf.geo.srid))
|
||||
mapbox_type: ClassVar[str] = 'line'
|
||||
|
|
@ -913,7 +916,6 @@ class GeoLineModel(GeoModel):
|
|||
|
||||
|
||||
class GeoLineModelZ(GeoLineModel):
|
||||
#__abstract__ = True
|
||||
shapefile_model: ClassVar[int] = POLYLINEZ
|
||||
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('LINESTRINGZ', dimension=3, srid=conf.geo.srid))
|
||||
|
||||
|
|
@ -930,7 +932,6 @@ class GeoLineModelZ(GeoLineModel):
|
|||
|
||||
|
||||
class GeoPolygonModel(GeoModel):
|
||||
#__abstract__ = True
|
||||
shapefile_model: ClassVar[int] = POLYGON
|
||||
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POLYGON', srid=conf.geo.srid))
|
||||
mapbox_type: ClassVar[str] = 'fill'
|
||||
|
|
@ -1002,7 +1003,6 @@ class GeoPolygonModel(GeoModel):
|
|||
|
||||
|
||||
class GeoPolygonModelZ(GeoPolygonModel):
|
||||
#__abstract__ = True
|
||||
shapefile_model: ClassVar[int] = POLYGONZ
|
||||
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POLYGONZ', dimension=3, srid=conf.geo.srid))
|
||||
|
||||
|
|
@ -1023,17 +1023,13 @@ class GeoPolygonModelZ(GeoPolygonModel):
|
|||
|
||||
|
||||
class GeoPointSurveyModel(SurveyModel, GeoPointMModel):
|
||||
#__abstract__ = True
|
||||
|
||||
## raw_model is set in category_models_maker.make_category_models
|
||||
raw_model: ClassVar['RawSurveyBaseModel'] = None
|
||||
raw_model: ClassVar['RawSurveyBaseModel']
|
||||
|
||||
|
||||
class LineWorkSurveyModel(SurveyModel):
|
||||
#__abstract__ = True
|
||||
|
||||
## raw_model is set in category_models_maker.make_category_models
|
||||
raw_model: ClassVar['RawSurveyBaseModel'] = None
|
||||
raw_model: ClassVar['RawSurveyBaseModel']
|
||||
|
||||
def match_raw_points(self):
|
||||
reprojected_geom = transform(reproject_func, self.shapely_geom)
|
||||
|
|
@ -1046,20 +1042,17 @@ class LineWorkSurveyModel(SurveyModel):
|
|||
|
||||
|
||||
class GeoLineSurveyModel(LineWorkSurveyModel, GeoLineModelZ):
|
||||
#__abstract__ = True
|
||||
pass
|
||||
|
||||
|
||||
class GeoPolygonSurveyModel(LineWorkSurveyModel, GeoPolygonModelZ):
|
||||
#__abstract__ = True
|
||||
pass
|
||||
|
||||
|
||||
class RawSurveyBaseModel(BaseSurveyModel, GeoPointMModel):
|
||||
class RawSurveyBaseModel(BaseSurveyModel, GeoPointModelNoStatus):
|
||||
"""
|
||||
Abstract base class for category based raw survey point models
|
||||
"""
|
||||
#__abstract__ = True
|
||||
metadata: ClassVar[MetaData] = raw_survey
|
||||
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINTZ', dimension=3,
|
||||
srid=conf.geo.raw_survey.srid))
|
||||
|
|
@ -1070,7 +1063,7 @@ class RawSurveyBaseModel(BaseSurveyModel, GeoPointMModel):
|
|||
|
||||
@classmethod
|
||||
async def get_geo_df(cls, *args, **kwargs):
|
||||
return await super().get_geo_df(crs=conf.raw_survey['spatial_sys_ref'],
|
||||
return await super().get_geo_df(crs=conf.raw_survey.spatial_sys_ref,
|
||||
*args, **kwargs)
|
||||
|
||||
|
||||
|
|
@ -1086,8 +1079,6 @@ class PlottableModel(Model):
|
|||
to be used (the first one being the default)
|
||||
* OR an ordereed dict of value => resampling method
|
||||
"""
|
||||
#__abstract__ = True
|
||||
|
||||
float_format: ClassVar[str] = '%.1f'
|
||||
values: ClassVar[list[dict[str, str]]] = []
|
||||
|
||||
|
|
@ -1117,8 +1108,6 @@ class PlottableModel(Model):
|
|||
|
||||
|
||||
class TimePlottableModel(PlottableModel):
|
||||
#__abstract__ = True
|
||||
|
||||
time: datetime
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue