From c1f229f805be8421747624dd848f4ce86b0ae986 Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 13 Feb 2024 12:47:07 +0530 Subject: [PATCH] Cosmetic: mostly typings --- src/gisaf/api/v2.py | 50 ++++++++++++++++++++++------- src/gisaf/config.py | 2 +- src/gisaf/models/authentication.py | 14 +++++--- src/gisaf/models/bootstrap.py | 2 +- src/gisaf/models/category.py | 44 ++++++++++++------------- src/gisaf/models/dashboard.py | 12 +++---- src/gisaf/models/geo_models_base.py | 8 ++--- src/gisaf/models/store.py | 8 +++-- src/gisaf/models/survey.py | 17 ++++++++-- src/gisaf/models/tags.py | 2 +- src/gisaf/redis_tools.py | 8 ++--- src/gisaf/security.py | 2 +- src/gisaf/utils.py | 20 ++++++------ 13 files changed, 120 insertions(+), 69 deletions(-) diff --git a/src/gisaf/api/v2.py b/src/gisaf/api/v2.py index dc20681..ed2e6ee 100644 --- a/src/gisaf/api/v2.py +++ b/src/gisaf/api/v2.py @@ -12,11 +12,14 @@ from gisaf.models.authentication import ( Role, RoleRead, ) from gisaf.models.category import Category, CategoryRead +from gisaf.models.geo_models_base import LineWorkSurveyModel from gisaf.models.to_migrate import DataProvider -from gisaf.config import conf +from gisaf.models.survey import Equipment, SurveyMeta, Surveyor +from gisaf.config import Survey, conf from gisaf.models.bootstrap import BootstrapData -from gisaf.models.store import Store +from gisaf.models.store import Store, StoreNameOnly from gisaf.models.project import Project +from gisaf.models.authentication import UserRoleLink #, ACL from gisaf.database import pandas_query, fastapi_db_session as db_session from gisaf.security import ( Token, @@ -69,17 +72,26 @@ async def login_for_access_token( async def get_users( db_session: db_session, ) -> list[UserRead]: - query = select(User).options(selectinload(User.roles)) + query = select(User).options(selectinload(User.roles)) # type: ignore[arg-type] data = await db_session.exec(query) - return data.all() + return data.all() # type: ignore[return-value] @api.get("/roles") async def get_roles( db_session: db_session, ) -> list[RoleRead]: - query = select(Role).options(selectinload(Role.users)) + query = select(Role).options(selectinload(Role.users)) # type: ignore[arg-type] data = await db_session.exec(query) - return data.all() + return data.all() # type: ignore[return-value] + +@api.get('/acls') +async def get_acls(db_session: db_session, + user: Annotated[User, Depends(get_current_user)]) -> list[UserRoleLink]: + """New: ACLs returned as UserRoleLink""" + if not user or not user.has_role('manager'): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED) + data = await db_session.exec(select(UserRoleLink)) + return data.all() # type: ignore[return-value] @api.get("/categories") async def get_categories( @@ -87,7 +99,7 @@ async def get_categories( ) -> list[CategoryRead]: query = select(Category) data = await db_session.exec(query) - return data.all() + return data.all() # type: ignore[return-value] @api.get("/categories_pandas") async def get_categories_p( @@ -95,7 +107,7 @@ async def get_categories_p( ) -> list[CategoryRead]: query = select(Category) df = await db_session.run_sync(pandas_query, query) - return df.to_dict(orient="records") + return df.to_dict(orient="records") # type: ignore[return-value] # @api.get("/list") @api.get("/data-providers") @@ -114,15 +126,31 @@ async def list_data_providers() -> list[DataProvider]: async def get_stores() -> list[Store]: df = registry.stores.reset_index().\ drop(columns=['model', 'raw_model', 'base_gis_type']) - return df.to_dict(orient="records") + return df.to_dict(orient="records") # type: ignore[return-value] @api.get("/projects") async def get_projects( db_session: db_session, ) -> list[Project]: query = select(Project) - df = await db_session.run_sync(pandas_query, query) - return df.to_dict(orient="records") + data = await db_session.exec(query) + return data.all() # type: ignore[return-value] + +@api.get("/survey_meta") +async def get_survey_meta( + db_session: db_session, + ) -> SurveyMeta: + return SurveyMeta( + projects=(await db_session.exec(select(Project))).all(), # type: ignore[arg-type] + surveyors=(await db_session.exec(select(Surveyor))).all(), # type: ignore[arg-type] + equipments=(await db_session.exec(select(Equipment))).all(), # type: ignore[arg-type] + statuses=conf.map.status, + stores_misc=[StoreNameOnly(name=name) + for name, model in registry.geom_custom.items()], + stores_line_work=[StoreNameOnly(name=name) + for name in registry.stores[registry.stores.is_line_work].index], + default=conf.admin.basket.default + ) @api.get("/feature-info/{store}/{id}") async def get_feature_info( diff --git a/src/gisaf/config.py b/src/gisaf/config.py index a0220f6..89c2032 100644 --- a/src/gisaf/config.py +++ b/src/gisaf/config.py @@ -6,7 +6,7 @@ from typing import Any, Type, Tuple from pydantic_settings import (BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict) -from pydantic import ConfigDict +#from pydantic import ConfigDict from pydantic.v1.utils import deep_update from yaml import safe_load diff --git a/src/gisaf/models/authentication.py b/src/gisaf/models/authentication.py index 986101e..a8f9f70 100644 --- a/src/gisaf/models/authentication.py +++ b/src/gisaf/models/authentication.py @@ -1,10 +1,11 @@ from sqlmodel import Field, SQLModel, Relationship +from pydantic import BaseModel from gisaf.models.metadata import gisaf_admin class UserRoleLink(SQLModel, table=True): - __tablename__ = 'roles_users' + __tablename__: str = 'roles_users' # type: ignore __table_args__ = gisaf_admin.table_args user_id: int | None = Field( default=None, @@ -56,7 +57,7 @@ class Role(RoleWithDescription, table=True): class UserReadNoRoles(UserBase): id: int - email: str | None + email: str | None # type: ignore class RoleRead(RoleBase): @@ -70,5 +71,10 @@ class RoleReadNoUsers(RoleBase): class UserRead(UserBase): id: int - email: str | None - roles: list[RoleReadNoUsers] = [] \ No newline at end of file + email: str | None # type: ignore + roles: list[RoleReadNoUsers] = [] + + +# class ACL(BaseModel): +# user_id: int +# role_ids: list[int] \ No newline at end of file diff --git a/src/gisaf/models/bootstrap.py b/src/gisaf/models/bootstrap.py index 9a48671..9455af2 100644 --- a/src/gisaf/models/bootstrap.py +++ b/src/gisaf/models/bootstrap.py @@ -16,4 +16,4 @@ class BootstrapData(BaseModel): geo: Geo = conf.geo measures: Measures = conf.measures redirect: str = conf.gisaf.redirect - user: UserRead | None = None \ No newline at end of file + user: UserRead | None = None # type: ignore \ No newline at end of file diff --git a/src/gisaf/models/category.py b/src/gisaf/models/category.py index 0e6c311..0100867 100644 --- a/src/gisaf/models/category.py +++ b/src/gisaf/models/category.py @@ -15,9 +15,9 @@ mapbox_type_mapping = { class CategoryGroup(BaseModel, table=True): - __tablename__ = 'category_group' + __tablename__: str = 'category_group' # type: ignore __table_args__ = gisaf_survey.table_args - name: str | None = Field(sa_type=String(4), default=None, primary_key=True) + name: str | None = Field(sa_type=String(4), default=None, primary_key=True) # type: ignore major: bool long_name: str categories: list['Category'] = Relationship(back_populates='category_group') @@ -28,7 +28,7 @@ class CategoryGroup(BaseModel, table=True): class CategoryModelType(BaseModel, table=True): - __tablename__ = 'category_model_type' + __tablename__: str = 'category_model_type' # type: ignore __table_args__ = gisaf_survey.table_args name: str | None = Field(default=None, primary_key=True) @@ -46,27 +46,27 @@ class CategoryBase(BaseModel): name: str | None = Field(default=None, primary_key=True) domain: ClassVar[str] = 'V' description: str | None - group: str = Field(sa_type=String(4), - foreign_key=gisaf_survey.table('category_group.name'), - index=True) - minor_group_1: str = Field(sa_type=String(4), default='----') - minor_group_2: str = Field(sa_type=String(4), default='----') - status: str = Field(sa_type=String(1)) + group: str = Field(sa_type=String(4), # type: ignore + foreign_key=gisaf_survey.table('category_group.name'), # type: ignore + index=True) # type: ignore + minor_group_1: str = Field(sa_type=String(4), default='----') # type: ignore + minor_group_2: str = Field(sa_type=String(4), default='----') # type: ignore + status: str = Field(sa_type=String(1)) # type: ignore custom: bool | None auto_import: bool = True - gis_type: str = Field(sa_type=String(50), - foreign_key=gisaf_survey.table('category_model_type.name'), - default='Point') - long_name: str | None = Field(sa_type=String(50)) + gis_type: str = Field(sa_type=String(50), # type: ignore + foreign_key=gisaf_survey.table('category_model_type.name'), # type: ignore + default='Point') # type: ignore + long_name: str | None = Field(sa_type=String(50)) # type: ignore style: str | None = Field(sa_type=TEXT) - symbol: str | None = Field(sa_type=String(1)) - mapbox_type_custom: str | None = Field(sa_type=String(12)) - mapbox_paint: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) - mapbox_layout: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) + symbol: str | None = Field(sa_type=String(1)) # type: ignore + mapbox_type_custom: str | None = Field(sa_type=String(12)) # type: ignore + mapbox_paint: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore + mapbox_layout: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore viewable_role: str | None - extra: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) + extra: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore - @computed_field + @computed_field # type: ignore @property def layer_name(self) -> str: """ @@ -75,7 +75,7 @@ class CategoryBase(BaseModel): """ return '{self.domain}-{self.group:4s}-{self.minor_group_1:4s}-{self.minor_group_2:4s}-{self.status:1s}'.format(self=self) - @computed_field + @computed_field # type: ignore @property def table_name(self) -> str: """ @@ -87,7 +87,7 @@ class CategoryBase(BaseModel): else: return '{self.domain}_{self.group:4s}_{self.minor_group_1:4s}_{self.minor_group_2:4s}'.format(self=self) - @computed_field + @computed_field # type: ignore @property def raw_survey_table_name(self) -> str: """ @@ -99,7 +99,7 @@ class CategoryBase(BaseModel): else: return 'RAW_{self.domain}_{self.group:4s}_{self.minor_group_1:4s}_{self.minor_group_2:4s}'.format(self=self) - @computed_field + @computed_field # type: ignore @property def mapbox_type(self) -> str: return self.mapbox_type_custom or mapbox_type_mapping[self.gis_type] diff --git a/src/gisaf/models/dashboard.py b/src/gisaf/models/dashboard.py index 183f000..53d8e40 100644 --- a/src/gisaf/models/dashboard.py +++ b/src/gisaf/models/dashboard.py @@ -18,11 +18,11 @@ logger = logging.getLogger(__name__) try: import matplotlib.pyplot as plt except ImportError: - plt = None + plt = None # type: ignore class DashboardPageSource(Model, table=True): - __tablename__ = 'dashboard_page_source' + __tablename__ = 'dashboard_page_source' # type: ignore __table_args__ = gisaf.table_args id: str = Field(primary_key=True) @@ -118,7 +118,7 @@ class DashboardPageCommon: class DashboardPage(Model, DashboardPageCommon, table=True): - __tablename__ = 'dashboard_page' + __tablename__ = 'dashboard_page' # type: ignore __table_args__ = gisaf.table_args class Admin: @@ -179,7 +179,7 @@ class DashboardPage(Model, DashboardPageCommon, table=True): class DashboardPageSection(Model, DashboardPageCommon, table=True): - __tablename__ = 'dashboard_page_section' + __tablename__ = 'dashboard_page_section' # type: ignore __table_args__ = gisaf.table_args class Admin: @@ -252,10 +252,10 @@ class DashboardPageSection(Model, DashboardPageCommon, table=True): class Widget(Model, table=True): - __tablename__ = 'widget' + __tablename__ = 'widget' # type: ignore __table_args__ = gisaf.table_args ## CREATE TABLE gisaf.widget (name char(50) not null PRIMARY KEY, title varchar, subtitle varchar, notebook varchar, content varchar, time timestamp); - name: str = Field(primary_key=True, sa_type=String(50)) + name: str = Field(primary_key=True, sa_type=String(50)) # type: ignore title: str subtitle: str content: str diff --git a/src/gisaf/models/geo_models_base.py b/src/gisaf/models/geo_models_base.py index e825fb9..96ecada 100644 --- a/src/gisaf/models/geo_models_base.py +++ b/src/gisaf/models/geo_models_base.py @@ -91,10 +91,10 @@ class BaseSurveyModel(BaseModel): @classmethod def selectinload(cls): return [ - cls.equipment, - cls.surveyor, - cls.accuracy, - cls.project, + cls.equipment, # type: ignore + cls.surveyor, # type: ignore + cls.accuracy, # type: ignore + cls.project, # type: ignore ] # @classmethod # def dyn_join_with(cls): diff --git a/src/gisaf/models/store.py b/src/gisaf/models/store.py index 4760bb3..2cfbd10 100644 --- a/src/gisaf/models/store.py +++ b/src/gisaf/models/store.py @@ -6,7 +6,12 @@ from gisaf.models.geo_models_base import GeoModel, RawSurveyBaseModel, GeoPointS class MapLibreStyle(BaseModel): ... -class Store(BaseModel): + +class StoreNameOnly(BaseModel): + name: str + + +class Store(StoreNameOnly): auto_import: bool # base_gis_type: str count: int | None = None @@ -29,7 +34,6 @@ class Store(BaseModel): minor_group_2: str | None #model: GeoModel gis_type: str - name: str #name_letter: str #name_number: int #raw_model: GeoPointSurveyModel diff --git a/src/gisaf/models/survey.py b/src/gisaf/models/survey.py index 7e98c0e..ea0c9a4 100644 --- a/src/gisaf/models/survey.py +++ b/src/gisaf/models/survey.py @@ -1,9 +1,13 @@ from enum import Enum from sqlmodel import Field, Relationship +from pydantic import BaseModel +from gisaf.config import BasketDefault from gisaf.models.models_base import Model from gisaf.models.metadata import gisaf_survey +from gisaf.models.project import Project +from gisaf.models.store import StoreNameOnly class Accuracy(Model, table=True): @@ -63,7 +67,7 @@ class GeometryType(str, Enum): class AccuracyEquimentSurveyorMapping(Model, table=True): __table_args__ = gisaf_survey.table_args - __tablename__ = 'accuracy_equiment_surveyor_mapping' + __tablename__: str = 'accuracy_equiment_surveyor_mapping' # type: ignore class Admin: menu = 'Other' @@ -91,4 +95,13 @@ class AccuracyEquimentSurveyorMapping(Model, table=True): # 'surveyor': Surveyor, # 'equipment': Equipment, # 'accuracy': Accuracy, - # } \ No newline at end of file + # } + +class SurveyMeta(BaseModel): + projects: list[Project] + surveyors: list[Surveyor] + equipments: list[Equipment] + statuses: list[str] + stores_misc: list[StoreNameOnly] + stores_line_work: list[StoreNameOnly] + default: BasketDefault diff --git a/src/gisaf/models/tags.py b/src/gisaf/models/tags.py index 5d0ef5a..3de7523 100644 --- a/src/gisaf/models/tags.py +++ b/src/gisaf/models/tags.py @@ -19,7 +19,7 @@ class Tags(GeoPointModel, table=True): id: int | None = Field(primary_key=True, default=None) store: str = Field(index=True) ref_id: int = Field(index=True, sa_type=BigInteger) - tags: dict = Field(sa_type=MutableDict.as_mutable(HSTORE)) + tags: dict = Field(sa_type=MutableDict.as_mutable(HSTORE)) # type: ignore def __str__(self): return '{self.store:s} {self.ref_id}: {self.tags}'.format(self=self) diff --git a/src/gisaf/redis_tools.py b/src/gisaf/redis_tools.py index 204e835..e99b7ac 100644 --- a/src/gisaf/redis_tools.py +++ b/src/gisaf/redis_tools.py @@ -7,7 +7,7 @@ from time import time import logging import pandas as pd -import geopandas as gpd +import geopandas as gpd # type: ignore[import-untyped] from asyncpg import connect from asyncpg.connection import Connection from asyncpg.exceptions import UndefinedTableError, InterfaceError @@ -167,8 +167,8 @@ class Store: Additionally, publish to the channel for websocket live updates to ws_clients """ if gdf is None: - gdf = gpd.GeoDataFrame(data={'geom': []}, geometry='geom') - if isinstance(gdf.index, pd.core.indexes.multi.MultiIndex): + gdf = gpd.GeoDataFrame(data={'geom': []}, geometry='geom') # type: ignore + if isinstance(gdf.index, pd.MultiIndex): raise ValueError('Gisaf live does not accept dataframes with multi index') return await self._store_live_to_redis(live_name, gdf, **kwargs) @@ -205,7 +205,7 @@ class Store: mapbox_layout['text-field'] = symbol if not symbol: symbol = gisTypeSymbolMap.get(gis_type, '\ue02e') - if properties == None: + if properties is None: properties = [] ## Add a column for json representation columns = {'status', 'popup', gdf.geometry.name, 'store', 'id'} diff --git a/src/gisaf/security.py b/src/gisaf/security.py index a03943d..c0c698b 100644 --- a/src/gisaf/security.py +++ b/src/gisaf/security.py @@ -108,7 +108,7 @@ def verify_password(user: User, plain_password): async def get_current_user( - token: str = Depends(oauth2_scheme)) -> UserRead | None: + token: str = Depends(oauth2_scheme)) -> User | None: if token is None: return None try: diff --git a/src/gisaf/utils.py b/src/gisaf/utils.py index 9fc43d5..0894374 100644 --- a/src/gisaf/utils.py +++ b/src/gisaf/utils.py @@ -1,11 +1,11 @@ import logging import asyncio from functools import wraps -from json import dumps, JSONEncoder +from json import JSONEncoder from math import isnan from time import time import datetime -import pyproj +from typing import Any from numpy import ndarray import pandas as pd @@ -27,7 +27,7 @@ SHAPELY_TYPE_TO_MAPBOX_TYPE = { 'MultiPolygon': 'fill', } -DEFAULT_MAPBOX_LAYOUT = { +DEFAULT_MAPBOX_LAYOUT: dict[str, dict[str, Any]] = { 'symbol': { 'text-line-height': 1, 'text-padding': 0, @@ -153,13 +153,13 @@ class NumpyEncoder(JSONEncoder): # recursive_joins[name] = join # return recursive_joins -def get_joined_query(cls): - """ - Helper function to get a query from a model with all the related tables loaded - :param cls: - :return: - """ - return cls.load(**get_join_with(cls)).query +# def get_joined_query(cls): +# """ +# Helper function to get a query from a model with all the related tables loaded +# :param cls: +# :return: +# """ +# return cls.load(**get_join_with(cls)).query def timeit(f):