From df5f67b79d54dfc3e2c3fd147a6ac6c933fd106d Mon Sep 17 00:00:00 2001 From: phil Date: Tue, 13 Feb 2024 12:46:24 +0530 Subject: [PATCH] Migrate core admin, baskets --- src/gisaf/_version.py | 2 +- src/gisaf/admin.py | 35 ++++----- src/gisaf/api/admin.py | 23 ++++++ src/gisaf/application.py | 6 +- src/gisaf/baskets.py | 12 ++- src/gisaf/custom_store_base.py | 131 +++++++++++++++++++++++++++++++++ src/gisaf/models/admin.py | 59 +++++++++++---- 7 files changed, 229 insertions(+), 39 deletions(-) create mode 100644 src/gisaf/api/admin.py create mode 100644 src/gisaf/custom_store_base.py diff --git a/src/gisaf/_version.py b/src/gisaf/_version.py index 84bd5f7..0e3a257 100644 --- a/src/gisaf/_version.py +++ b/src/gisaf/_version.py @@ -1 +1 @@ -__version__ = '2023.4.dev33+g7e9e266.d20240210' \ No newline at end of file +__version__: str = '2023.4.dev34+g5dacc90.d20240212' \ No newline at end of file diff --git a/src/gisaf/admin.py b/src/gisaf/admin.py index 1fde3d0..78adbe9 100644 --- a/src/gisaf/admin.py +++ b/src/gisaf/admin.py @@ -3,8 +3,11 @@ from importlib.metadata import entry_points import logging from gisaf.live import live_server +from gisaf.models.authentication import User from gisaf.redis_tools import Store -from gisaf.baskets import Basket +from gisaf.baskets import Basket, standard_baskets +from gisaf.redis_tools import store +from gisaf.registry import registry logger = logging.getLogger('Gisaf admin manager') @@ -15,17 +18,16 @@ class AdminManager: """ store: Store baskets: dict[str, Basket] - async def setup_admin(self, app): + async def setup_admin(self): """ Create the default baskets, scan and create baskets from the Python entry points. Runs at startup. """ - self.app = app - self.store = app['store'] + # self.app = app + # self.store = app['store'] ## Standard baskets - from gisaf.baskets import Basket, standard_baskets self.baskets = { basket.name: basket for basket in standard_baskets @@ -39,7 +41,7 @@ class AdminManager: continue if issubclass(basket_class, Basket): ## Get name, validity check - if basket_class.name == None: + if basket_class.name is None: name = entry_point.name else: name = basket_class.name @@ -48,7 +50,7 @@ class AdminManager: continue ## Instanciate basket = basket_class() - basket._custom_module = entry_point.name + basket._custom_module = entry_point.name # type: ignore ## Check base_dir, eventually create it if not basket.base_dir.exists(): try: @@ -64,23 +66,23 @@ class AdminManager: logger.info(f'Added Basket {entry_point.name} from {entry_point.module}') ## Give a reference to the application to the baskets - for basket in self.baskets.values(): - basket.app = app + # for basket in self.baskets.values(): + # basket.app = app ## Subscribe to admin redis channels - self.pub_categories = self.store.redis.pubsub() - self.pub_scheduler = self.store.redis.pubsub() + self.pub_categories = store.redis.pubsub() + self.pub_scheduler = store.redis.pubsub() await self.pub_categories.psubscribe('admin:categories:update') task1 = create_task(self._listen_to_redis_categories()) await self.pub_scheduler.psubscribe('admin:scheduler:json') task2 = create_task(self._listen_to_redis_scheduler()) - app['admin'] = self + # app['admin'] = self - async def baskets_for_role(self, request): + async def baskets_for_role(self, user: User) -> dict[str, Basket]: return { name: basket for name, basket in self.baskets.items() - if await basket.allowed_for(request) + if await basket.allowed_for(user) } async def _listen_to_redis_categories(self): @@ -91,7 +93,7 @@ class AdminManager: if msg['type'] == 'pmessage': ## XXX: Why the name isn't retrieved? #client = await self.app['store'].pub.client_getname() - client = self.app['store'].uuid + client = store.uuid ## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ## FIXME: pubsub admin:categories:update @@ -100,8 +102,7 @@ class AdminManager: ## Skip for the process which sent this message actually updated its registry #breakpoint() if client != msg['data'].decode(): - from gisaf.database import make_auto_models - await make_auto_models(self.app) + await registry.make_registry() async def _listen_to_redis_scheduler(self): """ diff --git a/src/gisaf/api/admin.py b/src/gisaf/api/admin.py new file mode 100644 index 0000000..1bcb047 --- /dev/null +++ b/src/gisaf/api/admin.py @@ -0,0 +1,23 @@ +import logging + +from fastapi import Depends, FastAPI, HTTPException, status, responses +from gisaf.models.admin import Basket, BasketNameOnly + +from gisaf.models.authentication import User +from gisaf.security import get_current_active_user +from gisaf.admin import manager + +logger = logging.getLogger(__name__) + +api = FastAPI( + default_response_class=responses.ORJSONResponse, +) + +@api.get('/basket') +async def get_baskets( + user: User = Depends(get_current_active_user), + ): + return [ + BasketNameOnly(name=name) + for name, basket in (await manager.baskets_for_role(user)).items() + ] diff --git a/src/gisaf/application.py b/src/gisaf/application.py index 1238f09..e3377a3 100644 --- a/src/gisaf/application.py +++ b/src/gisaf/application.py @@ -10,6 +10,8 @@ from gisaf.registry import registry from gisaf.redis_tools import setup_redis, setup_redis_cache, shutdown_redis from gisaf.tiles import registry as map_tile_registry from gisaf.live import setup_live +from gisaf.admin import manager as admin_manager +from gisaf.api.admin import api as admin_api logging.basicConfig(level=conf.gisaf.debugLevel) logger = logging.getLogger(__name__) @@ -21,6 +23,7 @@ async def lifespan(app: FastAPI): await setup_redis() await setup_redis_cache() await setup_live() + await admin_manager.setup_admin() await map_tile_registry.setup() yield await shutdown_redis() @@ -35,4 +38,5 @@ app = FastAPI( ) app.mount('/v2', api) -app.mount('/gj', geoapi) \ No newline at end of file +app.mount('/gj', geoapi) +app.mount('/admin', admin_api) \ No newline at end of file diff --git a/src/gisaf/baskets.py b/src/gisaf/baskets.py index e15776a..482f306 100644 --- a/src/gisaf/baskets.py +++ b/src/gisaf/baskets.py @@ -11,6 +11,7 @@ from typing import ClassVar from gisaf.config import conf from gisaf.models.admin import FileImport +from gisaf.models.authentication import User # from gisaf.models.graphql import AdminBasketFile, BasketImportResult from gisaf.models.survey import Surveyor, Accuracy, Equipment, AccuracyEquimentSurveyorMapping from gisaf.models.project import Project @@ -45,20 +46,17 @@ class Basket: self.importer = self.importer_class() self.importer.basket = self - async def allowed_for(self, request): + async def allowed_for(self, user: User): """ Return False if the basket is protected by a role Request: aiohttp.Request instance """ if not self.role: return True + if user is not None and user.has_role(self.role): + return True else: - try: - await check_permission(request, self.role) - except (HTTPUnauthorized, HTTPForbidden): - return False - else: - return True + return False async def get_files(self, convert_path=False): """ diff --git a/src/gisaf/custom_store_base.py b/src/gisaf/custom_store_base.py new file mode 100644 index 0000000..ff358d2 --- /dev/null +++ b/src/gisaf/custom_store_base.py @@ -0,0 +1,131 @@ +import geopandas as gpd +from shapely import from_wkb +from json import dumps + +from sqlmodel import SQLModel + +from gisaf.config import conf +from gisaf.models.to_migrate import MapboxPaint, MapboxLayout, FeatureInfo + + +class BaseStore(SQLModel): + mapbox_type: str = 'symbol' + name: str = '' + description: str = '' + icon: str | None = None + mapbox_paint: MapboxPaint | None = None + mapbox_layout: MapboxLayout | None = None + attribution: str | None = None + symbol: str = '\ue32b' + base_gis_type: str = 'Point' + z_index: int = 460 + cache_enabled: bool = False + can_get_features_as_df: bool = True + filtered_columns_on_map: list[str] = [] + status: str = 'E' + ## TODO: count and other Model-like interface + count: int = -1 + + @classmethod + async def get_popup(cls, df): + return cls.__name__ + ': ' + df.index.astype('U') + + @classmethod + async def get_properties(cls, df): + return {} + + ## XXX: Copied from GeoModel + ## TODO: Create a mixin for stores/models? Set Model as a subclass of Store? + @classmethod + async def get_geo_df(cls, where=None, crs=None, reproject=False, + filter_columns=False, with_popup=False, **kwargs): + """ + Return a Pandas dataframe of all records + :param where: where clause for the query (eg. Model.attr=='foo') + :param crs: coordinate system (eg. 'epsg:4326') (priority over the reproject parameter) + :param reproject: should reproject to conf.srid_for_proj + :return: + """ + df = await cls.get_df(where=where, **kwargs) + df.dropna(subset=['geom'], inplace=True) + #df.set_index('id', inplace=True) + df.sort_index(inplace=True) + df_clean = df[df.geom != None] + ## Drop coordinates + df_clean.drop(columns=set(df_clean.columns).intersection(['ST_X_1', 'ST_Y_1', 'ST_Z_1']), + inplace=True) + if not crs: + crs = conf.crs['geojson'] + + #if getattr(gpd.options, 'use_pygeos', False): + # geometry = from_wkb(df_clean.geom) + #else: + # geometry = [wkb.loads(geom) for geom in df_clean.geom] + ## XXX: There must be a vectorized way to do this + geom_bin = df_clean.geom.apply(lambda row: row.desc) + geometry = from_wkb(geom_bin) + # if not getattr(gpd.options, 'use_pygeos', False): + # geometry = pygeos.to_shapely(geometry) + + gdf = gpd.GeoDataFrame( + df_clean.drop('geom', axis=1), + crs=crs, + geometry=geometry + ) + + if hasattr(cls, 'simplify') and cls.simplify: + #shapely_geom = shapely_geom.simplify(simplify_tolerance / conf.geo.simplify_geom_factor, + #preserve_topology=conf.geo.simplify_preserve_topology) + gdf['geometry'] = gdf['geometry'].simplify( + float(cls.simplify) / conf.geo.simplify_geom_factor, + preserve_topology=conf.geo.simplify_preserve_topology) + + if reproject: + gdf.to_crs(crs=conf.crs['for_proj'], inplace=True) + + ## Filter out columns + if filter_columns: + gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map), + inplace=True) + + if with_popup: + gdf['popup'] = await cls.get_popup(gdf) + + return gdf + + @classmethod + async def get_df(cls, where=None, + with_related=None, recursive=True, + cast=True, + with_only_columns=None, + geom_as_ewkt=False, + **kwargs): + """ + Return a Pandas dataframe of all records + Optional arguments: + * an SQLAlchemy where clause + * with_related: automatically get data from related columns, following the foreign keys in the model definitions + * cast: automatically transform various data in their best python types (eg. with date, time...) + * with_only_columns: fetch only these columns (list of column names) + * geom_as_ewkt: convert geometry columns to EWKB (handy for eg. using upsert_df) + :return: + """ + raise NotImplementedError('Subclasses of BaseStore must implement get_df()') + + @classmethod + async def get_item_params(cls, id) -> FeatureInfo: + raise NotImplementedError('Subclasses of BaseStore must implement get_item_params()') + + @classmethod + async def get_mapbox_style(cls): + """ + Get the mapbox style (paint, layout, attribution...) + """ + style = {} + if cls.mapbox_paint is not None: + style['paint'] = dumps(cls.mapbox_paint) + if cls.mapbox_layout is not None: + style['layout'] = dumps(cls.mapbox_layout) + if cls.attribution is not None: + style['attribution'] = cls.attribution + return style diff --git a/src/gisaf/models/admin.py b/src/gisaf/models/admin.py index 3780730..df77310 100644 --- a/src/gisaf/models/admin.py +++ b/src/gisaf/models/admin.py @@ -1,11 +1,10 @@ import re -from datetime import datetime +from datetime import datetime, date -from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column +from pydantic import BaseModel +from sqlmodel import Field, Relationship import pandas as pd -# from graphene import ObjectType, Int, String, DateTime, List - from gisaf.models.models_base import Model from gisaf.models.survey import Surveyor, Equipment from gisaf.models.project import Project @@ -21,7 +20,7 @@ class BadSurveyFileName(Exception): pass -def get_file_import_date(record): +def get_file_import_date(record) -> date: """ Utility function that returns the date of survey from the file name, if it matches the convention for CSV survey files. @@ -34,9 +33,9 @@ def get_file_import_date(record): '(format should be: "PPP-DESCRIPTION-YYYY-MM-DD", ' 'PPP being the project name, DESCRITION is optional and discarded)' ) - return datetime.date(day=int(fname_search.group(4)), - month=int(fname_search.group(3)), - year=int(fname_search.group(2))) + return date(day=int(fname_search.group(4)), + month=int(fname_search.group(3)), + year=int(fname_search.group(2))) class FileImport(Model): @@ -44,7 +43,7 @@ class FileImport(Model): Files to import or imported in the DB. Give either url or path. """ - __tablename__ = 'file_import' + __tablename__: str = 'file_import' # type: ignore __table_args__ = gisaf_admin.table_args id: int | None = Field(default=None, primary_key=True) @@ -79,9 +78,10 @@ class FileImport(Model): def selectinload(cls): return [cls.project, cls.surveyor, cls.equipment] - def set_import_time(self): - self.time = datetime.now() - db.session.commit() + ## XXX: was used in Flask + # def set_import_time(self): + # self.time = datetime.now() + # db.session.commit() @classmethod async def get_df(cls, *args, **kwargs): @@ -116,7 +116,7 @@ class FeatureImportData(Model): """ Keep track of imported data, typically from shapefiles """ - __tablename__ = 'feature_import_data' + __tablename__: str = 'feature_import_data' # type: ignore __table_args__ = gisaf_admin.table_args id: int | None = Field(default=None, primary_key=True) @@ -127,3 +127,36 @@ class FeatureImportData(Model): origin: str file_path: str file_md5: str + + +class BasketFile(BaseModel): + id: int + dir: int + name: int + url: int + md5: int + time: datetime + comment: int + status: int + store: int + project: int + surveyor: int + equipment: int + import_result: int + + +class BasketNameOnly(BaseModel): + name: str + + +class Basket(BasketNameOnly): + files: list[BasketFile] + columns: list[str] + uploadFields: list[str] + projects: list[str] + + +class BasketImportResult(BaseModel): + time: datetime + message: str + details: str \ No newline at end of file