Migrate core admin, baskets

src/gisaf/_version.py

@@ -1 +1 @@
-__version__ = '2023.4.dev33+g7e9e266.d20240210'
+__version__: str = '2023.4.dev34+g5dacc90.d20240212'

src/gisaf/admin.py

@@ -3,8 +3,11 @@ from importlib.metadata import entry_points
 import logging
 
 from gisaf.live import live_server
+from gisaf.models.authentication import User
 from gisaf.redis_tools import Store
-from gisaf.baskets import Basket
+from gisaf.baskets import Basket, standard_baskets
+from gisaf.redis_tools import store
+from gisaf.registry import registry
 
 logger = logging.getLogger('Gisaf admin manager')
 
@@ -15,17 +18,16 @@ class AdminManager:
     """
     store: Store
     baskets: dict[str, Basket]
-    async def setup_admin(self, app):
+    async def setup_admin(self):
         """
         Create the default baskets, scan and create baskets
         from the Python entry points.
         Runs at startup.
         """
-        self.app = app
+        # self.app = app
-        self.store = app['store']
+        # self.store = app['store']
 
         ## Standard baskets
-        from gisaf.baskets import Basket, standard_baskets
         self.baskets = {
             basket.name: basket
             for basket in standard_baskets
@@ -39,7 +41,7 @@ class AdminManager:
                 continue
             if issubclass(basket_class, Basket):
                 ## Get name, validity check
-                if basket_class.name == None:
+                if basket_class.name is None:
                     name = entry_point.name
                 else:
                     name = basket_class.name
@@ -48,7 +50,7 @@ class AdminManager:
                     continue
                 ## Instanciate
                 basket = basket_class()
-                basket._custom_module = entry_point.name
+                basket._custom_module = entry_point.name # type: ignore
                 ## Check base_dir, eventually create it
                 if not basket.base_dir.exists():
                     try:
@@ -64,23 +66,23 @@ class AdminManager:
                 logger.info(f'Added Basket {entry_point.name} from {entry_point.module}')
 
         ## Give a reference to the application to the baskets
-        for basket in self.baskets.values():
+        # for basket in self.baskets.values():
-            basket.app = app
+        #     basket.app = app
 
         ## Subscribe to admin redis channels
-        self.pub_categories = self.store.redis.pubsub()
+        self.pub_categories = store.redis.pubsub()
-        self.pub_scheduler = self.store.redis.pubsub()
+        self.pub_scheduler = store.redis.pubsub()
         await self.pub_categories.psubscribe('admin:categories:update')
         task1 = create_task(self._listen_to_redis_categories())
         await self.pub_scheduler.psubscribe('admin:scheduler:json')
         task2 = create_task(self._listen_to_redis_scheduler())
 
-        app['admin'] = self
+        # app['admin'] = self
 
-    async def baskets_for_role(self, request):
+    async def baskets_for_role(self, user: User) -> dict[str, Basket]:
         return {
             name: basket for name, basket in self.baskets.items()
-            if await basket.allowed_for(request)
+            if await basket.allowed_for(user)
         }
 
     async def _listen_to_redis_categories(self):
@@ -91,7 +93,7 @@ class AdminManager:
             if msg['type'] == 'pmessage':
                 ## XXX: Why the name isn't retrieved?
                 #client = await self.app['store'].pub.client_getname()
-                client = self.app['store'].uuid
+                client = store.uuid
 
                 ## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
                 ## FIXME: pubsub admin:categories:update
@@ -100,8 +102,7 @@ class AdminManager:
                 ## Skip for the process which sent this message actually updated its registry
                 #breakpoint()
                 if client != msg['data'].decode():
-                    from gisaf.database import make_auto_models
+                    await registry.make_registry()
-                    await make_auto_models(self.app)
 
     async def _listen_to_redis_scheduler(self):
         """

src/gisaf/api/admin.py

@@ -0,0 +1,23 @@
+import logging
+
+from fastapi import Depends, FastAPI, HTTPException, status, responses
+from gisaf.models.admin import Basket, BasketNameOnly
+
+from gisaf.models.authentication import User
+from gisaf.security import get_current_active_user
+from gisaf.admin import manager
+
+logger = logging.getLogger(__name__)
+
+api = FastAPI(
+    default_response_class=responses.ORJSONResponse,
+)
+
+@api.get('/basket')
+async def get_baskets(
+    user: User = Depends(get_current_active_user),
+    ):
+    return [
+        BasketNameOnly(name=name)
+        for name, basket in (await manager.baskets_for_role(user)).items()
+    ]

src/gisaf/application.py

@@ -10,6 +10,8 @@ from gisaf.registry import registry
 from gisaf.redis_tools import setup_redis, setup_redis_cache, shutdown_redis
 from gisaf.tiles import registry as map_tile_registry
 from gisaf.live import setup_live
+from gisaf.admin import manager as admin_manager
+from gisaf.api.admin import api as admin_api
 
 logging.basicConfig(level=conf.gisaf.debugLevel)
 logger = logging.getLogger(__name__)
@@ -21,6 +23,7 @@ async def lifespan(app: FastAPI):
     await setup_redis()
     await setup_redis_cache()
     await setup_live()
+    await admin_manager.setup_admin()
     await map_tile_registry.setup()
     yield
     await shutdown_redis()
@@ -36,3 +39,4 @@ app = FastAPI(
 
 app.mount('/v2', api)
 app.mount('/gj', geoapi)
+app.mount('/admin', admin_api)

src/gisaf/baskets.py

@@ -11,6 +11,7 @@ from typing import ClassVar
 
 from gisaf.config import conf
 from gisaf.models.admin import FileImport
+from gisaf.models.authentication import User
 # from gisaf.models.graphql import AdminBasketFile, BasketImportResult
 from gisaf.models.survey import Surveyor, Accuracy, Equipment, AccuracyEquimentSurveyorMapping
 from gisaf.models.project import Project
@@ -45,20 +46,17 @@ class Basket:
             self.importer = self.importer_class()
             self.importer.basket = self
 
-    async def allowed_for(self, request):
+    async def allowed_for(self, user: User):
         """
         Return False if the basket is protected by a role
         Request: aiohttp.Request instance
         """
         if not self.role:
             return True
+        if user is not None and user.has_role(self.role):
+            return True
         else:
-            try:
+            return False
-                await check_permission(request, self.role)
-            except (HTTPUnauthorized, HTTPForbidden):
-                return False
-            else:
-                return True
 
     async def get_files(self, convert_path=False):
         """

src/gisaf/custom_store_base.py

@@ -0,0 +1,131 @@
+import geopandas as gpd
+from shapely import from_wkb
+from json import dumps
+
+from sqlmodel import SQLModel
+
+from gisaf.config import conf
+from gisaf.models.to_migrate import MapboxPaint, MapboxLayout, FeatureInfo
+
+
+class BaseStore(SQLModel):
+    mapbox_type: str = 'symbol'
+    name: str = '<Unnamed store>'
+    description: str = '<Description>'
+    icon: str | None = None
+    mapbox_paint: MapboxPaint | None = None
+    mapbox_layout: MapboxLayout | None = None
+    attribution: str | None = None
+    symbol: str = '\ue32b'
+    base_gis_type: str = 'Point'
+    z_index: int = 460
+    cache_enabled: bool = False
+    can_get_features_as_df: bool = True
+    filtered_columns_on_map: list[str] = []
+    status: str = 'E'
+    ## TODO: count and other Model-like interface
+    count: int = -1
+
+    @classmethod
+    async def get_popup(cls, df):
+        return cls.__name__ + ': ' + df.index.astype('U')
+
+    @classmethod
+    async def get_properties(cls, df):
+        return {}
+
+    ## XXX: Copied from GeoModel
+    ## TODO: Create a mixin for stores/models? Set Model as a subclass of Store?
+    @classmethod
+    async def get_geo_df(cls, where=None, crs=None, reproject=False,
+                         filter_columns=False, with_popup=False, **kwargs):
+        """
+        Return a Pandas dataframe of all records
+        :param where: where clause for the query (eg. Model.attr=='foo')
+        :param crs: coordinate system (eg. 'epsg:4326') (priority over the reproject parameter)
+        :param reproject: should reproject to conf.srid_for_proj
+        :return:
+        """
+        df = await cls.get_df(where=where, **kwargs)
+        df.dropna(subset=['geom'], inplace=True)
+        #df.set_index('id', inplace=True)
+        df.sort_index(inplace=True)
+        df_clean = df[df.geom != None]
+        ## Drop coordinates
+        df_clean.drop(columns=set(df_clean.columns).intersection(['ST_X_1', 'ST_Y_1', 'ST_Z_1']),
+                      inplace=True)
+        if not crs:
+            crs = conf.crs['geojson']
+
+        #if getattr(gpd.options, 'use_pygeos', False):
+        #    geometry = from_wkb(df_clean.geom)
+        #else:
+        #    geometry = [wkb.loads(geom) for geom in df_clean.geom]
+        ## XXX: There must be a vectorized way to do this
+        geom_bin = df_clean.geom.apply(lambda row: row.desc)
+        geometry = from_wkb(geom_bin)
+        # if not getattr(gpd.options, 'use_pygeos', False):
+        #     geometry = pygeos.to_shapely(geometry)
+
+        gdf = gpd.GeoDataFrame(
+            df_clean.drop('geom', axis=1),
+            crs=crs,
+            geometry=geometry
+        )
+
+        if hasattr(cls, 'simplify') and cls.simplify:
+            #shapely_geom = shapely_geom.simplify(simplify_tolerance / conf.geo.simplify_geom_factor,
+                                                 #preserve_topology=conf.geo.simplify_preserve_topology)
+            gdf['geometry'] = gdf['geometry'].simplify(
+                float(cls.simplify) / conf.geo.simplify_geom_factor,
+                preserve_topology=conf.geo.simplify_preserve_topology)
+
+        if reproject:
+            gdf.to_crs(crs=conf.crs['for_proj'], inplace=True)
+
+        ## Filter out columns
+        if filter_columns:
+            gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
+                     inplace=True)
+
+        if with_popup:
+            gdf['popup'] = await cls.get_popup(gdf)
+
+        return gdf
+
+    @classmethod
+    async def get_df(cls, where=None,
+                     with_related=None, recursive=True,
+                     cast=True,
+                     with_only_columns=None,
+                     geom_as_ewkt=False,
+                     **kwargs):
+        """
+        Return a Pandas dataframe of all records
+        Optional arguments:
+        * an SQLAlchemy where clause
+        * with_related: automatically get data from related columns, following the foreign keys in the model definitions
+        * cast: automatically transform various data in their best python types (eg. with date, time...)
+        * with_only_columns: fetch only these columns (list of column names)
+        * geom_as_ewkt: convert geometry columns to EWKB (handy for eg. using upsert_df)
+        :return:
+        """
+        raise NotImplementedError('Subclasses of BaseStore must implement get_df()')
+
+    @classmethod
+    async def get_item_params(cls, id) -> FeatureInfo:
+        raise NotImplementedError('Subclasses of BaseStore must implement get_item_params()')
+
+    @classmethod
+    async def get_mapbox_style(cls):
+        """
+        Get the mapbox style (paint, layout, attribution...)
+        """
+        style = {}
+        if cls.mapbox_paint is not None:
+            style['paint'] = dumps(cls.mapbox_paint)
+        if cls.mapbox_layout is not None:
+            style['layout'] = dumps(cls.mapbox_layout)
+        if cls.attribution is not None:
+            style['attribution'] = cls.attribution
+        return style

src/gisaf/models/admin.py

@@ -1,11 +1,10 @@
 import re
-from datetime import datetime
+from datetime import datetime, date
 
-from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
+from pydantic import BaseModel
+from sqlmodel import Field, Relationship
 import pandas as pd
 
-# from graphene import ObjectType, Int, String, DateTime, List
-
 from gisaf.models.models_base import Model
 from gisaf.models.survey import Surveyor, Equipment
 from gisaf.models.project import Project
@@ -21,7 +20,7 @@ class BadSurveyFileName(Exception):
     pass
 
 
-def get_file_import_date(record):
+def get_file_import_date(record) -> date:
     """
     Utility function that returns the date of survey from the file name,
     if it matches the convention for CSV survey files.
@@ -34,9 +33,9 @@ def get_file_import_date(record):
             '(format should be: "PPP-DESCRIPTION-YYYY-MM-DD", '
             'PPP being the project name, DESCRITION is optional and discarded)'
         )
-    return datetime.date(day=int(fname_search.group(4)),
+    return date(day=int(fname_search.group(4)),
-                         month=int(fname_search.group(3)),
+                month=int(fname_search.group(3)),
-                         year=int(fname_search.group(2)))
+                year=int(fname_search.group(2)))
 
 
 class FileImport(Model):
@@ -44,7 +43,7 @@ class FileImport(Model):
     Files to import or imported in the DB.
     Give either url or path.
     """
-    __tablename__ = 'file_import'
+    __tablename__: str = 'file_import' # type: ignore
     __table_args__ = gisaf_admin.table_args
 
     id: int | None = Field(default=None, primary_key=True)
@@ -79,9 +78,10 @@ class FileImport(Model):
     def selectinload(cls):
         return [cls.project, cls.surveyor, cls.equipment]
 
-    def set_import_time(self):
+    ## XXX: was used in Flask
-        self.time = datetime.now()
+    # def set_import_time(self):
-        db.session.commit()
+    #     self.time = datetime.now()
+    #     db.session.commit()
 
     @classmethod
     async def get_df(cls, *args, **kwargs):
@@ -116,7 +116,7 @@ class FeatureImportData(Model):
     """
     Keep track of imported data, typically from shapefiles
     """
-    __tablename__ = 'feature_import_data'
+    __tablename__: str = 'feature_import_data' # type: ignore
     __table_args__ = gisaf_admin.table_args
 
     id: int | None = Field(default=None, primary_key=True)
@@ -127,3 +127,36 @@ class FeatureImportData(Model):
     origin: str
     file_path: str
     file_md5: str
+
+
+class BasketFile(BaseModel):
+    id: int
+    dir: int
+    name: int
+    url: int
+    md5: int
+    time: datetime
+    comment: int
+    status: int
+    store: int
+    project: int
+    surveyor: int
+    equipment: int
+    import_result: int
+
+
+class BasketNameOnly(BaseModel):
+    name: str
+
+
+class Basket(BasketNameOnly):
+    files: list[BasketFile]
+    columns: list[str]
+    uploadFields: list[str]
+    projects: list[str]
+
+
+class BasketImportResult(BaseModel):
+    time: datetime
+    message: str
+    details: str