Add plugin management (WIP)

Refactoring some models, prevent circular deps

src/gisaf/api/main.py

@@ -12,7 +12,8 @@ from gisaf.models.authentication import (
   Role, RoleRead,
 )
 from gisaf.models.category import Category, CategoryRead
-from gisaf.models.to_migrate import DataProvider
+from gisaf.models.geo_models_base import GeoModel
+from gisaf.models.info import LegendItem, ModelAction, ModelInfo, DataProvider, ModelValue, TagActions
 from gisaf.models.survey import Equipment, SurveyMeta, Surveyor
 from gisaf.config import Survey, conf
 from gisaf.models.bootstrap import BootstrapData
@@ -26,10 +27,13 @@ from gisaf.security import (
     )
 from gisaf.registry import registry, NotInRegistry
 from gisaf.custom_store_base import BaseStore
-from gisaf.models.to_migrate import (
+from gisaf.redis_tools import store as redis_store
+from gisaf.models.info import (
   FeatureInfo, InfoItem, Attachment, InfoCategory
 )
 from gisaf.live_utils import get_live_feature_info
+from gisaf.plugins import manager as plugin_manager, NoSuchAction
+from gisaf.utils import gisTypeSymbolMap
 
 logger = logging.getLogger(__name__)
 
@@ -151,11 +155,11 @@ async def get_survey_meta(
 @api.get("/feature-info/{store}/{id}")
 async def get_feature_info(
     store: str, id: str,
-    ) -> FeatureInfo:
+    ) -> FeatureInfo | None:
     if store not in registry.stores.index:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
     store_record = registry.stores.loc[store]
-    model = store_record.model
+    model: type[GeoModel] = store_record.model
     if store_record.is_live:
         feature_info = await get_live_feature_info(store, id)
     elif issubclass(model, BaseStore):
@@ -168,6 +172,56 @@ async def get_feature_info(
             raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
     return feature_info
 
+
+@api.get("/model-info/{store}")
+async def get_model_info(
+    store: str
+    ) -> ModelInfo:
+    try:
+        store_record = registry.stores.loc[store]
+    except KeyError:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+    if store_record.is_live:
+        ## Get layer_defs from live redis and give symbol
+        layer_def = await redis_store.get_layer_def(store)
+        return ModelInfo(modelName=layer_def.pop('name'), **layer_def)
+    model = store_record.model
+    model_info = {
+        'store': store,
+        'modelName': model.__name__,
+        'symbol': model.symbol or gisTypeSymbolMap[model.base_gis_type],
+    }
+    ## Add information about the legend
+    if hasattr(model, 'get_legend'):
+        legend = await model.get_legend()
+        model_info['legend'] = [
+            LegendItem(key=k, value=v)
+            for k, v in legend.items()
+        ]
+    ## Add information about values
+    values_model = registry.values_for_model.get(model)
+    if hasattr(values_model, 'values'):
+        model_info['values'] = [ModelValue(**values) for values in values_model.values]
+    ## Add information about tags
+    ## TODO: add to plugin_manager a way to retrieve tag_store/tag_actions from a dict?
+    #tag_store = [tt for tt in plugin_manager.tagsStores.stores if tt.store==store][0]
+    model_info['tagActions'] = [
+        TagActions(key=key, domain=domain, actions=actions)
+        for domain, actions_keys in plugin_manager.tags_models[model].items()
+        for key, actions in actions_keys.items()
+    ]
+    model_info['actions'] = [
+        ModelAction(
+            name=name,
+            icon=action.icon,
+            formFields=action.formFields,
+        )
+        for name, actions in plugin_manager.actions_stores.get(store, {}).items()
+        for action in actions
+    ]
+    model_info['downloaders'] = plugin_manager.downloaders_stores[store]
+    return ModelInfo(**model_info)
+
 # @api.get("/user-role")
 # async def get_user_role_relation(
 #     *, db_session: AsyncSession = Depends(get_db_session)

src/gisaf/application.py

@@ -14,6 +14,7 @@ from gisaf.api.geoapi import api as geoapi
 from gisaf.api.admin import api as admin_api
 from gisaf.api.dashboard import api as dashboard_api
 from gisaf.api.map import api as map_api
+from gisaf.plugins import manager as plugin_manger
 
 logging.basicConfig(level=conf.gisaf.debugLevel)
 logger = logging.getLogger(__name__)
@@ -25,6 +26,7 @@ async def lifespan(app: FastAPI):
     await setup_redis()
     await setup_redis_cache()
     await setup_live()
+    await plugin_manger.scan_plugins()
     await admin_manager.setup_admin()
     await map_tile_registry.setup()
     yield

src/gisaf/custom_store_base.py

@@ -7,7 +7,7 @@ from sqlmodel import SQLModel
 
 from gisaf.config import conf
 from gisaf.models.map_bases import MaplibreStyle
-from gisaf.models.to_migrate import FeatureInfo
+from gisaf.models.info import FeatureInfo
 
 
 class BaseStore(SQLModel):

src/gisaf/live_utils.py

@@ -5,9 +5,7 @@ from shapely.geometry import (
 
 from gisaf.redis_tools import store as redis_store
 from gisaf.models.geo_models_base import reproject_func
-from gisaf.models.to_migrate import (
+from gisaf.models.info import FeatureInfo, InfoItem
-  FeatureInfo, InfoItem, Attachment, InfoCategory
-)
 
 async def get_live_feature_info(store: str, id: str) -> FeatureInfo:
         item = await redis_store.get_feature_info(store, id)

src/gisaf/models/geo_models_base.py

@@ -40,7 +40,7 @@ from gisaf.models.models_base import Model
 from gisaf.models.metadata import gisaf_survey, gisaf_admin, survey, raw_survey
 from gisaf.models.misc import Qml
 from gisaf.models.category import Category
-from gisaf.models.to_migrate import InfoItem
+from gisaf.models.info_item import InfoItem
 # from gisaf.models.survey import Equipment, Surveyor, Accuracy
 # from gisaf.models.project import Project
 

src/gisaf/models/info.py

@@ -0,0 +1,161 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from gisaf.models.info_item import InfoItem
+
+class ActionResult(BaseModel):
+    message: str
+
+
+class ActionResults(BaseModel):
+    name: str
+    message: str
+    actionResults: list[ActionResult]
+
+
+class DataProvider(BaseModel):
+    name: str
+    values: list[str]
+
+
+class InfoCategory(BaseModel):
+    name: str
+    infoItems: list[InfoItem]
+
+
+class PlotBgShape(BaseModel):
+    name: str
+    valueTop: float
+    valueBottom: float
+    color: str
+
+
+class PlotBaseLine(BaseModel):
+    name: str
+    value: float
+    color: str
+
+
+class PlotParams(BaseModel):
+    baseLines: list[PlotBaseLine]
+    bgShape: list[PlotBgShape]
+    barBase: float
+
+
+class Attachment(BaseModel):
+    name: str
+    path: str
+
+
+class FeatureInfo(BaseModel):
+    id: str
+    itemName: str
+    geoInfoItems: list[InfoItem] = []
+    surveyInfoItems: list[InfoItem] = []
+    infoItems: list[InfoItem] = []
+    categorizedInfoItems: list[InfoCategory] = []
+    tags: list[InfoItem] = []
+    graph: str | None = None
+    plotParams: PlotParams | None = None
+    files: list[Attachment] = []
+    images: list[Attachment] = []
+    externalRecordUrl: str | None = None
+
+
+class ModelValue(BaseModel):
+    name: str
+    title: str
+    unit: str
+    chartType: str = 'line'
+    chartColor: str = 'blue'
+
+
+class FormField(BaseModel):
+    name: str
+    type: str
+    dflt: str | None = None
+    value: str | None = None
+
+
+class ModelAction(BaseModel):
+    name: str
+    icon: str
+    formFields: list[FormField]
+
+
+class TagAction(BaseModel):
+    name: str
+    _plugin: Any
+    action: str
+    roles: list[str] | None = None
+    link: str | None = None
+    save: bool = False
+
+
+class TagActions(BaseModel):
+    domain: str
+    key: str
+    actions: list[TagAction]
+
+
+class ActionAction(BaseModel):
+    roles: list[str]
+    # plugin: Any
+    name: str
+    params: Any
+    icon: str
+    formFields: list[FormField]
+
+
+class Downloader(BaseModel):
+    # plugin: str
+    # downloader: str
+    roles: list[str] = []
+    name: str
+    icon: str | None = None
+
+
+class LegendItem(BaseModel):
+    key: str
+    value: str
+
+
+class ModelInfo(BaseModel):
+    store: str
+    modelName: str
+    symbol: str | None = None
+    values: list[ModelValue] = []
+    actions: list[ModelAction] = []
+    formName: str | None = None
+    formFields: list[FormField] = []
+    tagPlugins: list[str] = []
+    tagActions: list[TagActions] = []
+    downloaders: list[Downloader] = []
+    legend: list[LegendItem] = []
+
+
+class TagsStore(BaseModel):
+    store: str
+    tagActions: list[TagActions]
+
+
+class TagsStores(BaseModel):
+    stores: list[TagsStore]
+
+
+class ActionParam(BaseModel):
+    name: str
+    type: str
+    dflt: str
+
+
+class Action(BaseModel):
+    name: str
+    roles: list[str]
+    params: list[ActionParam]
+
+
+class ActionsStore(BaseModel):
+    store: str
+    actions: list[Action]

src/gisaf/models/info_item.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class InfoItem(BaseModel):
+    key: str
+    value: str | float | int

src/gisaf/models/tags.py

@@ -3,7 +3,7 @@ from sqlalchemy import BigInteger
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.dialects.postgresql import HSTORE
 from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
-from pydantic import computed_field
+from pydantic import BaseModel, computed_field
 
 from gisaf.models.metadata import gisaf
 from gisaf.models.geo_models_base import GeoPointModel
@@ -42,4 +42,5 @@ class TagKey(SQLModel, table=True):
         return self.key
 
     def __repr__(self):
-        return '<models.TagKey {self.key}>'.format(self=self)
+        return '<models.TagKey {self.key}>'.format(self=self)
+

src/gisaf/models/to_migrate.py

@@ -1,75 +0,0 @@
-from pydantic import BaseModel
-
-class ActionResult(BaseModel):
-    message: str
-
-
-class ActionResults(BaseModel):
-    name: str
-    message: str
-    actionResults: list[ActionResult]
-
-
-class FormField(BaseModel):
-    name: str
-    type: str
-
-
-class ModelAction(BaseModel):
-    name: str
-    icon: str
-    formFields: list[FormField]
-
-
-class DataProvider(BaseModel):
-    name: str
-    values: list[str]
-
-
-class InfoItem(BaseModel):
-    key: str
-    value: str | float | int
-
-
-class InfoCategory(BaseModel):
-    name: str
-    infoItems: list[InfoItem]
-
-
-class PlotBgShape(BaseModel):
-    name: str
-    valueTop: float
-    valueBottom: float
-    color: str
-
-
-class PlotBaseLine(BaseModel):
-    name: str
-    value: float
-    color: str
-
-
-class PlotParams(BaseModel):
-    baseLines: list[PlotBaseLine]
-    bgShape: list[PlotBgShape]
-    barBase: float
-
-
-class Attachment(BaseModel):
-    name: str
-    path: str
-
-
-class FeatureInfo(BaseModel):
-    id: str
-    itemName: str
-    geoInfoItems: list[InfoItem] = []
-    surveyInfoItems: list[InfoItem] = []
-    infoItems: list[InfoItem] = []
-    categorizedInfoItems: list[InfoCategory] = []
-    tags: list[InfoItem] = []
-    graph: str | None = None
-    plotParams: PlotParams | None = None
-    files: list[Attachment] = []
-    images: list[Attachment] = []
-    externalRecordUrl: str | None = None

src/gisaf/plugins.py

@@ -8,43 +8,30 @@ from datetime import datetime
 # from aiohttp.web_exceptions import HTTPUnauthorized
 # from aiohttp_security import check_permission
 
+from pydantic import BaseModel  # noqa: F401
 from sqlalchemy import or_, and_
 # from geoalchemy2.shape import to_shape, from_shape
 # from graphene import ObjectType, String, List, Boolean, Field, Float, InputObjectType
 
 import pandas as pd
-import shapely
+import shapely # type: ignore
 
 from gisaf.config import conf
+from gisaf.models.store import Store  # noqa: F401
 from gisaf.models.tags import Tags as TagsModel
 from gisaf.utils import upsert_df
 from gisaf.models.reconcile import StatusChange
-from gisaf.models.to_migrate import (
+from gisaf.models.info import (
     ActionResults,
+    Downloader,
+    TagAction, ActionAction,
+    TagActions,
+    TagsStore,
+    TagsStores,
+    ActionsStore,
+    Action
 )
-# from gisaf.models.graphql import (
+from gisaf.registry import NotInRegistry, registry
-#     Action,
-#     ActionAction,
-#     ActionParam,
-#     ActionParamInput,
-#     ActionResult,
-#     ActionResults,
-#     ActionsResults,
-#     ActionsStore,
-#     Downloader,
-#     FormField,
-#     Tag,
-#     TagAction,
-#     TagActions,
-#     TagKeyList,
-#     TaggedFeature,
-#     TaggedLayer,
-#     TagsStore,
-#     TagsStores,
-# )
-
-## GraphQL object types
-## TODO: move to models.graphql
 
 
 logger = logging.getLogger('Gisaf plugin manager')
@@ -54,6 +41,7 @@ class NoSuchAction(Exception):
     pass
 
 
+
 class ActionPlugin:
     """
     Base class for all actions plugins.
@@ -77,10 +65,24 @@ class TagPlugin:
     Keys might be reg exp.
     See Link (below) for a very basic example.
     """
+    key: str
+    domain: str
+    stores: list[str]
+    stores_by_re: list[str]
+    roles: list[str] | None
+    save: bool
+    link: str | None
+    action: str | None
 
-    def __init__(self, key='', domain='',
+    def __init__(self,
-                 stores=None, stores_by_re=None, roles=None,
+                 key: str = '',
-                 save=True, link=None, action=None):
+                 domain: str = '',
+                 stores: list[str] | None = None,
+                 stores_by_re: list[str] | None = None,
+                 roles: list[str] | None = None,
+                 save: bool = True,
+                 link: str | None = None,
+                 action: str | None = None):
         ## self._tags: instanciated tags
         self.key = key
         self.domain = domain
@@ -119,10 +121,12 @@ class DownloadPlugin:
 
 class DownloadCSVPlugin(DownloadPlugin):
     async def execute(self, model, item, request):
-        from gisaf.registry import registry
+        try:
-        values_models = registry.values_for_model.get(model)
+            values_models = registry.values_for_model[model]
+        except KeyError:
+            raise NotInRegistry
         for value_model in values_models:
-            df = await values_models.get_as_dataframe(model_id=item.id)
+            df = await value_model.get_as_dataframe(model_id=item.id)
             csv = df.to_csv(date_format='%d/%m/%Y %H:%M', float_format=value_model.float_format)
             ## TODO: implement multiple values for a model (search for values_for_model)
             break
@@ -138,18 +142,17 @@ class PluginManager:
     Application wide manager of the plugins.
     One instance only, handled by Gisaf's process.
     """
-    def setup(self, app):
+    def setup(self):
-        self.app = app
         for entry_point in entry_points().select(group='gisaf_extras.context'):
             try:
                 context = entry_point.load()
             except ModuleNotFoundError as err:
                 logger.warning(err)
                 continue
-            self.app.cleanup_ctx.append(context)
+            # self.app.cleanup_ctx.append(context)
             logger.info(f'Added context for {entry_point.name}')
 
-    async def scan_plugins(self, app):
+    async def scan_plugins(self) -> None:
         """
         Scan tag and action plugins from the Python entry points.
         Get all references of the tags defined in modules ad build a registry of:
@@ -161,27 +164,27 @@ class PluginManager:
         self.tags_domains = defaultdict(list)
 
         #self.actions_plugins = {}
-        self.actions_stores = defaultdict(lambda: defaultdict(list))
+        self.actions_stores: dict[str, dict[str, list[ActionAction]]] = {}
         self.executors = defaultdict(list)
         self.downloaders = defaultdict(list)
         self.downloaders_stores = defaultdict(list)
         self.actions_names = defaultdict(list)
 
-        registered_models = app['registry'].geom
+        registered_models = registry.geom
         registered_stores = registered_models.keys()
 
         for entry_point in entry_points().select(group='gisaf_extras.tags'):
             try:
-                tag = entry_point.load()
+                tagPlugin: TagPlugin = entry_point.load()
             except ModuleNotFoundError as err:
                 logger.warning(err)
                 continue
             ## Keys, domains
-            self.tags_domains[tag.domain].append(tag)
+            self.tags_domains[tagPlugin.domain].append(tagPlugin)
-            stores = tag.stores
+            stores = tagPlugin.stores
 
             ## Stores to which the tags apply
-            for _tag in tag.stores_by_re:
+            for _tag in tagPlugin.stores_by_re:
                 _re = re.compile(_tag)
                 for store in registered_stores:
                     if _re.match(store) and store not in stores:
@@ -195,16 +198,17 @@ class PluginManager:
                     continue
 
                 ## Actions
-                ## For graphql queries
+                if tagPlugin.action is not None:
-                self.tags_models[model][tag.domain][tag.key].append(
+                    self.tags_models[model][tagPlugin.domain][tagPlugin.key].append(
-                    TagAction(
+                        TagAction(
-                        #plugin=plugin.__class__.__name__,
+                            _plugin=tagPlugin,
-                        roles=tag.roles,
+                            name=entry_point.name,
-                        link=tag.link,
+                            roles=tagPlugin.roles,
-                        action=tag.action,
+                            link=tagPlugin.link,
-                        save=tag.save,
+                            action=tagPlugin.action,
+                            save=tagPlugin.save,
+                        )
                     )
-                )
             logger.info(f'Added tags plugin {entry_point.name} for {len(stores)} stores')
 
         for entry_point in entry_points().select(group='gisaf_extras.actions'):
@@ -227,9 +231,12 @@ class PluginManager:
                     logger.warn(f'Action plugin {entry_point.name}: skip model {store}'
                                     ', which is not found in registry')
                     continue
+                if store not in self.actions_stores:
+                    self.actions_stores[store] = {}
+                if action.name not in self.actions_stores[store]:
+                    self.actions_stores[store][action.name] = []
                 self.actions_stores[store][action.name].append(
                     ActionAction(
-                        action=action,
                         roles=action.roles,
                         #plugin=plugin.__class__.__name__,
                         name=action.name,
@@ -242,7 +249,7 @@ class PluginManager:
 
         for entry_point in entry_points().select(group='gisaf_extras.downloaders'):
             try:
-                downloader = entry_point.load()
+                downloader: DownloadPlugin = entry_point.load()
             except ModuleNotFoundError as err:
                 logger.warning(err)
                 continue
@@ -261,9 +268,8 @@ class PluginManager:
                     continue
                 self.downloaders_stores[store].append(
                     Downloader(
-                        downloader=downloader,
-                        roles=downloader.roles,
                         name=downloader.name,
+                        roles=downloader.roles,
                         icon=downloader.icon,
                     )
                 )
@@ -291,12 +297,12 @@ class PluginManager:
                         name=name,
                         roles=[rr for rr in set(
                             [r for r in chain.from_iterable(
-                                [aa._roles for aa in action_actions]
+                                [aa.roles for aa in action_actions]
                             )]
                         )],
                         params=[rr for rr in set(
                             [r for r in chain.from_iterable(
-                                [aa._params for aa in action_actions]
+                                [aa.params for aa in action_actions]
                             )]
                         )],
                     )
@@ -306,8 +312,6 @@ class PluginManager:
             for store, actions in self.actions_stores.items()
         ]
 
-        app['plugins'] = self
-
     async def do_tag_action(self, request, store, id, plugin_name, value, action):
         logger.warning('FIXME: in do_tag_action: self.tags_plugins is never populated!')
         ## FIXME: self.tags_plugins is never populated!

src/gisaf/registry.py

@@ -30,14 +30,13 @@ from gisaf.models.geo_models_base import (
     GeoLineSurveyModel,
     GeoPolygonSurveyModel,
 )
-from gisaf.models.survey import Equipment, Surveyor, Accuracy
-from gisaf.models.project import Project
-from gisaf.utils import ToMigrate
-from gisaf.models.category import Category, CategoryGroup
 from gisaf.database import db_session
 from gisaf import models
+from gisaf.models.survey import Equipment, Surveyor, Accuracy
+from gisaf.models.project import Project
+from gisaf.models.category import Category, CategoryGroup
 from gisaf.models.metadata import raw_survey, survey
-from gisaf.models.to_migrate import FeatureInfo, InfoCategory
+from gisaf.models.info import FeatureInfo
 
 logger = logging.getLogger(__name__)
 
@@ -79,7 +78,7 @@ class ModelRegistry:
     categories: pd.DataFrame
     primary_groups: list[CategoryGroup]
     values: dict[str, PlottableModel]
-    geom: dict[str, GeoModel]
+    geom: dict[str, GeoModel | SurveyModel]
     geom_live: dict[str, LiveGeoModel]
     geom_live_defs: dict[str, dict[str, Any]]
     geom_custom: dict[str, GeoModel]
@@ -131,9 +130,10 @@ class ModelRegistry:
         """
         logger.debug('make_category_models')
         async with db_session() as session:
-            query = select(Category).order_by(Category.long_name).options(selectinload(Category.category_group))
+            query = select(Category).order_by(Category.long_name).\
+                    options(selectinload(Category.category_group)) # type: ignore
             data = await session.exec(query)
-            categories: list[Category] = data.all()
+            categories: list[Category] = data.all() # type: ignore
         for category in categories:
             ## Several statuses can coexist for the same model, so
             ## consider only the ones with the 'E' (existing) status
@@ -156,7 +156,7 @@ class ModelRegistry:
             }
             ## Raw survey points
             try:
-                self.raw_survey_models[store_name] = create_model(
+                self.raw_survey_models[store_name] = create_model( # type: ignore
                     __base__=RawSurveyBaseModel,
                     __model_name=category.raw_survey_table_name,
                     __cls_kwargs__={
@@ -188,7 +188,7 @@ class ModelRegistry:
                         #'raw_model': (str, self.raw_survey_models.get(raw_store_name)),
                         # 'icon': (str, f'{survey.schema}-{category.table_name}'),
                     }
-                    self.survey_models[store_name] = create_model(
+                    self.survey_models[store_name] = create_model( # type: ignore
                         __base__= model_class,
                         __model_name=category.table_name,
                         __cls_kwargs__={
@@ -336,12 +336,12 @@ class ModelRegistry:
     #             for category in categories
     #             if self.raw_survey_models.get(category.table_name)}
 
-    async def get_model_id_params(self, model: SQLModel, id: int) -> FeatureInfo | None:
+    async def get_model_id_params(self, model: type[GeoModel], id: int) -> FeatureInfo:
         """
         Return the parameters for this item (table name, id), displayed in info pane
         """
         if not model:
-            return
+            raise NotInRegistry
         async with db_session() as session:
             query = select(model).where(model.id == id).options(
                 *(joinedload(jt) for jt in model.selectinload()))
@@ -350,11 +350,9 @@ class ModelRegistry:
                 item = result.one()
             except NoResultFound:
                 raise NotInRegistry
-
             # item = await model.load(**model.get_join_with()).query.where(model.id==id).gino.first()
         if not item:
-            return
+            raise NotInRegistry
-
         files, images = [], []
         externalRecordUrl, graph, categorizedInfoItems = (None, ) * 3
         if hasattr(item, 'get_categorized_info'):
@@ -368,7 +366,6 @@ class ModelRegistry:
                 images = await item.Attachments.images(item)
         if hasattr(item, 'get_external_record_url'):
             externalRecordUrl = item.get_external_record_url()
-
         return FeatureInfo(
             id=str(item.id),
             itemName=item.caption,