From 360a7a70f3a37c923d1129927942bc0b01a77c8c Mon Sep 17 00:00:00 2001 From: phil Date: Thu, 7 Mar 2024 12:13:47 +0530 Subject: [PATCH] Add plugin management (WIP) Refactoring some models, prevent circular deps --- src/gisaf/api/main.py | 62 ++++++++++- src/gisaf/application.py | 2 + src/gisaf/custom_store_base.py | 2 +- src/gisaf/live_utils.py | 4 +- src/gisaf/models/geo_models_base.py | 2 +- src/gisaf/models/info.py | 161 ++++++++++++++++++++++++++++ src/gisaf/models/info_item.py | 6 ++ src/gisaf/models/tags.py | 5 +- src/gisaf/models/to_migrate.py | 75 ------------- src/gisaf/plugins.py | 120 +++++++++++---------- src/gisaf/registry.py | 29 +++-- 11 files changed, 308 insertions(+), 160 deletions(-) create mode 100644 src/gisaf/models/info.py create mode 100644 src/gisaf/models/info_item.py delete mode 100644 src/gisaf/models/to_migrate.py diff --git a/src/gisaf/api/main.py b/src/gisaf/api/main.py index 2fa75bf..ddca321 100644 --- a/src/gisaf/api/main.py +++ b/src/gisaf/api/main.py @@ -12,7 +12,8 @@ from gisaf.models.authentication import ( Role, RoleRead, ) from gisaf.models.category import Category, CategoryRead -from gisaf.models.to_migrate import DataProvider +from gisaf.models.geo_models_base import GeoModel +from gisaf.models.info import LegendItem, ModelAction, ModelInfo, DataProvider, ModelValue, TagActions from gisaf.models.survey import Equipment, SurveyMeta, Surveyor from gisaf.config import Survey, conf from gisaf.models.bootstrap import BootstrapData @@ -26,10 +27,13 @@ from gisaf.security import ( ) from gisaf.registry import registry, NotInRegistry from gisaf.custom_store_base import BaseStore -from gisaf.models.to_migrate import ( +from gisaf.redis_tools import store as redis_store +from gisaf.models.info import ( FeatureInfo, InfoItem, Attachment, InfoCategory ) from gisaf.live_utils import get_live_feature_info +from gisaf.plugins import manager as plugin_manager, NoSuchAction +from gisaf.utils import gisTypeSymbolMap logger = logging.getLogger(__name__) @@ -151,11 +155,11 @@ async def get_survey_meta( @api.get("/feature-info/{store}/{id}") async def get_feature_info( store: str, id: str, - ) -> FeatureInfo: + ) -> FeatureInfo | None: if store not in registry.stores.index: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) store_record = registry.stores.loc[store] - model = store_record.model + model: type[GeoModel] = store_record.model if store_record.is_live: feature_info = await get_live_feature_info(store, id) elif issubclass(model, BaseStore): @@ -168,6 +172,56 @@ async def get_feature_info( raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) return feature_info + +@api.get("/model-info/{store}") +async def get_model_info( + store: str + ) -> ModelInfo: + try: + store_record = registry.stores.loc[store] + except KeyError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + if store_record.is_live: + ## Get layer_defs from live redis and give symbol + layer_def = await redis_store.get_layer_def(store) + return ModelInfo(modelName=layer_def.pop('name'), **layer_def) + model = store_record.model + model_info = { + 'store': store, + 'modelName': model.__name__, + 'symbol': model.symbol or gisTypeSymbolMap[model.base_gis_type], + } + ## Add information about the legend + if hasattr(model, 'get_legend'): + legend = await model.get_legend() + model_info['legend'] = [ + LegendItem(key=k, value=v) + for k, v in legend.items() + ] + ## Add information about values + values_model = registry.values_for_model.get(model) + if hasattr(values_model, 'values'): + model_info['values'] = [ModelValue(**values) for values in values_model.values] + ## Add information about tags + ## TODO: add to plugin_manager a way to retrieve tag_store/tag_actions from a dict? + #tag_store = [tt for tt in plugin_manager.tagsStores.stores if tt.store==store][0] + model_info['tagActions'] = [ + TagActions(key=key, domain=domain, actions=actions) + for domain, actions_keys in plugin_manager.tags_models[model].items() + for key, actions in actions_keys.items() + ] + model_info['actions'] = [ + ModelAction( + name=name, + icon=action.icon, + formFields=action.formFields, + ) + for name, actions in plugin_manager.actions_stores.get(store, {}).items() + for action in actions + ] + model_info['downloaders'] = plugin_manager.downloaders_stores[store] + return ModelInfo(**model_info) + # @api.get("/user-role") # async def get_user_role_relation( # *, db_session: AsyncSession = Depends(get_db_session) diff --git a/src/gisaf/application.py b/src/gisaf/application.py index 36681a1..ed79bff 100644 --- a/src/gisaf/application.py +++ b/src/gisaf/application.py @@ -14,6 +14,7 @@ from gisaf.api.geoapi import api as geoapi from gisaf.api.admin import api as admin_api from gisaf.api.dashboard import api as dashboard_api from gisaf.api.map import api as map_api +from gisaf.plugins import manager as plugin_manger logging.basicConfig(level=conf.gisaf.debugLevel) logger = logging.getLogger(__name__) @@ -25,6 +26,7 @@ async def lifespan(app: FastAPI): await setup_redis() await setup_redis_cache() await setup_live() + await plugin_manger.scan_plugins() await admin_manager.setup_admin() await map_tile_registry.setup() yield diff --git a/src/gisaf/custom_store_base.py b/src/gisaf/custom_store_base.py index d764be8..1cedfdc 100644 --- a/src/gisaf/custom_store_base.py +++ b/src/gisaf/custom_store_base.py @@ -7,7 +7,7 @@ from sqlmodel import SQLModel from gisaf.config import conf from gisaf.models.map_bases import MaplibreStyle -from gisaf.models.to_migrate import FeatureInfo +from gisaf.models.info import FeatureInfo class BaseStore(SQLModel): diff --git a/src/gisaf/live_utils.py b/src/gisaf/live_utils.py index d74ad5d..4bbadfd 100644 --- a/src/gisaf/live_utils.py +++ b/src/gisaf/live_utils.py @@ -5,9 +5,7 @@ from shapely.geometry import ( from gisaf.redis_tools import store as redis_store from gisaf.models.geo_models_base import reproject_func -from gisaf.models.to_migrate import ( - FeatureInfo, InfoItem, Attachment, InfoCategory -) +from gisaf.models.info import FeatureInfo, InfoItem async def get_live_feature_info(store: str, id: str) -> FeatureInfo: item = await redis_store.get_feature_info(store, id) diff --git a/src/gisaf/models/geo_models_base.py b/src/gisaf/models/geo_models_base.py index 0ea96db..e74be77 100644 --- a/src/gisaf/models/geo_models_base.py +++ b/src/gisaf/models/geo_models_base.py @@ -40,7 +40,7 @@ from gisaf.models.models_base import Model from gisaf.models.metadata import gisaf_survey, gisaf_admin, survey, raw_survey from gisaf.models.misc import Qml from gisaf.models.category import Category -from gisaf.models.to_migrate import InfoItem +from gisaf.models.info_item import InfoItem # from gisaf.models.survey import Equipment, Surveyor, Accuracy # from gisaf.models.project import Project diff --git a/src/gisaf/models/info.py b/src/gisaf/models/info.py new file mode 100644 index 0000000..098091b --- /dev/null +++ b/src/gisaf/models/info.py @@ -0,0 +1,161 @@ +from typing import Any + +from pydantic import BaseModel + +from gisaf.models.info_item import InfoItem + +class ActionResult(BaseModel): + message: str + + +class ActionResults(BaseModel): + name: str + message: str + actionResults: list[ActionResult] + + +class DataProvider(BaseModel): + name: str + values: list[str] + + +class InfoCategory(BaseModel): + name: str + infoItems: list[InfoItem] + + +class PlotBgShape(BaseModel): + name: str + valueTop: float + valueBottom: float + color: str + + +class PlotBaseLine(BaseModel): + name: str + value: float + color: str + + +class PlotParams(BaseModel): + baseLines: list[PlotBaseLine] + bgShape: list[PlotBgShape] + barBase: float + + +class Attachment(BaseModel): + name: str + path: str + + +class FeatureInfo(BaseModel): + id: str + itemName: str + geoInfoItems: list[InfoItem] = [] + surveyInfoItems: list[InfoItem] = [] + infoItems: list[InfoItem] = [] + categorizedInfoItems: list[InfoCategory] = [] + tags: list[InfoItem] = [] + graph: str | None = None + plotParams: PlotParams | None = None + files: list[Attachment] = [] + images: list[Attachment] = [] + externalRecordUrl: str | None = None + + +class ModelValue(BaseModel): + name: str + title: str + unit: str + chartType: str = 'line' + chartColor: str = 'blue' + + +class FormField(BaseModel): + name: str + type: str + dflt: str | None = None + value: str | None = None + + +class ModelAction(BaseModel): + name: str + icon: str + formFields: list[FormField] + + +class TagAction(BaseModel): + name: str + _plugin: Any + action: str + roles: list[str] | None = None + link: str | None = None + save: bool = False + + +class TagActions(BaseModel): + domain: str + key: str + actions: list[TagAction] + + +class ActionAction(BaseModel): + roles: list[str] + # plugin: Any + name: str + params: Any + icon: str + formFields: list[FormField] + + +class Downloader(BaseModel): + # plugin: str + # downloader: str + roles: list[str] = [] + name: str + icon: str | None = None + + +class LegendItem(BaseModel): + key: str + value: str + + +class ModelInfo(BaseModel): + store: str + modelName: str + symbol: str | None = None + values: list[ModelValue] = [] + actions: list[ModelAction] = [] + formName: str | None = None + formFields: list[FormField] = [] + tagPlugins: list[str] = [] + tagActions: list[TagActions] = [] + downloaders: list[Downloader] = [] + legend: list[LegendItem] = [] + + +class TagsStore(BaseModel): + store: str + tagActions: list[TagActions] + + +class TagsStores(BaseModel): + stores: list[TagsStore] + + +class ActionParam(BaseModel): + name: str + type: str + dflt: str + + +class Action(BaseModel): + name: str + roles: list[str] + params: list[ActionParam] + + +class ActionsStore(BaseModel): + store: str + actions: list[Action] diff --git a/src/gisaf/models/info_item.py b/src/gisaf/models/info_item.py new file mode 100644 index 0000000..b49df28 --- /dev/null +++ b/src/gisaf/models/info_item.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class InfoItem(BaseModel): + key: str + value: str | float | int diff --git a/src/gisaf/models/tags.py b/src/gisaf/models/tags.py index 3de7523..d262eab 100644 --- a/src/gisaf/models/tags.py +++ b/src/gisaf/models/tags.py @@ -3,7 +3,7 @@ from sqlalchemy import BigInteger from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.dialects.postgresql import HSTORE from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column -from pydantic import computed_field +from pydantic import BaseModel, computed_field from gisaf.models.metadata import gisaf from gisaf.models.geo_models_base import GeoPointModel @@ -42,4 +42,5 @@ class TagKey(SQLModel, table=True): return self.key def __repr__(self): - return ''.format(self=self) \ No newline at end of file + return ''.format(self=self) + diff --git a/src/gisaf/models/to_migrate.py b/src/gisaf/models/to_migrate.py deleted file mode 100644 index 853ccfb..0000000 --- a/src/gisaf/models/to_migrate.py +++ /dev/null @@ -1,75 +0,0 @@ -from pydantic import BaseModel - -class ActionResult(BaseModel): - message: str - - -class ActionResults(BaseModel): - name: str - message: str - actionResults: list[ActionResult] - - -class FormField(BaseModel): - name: str - type: str - - -class ModelAction(BaseModel): - name: str - icon: str - formFields: list[FormField] - - -class DataProvider(BaseModel): - name: str - values: list[str] - - -class InfoItem(BaseModel): - key: str - value: str | float | int - - -class InfoCategory(BaseModel): - name: str - infoItems: list[InfoItem] - - -class PlotBgShape(BaseModel): - name: str - valueTop: float - valueBottom: float - color: str - - -class PlotBaseLine(BaseModel): - name: str - value: float - color: str - - -class PlotParams(BaseModel): - baseLines: list[PlotBaseLine] - bgShape: list[PlotBgShape] - barBase: float - - -class Attachment(BaseModel): - name: str - path: str - - -class FeatureInfo(BaseModel): - id: str - itemName: str - geoInfoItems: list[InfoItem] = [] - surveyInfoItems: list[InfoItem] = [] - infoItems: list[InfoItem] = [] - categorizedInfoItems: list[InfoCategory] = [] - tags: list[InfoItem] = [] - graph: str | None = None - plotParams: PlotParams | None = None - files: list[Attachment] = [] - images: list[Attachment] = [] - externalRecordUrl: str | None = None diff --git a/src/gisaf/plugins.py b/src/gisaf/plugins.py index 464ce11..e6d4099 100644 --- a/src/gisaf/plugins.py +++ b/src/gisaf/plugins.py @@ -8,43 +8,30 @@ from datetime import datetime # from aiohttp.web_exceptions import HTTPUnauthorized # from aiohttp_security import check_permission +from pydantic import BaseModel # noqa: F401 from sqlalchemy import or_, and_ # from geoalchemy2.shape import to_shape, from_shape # from graphene import ObjectType, String, List, Boolean, Field, Float, InputObjectType import pandas as pd -import shapely +import shapely # type: ignore from gisaf.config import conf +from gisaf.models.store import Store # noqa: F401 from gisaf.models.tags import Tags as TagsModel from gisaf.utils import upsert_df from gisaf.models.reconcile import StatusChange -from gisaf.models.to_migrate import ( +from gisaf.models.info import ( ActionResults, + Downloader, + TagAction, ActionAction, + TagActions, + TagsStore, + TagsStores, + ActionsStore, + Action ) -# from gisaf.models.graphql import ( -# Action, -# ActionAction, -# ActionParam, -# ActionParamInput, -# ActionResult, -# ActionResults, -# ActionsResults, -# ActionsStore, -# Downloader, -# FormField, -# Tag, -# TagAction, -# TagActions, -# TagKeyList, -# TaggedFeature, -# TaggedLayer, -# TagsStore, -# TagsStores, -# ) - -## GraphQL object types -## TODO: move to models.graphql +from gisaf.registry import NotInRegistry, registry logger = logging.getLogger('Gisaf plugin manager') @@ -54,6 +41,7 @@ class NoSuchAction(Exception): pass + class ActionPlugin: """ Base class for all actions plugins. @@ -77,10 +65,24 @@ class TagPlugin: Keys might be reg exp. See Link (below) for a very basic example. """ + key: str + domain: str + stores: list[str] + stores_by_re: list[str] + roles: list[str] | None + save: bool + link: str | None + action: str | None - def __init__(self, key='', domain='', - stores=None, stores_by_re=None, roles=None, - save=True, link=None, action=None): + def __init__(self, + key: str = '', + domain: str = '', + stores: list[str] | None = None, + stores_by_re: list[str] | None = None, + roles: list[str] | None = None, + save: bool = True, + link: str | None = None, + action: str | None = None): ## self._tags: instanciated tags self.key = key self.domain = domain @@ -119,10 +121,12 @@ class DownloadPlugin: class DownloadCSVPlugin(DownloadPlugin): async def execute(self, model, item, request): - from gisaf.registry import registry - values_models = registry.values_for_model.get(model) + try: + values_models = registry.values_for_model[model] + except KeyError: + raise NotInRegistry for value_model in values_models: - df = await values_models.get_as_dataframe(model_id=item.id) + df = await value_model.get_as_dataframe(model_id=item.id) csv = df.to_csv(date_format='%d/%m/%Y %H:%M', float_format=value_model.float_format) ## TODO: implement multiple values for a model (search for values_for_model) break @@ -138,18 +142,17 @@ class PluginManager: Application wide manager of the plugins. One instance only, handled by Gisaf's process. """ - def setup(self, app): - self.app = app + def setup(self): for entry_point in entry_points().select(group='gisaf_extras.context'): try: context = entry_point.load() except ModuleNotFoundError as err: logger.warning(err) continue - self.app.cleanup_ctx.append(context) + # self.app.cleanup_ctx.append(context) logger.info(f'Added context for {entry_point.name}') - async def scan_plugins(self, app): + async def scan_plugins(self) -> None: """ Scan tag and action plugins from the Python entry points. Get all references of the tags defined in modules ad build a registry of: @@ -161,27 +164,27 @@ class PluginManager: self.tags_domains = defaultdict(list) #self.actions_plugins = {} - self.actions_stores = defaultdict(lambda: defaultdict(list)) + self.actions_stores: dict[str, dict[str, list[ActionAction]]] = {} self.executors = defaultdict(list) self.downloaders = defaultdict(list) self.downloaders_stores = defaultdict(list) self.actions_names = defaultdict(list) - registered_models = app['registry'].geom + registered_models = registry.geom registered_stores = registered_models.keys() for entry_point in entry_points().select(group='gisaf_extras.tags'): try: - tag = entry_point.load() + tagPlugin: TagPlugin = entry_point.load() except ModuleNotFoundError as err: logger.warning(err) continue ## Keys, domains - self.tags_domains[tag.domain].append(tag) - stores = tag.stores + self.tags_domains[tagPlugin.domain].append(tagPlugin) + stores = tagPlugin.stores ## Stores to which the tags apply - for _tag in tag.stores_by_re: + for _tag in tagPlugin.stores_by_re: _re = re.compile(_tag) for store in registered_stores: if _re.match(store) and store not in stores: @@ -195,16 +198,17 @@ class PluginManager: continue ## Actions - ## For graphql queries - self.tags_models[model][tag.domain][tag.key].append( - TagAction( - #plugin=plugin.__class__.__name__, - roles=tag.roles, - link=tag.link, - action=tag.action, - save=tag.save, + if tagPlugin.action is not None: + self.tags_models[model][tagPlugin.domain][tagPlugin.key].append( + TagAction( + _plugin=tagPlugin, + name=entry_point.name, + roles=tagPlugin.roles, + link=tagPlugin.link, + action=tagPlugin.action, + save=tagPlugin.save, + ) ) - ) logger.info(f'Added tags plugin {entry_point.name} for {len(stores)} stores') for entry_point in entry_points().select(group='gisaf_extras.actions'): @@ -227,9 +231,12 @@ class PluginManager: logger.warn(f'Action plugin {entry_point.name}: skip model {store}' ', which is not found in registry') continue + if store not in self.actions_stores: + self.actions_stores[store] = {} + if action.name not in self.actions_stores[store]: + self.actions_stores[store][action.name] = [] self.actions_stores[store][action.name].append( ActionAction( - action=action, roles=action.roles, #plugin=plugin.__class__.__name__, name=action.name, @@ -242,7 +249,7 @@ class PluginManager: for entry_point in entry_points().select(group='gisaf_extras.downloaders'): try: - downloader = entry_point.load() + downloader: DownloadPlugin = entry_point.load() except ModuleNotFoundError as err: logger.warning(err) continue @@ -261,9 +268,8 @@ class PluginManager: continue self.downloaders_stores[store].append( Downloader( - downloader=downloader, - roles=downloader.roles, name=downloader.name, + roles=downloader.roles, icon=downloader.icon, ) ) @@ -291,12 +297,12 @@ class PluginManager: name=name, roles=[rr for rr in set( [r for r in chain.from_iterable( - [aa._roles for aa in action_actions] + [aa.roles for aa in action_actions] )] )], params=[rr for rr in set( [r for r in chain.from_iterable( - [aa._params for aa in action_actions] + [aa.params for aa in action_actions] )] )], ) @@ -306,8 +312,6 @@ class PluginManager: for store, actions in self.actions_stores.items() ] - app['plugins'] = self - async def do_tag_action(self, request, store, id, plugin_name, value, action): logger.warning('FIXME: in do_tag_action: self.tags_plugins is never populated!') ## FIXME: self.tags_plugins is never populated! diff --git a/src/gisaf/registry.py b/src/gisaf/registry.py index 5189abb..7beeae4 100644 --- a/src/gisaf/registry.py +++ b/src/gisaf/registry.py @@ -30,14 +30,13 @@ from gisaf.models.geo_models_base import ( GeoLineSurveyModel, GeoPolygonSurveyModel, ) -from gisaf.models.survey import Equipment, Surveyor, Accuracy -from gisaf.models.project import Project -from gisaf.utils import ToMigrate -from gisaf.models.category import Category, CategoryGroup from gisaf.database import db_session from gisaf import models +from gisaf.models.survey import Equipment, Surveyor, Accuracy +from gisaf.models.project import Project +from gisaf.models.category import Category, CategoryGroup from gisaf.models.metadata import raw_survey, survey -from gisaf.models.to_migrate import FeatureInfo, InfoCategory +from gisaf.models.info import FeatureInfo logger = logging.getLogger(__name__) @@ -79,7 +78,7 @@ class ModelRegistry: categories: pd.DataFrame primary_groups: list[CategoryGroup] values: dict[str, PlottableModel] - geom: dict[str, GeoModel] + geom: dict[str, GeoModel | SurveyModel] geom_live: dict[str, LiveGeoModel] geom_live_defs: dict[str, dict[str, Any]] geom_custom: dict[str, GeoModel] @@ -131,9 +130,10 @@ class ModelRegistry: """ logger.debug('make_category_models') async with db_session() as session: - query = select(Category).order_by(Category.long_name).options(selectinload(Category.category_group)) + query = select(Category).order_by(Category.long_name).\ + options(selectinload(Category.category_group)) # type: ignore data = await session.exec(query) - categories: list[Category] = data.all() + categories: list[Category] = data.all() # type: ignore for category in categories: ## Several statuses can coexist for the same model, so ## consider only the ones with the 'E' (existing) status @@ -156,7 +156,7 @@ class ModelRegistry: } ## Raw survey points try: - self.raw_survey_models[store_name] = create_model( + self.raw_survey_models[store_name] = create_model( # type: ignore __base__=RawSurveyBaseModel, __model_name=category.raw_survey_table_name, __cls_kwargs__={ @@ -188,7 +188,7 @@ class ModelRegistry: #'raw_model': (str, self.raw_survey_models.get(raw_store_name)), # 'icon': (str, f'{survey.schema}-{category.table_name}'), } - self.survey_models[store_name] = create_model( + self.survey_models[store_name] = create_model( # type: ignore __base__= model_class, __model_name=category.table_name, __cls_kwargs__={ @@ -336,12 +336,12 @@ class ModelRegistry: # for category in categories # if self.raw_survey_models.get(category.table_name)} - async def get_model_id_params(self, model: SQLModel, id: int) -> FeatureInfo | None: + async def get_model_id_params(self, model: type[GeoModel], id: int) -> FeatureInfo: """ Return the parameters for this item (table name, id), displayed in info pane """ if not model: - return + raise NotInRegistry async with db_session() as session: query = select(model).where(model.id == id).options( *(joinedload(jt) for jt in model.selectinload())) @@ -350,11 +350,9 @@ class ModelRegistry: item = result.one() except NoResultFound: raise NotInRegistry - # item = await model.load(**model.get_join_with()).query.where(model.id==id).gino.first() if not item: - return - + raise NotInRegistry files, images = [], [] externalRecordUrl, graph, categorizedInfoItems = (None, ) * 3 if hasattr(item, 'get_categorized_info'): @@ -368,7 +366,6 @@ class ModelRegistry: images = await item.Attachments.images(item) if hasattr(item, 'get_external_record_url'): externalRecordUrl = item.get_external_record_url() - return FeatureInfo( id=str(item.id), itemName=item.caption,