philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Add plugin management (WIP)

Browse source 
Refactoring some models, prevent circular deps
This commit is contained in:
phil 2024-03-07 12:13:47 +05:30
parent f1534dfed7
commit 360a7a70f3
11 changed files with 308 additions and 160 deletions
Download patch file Download diff file Expand all files Collapse all files

62
src/gisaf/api/main.py
View file

@ -12,7 +12,8 @@ from gisaf.models.authentication import (
Role, RoleRead,
)
from gisaf.models.category import Category, CategoryRead
from gisaf.models.to_migrate import DataProvider
from gisaf.models.geo_models_base import GeoModel
from gisaf.models.info import LegendItem, ModelAction, ModelInfo, DataProvider, ModelValue, TagActions
from gisaf.models.survey import Equipment, SurveyMeta, Surveyor
from gisaf.config import Survey, conf
from gisaf.models.bootstrap import BootstrapData
@ -26,10 +27,13 @@ from gisaf.security import (
)
from gisaf.registry import registry, NotInRegistry
from gisaf.custom_store_base import BaseStore
from gisaf.models.to_migrate import (
from gisaf.redis_tools import store as redis_store
from gisaf.models.info import (
FeatureInfo, InfoItem, Attachment, InfoCategory
)
from gisaf.live_utils import get_live_feature_info
from gisaf.plugins import manager as plugin_manager, NoSuchAction
from gisaf.utils import gisTypeSymbolMap
logger = logging.getLogger(__name__)
@ -151,11 +155,11 @@ async def get_survey_meta(
@api.get("/feature-info/{store}/{id}")
async def get_feature_info(
store: str, id: str,
) -> FeatureInfo:
) -> FeatureInfo | None:
if store not in registry.stores.index:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
store_record = registry.stores.loc[store]
model = store_record.model
model: type[GeoModel] = store_record.model
if store_record.is_live:
feature_info = await get_live_feature_info(store, id)
elif issubclass(model, BaseStore):
@ -168,6 +172,56 @@ async def get_feature_info(
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
return feature_info
@api.get("/model-info/{store}")
async def get_model_info(
store: str
) -> ModelInfo:
try:
store_record = registry.stores.loc[store]
except KeyError:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
if store_record.is_live:
## Get layer_defs from live redis and give symbol
layer_def = await redis_store.get_layer_def(store)
return ModelInfo(modelName=layer_def.pop('name'), **layer_def)
model = store_record.model
model_info = {
'store': store,
'modelName': model.__name__,
'symbol': model.symbol or gisTypeSymbolMap[model.base_gis_type],
}
## Add information about the legend
if hasattr(model, 'get_legend'):
legend = await model.get_legend()
model_info['legend'] = [
LegendItem(key=k, value=v)
for k, v in legend.items()
]
## Add information about values
values_model = registry.values_for_model.get(model)
if hasattr(values_model, 'values'):
model_info['values'] = [ModelValue(**values) for values in values_model.values]
## Add information about tags
## TODO: add to plugin_manager a way to retrieve tag_store/tag_actions from a dict?
#tag_store = [tt for tt in plugin_manager.tagsStores.stores if tt.store==store][0]
model_info['tagActions'] = [
TagActions(key=key, domain=domain, actions=actions)
for domain, actions_keys in plugin_manager.tags_models[model].items()
for key, actions in actions_keys.items()
]
model_info['actions'] = [
ModelAction(
name=name,
icon=action.icon,
formFields=action.formFields,
)
for name, actions in plugin_manager.actions_stores.get(store, {}).items()
for action in actions
]
model_info['downloaders'] = plugin_manager.downloaders_stores[store]
return ModelInfo(**model_info)
# @api.get("/user-role")
# async def get_user_role_relation(
# *, db_session: AsyncSession = Depends(get_db_session)

2
src/gisaf/application.py
View file

@ -14,6 +14,7 @@ from gisaf.api.geoapi import api as geoapi
from gisaf.api.admin import api as admin_api
from gisaf.api.dashboard import api as dashboard_api
from gisaf.api.map import api as map_api
from gisaf.plugins import manager as plugin_manger
logging.basicConfig(level=conf.gisaf.debugLevel)
logger = logging.getLogger(__name__)
@ -25,6 +26,7 @@ async def lifespan(app: FastAPI):
await setup_redis()
await setup_redis_cache()
await setup_live()
await plugin_manger.scan_plugins()
await admin_manager.setup_admin()
await map_tile_registry.setup()
yield

2
src/gisaf/custom_store_base.py
View file

@ -7,7 +7,7 @@ from sqlmodel import SQLModel
from gisaf.config import conf
from gisaf.models.map_bases import MaplibreStyle
from gisaf.models.to_migrate import FeatureInfo
from gisaf.models.info import FeatureInfo
class BaseStore(SQLModel):

4
src/gisaf/live_utils.py
View file

@ -5,9 +5,7 @@ from shapely.geometry import (
from gisaf.redis_tools import store as redis_store
from gisaf.models.geo_models_base import reproject_func
from gisaf.models.to_migrate import (
FeatureInfo, InfoItem, Attachment, InfoCategory
)
from gisaf.models.info import FeatureInfo, InfoItem
async def get_live_feature_info(store: str, id: str) -> FeatureInfo:
item = await redis_store.get_feature_info(store, id)

2
src/gisaf/models/geo_models_base.py
View file

@ -40,7 +40,7 @@ from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_survey, gisaf_admin, survey, raw_survey
from gisaf.models.misc import Qml
from gisaf.models.category import Category
from gisaf.models.to_migrate import InfoItem
from gisaf.models.info_item import InfoItem
# from gisaf.models.survey import Equipment, Surveyor, Accuracy
# from gisaf.models.project import Project

161
src/gisaf/models/info.py Normal file
View file

@ -0,0 +1,161 @@
from typing import Any
from pydantic import BaseModel
from gisaf.models.info_item import InfoItem
class ActionResult(BaseModel):
message: str
class ActionResults(BaseModel):
name: str
message: str
actionResults: list[ActionResult]
class DataProvider(BaseModel):
name: str
values: list[str]
class InfoCategory(BaseModel):
name: str
infoItems: list[InfoItem]
class PlotBgShape(BaseModel):
name: str
valueTop: float
valueBottom: float
color: str
class PlotBaseLine(BaseModel):
name: str
value: float
color: str
class PlotParams(BaseModel):
baseLines: list[PlotBaseLine]
bgShape: list[PlotBgShape]
barBase: float
class Attachment(BaseModel):
name: str
path: str
class FeatureInfo(BaseModel):
id: str
itemName: str
geoInfoItems: list[InfoItem] = []
surveyInfoItems: list[InfoItem] = []
infoItems: list[InfoItem] = []
categorizedInfoItems: list[InfoCategory] = []
tags: list[InfoItem] = []
graph: str | None = None
plotParams: PlotParams | None = None
files: list[Attachment] = []
images: list[Attachment] = []
externalRecordUrl: str | None = None
class ModelValue(BaseModel):
name: str
title: str
unit: str
chartType: str = 'line'
chartColor: str = 'blue'
class FormField(BaseModel):
name: str
type: str
dflt: str | None = None
value: str | None = None
class ModelAction(BaseModel):
name: str
icon: str
formFields: list[FormField]
class TagAction(BaseModel):
name: str
_plugin: Any
action: str
roles: list[str] | None = None
link: str | None = None
save: bool = False
class TagActions(BaseModel):
domain: str
key: str
actions: list[TagAction]
class ActionAction(BaseModel):
roles: list[str]
# plugin: Any
name: str
params: Any
icon: str
formFields: list[FormField]
class Downloader(BaseModel):
# plugin: str
# downloader: str
roles: list[str] = []
name: str
icon: str | None = None
class LegendItem(BaseModel):
key: str
value: str
class ModelInfo(BaseModel):
store: str
modelName: str
symbol: str | None = None
values: list[ModelValue] = []
actions: list[ModelAction] = []
formName: str | None = None
formFields: list[FormField] = []
tagPlugins: list[str] = []
tagActions: list[TagActions] = []
downloaders: list[Downloader] = []
legend: list[LegendItem] = []
class TagsStore(BaseModel):
store: str
tagActions: list[TagActions]
class TagsStores(BaseModel):
stores: list[TagsStore]
class ActionParam(BaseModel):
name: str
type: str
dflt: str
class Action(BaseModel):
name: str
roles: list[str]
params: list[ActionParam]
class ActionsStore(BaseModel):
store: str
actions: list[Action]

6
src/gisaf/models/info_item.py Normal file
View file

@ -0,0 +1,6 @@
from pydantic import BaseModel
class InfoItem(BaseModel):
key: str
value: str | float | int

5
src/gisaf/models/tags.py
View file

@ -3,7 +3,7 @@ from sqlalchemy import BigInteger
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy.dialects.postgresql import HSTORE
from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
from pydantic import computed_field
from pydantic import BaseModel, computed_field
from gisaf.models.metadata import gisaf
from gisaf.models.geo_models_base import GeoPointModel
@ -42,4 +42,5 @@ class TagKey(SQLModel, table=True):
return self.key
def __repr__(self):
return '<models.TagKey {self.key}>'.format(self=self)
return '<models.TagKey {self.key}>'.format(self=self)

75
src/gisaf/models/to_migrate.py
View file

@ -1,75 +0,0 @@
from pydantic import BaseModel
class ActionResult(BaseModel):
message: str
class ActionResults(BaseModel):
name: str
message: str
actionResults: list[ActionResult]
class FormField(BaseModel):
name: str
type: str
class ModelAction(BaseModel):
name: str
icon: str
formFields: list[FormField]
class DataProvider(BaseModel):
name: str
values: list[str]
class InfoItem(BaseModel):
key: str
value: str | float | int
class InfoCategory(BaseModel):
name: str
infoItems: list[InfoItem]
class PlotBgShape(BaseModel):
name: str
valueTop: float
valueBottom: float
color: str
class PlotBaseLine(BaseModel):
name: str
value: float
color: str
class PlotParams(BaseModel):
baseLines: list[PlotBaseLine]
bgShape: list[PlotBgShape]
barBase: float
class Attachment(BaseModel):
name: str
path: str
class FeatureInfo(BaseModel):
id: str
itemName: str
geoInfoItems: list[InfoItem] = []
surveyInfoItems: list[InfoItem] = []
infoItems: list[InfoItem] = []
categorizedInfoItems: list[InfoCategory] = []
tags: list[InfoItem] = []
graph: str | None = None
plotParams: PlotParams | None = None
files: list[Attachment] = []
images: list[Attachment] = []
externalRecordUrl: str | None = None

120
src/gisaf/plugins.py
View file

@ -8,43 +8,30 @@ from datetime import datetime
# from aiohttp.web_exceptions import HTTPUnauthorized
# from aiohttp_security import check_permission
from pydantic import BaseModel # noqa: F401
from sqlalchemy import or_, and_
# from geoalchemy2.shape import to_shape, from_shape
# from graphene import ObjectType, String, List, Boolean, Field, Float, InputObjectType
import pandas as pd
import shapely
import shapely # type: ignore
from gisaf.config import conf
from gisaf.models.store import Store # noqa: F401
from gisaf.models.tags import Tags as TagsModel
from gisaf.utils import upsert_df
from gisaf.models.reconcile import StatusChange
from gisaf.models.to_migrate import (
from gisaf.models.info import (
ActionResults,
Downloader,
TagAction, ActionAction,
TagActions,
TagsStore,
TagsStores,
ActionsStore,
Action
)
# from gisaf.models.graphql import (
# Action,
# ActionAction,
# ActionParam,
# ActionParamInput,
# ActionResult,
# ActionResults,
# ActionsResults,
# ActionsStore,
# Downloader,
# FormField,
# Tag,
# TagAction,
# TagActions,
# TagKeyList,
# TaggedFeature,
# TaggedLayer,
# TagsStore,
# TagsStores,
# )
## GraphQL object types
## TODO: move to models.graphql
from gisaf.registry import NotInRegistry, registry
logger = logging.getLogger('Gisaf plugin manager')
@ -54,6 +41,7 @@ class NoSuchAction(Exception):
pass
class ActionPlugin:
"""
Base class for all actions plugins.
@ -77,10 +65,24 @@ class TagPlugin:
Keys might be reg exp.
See Link (below) for a very basic example.
"""
key: str
domain: str
stores: list[str]
stores_by_re: list[str]
roles: list[str] | None
save: bool
link: str | None
action: str | None
def __init__(self, key='', domain='',
stores=None, stores_by_re=None, roles=None,
save=True, link=None, action=None):
def __init__(self,
key: str = '',
domain: str = '',
stores: list[str] | None = None,
stores_by_re: list[str] | None = None,
roles: list[str] | None = None,
save: bool = True,
link: str | None = None,
action: str | None = None):
## self._tags: instanciated tags
self.key = key
self.domain = domain
@ -119,10 +121,12 @@ class DownloadPlugin:
class DownloadCSVPlugin(DownloadPlugin):
async def execute(self, model, item, request):
from gisaf.registry import registry
values_models = registry.values_for_model.get(model)
try:
values_models = registry.values_for_model[model]
except KeyError:
raise NotInRegistry
for value_model in values_models:
df = await values_models.get_as_dataframe(model_id=item.id)
df = await value_model.get_as_dataframe(model_id=item.id)
csv = df.to_csv(date_format='%d/%m/%Y %H:%M', float_format=value_model.float_format)
## TODO: implement multiple values for a model (search for values_for_model)
break
@ -138,18 +142,17 @@ class PluginManager:
Application wide manager of the plugins.
One instance only, handled by Gisaf's process.
"""
def setup(self, app):
self.app = app
def setup(self):
for entry_point in entry_points().select(group='gisaf_extras.context'):
try:
context = entry_point.load()
except ModuleNotFoundError as err:
logger.warning(err)
continue
self.app.cleanup_ctx.append(context)
# self.app.cleanup_ctx.append(context)
logger.info(f'Added context for {entry_point.name}')
async def scan_plugins(self, app):
async def scan_plugins(self) -> None:
"""
Scan tag and action plugins from the Python entry points.
Get all references of the tags defined in modules ad build a registry of:
@ -161,27 +164,27 @@ class PluginManager:
self.tags_domains = defaultdict(list)
#self.actions_plugins = {}
self.actions_stores = defaultdict(lambda: defaultdict(list))
self.actions_stores: dict[str, dict[str, list[ActionAction]]] = {}
self.executors = defaultdict(list)
self.downloaders = defaultdict(list)
self.downloaders_stores = defaultdict(list)
self.actions_names = defaultdict(list)
registered_models = app['registry'].geom
registered_models = registry.geom
registered_stores = registered_models.keys()
for entry_point in entry_points().select(group='gisaf_extras.tags'):
try:
tag = entry_point.load()
tagPlugin: TagPlugin = entry_point.load()
except ModuleNotFoundError as err:
logger.warning(err)
continue
## Keys, domains
self.tags_domains[tag.domain].append(tag)
stores = tag.stores
self.tags_domains[tagPlugin.domain].append(tagPlugin)
stores = tagPlugin.stores
## Stores to which the tags apply
for _tag in tag.stores_by_re:
for _tag in tagPlugin.stores_by_re:
_re = re.compile(_tag)
for store in registered_stores:
if _re.match(store) and store not in stores:
@ -195,16 +198,17 @@ class PluginManager:
continue
## Actions
## For graphql queries
self.tags_models[model][tag.domain][tag.key].append(
TagAction(
#plugin=plugin.__class__.__name__,
roles=tag.roles,
link=tag.link,
action=tag.action,
save=tag.save,
if tagPlugin.action is not None:
self.tags_models[model][tagPlugin.domain][tagPlugin.key].append(
TagAction(
_plugin=tagPlugin,
name=entry_point.name,
roles=tagPlugin.roles,
link=tagPlugin.link,
action=tagPlugin.action,
save=tagPlugin.save,
)
)
)
logger.info(f'Added tags plugin {entry_point.name} for {len(stores)} stores')
for entry_point in entry_points().select(group='gisaf_extras.actions'):
@ -227,9 +231,12 @@ class PluginManager:
logger.warn(f'Action plugin {entry_point.name}: skip model {store}'
', which is not found in registry')
continue
if store not in self.actions_stores:
self.actions_stores[store] = {}
if action.name not in self.actions_stores[store]:
self.actions_stores[store][action.name] = []
self.actions_stores[store][action.name].append(
ActionAction(
action=action,
roles=action.roles,
#plugin=plugin.__class__.__name__,
name=action.name,
@ -242,7 +249,7 @@ class PluginManager:
for entry_point in entry_points().select(group='gisaf_extras.downloaders'):
try:
downloader = entry_point.load()
downloader: DownloadPlugin = entry_point.load()
except ModuleNotFoundError as err:
logger.warning(err)
continue
@ -261,9 +268,8 @@ class PluginManager:
continue
self.downloaders_stores[store].append(
Downloader(
downloader=downloader,
roles=downloader.roles,
name=downloader.name,
roles=downloader.roles,
icon=downloader.icon,
)
)
@ -291,12 +297,12 @@ class PluginManager:
name=name,
roles=[rr for rr in set(
[r for r in chain.from_iterable(
[aa._roles for aa in action_actions]
[aa.roles for aa in action_actions]
)]
)],
params=[rr for rr in set(
[r for r in chain.from_iterable(
[aa._params for aa in action_actions]
[aa.params for aa in action_actions]
)]
)],
)
@ -306,8 +312,6 @@ class PluginManager:
for store, actions in self.actions_stores.items()
]
app['plugins'] = self
async def do_tag_action(self, request, store, id, plugin_name, value, action):
logger.warning('FIXME: in do_tag_action: self.tags_plugins is never populated!')
## FIXME: self.tags_plugins is never populated!

29
src/gisaf/registry.py
View file

@ -30,14 +30,13 @@ from gisaf.models.geo_models_base import (
GeoLineSurveyModel,
GeoPolygonSurveyModel,
)
from gisaf.models.survey import Equipment, Surveyor, Accuracy
from gisaf.models.project import Project
from gisaf.utils import ToMigrate
from gisaf.models.category import Category, CategoryGroup
from gisaf.database import db_session
from gisaf import models
from gisaf.models.survey import Equipment, Surveyor, Accuracy
from gisaf.models.project import Project
from gisaf.models.category import Category, CategoryGroup
from gisaf.models.metadata import raw_survey, survey
from gisaf.models.to_migrate import FeatureInfo, InfoCategory
from gisaf.models.info import FeatureInfo
logger = logging.getLogger(__name__)
@ -79,7 +78,7 @@ class ModelRegistry:
categories: pd.DataFrame
primary_groups: list[CategoryGroup]
values: dict[str, PlottableModel]
geom: dict[str, GeoModel]
geom: dict[str, GeoModel | SurveyModel]
geom_live: dict[str, LiveGeoModel]
geom_live_defs: dict[str, dict[str, Any]]
geom_custom: dict[str, GeoModel]
@ -131,9 +130,10 @@ class ModelRegistry:
"""
logger.debug('make_category_models')
async with db_session() as session:
query = select(Category).order_by(Category.long_name).options(selectinload(Category.category_group))
query = select(Category).order_by(Category.long_name).\
options(selectinload(Category.category_group)) # type: ignore
data = await session.exec(query)
categories: list[Category] = data.all()
categories: list[Category] = data.all() # type: ignore
for category in categories:
## Several statuses can coexist for the same model, so
## consider only the ones with the 'E' (existing) status
@ -156,7 +156,7 @@ class ModelRegistry:
}
## Raw survey points
try:
self.raw_survey_models[store_name] = create_model(
self.raw_survey_models[store_name] = create_model( # type: ignore
__base__=RawSurveyBaseModel,
__model_name=category.raw_survey_table_name,
__cls_kwargs__={
@ -188,7 +188,7 @@ class ModelRegistry:
#'raw_model': (str, self.raw_survey_models.get(raw_store_name)),
# 'icon': (str, f'{survey.schema}-{category.table_name}'),
}
self.survey_models[store_name] = create_model(
self.survey_models[store_name] = create_model( # type: ignore
__base__= model_class,
__model_name=category.table_name,
__cls_kwargs__={
@ -336,12 +336,12 @@ class ModelRegistry:
# for category in categories
# if self.raw_survey_models.get(category.table_name)}
async def get_model_id_params(self, model: SQLModel, id: int) -> FeatureInfo | None:
async def get_model_id_params(self, model: type[GeoModel], id: int) -> FeatureInfo:
"""
Return the parameters for this item (table name, id), displayed in info pane
"""
if not model:
return
raise NotInRegistry
async with db_session() as session:
query = select(model).where(model.id == id).options(
*(joinedload(jt) for jt in model.selectinload()))
@ -350,11 +350,9 @@ class ModelRegistry:
item = result.one()
except NoResultFound:
raise NotInRegistry
# item = await model.load(**model.get_join_with()).query.where(model.id==id).gino.first()
if not item:
return
raise NotInRegistry
files, images = [], []
externalRecordUrl, graph, categorizedInfoItems = (None, ) * 3
if hasattr(item, 'get_categorized_info'):
@ -368,7 +366,6 @@ class ModelRegistry:
images = await item.Attachments.images(item)
if hasattr(item, 'get_external_record_url'):
externalRecordUrl = item.get_external_record_url()
return FeatureInfo(
id=str(item.id),
itemName=item.caption,