diff --git a/src/gisaf/api/main.py b/src/gisaf/api/main.py index ddca321..3e42922 100644 --- a/src/gisaf/api/main.py +++ b/src/gisaf/api/main.py @@ -1,9 +1,12 @@ import logging from datetime import timedelta from typing import Annotated +from json import loads -from fastapi import Depends, APIRouter, HTTPException, status, responses -from sqlalchemy.orm import selectinload +from fastapi import Depends, APIRouter, HTTPException, status, Response +from sqlalchemy import func +from sqlalchemy.orm import selectinload, joinedload +from sqlalchemy.orm.attributes import QueryableAttribute from fastapi.security import OAuth2PasswordRequestForm from sqlmodel import select @@ -12,8 +15,9 @@ from gisaf.models.authentication import ( Role, RoleRead, ) from gisaf.models.category import Category, CategoryRead -from gisaf.models.geo_models_base import GeoModel +from gisaf.models.geo_models_base import GeoModel, PlottableModel from gisaf.models.info import LegendItem, ModelAction, ModelInfo, DataProvider, ModelValue, TagActions +from gisaf.models.measures import MeasuresItem from gisaf.models.survey import Equipment, SurveyMeta, Surveyor from gisaf.config import Survey, conf from gisaf.models.bootstrap import BootstrapData @@ -118,10 +122,137 @@ async def list_data_providers() -> list[DataProvider]: """ return [ DataProvider( - name=model.get_store_name(), + store=model.get_store_name(), + name=model.__name__, values=[value.get_store_name() for value in values] ) for model, values in registry.values_for_model.items()] +@api.get("/data-provider/{store}") +async def get_model_list( + store: str, + db_session: db_session, + ) -> list[MeasuresItem]: + """ + Json REST store API compatible with Flask Potion and Angular + Get the list of items (used for making the list of items in measures) + Filter only items with at least one measure + """ + try: + store_record = registry.stores.loc[store] + except KeyError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + model: type[PlottableModel] = store_record.model + # FIXME: get only the first model of values + values_models = registry.values_for_model.get(model) # type: ignore + if values_models is None or len(values_models) == 0: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + values_model = values_models[0] + try: + ref_id_attr: QueryableAttribute = getattr(values_model, 'ref_id') + except AttributeError: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'No ref_id defined for {values_model.__name__}') + data = await db_session.exec( + select(ref_id_attr, func.count(ref_id_attr)).group_by(ref_id_attr) + ) + counts = dict(data.all()) + objs = await db_session.exec(select(model).options( + *(joinedload(jt) for jt in model.selectinload())) + ) + resp = [ + MeasuresItem( + # uri=f'/data-provider/{store}/{obj.id}', + id=obj.id, + name=obj.caption, + ) + for obj in objs.all() + if obj.id in counts + ] + return resp + +@api.get('/{store_name}/values/{value}') +async def get_model_values(store_name: str, value: str, + response: Response, + where: str, + resample: str | None = None, + ): + """ + Get values + """ + comment = '' + ## Get the request's args, i.e. the where clause of the DB query + model_query = loads(where) + # store_name = [k for k in model_query.keys()][0] + model_id = model_query[store_name] + model: GeoModel + model = registry.geom.get(store_name) # type: ignore + if model is None: + raise HTTPException(status.HTTP_404_NOT_FOUND) + values_model = registry.values_for_model.get(model)[0] + + ## Allow custom getter + getter = getattr(values_model, f'get_{value}', None) + if getter: + df = await getter(model_id) + else: + df = await values_model.get_as_dataframe(model_id=model_id, + with_only_columns=[value]) + + if len(df) == 0: + return [] + + if resample is not None and resample != '0': + ## Model defines how to resample + value_defs = [v for v in values_model.values if v['name'] == value] + rule = request.query['resample'] + if len(value_defs) > 0: + value_defs = value_defs[0] + else: + value_defs = {} + if hasattr(values_model, 'resampling_args') \ + and value in values_model.resampling_args \ + and rule in values_model.resampling_args[value]: + resampling_args = values_model.resampling_args[value][rule].copy() + comment = resampling_args.pop('comment', '') + else: + resampling_args = {} + resampling_agg_method = value_defs.get('agg', 'mean') + ## If the resampling method is sum, set the date as the end of each period + #if resampling_agg_method == 'sum': + #resampling_args['loffset'] = rule + ## loffset was deprecated in Pandas 1.1.0 + loffset = resampling_args.pop('loffset', None) + df = df.resample(rule, **resampling_args).agg(resampling_agg_method) + if loffset is not None: + df.index = df.index + to_offset(loffset) + if len(df) > 0: + df.reset_index(inplace=True) + elif len(df) > conf.plot.maxDataSize: + msg ='Too much data to display in the graph, automatically switching to daily resampling. ' \ + 'Note that you can download raw data anyway as CSV in the "Tools" tab.', + raise HTTPException(status.HTTP_502_BAD_GATEWAY, # FIXME: 502 status code + detail=msg, + headers={'resampling': 'D'} + ) + else: + df.reset_index(inplace=True) + + df.dropna(inplace=True) + + ## Round values + values_dict = {value['name']: value for value in values_model.values} + for column in df.columns: + if column in values_dict: + ## XXX: workaround for https://github.com/pandas-dev/pandas/issues/38844: + ## convert column to float. + ## Revert back to the commented out line below when the + ## bug fix is applied: in Pandas 1.3 + #df[column] = df[column].round(values_dict[column].get('round', 1)) + df[column] = df[column].astype(float).round(values_dict[column].get('round', 1)) + + response.headers["comment"] = comment + return df.to_json(orient='records', date_format='iso'), + @api.get("/stores") async def get_stores() -> list[Store]: df = registry.stores.reset_index().\ @@ -200,8 +331,10 @@ async def get_model_info( ] ## Add information about values values_model = registry.values_for_model.get(model) - if hasattr(values_model, 'values'): - model_info['values'] = [ModelValue(**values) for values in values_model.values] + assert values_model is not None + # FIXME: one the first values_model is managed + if len(values_model) > 0 and hasattr(values_model[0], 'values'): + model_info['values'] = [ModelValue(**values) for values in values_model[0].values] ## Add information about tags ## TODO: add to plugin_manager a way to retrieve tag_store/tag_actions from a dict? #tag_store = [tt for tt in plugin_manager.tagsStores.stores if tt.store==store][0] diff --git a/src/gisaf/models/geo_models_base.py b/src/gisaf/models/geo_models_base.py index a05a5ec..0680ce8 100644 --- a/src/gisaf/models/geo_models_base.py +++ b/src/gisaf/models/geo_models_base.py @@ -212,8 +212,8 @@ class SurveyModel(BaseSurveyModel): } @property - def caption(self): - return '{self.category.description} [{self.category.name}: {self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self) + def caption(self) -> str: + return f'{self.category.description} [{self.category.name}: {self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self) @classmethod async def get_popup(cls, df): @@ -418,7 +418,7 @@ class GeoModelNoStatus(Model): } @property - def caption(self): + def caption(self) -> str: """ Subclass me! :return: str @@ -815,12 +815,12 @@ class GeoPointModelNoStatus(GeoModelNoStatus): return self.shapely_geom.z @property - def caption(self): + def caption(self) -> str: """ Return user friendly name (used in menu, etc) :return: """ - return '{self.__class__.__name__}: {self.id:d}'.format(self=self) + return f'{self.__class__.__name__}: {self.id:d}' def get_coords(self): return (self.shapely_geom.x, self.shapely_geom.y) @@ -893,12 +893,12 @@ class GeoLineModel(GeoModel): return transform(reproject_func, self.shapely_geom).length @property - def caption(self): + def caption(self) -> str: """ Return user friendly name (used in menu, etc) :return: """ - return '{self.__class__.__name__}: {self.id:d}'.format(self=self) + return f'{self.__class__.__name__}: {self.id:d}' @classmethod async def get_shapefile_writer(cls): @@ -980,12 +980,12 @@ class GeoPolygonModel(GeoModel): return transform(reproject_func, self.shapely_geom).length @property - def caption(self): + def caption(self) -> str: """ Return user friendly name (used in menu, etc) :return: """ - return '{self.__class__.__name__}: {self.id:d}'.format(self=self) + return f'{self.__class__.__name__}: {self.id:d}' @classmethod async def get_shapefile_writer(cls): diff --git a/src/gisaf/models/info.py b/src/gisaf/models/info.py index 18a173a..05e8e37 100644 --- a/src/gisaf/models/info.py +++ b/src/gisaf/models/info.py @@ -15,6 +15,7 @@ class ActionResults(BaseModel): class DataProvider(BaseModel): + store: str name: str values: list[str] diff --git a/src/gisaf/models/measures.py b/src/gisaf/models/measures.py new file mode 100644 index 0000000..6b623de --- /dev/null +++ b/src/gisaf/models/measures.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + +class MeasuresItem(BaseModel): + id: int + name: str