Migrate info/measures: add data-provider

src/gisaf/api/main.py

@@ -1,9 +1,12 @@
 import logging
 from datetime import timedelta
 from typing import Annotated
+from json import loads
 
-from fastapi import Depends, APIRouter, HTTPException, status, responses
+from fastapi import Depends, APIRouter, HTTPException, status, Response
-from sqlalchemy.orm import selectinload
+from sqlalchemy import func
+from sqlalchemy.orm import selectinload, joinedload
+from sqlalchemy.orm.attributes import QueryableAttribute
 from fastapi.security import OAuth2PasswordRequestForm
 from sqlmodel import select
 
@@ -12,8 +15,9 @@ from gisaf.models.authentication import (
   Role, RoleRead,
 )
 from gisaf.models.category import Category, CategoryRead
-from gisaf.models.geo_models_base import GeoModel
+from gisaf.models.geo_models_base import GeoModel, PlottableModel
 from gisaf.models.info import LegendItem, ModelAction, ModelInfo, DataProvider, ModelValue, TagActions
+from gisaf.models.measures import MeasuresItem
 from gisaf.models.survey import Equipment, SurveyMeta, Surveyor
 from gisaf.config import Survey, conf
 from gisaf.models.bootstrap import BootstrapData
@@ -118,10 +122,137 @@ async def list_data_providers() -> list[DataProvider]:
     """
     return [
         DataProvider(
-            name=model.get_store_name(),
+            store=model.get_store_name(),
+            name=model.__name__,
             values=[value.get_store_name() for value in values]
         ) for model, values in registry.values_for_model.items()]
 
+@api.get("/data-provider/{store}")
+async def get_model_list(
+    store: str,
+    db_session: db_session,
+    ) -> list[MeasuresItem]:
+    """
+    Json REST store API compatible with Flask Potion and Angular
+    Get the list of items (used for making the list of items in measures)
+    Filter only items with at least one measure
+    """
+    try:
+        store_record = registry.stores.loc[store]
+    except KeyError:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+    model: type[PlottableModel] = store_record.model
+    # FIXME: get only the first model of values
+    values_models = registry.values_for_model.get(model) # type: ignore
+    if values_models is None or len(values_models) == 0:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+    values_model = values_models[0]
+    try:
+        ref_id_attr: QueryableAttribute = getattr(values_model, 'ref_id')
+    except AttributeError:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                            detail=f'No ref_id defined for {values_model.__name__}')
+    data = await db_session.exec(
+        select(ref_id_attr, func.count(ref_id_attr)).group_by(ref_id_attr)
+    )
+    counts = dict(data.all())
+    objs = await db_session.exec(select(model).options(
+        *(joinedload(jt) for jt in model.selectinload()))
+    )
+    resp = [
+        MeasuresItem(
+            # uri=f'/data-provider/{store}/{obj.id}',
+            id=obj.id,
+            name=obj.caption,
+        )
+        for obj in objs.all()
+        if obj.id in counts
+    ]
+    return resp
+
+@api.get('/{store_name}/values/{value}')
+async def get_model_values(store_name: str, value: str,
+                           response: Response,
+                           where: str,
+                           resample: str | None = None,
+                           ):
+    """
+    Get values
+    """
+    comment = ''
+    ## Get the request's args, i.e. the where clause of the DB query
+    model_query = loads(where)
+    # store_name = [k for k in model_query.keys()][0]
+    model_id = model_query[store_name]
+    model: GeoModel
+    model = registry.geom.get(store_name) # type: ignore
+    if model is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND)
+    values_model = registry.values_for_model.get(model)[0]
+
+    ## Allow custom getter
+    getter = getattr(values_model, f'get_{value}', None)
+    if getter:
+        df = await getter(model_id)
+    else:
+        df = await values_model.get_as_dataframe(model_id=model_id,
+                                                 with_only_columns=[value])
+
+    if len(df) == 0:
+        return []
+
+    if resample is not None and resample != '0':
+        ## Model defines how to resample
+        value_defs = [v for v in values_model.values if v['name'] == value]
+        rule = request.query['resample']
+        if len(value_defs) > 0:
+            value_defs = value_defs[0]
+        else:
+            value_defs = {}
+        if hasattr(values_model, 'resampling_args') \
+            and value in values_model.resampling_args \
+            and rule in values_model.resampling_args[value]:
+            resampling_args = values_model.resampling_args[value][rule].copy()
+            comment = resampling_args.pop('comment', '')
+        else:
+            resampling_args = {}
+        resampling_agg_method = value_defs.get('agg', 'mean')
+        ## If the resampling method is sum, set the date as the end of each period
+        #if resampling_agg_method == 'sum':
+            #resampling_args['loffset'] = rule
+        ## loffset was deprecated in Pandas 1.1.0
+        loffset = resampling_args.pop('loffset', None)
+        df = df.resample(rule, **resampling_args).agg(resampling_agg_method)
+        if loffset is not None:
+            df.index = df.index + to_offset(loffset)
+        if len(df) > 0:
+            df.reset_index(inplace=True)
+    elif len(df) > conf.plot.maxDataSize:
+        msg ='Too much data to display in the graph, automatically switching to daily resampling. ' \
+             'Note that you can download raw data anyway as CSV in the "Tools" tab.',
+        raise HTTPException(status.HTTP_502_BAD_GATEWAY, # FIXME: 502 status code
+                            detail=msg,
+                            headers={'resampling': 'D'}
+        )
+    else:
+        df.reset_index(inplace=True)
+
+    df.dropna(inplace=True)
+
+    ## Round values
+    values_dict = {value['name']: value for value in values_model.values}
+    for column in df.columns:
+        if column in values_dict:
+            ## XXX: workaround for https://github.com/pandas-dev/pandas/issues/38844:
+            ##      convert column to float.
+            ##      Revert back to the commented out line below when the
+            ##      bug fix is applied: in Pandas 1.3
+            #df[column] = df[column].round(values_dict[column].get('round', 1))
+            df[column] = df[column].astype(float).round(values_dict[column].get('round', 1))
+
+    response.headers["comment"] = comment
+    return df.to_json(orient='records', date_format='iso'),
+
 @api.get("/stores")
 async def get_stores() -> list[Store]:
     df = registry.stores.reset_index().\
@@ -200,8 +331,10 @@ async def get_model_info(
         ]
     ## Add information about values
     values_model = registry.values_for_model.get(model)
-    if hasattr(values_model, 'values'):
+    assert values_model is not None
-        model_info['values'] = [ModelValue(**values) for values in values_model.values]
+    # FIXME: one the first values_model is managed
+    if len(values_model) > 0 and hasattr(values_model[0], 'values'):
+        model_info['values'] = [ModelValue(**values) for values in values_model[0].values]
     ## Add information about tags
     ## TODO: add to plugin_manager a way to retrieve tag_store/tag_actions from a dict?
     #tag_store = [tt for tt in plugin_manager.tagsStores.stores if tt.store==store][0]

src/gisaf/models/geo_models_base.py

@@ -212,8 +212,8 @@ class SurveyModel(BaseSurveyModel):
         }
 
     @property
-    def caption(self):
+    def caption(self) -> str:
-        return '{self.category.description} [{self.category.name}: {self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self)
+        return f'{self.category.description} [{self.category.name}: {self.category.group}-{self.category.minor_group_1}] #{self.id:d}'.format(self=self)
 
     @classmethod
     async def get_popup(cls, df):
@@ -418,7 +418,7 @@ class GeoModelNoStatus(Model):
         }
 
     @property
-    def caption(self):
+    def caption(self) -> str:
         """
         Subclass me!
         :return: str
@@ -815,12 +815,12 @@ class GeoPointModelNoStatus(GeoModelNoStatus):
         return self.shapely_geom.z
 
     @property
-    def caption(self):
+    def caption(self) -> str:
         """
         Return user friendly name (used in menu, etc)
         :return:
         """
-        return '{self.__class__.__name__}: {self.id:d}'.format(self=self)
+        return f'{self.__class__.__name__}: {self.id:d}'
 
     def get_coords(self):
         return (self.shapely_geom.x, self.shapely_geom.y)
@@ -893,12 +893,12 @@ class GeoLineModel(GeoModel):
         return transform(reproject_func, self.shapely_geom).length
 
     @property
-    def caption(self):
+    def caption(self) -> str:
         """
         Return user friendly name (used in menu, etc)
         :return:
         """
-        return '{self.__class__.__name__}: {self.id:d}'.format(self=self)
+        return f'{self.__class__.__name__}: {self.id:d}'
 
     @classmethod
     async def get_shapefile_writer(cls):
@@ -980,12 +980,12 @@ class GeoPolygonModel(GeoModel):
         return transform(reproject_func, self.shapely_geom).length
 
     @property
-    def caption(self):
+    def caption(self) -> str:
         """
         Return user friendly name (used in menu, etc)
         :return:
         """
-        return '{self.__class__.__name__}: {self.id:d}'.format(self=self)
+        return f'{self.__class__.__name__}: {self.id:d}'
 
     @classmethod
     async def get_shapefile_writer(cls):

src/gisaf/models/info.py

@@ -15,6 +15,7 @@ class ActionResults(BaseModel):
 
 
 class DataProvider(BaseModel):
+    store: str
     name: str
     values: list[str]
 

src/gisaf/models/measures.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel
+
+class MeasuresItem(BaseModel):
+    id: int
+    name: str