Migrate info/measures: add data-provider
This commit is contained in:
parent
9bf78dd421
commit
9c328642cb
4 changed files with 154 additions and 15 deletions
|
|
@ -1,9 +1,12 @@
|
|||
import logging
|
||||
from datetime import timedelta
|
||||
from typing import Annotated
|
||||
from json import loads
|
||||
|
||||
from fastapi import Depends, APIRouter, HTTPException, status, responses
|
||||
from sqlalchemy.orm import selectinload
|
||||
from fastapi import Depends, APIRouter, HTTPException, status, Response
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import selectinload, joinedload
|
||||
from sqlalchemy.orm.attributes import QueryableAttribute
|
||||
from fastapi.security import OAuth2PasswordRequestForm
|
||||
from sqlmodel import select
|
||||
|
||||
|
|
@ -12,8 +15,9 @@ from gisaf.models.authentication import (
|
|||
Role, RoleRead,
|
||||
)
|
||||
from gisaf.models.category import Category, CategoryRead
|
||||
from gisaf.models.geo_models_base import GeoModel
|
||||
from gisaf.models.geo_models_base import GeoModel, PlottableModel
|
||||
from gisaf.models.info import LegendItem, ModelAction, ModelInfo, DataProvider, ModelValue, TagActions
|
||||
from gisaf.models.measures import MeasuresItem
|
||||
from gisaf.models.survey import Equipment, SurveyMeta, Surveyor
|
||||
from gisaf.config import Survey, conf
|
||||
from gisaf.models.bootstrap import BootstrapData
|
||||
|
|
@ -118,10 +122,137 @@ async def list_data_providers() -> list[DataProvider]:
|
|||
"""
|
||||
return [
|
||||
DataProvider(
|
||||
name=model.get_store_name(),
|
||||
store=model.get_store_name(),
|
||||
name=model.__name__,
|
||||
values=[value.get_store_name() for value in values]
|
||||
) for model, values in registry.values_for_model.items()]
|
||||
|
||||
@api.get("/data-provider/{store}")
|
||||
async def get_model_list(
|
||||
store: str,
|
||||
db_session: db_session,
|
||||
) -> list[MeasuresItem]:
|
||||
"""
|
||||
Json REST store API compatible with Flask Potion and Angular
|
||||
Get the list of items (used for making the list of items in measures)
|
||||
Filter only items with at least one measure
|
||||
"""
|
||||
try:
|
||||
store_record = registry.stores.loc[store]
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
model: type[PlottableModel] = store_record.model
|
||||
# FIXME: get only the first model of values
|
||||
values_models = registry.values_for_model.get(model) # type: ignore
|
||||
if values_models is None or len(values_models) == 0:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
values_model = values_models[0]
|
||||
try:
|
||||
ref_id_attr: QueryableAttribute = getattr(values_model, 'ref_id')
|
||||
except AttributeError:
|
||||
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f'No ref_id defined for {values_model.__name__}')
|
||||
data = await db_session.exec(
|
||||
select(ref_id_attr, func.count(ref_id_attr)).group_by(ref_id_attr)
|
||||
)
|
||||
counts = dict(data.all())
|
||||
objs = await db_session.exec(select(model).options(
|
||||
*(joinedload(jt) for jt in model.selectinload()))
|
||||
)
|
||||
resp = [
|
||||
MeasuresItem(
|
||||
# uri=f'/data-provider/{store}/{obj.id}',
|
||||
id=obj.id,
|
||||
name=obj.caption,
|
||||
)
|
||||
for obj in objs.all()
|
||||
if obj.id in counts
|
||||
]
|
||||
return resp
|
||||
|
||||
@api.get('/{store_name}/values/{value}')
|
||||
async def get_model_values(store_name: str, value: str,
|
||||
response: Response,
|
||||
where: str,
|
||||
resample: str | None = None,
|
||||
):
|
||||
"""
|
||||
Get values
|
||||
"""
|
||||
comment = ''
|
||||
## Get the request's args, i.e. the where clause of the DB query
|
||||
model_query = loads(where)
|
||||
# store_name = [k for k in model_query.keys()][0]
|
||||
model_id = model_query[store_name]
|
||||
model: GeoModel
|
||||
model = registry.geom.get(store_name) # type: ignore
|
||||
if model is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND)
|
||||
values_model = registry.values_for_model.get(model)[0]
|
||||
|
||||
## Allow custom getter
|
||||
getter = getattr(values_model, f'get_{value}', None)
|
||||
if getter:
|
||||
df = await getter(model_id)
|
||||
else:
|
||||
df = await values_model.get_as_dataframe(model_id=model_id,
|
||||
with_only_columns=[value])
|
||||
|
||||
if len(df) == 0:
|
||||
return []
|
||||
|
||||
if resample is not None and resample != '0':
|
||||
## Model defines how to resample
|
||||
value_defs = [v for v in values_model.values if v['name'] == value]
|
||||
rule = request.query['resample']
|
||||
if len(value_defs) > 0:
|
||||
value_defs = value_defs[0]
|
||||
else:
|
||||
value_defs = {}
|
||||
if hasattr(values_model, 'resampling_args') \
|
||||
and value in values_model.resampling_args \
|
||||
and rule in values_model.resampling_args[value]:
|
||||
resampling_args = values_model.resampling_args[value][rule].copy()
|
||||
comment = resampling_args.pop('comment', '')
|
||||
else:
|
||||
resampling_args = {}
|
||||
resampling_agg_method = value_defs.get('agg', 'mean')
|
||||
## If the resampling method is sum, set the date as the end of each period
|
||||
#if resampling_agg_method == 'sum':
|
||||
#resampling_args['loffset'] = rule
|
||||
## loffset was deprecated in Pandas 1.1.0
|
||||
loffset = resampling_args.pop('loffset', None)
|
||||
df = df.resample(rule, **resampling_args).agg(resampling_agg_method)
|
||||
if loffset is not None:
|
||||
df.index = df.index + to_offset(loffset)
|
||||
if len(df) > 0:
|
||||
df.reset_index(inplace=True)
|
||||
elif len(df) > conf.plot.maxDataSize:
|
||||
msg ='Too much data to display in the graph, automatically switching to daily resampling. ' \
|
||||
'Note that you can download raw data anyway as CSV in the "Tools" tab.',
|
||||
raise HTTPException(status.HTTP_502_BAD_GATEWAY, # FIXME: 502 status code
|
||||
detail=msg,
|
||||
headers={'resampling': 'D'}
|
||||
)
|
||||
else:
|
||||
df.reset_index(inplace=True)
|
||||
|
||||
df.dropna(inplace=True)
|
||||
|
||||
## Round values
|
||||
values_dict = {value['name']: value for value in values_model.values}
|
||||
for column in df.columns:
|
||||
if column in values_dict:
|
||||
## XXX: workaround for https://github.com/pandas-dev/pandas/issues/38844:
|
||||
## convert column to float.
|
||||
## Revert back to the commented out line below when the
|
||||
## bug fix is applied: in Pandas 1.3
|
||||
#df[column] = df[column].round(values_dict[column].get('round', 1))
|
||||
df[column] = df[column].astype(float).round(values_dict[column].get('round', 1))
|
||||
|
||||
response.headers["comment"] = comment
|
||||
return df.to_json(orient='records', date_format='iso'),
|
||||
|
||||
@api.get("/stores")
|
||||
async def get_stores() -> list[Store]:
|
||||
df = registry.stores.reset_index().\
|
||||
|
|
@ -200,8 +331,10 @@ async def get_model_info(
|
|||
]
|
||||
## Add information about values
|
||||
values_model = registry.values_for_model.get(model)
|
||||
if hasattr(values_model, 'values'):
|
||||
model_info['values'] = [ModelValue(**values) for values in values_model.values]
|
||||
assert values_model is not None
|
||||
# FIXME: one the first values_model is managed
|
||||
if len(values_model) > 0 and hasattr(values_model[0], 'values'):
|
||||
model_info['values'] = [ModelValue(**values) for values in values_model[0].values]
|
||||
## Add information about tags
|
||||
## TODO: add to plugin_manager a way to retrieve tag_store/tag_actions from a dict?
|
||||
#tag_store = [tt for tt in plugin_manager.tagsStores.stores if tt.store==store][0]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue