Restructure apis, to the dedicated directory
Implement dashboards
This commit is contained in:
parent
aed84e0f36
commit
581598c208
7 changed files with 456 additions and 20 deletions
137
src/gisaf/api/dashboard.py
Normal file
137
src/gisaf/api/dashboard.py
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import logging
|
||||
from pathlib import Path
|
||||
from fastapi import Depends, FastAPI, HTTPException, status, responses
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlmodel import select
|
||||
|
||||
from gisaf.config import conf
|
||||
from gisaf.database import pandas_query, fastapi_db_session as db_session
|
||||
from gisaf.models.authentication import User
|
||||
from gisaf.models.dashboard import (
|
||||
DashboardPage, DashboardPageSection,
|
||||
DashboadPageSectionType, DashboardPage_,
|
||||
DashboardGroup, DashboardHome,
|
||||
)
|
||||
from gisaf.models.misc import NotADataframeError
|
||||
from gisaf.security import get_current_active_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
api = FastAPI(
|
||||
default_response_class=responses.ORJSONResponse,
|
||||
)
|
||||
|
||||
@api.get('/groups')
|
||||
async def get_groups(
|
||||
db_session: db_session,
|
||||
) -> list[DashboardGroup]:
|
||||
query = select(DashboardPage)
|
||||
data = await db_session.exec(query)
|
||||
groups: dict[str, DashboardPage_] = {}
|
||||
for page in data.all():
|
||||
page_field = DashboardPage_(
|
||||
name=page.name,
|
||||
group=page.group,
|
||||
description=page.description,
|
||||
)
|
||||
group = groups.get(page.group)
|
||||
if group is None:
|
||||
group = DashboardGroup(
|
||||
name=page.group,
|
||||
pages=[page_field]
|
||||
)
|
||||
groups[page.group] = group
|
||||
else:
|
||||
group.pages.append(page_field)
|
||||
return groups.values()
|
||||
|
||||
|
||||
@api.get('/home')
|
||||
async def get_home() -> DashboardHome:
|
||||
content_path = Path(conf.gisaf.dashboard_home.content_file).expanduser()
|
||||
footer_path = Path(conf.gisaf.dashboard_home.footer_file).expanduser()
|
||||
if content_path.is_file():
|
||||
content = content_path.read_text()
|
||||
else:
|
||||
content = 'Gisaf is free, open source software for geomatics and GIS: <a href="http://redmine.auroville.org.in/projects/gisaf">Gisaf</a>.'
|
||||
if footer_path.is_file():
|
||||
footer = footer_path.read_text()
|
||||
else:
|
||||
footer = '<a rel="license" href="https://www.gnu.org/licenses/gpl.html"><img alt="GNU GPL v3 license"style="border-width:0" src="/static/icons/gplv3-88x31.png" title="GPL Open Source license"/></a>'
|
||||
return DashboardHome(
|
||||
title=conf.gisaf.dashboard_home.title,
|
||||
content=content,
|
||||
footer=footer,
|
||||
)
|
||||
|
||||
|
||||
@api.get('/page/{group}/{name}')
|
||||
async def get_dashboard_page(group: str, name: str,
|
||||
db_session: db_session,
|
||||
user: User = Depends(get_current_active_user),
|
||||
) -> DashboardPage_:
|
||||
query1 = select(DashboardPage).where((DashboardPage.name==name)
|
||||
& (DashboardPage.group==group))
|
||||
data1 = await db_session.exec(query1)
|
||||
page = data1.one_or_none()
|
||||
if not page:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND)
|
||||
query2 = select(DashboardPageSection)\
|
||||
.where(DashboardPageSection.dashboard_page_id==page.id)\
|
||||
.options(selectinload(DashboardPageSection.dashboard_page))\
|
||||
.order_by(DashboardPageSection.name)
|
||||
data2 = await db_session.exec(query2)
|
||||
sections = data2.all()
|
||||
if page.viewable_role:
|
||||
if not(user and user.has_role(page.viewable_role)):
|
||||
username = user.username if user is not None else "Anonymous"
|
||||
logger.info(f'{username} tried to access dashboard page {name}')
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED)
|
||||
dashboard_page = DashboardPage_(
|
||||
name=page.name,
|
||||
group=page.group,
|
||||
description=page.description,
|
||||
html=page.html,
|
||||
time=page.time,
|
||||
notebook=page.get_notebook_url(),
|
||||
attachment=page.get_attachment_url(),
|
||||
expandedPanes=[
|
||||
p.strip()
|
||||
for p in page.expanded_panes.split(',')
|
||||
] if page.expanded_panes else [],
|
||||
sections=[
|
||||
DashboadPageSectionType(
|
||||
name=dps.name,
|
||||
plot=dps.get_plot_url()
|
||||
)
|
||||
for dps in sections
|
||||
]
|
||||
)
|
||||
try:
|
||||
df = page.get_page_df()
|
||||
if df is not None:
|
||||
## TODO: plot as external file, like for sections
|
||||
## Convert Geopandas dataframe to Pandas
|
||||
if isinstance(df, gpd.GeoDataFrame):
|
||||
gdf = pd.DataFrame(df.drop(columns=['geometry']))
|
||||
df = gdf
|
||||
dashboard_page.dfData = df.to_json(orient='table', double_precision=2)
|
||||
except NotADataframeError:
|
||||
logger.warning(f'Dashboard: cannot read dataframe for page {page.name}')
|
||||
except Exception as err:
|
||||
logger.warning(f'Dashboard: cannot add dataframe for page {page.name}, see debug message')
|
||||
logger.exception(err)
|
||||
if page.plot:
|
||||
try:
|
||||
plot = page.get_plot()
|
||||
plotData = {
|
||||
'data': [d.to_plotly_json() for d in plot.data],
|
||||
'layout': plot.layout.to_plotly_json(),
|
||||
}
|
||||
except Exception as err:
|
||||
logger.warning(f'Dashboard: cannot add plot for page {page.name}, see debug message')
|
||||
logger.exception(err)
|
||||
else:
|
||||
dashboard_page.plotData = dumps(plotData, cls=NumpyEncoder)
|
||||
return dashboard_page
|
||||
|
|
@ -28,6 +28,7 @@ from gisaf.models.to_migrate import (
|
|||
FeatureInfo, InfoItem, Attachment, InfoCategory
|
||||
)
|
||||
from gisaf.live_utils import get_live_feature_info
|
||||
from gisaf.api.dashboard import api as dashboard_api
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -36,8 +37,7 @@ api = FastAPI(
|
|||
default_response_class=responses.ORJSONResponse,
|
||||
)
|
||||
#api.add_middleware(SessionMiddleware, secret_key=conf.crypto.secret)
|
||||
|
||||
#db_session = Annotated[AsyncSession, Depends(get_db_session)]
|
||||
api.mount('/dashboard', dashboard_api)
|
||||
|
||||
|
||||
@api.get('/bootstrap')
|
||||
|
|
@ -62,18 +62,6 @@ async def login_for_access_token(
|
|||
expires_delta=timedelta(seconds=conf.crypto.expire))
|
||||
return Token(access_token=access_token, token_type='bearer')
|
||||
|
||||
@api.get("/list")
|
||||
async def list_data_providers() -> list[DataProvider]:
|
||||
"""
|
||||
Return a list of data providers, for use with the api (graphs, etc)
|
||||
:return:
|
||||
"""
|
||||
return [
|
||||
DataProvider(
|
||||
name=model.get_store_name(),
|
||||
values=[value.get_store_name() for value in values]
|
||||
) for model, values in registry.values_for_model.items()]
|
||||
|
||||
|
||||
@api.get("/users")
|
||||
async def get_users(
|
||||
|
|
@ -107,6 +95,19 @@ async def get_categories_p(
|
|||
df = await db_session.run_sync(pandas_query, query)
|
||||
return df.to_dict(orient="records")
|
||||
|
||||
# @api.get("/list")
|
||||
@api.get("/data-providers")
|
||||
async def list_data_providers() -> list[DataProvider]:
|
||||
"""
|
||||
Return a list of data providers, for use with the api (graphs, etc)
|
||||
:return:
|
||||
"""
|
||||
return [
|
||||
DataProvider(
|
||||
name=model.get_store_name(),
|
||||
values=[value.get_store_name() for value in values]
|
||||
) for model, values in registry.values_for_model.items()]
|
||||
|
||||
@api.get("/stores")
|
||||
async def get_stores() -> list[Store]:
|
||||
df = registry.stores.reset_index().\
|
||||
|
|
@ -3,8 +3,8 @@ from contextlib import asynccontextmanager
|
|||
|
||||
from fastapi import FastAPI, responses
|
||||
|
||||
from gisaf.api import api
|
||||
from gisaf.geoapi import api as geoapi
|
||||
from gisaf.api.v2 import api
|
||||
from gisaf.api.geoapi import api as geoapi
|
||||
from gisaf.config import conf
|
||||
from gisaf.registry import registry
|
||||
from gisaf.redis_tools import setup_redis, setup_redis_cache, shutdown_redis
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ class Plot(BaseSettings):
|
|||
class Dashboard(BaseSettings):
|
||||
base_source_url: str
|
||||
base_storage_dir: str
|
||||
base_storage_url: str
|
||||
base_storage_url: str = '/dashboard-attachment/'
|
||||
|
||||
class Widgets(BaseSettings):
|
||||
footer: str
|
||||
|
|
|
|||
|
|
@ -32,12 +32,14 @@ class User(UserBase, table=True):
|
|||
password: str | None = None
|
||||
|
||||
def can_view(self, model) -> bool:
|
||||
viewable_role = getattr(model, 'viewable_role', None)
|
||||
if viewable_role:
|
||||
return viewable_role in (role.name for role in self.roles)
|
||||
role = getattr(model, 'viewable_role', None)
|
||||
if role:
|
||||
return self.has_role(role)
|
||||
else:
|
||||
return True
|
||||
|
||||
def has_role(self, role: str) -> bool:
|
||||
return role in (role.name for role in self.roles)
|
||||
|
||||
class RoleBase(SQLModel):
|
||||
name: str = Field(unique=True)
|
||||
|
|
|
|||
296
src/gisaf/models/dashboard.py
Normal file
296
src/gisaf/models/dashboard.py
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
from io import BytesIO
|
||||
from pickle import loads
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
from sqlmodel import Field, Relationship, String
|
||||
from pydantic import BaseModel
|
||||
import pandas as pd
|
||||
|
||||
from gisaf.config import conf
|
||||
from gisaf.models.metadata import gisaf
|
||||
from gisaf.models.models_base import Model
|
||||
from gisaf.models.misc import NotADataframeError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
plt = None
|
||||
|
||||
|
||||
class DashboardPageSource(Model, table=True):
|
||||
__tablename__ = 'dashboard_page_source'
|
||||
__table_args__ = gisaf.table_args
|
||||
|
||||
id: str = Field(primary_key=True)
|
||||
name: str
|
||||
|
||||
|
||||
class DashboardPageCommon:
|
||||
"""
|
||||
Base class for DashboardPage and DashboardPageSection, where some methods
|
||||
are common, eg. attachments
|
||||
"""
|
||||
|
||||
def get_attachment_url(self):
|
||||
## Serve through web front-end (nginx static file)
|
||||
if not self.attachment:
|
||||
return
|
||||
base_storage_url = conf.dashboard.base_storage_url
|
||||
if not base_storage_url:
|
||||
base_storage_url = '/dashboard-attachment/'
|
||||
return f'{base_storage_url}{self.group}/{self.attachment}'
|
||||
|
||||
def save_plot(self, plot):
|
||||
"""
|
||||
Render the matplotlib plot (or figure) and save it in the filesystem
|
||||
:param plot: matplotlib plot or figure...
|
||||
:return:
|
||||
"""
|
||||
self.ensure_dir_exists()
|
||||
|
||||
## Different types of figures supported
|
||||
fig = None
|
||||
if plt:
|
||||
if isinstance(plot, plt.Axes):
|
||||
fig = plot.figure
|
||||
elif isinstance(plot, plt.Figure):
|
||||
fig = plot
|
||||
if fig:
|
||||
fig.savefig(self.get_plot_file_path(), bbox_inches='tight')
|
||||
plt.close(fig)
|
||||
|
||||
if plot and not fig:
|
||||
logger.warning('Cannot save dashboard attachment (unknown attachment type)')
|
||||
return
|
||||
|
||||
#logger.info(f'Saved attachment of dashboard page {self.group}/{self.name} '
|
||||
#f'in {self.get_attachment_file_name()}')
|
||||
return self.get_plot_file_name()
|
||||
|
||||
def save_attachment(self, attached, name=None):
|
||||
"""
|
||||
Save the attachment in the filesystem
|
||||
:param attached: matplotlib plot or figure...
|
||||
:return:
|
||||
"""
|
||||
if not self.attachment:
|
||||
## Not set yet (creation)
|
||||
self.attachment = f'{self.name}.png'
|
||||
|
||||
self.ensure_dir_exists()
|
||||
|
||||
## Different types of figures supported
|
||||
fig = None
|
||||
if plt:
|
||||
if isinstance(attached, plt.Axes):
|
||||
fig = attached.figure
|
||||
elif isinstance(attached, plt.Figure):
|
||||
fig = attached
|
||||
if fig:
|
||||
fig.savefig(self.get_attachment_file_name(), bbox_inches='tight')
|
||||
plt.close(fig)
|
||||
|
||||
if attached and not fig:
|
||||
logger.warning('Cannot save dashboard attachment (unknown attachment type)')
|
||||
return
|
||||
|
||||
#logger.info(f'Saved attachment of dashboard page {self.group}/{self.name} '
|
||||
#f'in {self.get_attachment_file_name()}')
|
||||
return self.attachment
|
||||
|
||||
def get_page_df(self):
|
||||
"""
|
||||
Get the dataframe of the page
|
||||
"""
|
||||
if not self.df:
|
||||
return
|
||||
try:
|
||||
return pd.read_pickle(BytesIO(self.df), compression=None)
|
||||
except KeyError:
|
||||
raise NotADataframeError()
|
||||
|
||||
def get_plot(self):
|
||||
return loads(self.plot)
|
||||
|
||||
|
||||
class DashboardPage(Model, DashboardPageCommon, table=True):
|
||||
__tablename__ = 'dashboard_page'
|
||||
__table_args__ = gisaf.table_args
|
||||
|
||||
class Admin:
|
||||
menu = 'Dashboard'
|
||||
|
||||
id: int = Field(primary_key=True)
|
||||
name: str
|
||||
notebook: str
|
||||
group: str
|
||||
description: str
|
||||
attachment: str
|
||||
html: str
|
||||
viewable_role: str
|
||||
df: bytes
|
||||
plot: bytes
|
||||
source_id: int = Field(foreign_key=gisaf.table('dashboard_page_source.id'))
|
||||
time: datetime
|
||||
expanded_panes: str
|
||||
source: DashboardPageSource = Relationship()
|
||||
|
||||
def __str__(self):
|
||||
return f'{self.group:s}/{self.name:s}'
|
||||
|
||||
def __repr__(self):
|
||||
return f'<models.DashboardPage {self.group:s}/{self.name:s}>'
|
||||
|
||||
@classmethod
|
||||
def selectinload(cls):
|
||||
return [cls.source]
|
||||
|
||||
def ensure_dir_exists(self):
|
||||
"""
|
||||
Make sure the directory exists, before saving the file
|
||||
"""
|
||||
dir_name = Path(conf.dashboard.base_storage_dir) / self.group
|
||||
dir_name.mkdir(exist_ok=True)
|
||||
return dir_name
|
||||
|
||||
def get_attachment_file_name(self):
|
||||
"""
|
||||
Get the file name of the attachment
|
||||
:return:
|
||||
"""
|
||||
if self.attachment:
|
||||
base_dir = Path(conf.dashboard.base_storage_dir)
|
||||
if base_dir:
|
||||
return base_dir/self.group/self.attachment
|
||||
else:
|
||||
raise UserWarning('Cannot save attachment: no notebook/base_storage_dir in gisaf config')
|
||||
|
||||
def get_notebook_url(self):
|
||||
if self.notebook:
|
||||
base_url = conf.dashboard.base_source_url
|
||||
if base_url:
|
||||
return f'{base_url}{self.notebook}'
|
||||
else:
|
||||
logger.debug('Notebook: no base_url in gisaf config')
|
||||
|
||||
|
||||
class DashboardPageSection(Model, DashboardPageCommon, table=True):
|
||||
__tablename__ = 'dashboard_page_section'
|
||||
__table_args__ = gisaf.table_args
|
||||
|
||||
class Admin:
|
||||
menu = 'Dashboard'
|
||||
|
||||
id: str = Field(primary_key=True)
|
||||
name: str
|
||||
dashboard_page_id: int = Field(foreign_key=gisaf.table('dashboard_page.id'))
|
||||
dashboard_page: DashboardPage = Relationship()
|
||||
|
||||
description: str
|
||||
attachment: str
|
||||
html: str
|
||||
df: bytes
|
||||
plot: str
|
||||
|
||||
def __str__(self):
|
||||
return f'{self.name} for dashboard page #{self.dashboard_page_id}'
|
||||
|
||||
def __repr__(self):
|
||||
return f'<models.DashboardPageSection #{self.id}>'
|
||||
|
||||
@classmethod
|
||||
def selectinload(cls):
|
||||
return [cls.dashboard_page]
|
||||
|
||||
def get_plot_url(self):
|
||||
## Serve through web front-end (nginx static file)
|
||||
if not self.plot:
|
||||
return
|
||||
return conf.dashboard.base_storage_url \
|
||||
+ self.dashboard_page.group + '/' \
|
||||
+ self.dashboard_page.name + '/' \
|
||||
+ self.name + '.png'
|
||||
|
||||
def ensure_dir_exists(self):
|
||||
"""
|
||||
Make sure the directory exists, before saving the file
|
||||
"""
|
||||
dir_name = Path(conf.dashboard.base_storage_dir) / self.page.group / self.page.name
|
||||
dir_name.mkdir(exist_ok=True)
|
||||
return dir_name
|
||||
|
||||
def get_attachment_file_name(self):
|
||||
"""
|
||||
Get the file name of the attachment
|
||||
:return:
|
||||
"""
|
||||
if not self.attachment:
|
||||
return
|
||||
base_dir = Path(conf.dashboard.base_storage_dir)
|
||||
if not base_dir:
|
||||
raise UserWarning('Cannot save attachment: no notebook/base_storage_dir in gisaf config')
|
||||
return base_dir/self.page.group/self.page.name/self.attachment
|
||||
|
||||
def get_plot_file_name(self):
|
||||
return f'{self.name}.png'
|
||||
|
||||
def get_plot_file_path(self):
|
||||
"""
|
||||
Get the file name of the plot
|
||||
:return:
|
||||
"""
|
||||
if not self.plot:
|
||||
return
|
||||
base_dir = Path(conf.dashboard.base_storage_dir)
|
||||
if not base_dir:
|
||||
raise UserWarning('Cannot save attachment: no notebook/base_storage_dir in gisaf config')
|
||||
return base_dir/self.page.group/self.page.name/self.get_plot_file_name()
|
||||
|
||||
|
||||
class Widget(Model, table=True):
|
||||
__tablename__ = 'widget'
|
||||
__table_args__ = gisaf.table_args
|
||||
## CREATE TABLE gisaf.widget (name char(50) not null PRIMARY KEY, title varchar, subtitle varchar, notebook varchar, content varchar, time timestamp);
|
||||
name: str = Field(primary_key=True, sa_type=String(50))
|
||||
title: str
|
||||
subtitle: str
|
||||
content: str
|
||||
time: datetime
|
||||
notebook: str
|
||||
|
||||
class Admin:
|
||||
menu = 'Dashboard'
|
||||
|
||||
|
||||
class DashboadPageSectionType(BaseModel):
|
||||
name: str
|
||||
plot: str
|
||||
|
||||
|
||||
class DashboardPage_(BaseModel):
|
||||
name: str
|
||||
group: str
|
||||
description: str
|
||||
time: datetime | None = Field(default_factory=datetime.now)
|
||||
html: str | None = None
|
||||
attachment: str | None = None
|
||||
dfData: str | None = None
|
||||
plotData: str | None = None
|
||||
notebook: str | None = None
|
||||
expandedPanes: list[str] | None = None
|
||||
sections: list[DashboadPageSectionType] | None = None
|
||||
|
||||
|
||||
class DashboardGroup(BaseModel):
|
||||
name: str
|
||||
pages: list[DashboardPage_]
|
||||
|
||||
|
||||
class DashboardHome(BaseModel):
|
||||
title: str
|
||||
content: str
|
||||
footer: str
|
||||
Loading…
Add table
Add a link
Reference in a new issue