Restructure apis, to the dedicated directory

Implement dashboards

src/gisaf/api/dashboard.py

@@ -0,0 +1,137 @@
+import logging
+from pathlib import Path
+from fastapi import Depends, FastAPI, HTTPException, status, responses
+from sqlalchemy.orm import selectinload
+from sqlmodel import select
+
+from gisaf.config import conf
+from gisaf.database import pandas_query, fastapi_db_session as db_session
+from gisaf.models.authentication import User
+from gisaf.models.dashboard import (
+    DashboardPage, DashboardPageSection,
+    DashboadPageSectionType, DashboardPage_,
+    DashboardGroup, DashboardHome,
+)
+from gisaf.models.misc import NotADataframeError
+from gisaf.security import get_current_active_user
+
+logger = logging.getLogger(__name__)
+
+
+api = FastAPI(
+    default_response_class=responses.ORJSONResponse,
+)
+
+@api.get('/groups')
+async def get_groups(
+    db_session: db_session,
+    ) -> list[DashboardGroup]:
+    query = select(DashboardPage)
+    data = await db_session.exec(query)
+    groups: dict[str, DashboardPage_] = {}
+    for page in data.all():
+        page_field = DashboardPage_(
+            name=page.name,
+            group=page.group,
+            description=page.description,
+        )
+        group = groups.get(page.group)
+        if group is None:
+            group = DashboardGroup(
+                name=page.group,
+                pages=[page_field]
+            )
+            groups[page.group] = group
+        else:
+            group.pages.append(page_field)
+    return groups.values()
+
+
+@api.get('/home')
+async def get_home() -> DashboardHome:
+    content_path = Path(conf.gisaf.dashboard_home.content_file).expanduser()
+    footer_path = Path(conf.gisaf.dashboard_home.footer_file).expanduser()
+    if content_path.is_file():
+        content = content_path.read_text()
+    else:
+        content = 'Gisaf is free, open source software for geomatics and GIS: <a href="http://redmine.auroville.org.in/projects/gisaf">Gisaf</a>.'
+    if footer_path.is_file():
+        footer = footer_path.read_text()
+    else:
+        footer = '<a rel="license" href="https://www.gnu.org/licenses/gpl.html"><img alt="GNU GPL v3 license"style="border-width:0" src="/static/icons/gplv3-88x31.png" title="GPL Open Source license"/></a>'
+    return DashboardHome(
+        title=conf.gisaf.dashboard_home.title,
+        content=content,
+        footer=footer,
+    )
+
+
+@api.get('/page/{group}/{name}')
+async def get_dashboard_page(group: str, name: str,
+    db_session: db_session,
+    user: User = Depends(get_current_active_user),
+    ) -> DashboardPage_:
+    query1 = select(DashboardPage).where((DashboardPage.name==name)
+                                        & (DashboardPage.group==group))
+    data1 = await db_session.exec(query1)
+    page = data1.one_or_none()
+    if not page:
+        raise HTTPException(status.HTTP_404_NOT_FOUND)
+    query2 = select(DashboardPageSection)\
+        .where(DashboardPageSection.dashboard_page_id==page.id)\
+        .options(selectinload(DashboardPageSection.dashboard_page))\
+        .order_by(DashboardPageSection.name)
+    data2 = await db_session.exec(query2)
+    sections = data2.all()
+    if page.viewable_role:
+        if not(user and user.has_role(page.viewable_role)):
+            username = user.username if user is not None else "Anonymous"
+            logger.info(f'{username} tried to access dashboard page {name}')
+            raise HTTPException(status.HTTP_401_UNAUTHORIZED)
+    dashboard_page = DashboardPage_(
+        name=page.name,
+        group=page.group,
+        description=page.description,
+        html=page.html,
+        time=page.time,
+        notebook=page.get_notebook_url(),
+        attachment=page.get_attachment_url(),
+        expandedPanes=[
+            p.strip()
+            for p in page.expanded_panes.split(',')
+        ] if page.expanded_panes else [],
+        sections=[
+            DashboadPageSectionType(
+                name=dps.name,
+                plot=dps.get_plot_url()
+            )
+            for dps in sections
+        ]
+    )
+    try:
+        df = page.get_page_df()
+        if df is not None:
+            ## TODO: plot as external file, like for sections
+            ## Convert Geopandas dataframe to Pandas
+            if isinstance(df, gpd.GeoDataFrame):
+                gdf = pd.DataFrame(df.drop(columns=['geometry']))
+                df = gdf
+            dashboard_page.dfData = df.to_json(orient='table', double_precision=2)
+    except NotADataframeError:
+        logger.warning(f'Dashboard: cannot read dataframe for page {page.name}')
+    except Exception as err:
+        logger.warning(f'Dashboard: cannot add dataframe for page {page.name}, see debug message')
+        logger.exception(err)
+    if page.plot:
+        try:
+            plot = page.get_plot()
+            plotData = {
+                'data': [d.to_plotly_json() for d in plot.data],
+                'layout': plot.layout.to_plotly_json(),
+            }
+        except Exception as err:
+            logger.warning(f'Dashboard: cannot add plot for page {page.name}, see debug message')
+            logger.exception(err)
+        else:
+            dashboard_page.plotData = dumps(plotData, cls=NumpyEncoder)
+    return dashboard_page

src/gisaf/api/v2.py

@@ -28,6 +28,7 @@ from gisaf.models.to_migrate import (
   FeatureInfo, InfoItem, Attachment, InfoCategory
 )
 from gisaf.live_utils import get_live_feature_info
+from gisaf.api.dashboard import api as dashboard_api
 
 logger = logging.getLogger(__name__)
 
@@ -36,8 +37,7 @@ api = FastAPI(
     default_response_class=responses.ORJSONResponse,
 )
 #api.add_middleware(SessionMiddleware, secret_key=conf.crypto.secret)
-
+api.mount('/dashboard', dashboard_api)
-#db_session = Annotated[AsyncSession, Depends(get_db_session)]
 
 
 @api.get('/bootstrap')
@@ -62,18 +62,6 @@ async def login_for_access_token(
         expires_delta=timedelta(seconds=conf.crypto.expire))
     return Token(access_token=access_token, token_type='bearer')
 
-@api.get("/list")
-async def list_data_providers() -> list[DataProvider]:
-    """
-    Return a list of data providers, for use with the api (graphs, etc)
-    :return:
-    """
-    return [
-        DataProvider(
-            name=model.get_store_name(),
-            values=[value.get_store_name() for value in values]
-        ) for model, values in registry.values_for_model.items()]
-
 
 @api.get("/users")
 async def get_users(
@@ -107,6 +95,19 @@ async def get_categories_p(
     df = await db_session.run_sync(pandas_query, query)
     return df.to_dict(orient="records")
 
+# @api.get("/list")
+@api.get("/data-providers")
+async def list_data_providers() -> list[DataProvider]:
+    """
+    Return a list of data providers, for use with the api (graphs, etc)
+    :return:
+    """
+    return [
+        DataProvider(
+            name=model.get_store_name(),
+            values=[value.get_store_name() for value in values]
+        ) for model, values in registry.values_for_model.items()]
+
 @api.get("/stores")
 async def get_stores() -> list[Store]:
     df = registry.stores.reset_index().\

src/gisaf/application.py

@@ -3,8 +3,8 @@ from contextlib import asynccontextmanager
 
 from fastapi import FastAPI, responses
 
-from gisaf.api import api
+from gisaf.api.v2 import api
-from gisaf.geoapi import api as geoapi
+from gisaf.api.geoapi import api as geoapi
 from gisaf.config import conf
 from gisaf.registry import registry
 from gisaf.redis_tools import setup_redis, setup_redis_cache, shutdown_redis

src/gisaf/config.py

@@ -213,7 +213,7 @@ class Plot(BaseSettings):
 class Dashboard(BaseSettings):
     base_source_url: str
     base_storage_dir: str
-    base_storage_url: str
+    base_storage_url: str = '/dashboard-attachment/'
 
 class Widgets(BaseSettings):
     footer: str

src/gisaf/models/authentication.py

@@ -32,12 +32,14 @@ class User(UserBase, table=True):
     password: str | None = None
 
     def can_view(self, model) -> bool:
-        viewable_role = getattr(model, 'viewable_role', None)
+        role = getattr(model, 'viewable_role', None)
-        if viewable_role:
+        if role:
-            return viewable_role in (role.name for role in self.roles)
+            return self.has_role(role)
         else:
             return True
 
+    def has_role(self, role: str) -> bool:
+        return role in (role.name for role in self.roles)
 
 class RoleBase(SQLModel):
     name: str = Field(unique=True)

src/gisaf/models/dashboard.py

@@ -0,0 +1,296 @@
+from io import BytesIO
+from pickle import loads
+from pathlib import Path
+from datetime import datetime
+import logging
+
+from sqlmodel import Field, Relationship, String
+from pydantic import BaseModel
+import pandas as pd
+
+from gisaf.config import conf
+from gisaf.models.metadata import gisaf
+from gisaf.models.models_base import Model
+from gisaf.models.misc import NotADataframeError
+
+logger = logging.getLogger(__name__)
+
+try:
+    import matplotlib.pyplot as plt
+except ImportError:
+    plt = None
+
+
+class DashboardPageSource(Model, table=True):
+    __tablename__ = 'dashboard_page_source'
+    __table_args__ = gisaf.table_args
+
+    id: str = Field(primary_key=True)
+    name: str
+
+
+class DashboardPageCommon:
+    """
+    Base class for DashboardPage and DashboardPageSection, where some methods
+    are common, eg. attachments
+    """
+
+    def get_attachment_url(self):
+        ## Serve through web front-end (nginx static file)
+        if not self.attachment:
+            return
+        base_storage_url = conf.dashboard.base_storage_url
+        if not base_storage_url:
+            base_storage_url = '/dashboard-attachment/'
+        return f'{base_storage_url}{self.group}/{self.attachment}'
+
+    def save_plot(self, plot):
+        """
+        Render the matplotlib plot (or figure) and save it in the filesystem
+        :param plot: matplotlib plot or figure...
+        :return:
+        """
+        self.ensure_dir_exists()
+
+        ## Different types of figures supported
+        fig = None
+        if plt:
+            if isinstance(plot, plt.Axes):
+                fig = plot.figure
+            elif isinstance(plot, plt.Figure):
+                fig = plot
+            if fig:
+                fig.savefig(self.get_plot_file_path(), bbox_inches='tight')
+                plt.close(fig)
+
+        if plot and not fig:
+            logger.warning('Cannot save dashboard attachment (unknown attachment type)')
+            return
+
+        #logger.info(f'Saved attachment of dashboard page {self.group}/{self.name} '
+                    #f'in {self.get_attachment_file_name()}')
+        return self.get_plot_file_name()
+
+    def save_attachment(self, attached, name=None):
+        """
+        Save the attachment in the filesystem
+        :param attached: matplotlib plot or figure...
+        :return:
+        """
+        if not self.attachment:
+            ## Not set yet (creation)
+            self.attachment = f'{self.name}.png'
+
+        self.ensure_dir_exists()
+
+        ## Different types of figures supported
+        fig = None
+        if plt:
+            if isinstance(attached, plt.Axes):
+                fig = attached.figure
+            elif isinstance(attached, plt.Figure):
+                fig = attached
+            if fig:
+                fig.savefig(self.get_attachment_file_name(), bbox_inches='tight')
+                plt.close(fig)
+
+        if attached and not fig:
+            logger.warning('Cannot save dashboard attachment (unknown attachment type)')
+            return
+
+        #logger.info(f'Saved attachment of dashboard page {self.group}/{self.name} '
+                    #f'in {self.get_attachment_file_name()}')
+        return self.attachment
+
+    def get_page_df(self):
+        """
+        Get the dataframe of the page
+        """
+        if not self.df:
+            return
+        try:
+            return pd.read_pickle(BytesIO(self.df), compression=None)
+        except KeyError:
+            raise NotADataframeError()
+
+    def get_plot(self):
+        return loads(self.plot)
+
+
+class DashboardPage(Model, DashboardPageCommon, table=True):
+    __tablename__ = 'dashboard_page'
+    __table_args__ = gisaf.table_args
+
+    class Admin:
+        menu = 'Dashboard'
+
+    id: int = Field(primary_key=True)
+    name: str
+    notebook: str
+    group: str
+    description: str
+    attachment: str
+    html: str
+    viewable_role: str
+    df: bytes
+    plot: bytes
+    source_id: int = Field(foreign_key=gisaf.table('dashboard_page_source.id'))
+    time: datetime
+    expanded_panes: str
+    source: DashboardPageSource = Relationship()
+
+    def __str__(self):
+        return f'{self.group:s}/{self.name:s}'
+
+    def __repr__(self):
+        return f'<models.DashboardPage {self.group:s}/{self.name:s}>'
+
+    @classmethod
+    def selectinload(cls):
+        return [cls.source]
+
+    def ensure_dir_exists(self):
+        """
+        Make sure the directory exists, before saving the file
+        """
+        dir_name = Path(conf.dashboard.base_storage_dir) / self.group
+        dir_name.mkdir(exist_ok=True)
+        return dir_name
+
+    def get_attachment_file_name(self):
+        """
+        Get the file name of the attachment
+        :return:
+        """
+        if self.attachment:
+            base_dir = Path(conf.dashboard.base_storage_dir)
+            if base_dir:
+                return base_dir/self.group/self.attachment
+            else:
+                raise UserWarning('Cannot save attachment: no notebook/base_storage_dir in gisaf config')
+
+    def get_notebook_url(self):
+        if self.notebook:
+            base_url = conf.dashboard.base_source_url
+            if base_url:
+                return f'{base_url}{self.notebook}'
+            else:
+                logger.debug('Notebook: no base_url in gisaf config')
+
+
+class DashboardPageSection(Model, DashboardPageCommon, table=True):
+    __tablename__ = 'dashboard_page_section'
+    __table_args__ = gisaf.table_args
+
+    class Admin:
+        menu = 'Dashboard'
+
+    id: str = Field(primary_key=True)
+    name: str
+    dashboard_page_id: int = Field(foreign_key=gisaf.table('dashboard_page.id'))
+    dashboard_page: DashboardPage = Relationship()
+
+    description: str
+    attachment: str
+    html: str
+    df: bytes
+    plot: str
+
+    def __str__(self):
+        return f'{self.name} for dashboard page #{self.dashboard_page_id}'
+
+    def __repr__(self):
+        return f'<models.DashboardPageSection #{self.id}>'
+
+    @classmethod
+    def selectinload(cls):
+        return [cls.dashboard_page]
+
+    def get_plot_url(self):
+        ## Serve through web front-end (nginx static file)
+        if not self.plot:
+            return
+        return conf.dashboard.base_storage_url \
+            + self.dashboard_page.group + '/' \
+            + self.dashboard_page.name + '/' \
+            + self.name + '.png'
+
+    def ensure_dir_exists(self):
+        """
+        Make sure the directory exists, before saving the file
+        """
+        dir_name = Path(conf.dashboard.base_storage_dir) / self.page.group / self.page.name
+        dir_name.mkdir(exist_ok=True)
+        return dir_name
+
+    def get_attachment_file_name(self):
+        """
+        Get the file name of the attachment
+        :return:
+        """
+        if not self.attachment:
+            return
+        base_dir = Path(conf.dashboard.base_storage_dir)
+        if not base_dir:
+            raise UserWarning('Cannot save attachment: no notebook/base_storage_dir in gisaf config')
+        return base_dir/self.page.group/self.page.name/self.attachment
+
+    def get_plot_file_name(self):
+        return f'{self.name}.png'
+
+    def get_plot_file_path(self):
+        """
+        Get the file name of the plot
+        :return:
+        """
+        if not self.plot:
+            return
+        base_dir = Path(conf.dashboard.base_storage_dir)
+        if not base_dir:
+            raise UserWarning('Cannot save attachment: no notebook/base_storage_dir in gisaf config')
+        return base_dir/self.page.group/self.page.name/self.get_plot_file_name()
+
+
+class Widget(Model, table=True):
+    __tablename__ = 'widget'
+    __table_args__ = gisaf.table_args
+    ## CREATE TABLE gisaf.widget (name char(50) not null PRIMARY KEY, title varchar, subtitle varchar, notebook varchar, content varchar, time timestamp);
+    name: str = Field(primary_key=True, sa_type=String(50))
+    title: str
+    subtitle: str
+    content: str
+    time: datetime
+    notebook: str
+
+    class Admin:
+        menu = 'Dashboard'
+
+
+class DashboadPageSectionType(BaseModel):
+    name: str
+    plot: str
+
+
+class DashboardPage_(BaseModel):
+    name: str
+    group: str
+    description: str
+    time: datetime | None = Field(default_factory=datetime.now)
+    html: str | None = None
+    attachment: str | None = None
+    dfData: str | None = None
+    plotData: str | None = None
+    notebook: str | None = None
+    expandedPanes: list[str] | None = None
+    sections: list[DashboadPageSectionType] | None = None
+
+
+class DashboardGroup(BaseModel):
+    name: str
+    pages: list[DashboardPage_]
+
+
+class DashboardHome(BaseModel):
+    title: str
+    content: str
+    footer: str