philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

ipynbtools: remove the gisaf "proxy"

Browse source 
Move methods to module level
Fix DashboardPage model definitions
Fix RawSurveyModel (missing geom field)
Cleanup
This commit is contained in:
phil 2024-05-02 23:42:45 +02:00
parent 00a5ae2d4e
commit dedb01b712
6 changed files with 284 additions and 256 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/gisaf/_version.py
View file

@ -1 +1 @@
__version__: str = '0.1.dev70+g53c2e35.d20240422' __version__: str = '0.1.dev74+gd3fa462.d20240430'

364
src/gisaf/ipynb_tools.py
View file

@ -9,7 +9,7 @@ from urllib.error import URLError
from datetime import datetime from datetime import datetime
from io import BytesIO from io import BytesIO
from pickle import dump, HIGHEST_PROTOCOL from pickle import dump, HIGHEST_PROTOCOL
# from aiohttp import ClientSession, MultipartWriter from aiohttp import ClientSession, MultipartWriter
import pandas as pd import pandas as pd
import geopandas as gpd import geopandas as gpd
@ -18,13 +18,16 @@ from geoalchemy2 import WKTElement
# from geoalchemy2.shape import from_shape # from geoalchemy2.shape import from_shape
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlmodel import select
# from shapely import wkb # from shapely import wkb
from gisaf.config import conf from gisaf.config import conf
from gisaf.database import db_session
from gisaf.redis_tools import store as redis_store from gisaf.redis_tools import store as redis_store
from gisaf.live import live_server from gisaf.live import live_server
from gisaf.registry import registry from gisaf.registry import registry
from gisaf.models.dashboard import Widget, DashboardPage, DashboardPageSection
## For base maps: contextily ## For base maps: contextily
try: try:
@ -35,79 +38,45 @@ except ImportError:
logger = logging.getLogger('Gisaf tools') logger = logging.getLogger('Gisaf tools')
class Notebook: async def remove_live_layer(channel):
""" """
Proof of concept? Gisaf could control notebook execution. Remove the channel from Gisaf Live
""" """
def __init__(self, path: str): async with ClientSession() as session:
self.path = path async with session.get('{}://{}:{}/api/remove-live/{}'.format(
conf.gisaf_live.scheme,
conf.gisaf_live.hostname,
conf.gisaf_live.port,
channel
)) as resp:
return await resp.text()
async def to_live_layer(gdf, channel, mapbox_paint=None, mapbox_layout=None, properties=None):
"""
Send a geodataframe to a gisaf server with an HTTP POST request for live map display
"""
with BytesIO() as buf:
dump(gdf, buf, protocol=HIGHEST_PROTOCOL)
buf.seek(0)
class Gisaf: async with ClientSession() as session:
""" with MultipartWriter('mixed') as mpwriter:
Gisaf tool for ipython/Jupyter notebooks mpwriter.append(buf)
""" if mapbox_paint != None:
def __init__(self): mpwriter.append_json(mapbox_paint, {'name': 'mapbox_paint'})
# self.db = db if mapbox_layout != None:
self.conf = conf mpwriter.append_json(mapbox_layout, {'name': 'mapbox_layout'})
self.store = redis_store if properties != None:
self.live_server = live_server mpwriter.append_json(properties, {'name': 'properties'})
if ctx: async with session.post('{}://{}:{}/api/live/{}'.format(
## Contextily newer version deprecated ctx.sources conf.gisaf_live.scheme,
self.basemaps = ctx.providers conf.gisaf_live.hostname,
else: conf.gisaf_live.port,
self.basemaps = None channel,
), data=mpwriter) as resp:
return await resp.text()
async def setup(self, with_mqtt=False): async def set_dashboard(name, group,
await self.store.create_connections()
if with_mqtt:
logger.warning('Gisaf live_server does not support with_mqtt anymore: ignoring')
try:
await self.live_server.setup()
except Exception as err:
logger.warn(f'Cannot setup live_server: {err}')
logger.exception(err)
async def make_models(self, **kwargs):
"""
Populate the model registry.
By default, all models will be added, including the those defined in categories (full registry).
Set with_categories=False to skip them and speed up the registry initialization.
:return:
"""
await registry.make_registry()
if 'with_categories' in kwargs:
logger.warning(f'{self.__class__}.make_models() does not support argument with_categories anymore')
self.registry = registry
## TODO: Compatibility: mark "models" deprecated, replaced by "registry"
# self.models = registry
def get_layer_list(self):
"""
Get a list of the names of all layers (ie. models with a geometry).
See get_all_geo for fetching data for a layer.
:return: list of strings
"""
return self.registry.geom.keys()
async def get_query(self, query):
"""
Return a dataframe for the query
"""
async with query.bind.raw_pool.acquire() as conn:
compiled = query.compile()
columns = [a.name for a in compiled.statement.columns]
stmt = await conn.prepare(compiled.string)
data = await stmt.fetch(*[compiled.params.get(param) for param in compiled.positiontup])
return pd.DataFrame(data, columns=columns)
async def get_all(self, model, **kwargs):
"""
Return a dataframe with all records for the model
"""
return await self.get_query(model.query)
async def set_dashboard(self, name, group,
notebook=None, notebook=None,
description=None, description=None,
html=None, html=None,
@ -126,8 +95,7 @@ class Gisaf:
:param sections: a list of DashboardPageSection :param sections: a list of DashboardPageSection
:return: :return:
""" """
from gisaf.models.dashboard import DashboardPage, DashboardPageSection async with db_session() as session:
expanded_panes = expanded_panes or [] expanded_panes = expanded_panes or []
sections = sections or [] sections = sections or []
now = datetime.now() now = datetime.now()
@ -150,9 +118,12 @@ class Gisaf:
else: else:
plot_blob = None plot_blob = None
page = await DashboardPage.query.where((DashboardPage.name==name) & (DashboardPage.group==group)).gino.first() request = select(DashboardPage).where((DashboardPage.name==name) & (DashboardPage.group==group))
if not page: res = await session.exec(request)
page: DashboardPage | None = res.one_or_none()
if page is None:
page = DashboardPage( page = DashboardPage(
id=None,
name=name, name=name,
group=group, group=group,
description=description, description=description,
@ -161,6 +132,7 @@ class Gisaf:
df=df_blob, df=df_blob,
plot=plot_blob, plot=plot_blob,
html=html, html=html,
source_id=None, # TODO: DashoardPage source
expanded_panes=','.join(expanded_panes) expanded_panes=','.join(expanded_panes)
) )
if attached: if attached:
@ -169,16 +141,14 @@ class Gisaf:
else: else:
if attached: if attached:
page.attachment = page.save_attachment(attached) page.attachment = page.save_attachment(attached)
await page.update( page.description=description
description=description, page.notebook=notebook
notebook=notebook, page.html=html
html=html, page.attachment=page.attachment
attachment=page.attachment, page.time=now
time=now, page.df=df_blob
df=df_blob, page.plot=plot_blob
plot=plot_blob, page.expanded_panes=','.join(expanded_panes)
expanded_panes=','.join(expanded_panes)
).apply()
for section in sections: for section in sections:
#print(section) #print(section)
@ -186,123 +156,101 @@ class Gisaf:
## Replace section.plot (matplotlib plot or figure) ## Replace section.plot (matplotlib plot or figure)
## by the name of the rendered pic inthe filesystem ## by the name of the rendered pic inthe filesystem
section.plot = section.save_plot(section.plot) section.plot = section.save_plot(section.plot)
section_record = await DashboardPageSection.query.where( query = select(DashboardPageSection).where(
(DashboardPageSection.dashboard_page_id==page.id) & (DashboardPageSection.name==section.name) (DashboardPageSection.dashboard_page_id==page.id) & (DashboardPageSection.name==section.name)
).gino.first() )
if not section_record: res = await session.exec(query)
section_record = res.one_or_none()
if section_record is None:
section.dashboard_page_id = page.id section.dashboard_page_id = page.id
await section.create() section.add(section)
else: else:
logger.warn('TODO: set_dashboard section update') logger.warn('TODO: set_dashboard section update')
logger.warn('TODO: set_dashboard section remove') logger.warn('TODO: set_dashboard section remove')
await session.commit()
async def set_widget(self, name, title, subtitle, content, notebook=None): async def set_widget(name, title, subtitle, content, notebook=None):
""" """
Create a web widget, that is served by /embed/<name>. Create a web widget, that is served by /embed/<name>.
""" """
from gisaf.models.dashboard import Widget
now = datetime.now() now = datetime.now()
widget = await Widget.query.where(Widget.name==name).gino.first() async with db_session() as session:
query = select(Widget).where(Widget.name==name)
res = await session.exec(query)
widget = res.one_or_none()
kwargs = dict( kwargs = dict(
)
if widget is None:
widget = Widget(
name=name,
title=title, title=title,
subtitle=subtitle, subtitle=subtitle,
content=content, content=content,
notebook=notebook, notebook=notebook,
time=now, time=now
) )
if widget:
await widget.update(**kwargs).apply()
else: else:
await Widget(name=name, **kwargs).create() widget.title=title
widget.subtitle=subtitle
widget.content=content
widget.notebook=notebook
widget.time=now
await session.commit()
async def to_live_layer(self, gdf, channel, mapbox_paint=None, mapbox_layout=None, properties=None):
"""
Send a geodataframe to a gisaf server with an HTTP POST request for live map display
"""
with BytesIO() as buf:
dump(gdf, buf, protocol=HIGHEST_PROTOCOL)
buf.seek(0)
async with ClientSession() as session:
with MultipartWriter('mixed') as mpwriter:
mpwriter.append(buf)
if mapbox_paint != None:
mpwriter.append_json(mapbox_paint, {'name': 'mapbox_paint'})
if mapbox_layout != None:
mpwriter.append_json(mapbox_layout, {'name': 'mapbox_layout'})
if properties != None:
mpwriter.append_json(properties, {'name': 'properties'})
async with session.post('{}://{}:{}/api/live/{}'.format(
self.conf.gisaf_live['scheme'],
self.conf.gisaf_live['hostname'],
self.conf.gisaf_live['port'],
channel,
), data=mpwriter) as resp:
return await resp.text()
async def remove_live_layer(self, channel): ## Below: old stuf, to delete
"""
Remove the channel from Gisaf Live
"""
async with ClientSession() as session:
async with session.get('{}://{}:{}/api/remove-live/{}'.format(
self.conf.gisaf_live['scheme'],
self.conf.gisaf_live['hostname'],
self.conf.gisaf_live['port'],
channel
)) as resp:
return await resp.text()
def to_layer(self, gdf: gpd.GeoDataFrame, model, project_id=None, # def to_layer(self, gdf: gpd.GeoDataFrame, model, project_id=None,
skip_columns=None, replace_all=True, # skip_columns=None, replace_all=True,
chunksize=100): # chunksize=100):
""" # """
Save the geodataframe gdf to the Gisaf model, using pandas' to_sql dataframes' method. # Save the geodataframe gdf to the Gisaf model, using pandas' to_sql dataframes' method.
Note that it's NOT an async call. Explanations: # Note that it's NOT an async call. Explanations:
* to_sql doesn't seems to work with gino/asyncpg # * to_sql doesn't seems to work with gino/asyncpg
* using Gisaf models is few magnitude orders slower # * using Gisaf models is few magnitude orders slower
(the async code using this technique is left commented out, for reference) # (the async code using this technique is left commented out, for reference)
""" # """
if skip_columns == None: # if skip_columns == None:
skip_columns = [] # skip_columns = []
## Filter empty geometries, and reproject # ## Filter empty geometries, and reproject
_gdf: gpd.GeoDataFrame = gdf[~gdf.geometry.is_empty].to_crs(self.conf.crs['geojson']) # _gdf: gpd.GeoDataFrame = gdf[~gdf.geometry.is_empty].to_crs(conf.crs.geojson)
## Remove the empty geometries # ## Remove the empty geometries
_gdf.dropna(inplace=True, subset=['geometry']) # _gdf.dropna(inplace=True, subset=['geometry'])
#_gdf['geom'] = _gdf.geom1.apply(lambda geom: from_shape(geom, srid=self.conf.srid)) # #_gdf['geom'] = _gdf.geom1.apply(lambda geom: from_shape(geom, srid=conf.geo.srid))
for col in skip_columns: # for col in skip_columns:
if col in _gdf.columns: # if col in _gdf.columns:
_gdf.drop(columns=[col], inplace=True) # _gdf.drop(columns=[col], inplace=True)
_gdf['geom'] = _gdf['geometry'].apply(lambda geom: WKTElement(geom.wkt, srid=self.conf.srid)) # _gdf['geom'] = _gdf['geometry'].apply(lambda geom: WKTElement(geom.wkt, srid=conf.geo.srid))
_gdf.drop(columns=['geometry'], inplace=True) # _gdf.drop(columns=['geometry'], inplace=True)
engine = create_engine(self.conf.db['uri'], echo=False) # engine = create_engine(conf.db.get_sqla_url(), echo=False)
## Drop existing # ## Drop existing
if replace_all: # if replace_all:
engine.execute('DELETE FROM "{}"'.format(model.__table__.fullname)) # engine.execute('DELETE FROM "{}"'.format(model.__table__.fullname))
else: # else:
raise NotImplementedError('ipynb_tools.Gisaf.to_layer does not support updates yet') # raise NotImplementedError('ipynb_tools.Gisaf.to_layer does not support updates yet')
## See https://stackoverflow.com/questions/38361336/write-geodataframe-into-sql-database # ## See https://stackoverflow.com/questions/38361336/write-geodataframe-into-sql-database
# Use 'dtype' to specify column's type # # Use 'dtype' to specify column's type
_gdf.to_sql( # _gdf.to_sql(
name=model.__tablename__, # name=model.__tablename__,
con=engine, # con=engine,
schema=model.__table_args__['schema'], # schema=model.__table_args__['schema'],
if_exists='append', # if_exists='append',
index=False, # index=False,
dtype={ # dtype={
'geom': model.geom.type, # 'geom': model.geom.type,
}, # },
method='multi', # method='multi',
chunksize=chunksize, # chunksize=chunksize,
) # )
#async with self.db.transaction() as tx: #async with self.db.transaction() as tx:
# if replace_all: # if replace_all:
@ -354,4 +302,76 @@ class Gisaf:
# await feature.create() # await feature.create()
# #db.session.commit() # #db.session.commit()
gisaf = Gisaf() # class Notebook:
# """
# Proof of concept? Gisaf could control notebook execution.
# """
# def __init__(self, path: str):
# self.path = path
# class Gisaf:
# """
# Gisaf tool for ipython/Jupyter notebooks
# """
# def __init__(self):
# # self.db = db
# self.conf = conf
# self.store = redis_store
# self.live_server = live_server
# if ctx:
# ## Contextily newer version deprecated ctx.sources
# self.basemaps = ctx.providers
# else:
# self.basemaps = None
# async def setup(self, with_mqtt=False):
# await self.store.create_connections()
# if with_mqtt:
# logger.warning('Gisaf live_server does not support with_mqtt anymore: ignoring')
# try:
# await self.live_server.setup()
# except Exception as err:
# logger.warn(f'Cannot setup live_server: {err}')
# logger.exception(err)
# async def make_models(self, **kwargs):
# """
# Populate the model registry.
# By default, all models will be added, including the those defined in categories (full registry).
# Set with_categories=False to skip them and speed up the registry initialization.
# :return:
# """
# await registry.make_registry()
# if 'with_categories' in kwargs:
# logger.warning(f'{self.__class__}.make_models() does not support argument with_categories anymore')
# self.registry = registry
# ## TODO: Compatibility: mark "models" deprecated, replaced by "registry"
# # self.models = registry
# def get_layer_list(self):
# """
# Get a list of the names of all layers (ie. models with a geometry).
# See get_all_geo for fetching data for a layer.
# :return: list of strings
# """
# return self.registry.geom.keys()
# async def get_query(self, query):
# """
# Return a dataframe for the query
# """
# async with query.bind.raw_pool.acquire() as conn:
# compiled = query.compile()
# columns = [a.name for a in compiled.statement.columns]
# stmt = await conn.prepare(compiled.string)
# data = await stmt.fetch(*[compiled.params.get(param) for param in compiled.positiontup])
# return pd.DataFrame(data, columns=columns)
# async def get_all(self, model, **kwargs):
# """
# Return a dataframe with all records for the model
# """
# return await self.get_query(model.query)
# gisaf = Gisaf()

14
src/gisaf/models/dashboard.py
View file

@ -31,16 +31,16 @@ class DashboardPageSource(Model, table=True):
name: str name: str
class DashboardPageCommon: class DashboardPageCommon(Model):
""" """
Base class for DashboardPage and DashboardPageSection, where some methods Base class for DashboardPage and DashboardPageSection, where some methods
are common, eg. attachments are common, eg. attachments
""" """
name: str name: str
df: bytes df: bytes | None = None
plot: bytes plot: bytes | None = None
#plot: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore #plot: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore
attachment: str | None attachment: str | None = None
html: str | None = None html: str | None = None
def ensure_dir_exists(self): def ensure_dir_exists(self):
@ -139,7 +139,7 @@ class DashboardPageMetaData(BaseModel):
viewable_role: str | None = None viewable_role: str | None = None
class DashboardPage(Model, DashboardPageCommon, DashboardPageMetaData, table=True): class DashboardPage(DashboardPageCommon, DashboardPageMetaData, table=True):
__tablename__ = 'dashboard_page' # type: ignore __tablename__ = 'dashboard_page' # type: ignore
__table_args__ = gisaf.table_args __table_args__ = gisaf.table_args
@ -202,7 +202,7 @@ class DashboardPage(Model, DashboardPageCommon, DashboardPageMetaData, table=Tru
logger.debug('Notebook: no base_url in gisaf config') logger.debug('Notebook: no base_url in gisaf config')
class DashboardPageSection(Model, DashboardPageCommon, table=True): class DashboardPageSection(DashboardPageCommon, table=True):
__tablename__ = 'dashboard_page_section' # type: ignore __tablename__ = 'dashboard_page_section' # type: ignore
__table_args__ = gisaf.table_args __table_args__ = gisaf.table_args
@ -280,7 +280,7 @@ class Widget(Model, table=True):
subtitle: str subtitle: str
content: str content: str
time: datetime time: datetime
notebook: str notebook: str | None = None
class Admin: class Admin:
menu = 'Dashboard' menu = 'Dashboard'

1
src/gisaf/models/project.py
View file

@ -179,7 +179,6 @@ class Project(Model, table=True):
return result return result
# def download_raw_survey_data(self, session=None): # def download_raw_survey_data(self, session=None):
# from gisaf.models.raw_survey_models import RawSurvey # from gisaf.models.raw_survey_models import RawSurvey
# from gisaf.registry import registry # from gisaf.registry import registry

27
src/gisaf/models/raw_survey.py
View file

@ -1,24 +1,31 @@
from typing import ClassVar from typing import Annotated, ClassVar
from sqlmodel import Field, BigInteger from geoalchemy2 import Geometry, WKBElement
from sqlmodel import Field, BigInteger, Relationship
from gisaf.config import conf
from gisaf.models.models_base import Model from gisaf.models.models_base import Model
from gisaf.models.geo_models_base import GeoPointMModel, BaseSurveyModel from gisaf.models.geo_models_base import GeoPointMModel, BaseSurveyModel
from gisaf.models.project import Project from gisaf.models.project import Project
from gisaf.models.category import Category from gisaf.models.category import Category
from gisaf.models.metadata import gisaf_survey from gisaf.models.metadata import gisaf_survey, gisaf_admin
class RawSurveyModel(BaseSurveyModel, GeoPointMModel): class RawSurveyModel(BaseSurveyModel, GeoPointMModel, table=True):
__table_args__ = gisaf_survey.table_args __table_args__ = gisaf_survey.table_args
__tablename__ = 'raw_survey' __tablename__ = 'raw_survey'
hidden: ClassVar[bool] = True hidden: ClassVar[bool] = True
geom: Annotated[str, WKBElement] = Field(
sa_type=Geometry('POINTZ', dimension=3, srid=conf.geo.raw_survey.srid))
id: int | None = Field(default=None, primary_key=True) id: int | None = Field(default=None, primary_key=True)
project_id: int | None = Field(foreign_key='project.id') project_id: int | None = Field(foreign_key=gisaf_admin.table('project.id'))
category: str = Field(foreign_key='category.name') category: str = Field(foreign_key=gisaf_survey.table('category.name'))
in_menu: bool = False #in_menu: bool = False
# Subclasses must include: project: Project = Relationship()
# project: Project = Relationship() category_info: Category = Relationship()
# category_info: Project = Relationship()
## XXX: Unused - calls to get_gdf have to provide this
## if the CRS is not standard, maybe due to an update of shapely?
_crs = conf.geo.raw_survey.spatial_sys_ref
@classmethod @classmethod
def selectinload(cls): def selectinload(cls):

4
src/gisaf/redis_tools.py
View file

@ -243,7 +243,9 @@ class Store:
await self.redis.set(self.get_layer_def_channel(store_name), layer_def_data) await self.redis.set(self.get_layer_def_channel(store_name), layer_def_data)
## Update the layers/stores registry ## Update the layers/stores registry
await self.get_live_layer_defs() ## XXX: Commentinhg out the update of live layers:
## This should be triggerred from a redis listener
#await self.get_live_layer_defs()
return geojson return geojson