philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Migrate core admin, baskets

Browse source
This commit is contained in:
phil 2024-02-13 12:46:24 +05:30
parent 5dacc908f2
commit df5f67b79d
7 changed files with 229 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/gisaf/_version.py
View file

@ -1 +1 @@
__version__ = '2023.4.dev33+g7e9e266.d20240210'
__version__: str = '2023.4.dev34+g5dacc90.d20240212'

35
src/gisaf/admin.py
View file

@ -3,8 +3,11 @@ from importlib.metadata import entry_points
import logging
from gisaf.live import live_server
from gisaf.models.authentication import User
from gisaf.redis_tools import Store
from gisaf.baskets import Basket
from gisaf.baskets import Basket, standard_baskets
from gisaf.redis_tools import store
from gisaf.registry import registry
logger = logging.getLogger('Gisaf admin manager')
@ -15,17 +18,16 @@ class AdminManager:
"""
store: Store
baskets: dict[str, Basket]
async def setup_admin(self, app):
async def setup_admin(self):
"""
Create the default baskets, scan and create baskets
from the Python entry points.
Runs at startup.
"""
self.app = app
self.store = app['store']
# self.app = app
# self.store = app['store']
## Standard baskets
from gisaf.baskets import Basket, standard_baskets
self.baskets = {
basket.name: basket
for basket in standard_baskets
@ -39,7 +41,7 @@ class AdminManager:
continue
if issubclass(basket_class, Basket):
## Get name, validity check
if basket_class.name == None:
if basket_class.name is None:
name = entry_point.name
else:
name = basket_class.name
@ -48,7 +50,7 @@ class AdminManager:
continue
## Instanciate
basket = basket_class()
basket._custom_module = entry_point.name
basket._custom_module = entry_point.name # type: ignore
## Check base_dir, eventually create it
if not basket.base_dir.exists():
try:
@ -64,23 +66,23 @@ class AdminManager:
logger.info(f'Added Basket {entry_point.name} from {entry_point.module}')
## Give a reference to the application to the baskets
for basket in self.baskets.values():
basket.app = app
# for basket in self.baskets.values():
# basket.app = app
## Subscribe to admin redis channels
self.pub_categories = self.store.redis.pubsub()
self.pub_scheduler = self.store.redis.pubsub()
self.pub_categories = store.redis.pubsub()
self.pub_scheduler = store.redis.pubsub()
await self.pub_categories.psubscribe('admin:categories:update')
task1 = create_task(self._listen_to_redis_categories())
await self.pub_scheduler.psubscribe('admin:scheduler:json')
task2 = create_task(self._listen_to_redis_scheduler())
app['admin'] = self
# app['admin'] = self
async def baskets_for_role(self, request):
async def baskets_for_role(self, user: User) -> dict[str, Basket]:
return {
name: basket for name, basket in self.baskets.items()
if await basket.allowed_for(request)
if await basket.allowed_for(user)
}
async def _listen_to_redis_categories(self):
@ -91,7 +93,7 @@ class AdminManager:
if msg['type'] == 'pmessage':
## XXX: Why the name isn't retrieved?
#client = await self.app['store'].pub.client_getname()
client = self.app['store'].uuid
client = store.uuid
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## FIXME: pubsub admin:categories:update
@ -100,8 +102,7 @@ class AdminManager:
## Skip for the process which sent this message actually updated its registry
#breakpoint()
if client != msg['data'].decode():
from gisaf.database import make_auto_models
await make_auto_models(self.app)
await registry.make_registry()
async def _listen_to_redis_scheduler(self):
"""

23
src/gisaf/api/admin.py Normal file
View file

@ -0,0 +1,23 @@
import logging
from fastapi import Depends, FastAPI, HTTPException, status, responses
from gisaf.models.admin import Basket, BasketNameOnly
from gisaf.models.authentication import User
from gisaf.security import get_current_active_user
from gisaf.admin import manager
logger = logging.getLogger(__name__)
api = FastAPI(
default_response_class=responses.ORJSONResponse,
)
@api.get('/basket')
async def get_baskets(
user: User = Depends(get_current_active_user),
):
return [
BasketNameOnly(name=name)
for name, basket in (await manager.baskets_for_role(user)).items()
]

6
src/gisaf/application.py
View file

@ -10,6 +10,8 @@ from gisaf.registry import registry
from gisaf.redis_tools import setup_redis, setup_redis_cache, shutdown_redis
from gisaf.tiles import registry as map_tile_registry
from gisaf.live import setup_live
from gisaf.admin import manager as admin_manager
from gisaf.api.admin import api as admin_api
logging.basicConfig(level=conf.gisaf.debugLevel)
logger = logging.getLogger(__name__)
@ -21,6 +23,7 @@ async def lifespan(app: FastAPI):
await setup_redis()
await setup_redis_cache()
await setup_live()
await admin_manager.setup_admin()
await map_tile_registry.setup()
yield
await shutdown_redis()
@ -35,4 +38,5 @@ app = FastAPI(
)
app.mount('/v2', api)
app.mount('/gj', geoapi)
app.mount('/gj', geoapi)
app.mount('/admin', admin_api)

12
src/gisaf/baskets.py
View file

@ -11,6 +11,7 @@ from typing import ClassVar
from gisaf.config import conf
from gisaf.models.admin import FileImport
from gisaf.models.authentication import User
# from gisaf.models.graphql import AdminBasketFile, BasketImportResult
from gisaf.models.survey import Surveyor, Accuracy, Equipment, AccuracyEquimentSurveyorMapping
from gisaf.models.project import Project
@ -45,20 +46,17 @@ class Basket:
self.importer = self.importer_class()
self.importer.basket = self
async def allowed_for(self, request):
async def allowed_for(self, user: User):
"""
Return False if the basket is protected by a role
Request: aiohttp.Request instance
"""
if not self.role:
return True
if user is not None and user.has_role(self.role):
return True
else:
try:
await check_permission(request, self.role)
except (HTTPUnauthorized, HTTPForbidden):
return False
else:
return True
return False
async def get_files(self, convert_path=False):
"""

131
src/gisaf/custom_store_base.py Normal file
View file

@ -0,0 +1,131 @@
import geopandas as gpd
from shapely import from_wkb
from json import dumps
from sqlmodel import SQLModel
from gisaf.config import conf
from gisaf.models.to_migrate import MapboxPaint, MapboxLayout, FeatureInfo
class BaseStore(SQLModel):
mapbox_type: str = 'symbol'
name: str = '<Unnamed store>'
description: str = '<Description>'
icon: str | None = None
mapbox_paint: MapboxPaint | None = None
mapbox_layout: MapboxLayout | None = None
attribution: str | None = None
symbol: str = '\ue32b'
base_gis_type: str = 'Point'
z_index: int = 460
cache_enabled: bool = False
can_get_features_as_df: bool = True
filtered_columns_on_map: list[str] = []
status: str = 'E'
## TODO: count and other Model-like interface
count: int = -1
@classmethod
async def get_popup(cls, df):
return cls.__name__ + ': ' + df.index.astype('U')
@classmethod
async def get_properties(cls, df):
return {}
## XXX: Copied from GeoModel
## TODO: Create a mixin for stores/models? Set Model as a subclass of Store?
@classmethod
async def get_geo_df(cls, where=None, crs=None, reproject=False,
filter_columns=False, with_popup=False, **kwargs):
"""
Return a Pandas dataframe of all records
:param where: where clause for the query (eg. Model.attr=='foo')
:param crs: coordinate system (eg. 'epsg:4326') (priority over the reproject parameter)
:param reproject: should reproject to conf.srid_for_proj
:return:
"""
df = await cls.get_df(where=where, **kwargs)
df.dropna(subset=['geom'], inplace=True)
#df.set_index('id', inplace=True)
df.sort_index(inplace=True)
df_clean = df[df.geom != None]
## Drop coordinates
df_clean.drop(columns=set(df_clean.columns).intersection(['ST_X_1', 'ST_Y_1', 'ST_Z_1']),
inplace=True)
if not crs:
crs = conf.crs['geojson']
#if getattr(gpd.options, 'use_pygeos', False):
# geometry = from_wkb(df_clean.geom)
#else:
# geometry = [wkb.loads(geom) for geom in df_clean.geom]
## XXX: There must be a vectorized way to do this
geom_bin = df_clean.geom.apply(lambda row: row.desc)
geometry = from_wkb(geom_bin)
# if not getattr(gpd.options, 'use_pygeos', False):
# geometry = pygeos.to_shapely(geometry)
gdf = gpd.GeoDataFrame(
df_clean.drop('geom', axis=1),
crs=crs,
geometry=geometry
)
if hasattr(cls, 'simplify') and cls.simplify:
#shapely_geom = shapely_geom.simplify(simplify_tolerance / conf.geo.simplify_geom_factor,
#preserve_topology=conf.geo.simplify_preserve_topology)
gdf['geometry'] = gdf['geometry'].simplify(
float(cls.simplify) / conf.geo.simplify_geom_factor,
preserve_topology=conf.geo.simplify_preserve_topology)
if reproject:
gdf.to_crs(crs=conf.crs['for_proj'], inplace=True)
## Filter out columns
if filter_columns:
gdf.drop(columns=set(gdf.columns).intersection(cls.filtered_columns_on_map),
inplace=True)
if with_popup:
gdf['popup'] = await cls.get_popup(gdf)
return gdf
@classmethod
async def get_df(cls, where=None,
with_related=None, recursive=True,
cast=True,
with_only_columns=None,
geom_as_ewkt=False,
**kwargs):
"""
Return a Pandas dataframe of all records
Optional arguments:
* an SQLAlchemy where clause
* with_related: automatically get data from related columns, following the foreign keys in the model definitions
* cast: automatically transform various data in their best python types (eg. with date, time...)
* with_only_columns: fetch only these columns (list of column names)
* geom_as_ewkt: convert geometry columns to EWKB (handy for eg. using upsert_df)
:return:
"""
raise NotImplementedError('Subclasses of BaseStore must implement get_df()')
@classmethod
async def get_item_params(cls, id) -> FeatureInfo:
raise NotImplementedError('Subclasses of BaseStore must implement get_item_params()')
@classmethod
async def get_mapbox_style(cls):
"""
Get the mapbox style (paint, layout, attribution...)
"""
style = {}
if cls.mapbox_paint is not None:
style['paint'] = dumps(cls.mapbox_paint)
if cls.mapbox_layout is not None:
style['layout'] = dumps(cls.mapbox_layout)
if cls.attribution is not None:
style['attribution'] = cls.attribution
return style

59
src/gisaf/models/admin.py
View file

@ -1,11 +1,10 @@
import re
from datetime import datetime
from datetime import datetime, date
from sqlmodel import Field, SQLModel, MetaData, JSON, TEXT, Relationship, Column
from pydantic import BaseModel
from sqlmodel import Field, Relationship
import pandas as pd
# from graphene import ObjectType, Int, String, DateTime, List
from gisaf.models.models_base import Model
from gisaf.models.survey import Surveyor, Equipment
from gisaf.models.project import Project
@ -21,7 +20,7 @@ class BadSurveyFileName(Exception):
pass
def get_file_import_date(record):
def get_file_import_date(record) -> date:
"""
Utility function that returns the date of survey from the file name,
if it matches the convention for CSV survey files.
@ -34,9 +33,9 @@ def get_file_import_date(record):
'(format should be: "PPP-DESCRIPTION-YYYY-MM-DD", '
'PPP being the project name, DESCRITION is optional and discarded)'
)
return datetime.date(day=int(fname_search.group(4)),
month=int(fname_search.group(3)),
year=int(fname_search.group(2)))
return date(day=int(fname_search.group(4)),
month=int(fname_search.group(3)),
year=int(fname_search.group(2)))
class FileImport(Model):
@ -44,7 +43,7 @@ class FileImport(Model):
Files to import or imported in the DB.
Give either url or path.
"""
__tablename__ = 'file_import'
__tablename__: str = 'file_import' # type: ignore
__table_args__ = gisaf_admin.table_args
id: int | None = Field(default=None, primary_key=True)
@ -79,9 +78,10 @@ class FileImport(Model):
def selectinload(cls):
return [cls.project, cls.surveyor, cls.equipment]
def set_import_time(self):
self.time = datetime.now()
db.session.commit()
## XXX: was used in Flask
# def set_import_time(self):
# self.time = datetime.now()
# db.session.commit()
@classmethod
async def get_df(cls, *args, **kwargs):
@ -116,7 +116,7 @@ class FeatureImportData(Model):
"""
Keep track of imported data, typically from shapefiles
"""
__tablename__ = 'feature_import_data'
__tablename__: str = 'feature_import_data' # type: ignore
__table_args__ = gisaf_admin.table_args
id: int | None = Field(default=None, primary_key=True)
@ -127,3 +127,36 @@ class FeatureImportData(Model):
origin: str
file_path: str
file_md5: str
class BasketFile(BaseModel):
id: int
dir: int
name: int
url: int
md5: int
time: datetime
comment: int
status: int
store: int
project: int
surveyor: int
equipment: int
import_result: int
class BasketNameOnly(BaseModel):
name: str
class Basket(BasketNameOnly):
files: list[BasketFile]
columns: list[str]
uploadFields: list[str]
projects: list[str]
class BasketImportResult(BaseModel):
time: datetime
message: str
details: str