philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Restructure api

Browse source 
Fixes in geo api, registry
Cleanups
This commit is contained in:
phil 2024-02-27 05:05:33 +05:30
parent c84dd61f6a
commit 8c299f0041
17 changed files with 212 additions and 162 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/gisaf/_version.py
View file

@ -1 +1 @@
__version__: str = '2023.4.dev34+g5dacc90.d20240212'
__version__: str = '2023.4.dev37+gb00bf1f.d20240226'

8
src/gisaf/api/admin.py
View file

@ -1,6 +1,6 @@
import logging
from fastapi import Depends, FastAPI, HTTPException, status, responses
from fastapi import Depends, APIRouter, HTTPException, status, responses
from gisaf.models.admin import AdminBasket, BasketNameOnly
from gisaf.models.authentication import User
@ -9,8 +9,10 @@ from gisaf.admin import manager
logger = logging.getLogger(__name__)
api = FastAPI(
default_response_class=responses.ORJSONResponse,
api = APIRouter(
tags=["admin"],
# dependencies=[Depends(get_token_header)],
responses={404: {"description": "Not found"}},
)
@api.get('/basket')

8
src/gisaf/api/dashboard.py
View file

@ -2,7 +2,7 @@ import logging
from pathlib import Path
from json import dumps
from fastapi import Depends, FastAPI, HTTPException, status, responses
from fastapi import Depends, APIRouter, HTTPException, status, responses
from sqlalchemy.orm import selectinload
from sqlmodel import select
import pandas as pd
@ -34,8 +34,10 @@ Gisaf is free, open source software for geomatics and GIS:
<a href="http://redmine.auroville.org.in/projects/gisaf">Gisaf</a>.
'''
api = FastAPI(
default_response_class=responses.ORJSONResponse,
api = APIRouter(
tags=["dashboard"],
# dependencies=[Depends(get_token_header)],
responses={404: {"description": "Not found"}},
)
@api.get('/groups')

13
src/gisaf/api/geoapi.py
View file

@ -7,7 +7,7 @@ import logging
from typing import Annotated
from asyncio import CancelledError
from fastapi import (Depends, FastAPI, HTTPException, Response, Header,
from fastapi import (Depends, APIRouter, HTTPException, Response, Header,
WebSocket, WebSocketDisconnect,
status, responses)
@ -20,8 +20,10 @@ from gisaf.security import get_current_active_user
logger = logging.getLogger(__name__)
api = FastAPI(
default_response_class=responses.ORJSONResponse,
api = APIRouter(
tags=["geoapi"],
# dependencies=[Depends(get_token_header)],
responses={404: {"description": "Not found"}},
)
class ConnectionManager:
@ -103,11 +105,10 @@ async def get_geojson(store_name,
if model.cache_enabled:
ttag = await redis_store.get_ttag(store_name)
if ttag and If_None_Match == ttag:
return status.HTTP_304_NOT_MODIFIED
raise HTTPException(status.HTTP_304_NOT_MODIFIED)
if hasattr(model, 'get_geojson'):
geojson = await model.get_geojson(simplify_tolerance=simplify,
preserve_topology=preserveTopology,
registry=registry)
preserve_topology=preserveTopology)
## Store to redis for caching
if use_cache:
await redis_store.store_json(model, geojson)

14
src/gisaf/api/v2.py → src/gisaf/api/main.py
View file

@ -2,7 +2,7 @@ import logging
from datetime import timedelta
from typing import Annotated
from fastapi import Depends, FastAPI, HTTPException, status, responses
from fastapi import Depends, APIRouter, HTTPException, status, responses
from sqlalchemy.orm import selectinload
from fastapi.security import OAuth2PasswordRequestForm
from sqlmodel import select
@ -12,7 +12,6 @@ from gisaf.models.authentication import (
Role, RoleRead,
)
from gisaf.models.category import Category, CategoryRead
from gisaf.models.geo_models_base import LineWorkSurveyModel
from gisaf.models.to_migrate import DataProvider
from gisaf.models.survey import Equipment, SurveyMeta, Surveyor
from gisaf.config import Survey, conf
@ -31,19 +30,16 @@ from gisaf.models.to_migrate import (
FeatureInfo, InfoItem, Attachment, InfoCategory
)
from gisaf.live_utils import get_live_feature_info
from gisaf.api.dashboard import api as dashboard_api
from gisaf.api.map import api as map_api
logger = logging.getLogger(__name__)
api = FastAPI(
default_response_class=responses.ORJSONResponse,
api = APIRouter(
tags=["api"],
# dependencies=[Depends(get_token_header)],
responses={404: {"description": "Not found"}},
)
#api.add_middleware(SessionMiddleware, secret_key=conf.crypto.secret)
api.mount('/dashboard', dashboard_api)
api.mount('/map', map_api)
@api.get('/bootstrap')
async def bootstrap(

57
src/gisaf/api/map.py
View file

@ -2,21 +2,26 @@ from collections import OrderedDict, defaultdict
import logging
from json import dumps
from fastapi import FastAPI, Request, HTTPException, status, responses
from fastapi import (APIRouter, Request, HTTPException,
Response, status)
from sqlalchemy.orm import selectinload
from sqlalchemy.exc import NoResultFound
from sqlmodel import select
from gisaf.models.map_bases import BaseMap, BaseMapLayer, BaseStyle, MapInitData
from gisaf.models.map_bases import (BaseMap, BaseMapLayer, BaseMapWithStores,
BaseStyle, MapInitData, MaplibreStyle)
from gisaf.models.misc import Qml
from gisaf.registry import registry
from gisaf.database import db_session
from gisaf.database import fastapi_db_session
from gisaf.database import db_session, fastapi_db_session
from gisaf.tiles import registry as tiles_registry
from gisaf.redis_tools import store as redis_store
logger = logging.getLogger(__name__)
api = FastAPI(
default_response_class=responses.ORJSONResponse,
api = APIRouter(
tags=["map"],
# dependencies=[Depends(get_token_header)],
# responses={404: {"description": "Not found"}},
)
async def get_base_styles():
@ -31,23 +36,21 @@ async def get_base_styles():
# base_styles.extend(tiles_registry.mbtiles.values())
return [BaseStyle(name=bs) for bs in base_styles] # type: ignore
async def get_base_maps() -> list[BaseMap]:
async def get_base_maps() -> list[BaseMapWithStores]:
async with db_session() as session:
query1 = select(BaseMap).options(selectinload(BaseMap.layers)) # type: ignore
query1 = select(BaseMap) #.options(selectinload(BaseMap.layers)) # type: ignore
data1 = await session.exec(query1)
base_maps = data1.all()
return base_maps # type: ignore
base_map_dict = {bm.id: bm.name for bm in base_maps}
query2 = select(BaseMapLayer).options(selectinload(BaseMapLayer.base_map))
query2 = select(BaseMapLayer).options(selectinload(BaseMapLayer.base_map)) # type: ignore
data2 = await session.exec(query2)
base_map_layers = data2.all()
bms = defaultdict(list)
bms: dict[str, list] = defaultdict(list)
for bml in base_map_layers:
breakpoint()
if bml.store:
bms[base_map_dict[bml.base_map_id]].append(name=bml.store)
bms[base_map_dict[bml.base_map_id]].append(bml.store)
return [
BaseMap(name=bm, stores=bmls)
BaseMapWithStores(name=bm, stores=bmls)
for bm, bmls in OrderedDict(sorted(bms.items())).items()
]
@ -84,3 +87,29 @@ async def get_base_style(request: Request, name: str,
else:
style = base_style.style # type: ignore
return BaseStyle(name=name, style=style) # type: ignore
@api.get('/layer_style/{store}')
async def get_layer_style(request: Request, store: str,
response: Response,
) -> MaplibreStyle | None:
store_record = registry.stores.loc[store]
if store_record.is_live:
## No ttag for live layers' style (could be added?)
## Get layer_defs from live redis and give symbol
return await redis_store.get_maplibre_style(store)
## Set the etag based on the last modification of the model's style.
#if store in info.schema.app['registry'].geom_auto:
if store_record.custom:
## The style is in Qml
ttag_channel = 'gisaf_map.qml'
else:
## The style is in Category
ttag_channel = 'gisaf_survey.category'
## Check if the request was etagged:
ttag = await redis_store.get_ttag(ttag_channel)
if ttag and request.headers.get('If-None-Match') == ttag:
raise HTTPException(status_code=status.HTTP_304_NOT_MODIFIED)
# request.not_modified = True
# return MaplibreStyle()
response.headers['ETag'] = ttag
return await store_record.model.get_maplibre_style()

14
src/gisaf/application.py
View file

@ -3,15 +3,17 @@ from contextlib import asynccontextmanager
from fastapi import FastAPI, responses
from gisaf.api.v2 import api
from gisaf.api.geoapi import api as geoapi
from gisaf.config import conf
from gisaf.registry import registry
from gisaf.redis_tools import setup_redis, setup_redis_cache, shutdown_redis
from gisaf.tiles import registry as map_tile_registry
from gisaf.live import setup_live
from gisaf.admin import manager as admin_manager
from gisaf.api.main import api
from gisaf.api.geoapi import api as geoapi
from gisaf.api.admin import api as admin_api
from gisaf.api.dashboard import api as dashboard_api
from gisaf.api.map import api as map_api
logging.basicConfig(level=conf.gisaf.debugLevel)
logger = logging.getLogger(__name__)
@ -37,6 +39,8 @@ app = FastAPI(
default_response_class=responses.ORJSONResponse,
)
app.mount('/v2', api)
app.mount('/gj', geoapi)
app.mount('/admin', admin_api)
app.include_router(api, prefix="/api")
app.include_router(geoapi, prefix="/api/gj")
app.include_router(admin_api, prefix="/api/admin")
app.include_router(dashboard_api, prefix="/api/dashboard")
app.include_router(map_api, prefix='/api/map')

23
src/gisaf/custom_store_base.py
View file

@ -1,3 +1,4 @@
from re import A
import geopandas as gpd
from shapely import from_wkb
from json import dumps
@ -5,7 +6,8 @@ from json import dumps
from sqlmodel import SQLModel
from gisaf.config import conf
from gisaf.models.to_migrate import MapboxPaint, MapboxLayout, FeatureInfo
from gisaf.models.map_bases import MaplibreStyle
from gisaf.models.to_migrate import FeatureInfo
class BaseStore(SQLModel):
@ -13,8 +15,8 @@ class BaseStore(SQLModel):
name: str = '<Unnamed store>'
description: str = '<Description>'
icon: str | None = None
mapbox_paint: MapboxPaint | None = None
mapbox_layout: MapboxLayout | None = None
mapbox_paint: dict[str, dict] | None = None
mapbox_layout: dict[str, dict] | None = None
attribution: str | None = None
symbol: str = '\ue32b'
base_gis_type: str = 'Point'
@ -117,15 +119,12 @@ class BaseStore(SQLModel):
raise NotImplementedError('Subclasses of BaseStore must implement get_item_params()')
@classmethod
async def get_mapbox_style(cls):
async def get_maplibre_style(cls) -> MaplibreStyle:
"""
Get the mapbox style (paint, layout, attribution...)
"""
style = {}
if cls.mapbox_paint is not None:
style['paint'] = dumps(cls.mapbox_paint)
if cls.mapbox_layout is not None:
style['layout'] = dumps(cls.mapbox_layout)
if cls.attribution is not None:
style['attribution'] = cls.attribution
return style
return MaplibreStyle(
paint=cls.mapbox_paint,
layout=cls.mapbox_layout,
attribution=cls.attribution
)

2
src/gisaf/models/authentication.py
View file

@ -27,7 +27,7 @@ class UserBase(SQLModel):
class User(UserBase, table=True):
__table_args__ = gisaf_admin.table_args
id: int | None = Field(default=None, primary_key=True)
id: str | None = Field(default=None, primary_key=True)
roles: list["Role"] = Relationship(back_populates="users",
link_model=UserRoleLink)
password: str | None = None

4
src/gisaf/models/category.py
View file

@ -61,8 +61,8 @@ class CategoryBase(BaseModel):
style: str | None = Field(sa_type=TEXT)
symbol: str | None = Field(sa_type=String(1)) # type: ignore
mapbox_type_custom: str | None = Field(sa_type=String(12)) # type: ignore
mapbox_paint: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore
mapbox_layout: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore
mapbox_paint: dict[str, dict | list | float | int | str] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore
mapbox_layout: dict[str, dict | list | float | int | str] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore
viewable_role: str | None
extra: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore

75
src/gisaf/models/geo_models_base.py
View file

@ -14,20 +14,20 @@ import shapely # type: ignore
import pyproj
from pydantic import BaseModel
from sqlmodel import select, Field, Relationship
from sqlmodel import select, Field
from sqlmodel.ext.asyncio.session import AsyncSession
from sqlalchemy import BigInteger, MetaData, String, func, and_, text
from sqlalchemy import BigInteger, String, func, and_, text
from sqlalchemy.sql import sqltypes
from sqlalchemy.orm import declared_attr
from psycopg2.extensions import adapt
from geoalchemy2.shape import from_shape
from geoalchemy2.types import Geometry, WKBElement
from shapely import wkb, from_wkb
from shapely.geometry import mapping
from shapely.geometry import mapping # type: ignore
from shapely.ops import transform # type: ignore
from shapefile import (
Writer as ShapeFileWriter, # type: ignore
from shapefile import ( # type: ignore
Writer as ShapeFileWriter,
POINT, POINTZ,
POLYLINE, POLYLINEZ,
POLYGON, POLYGONZ,
@ -35,8 +35,9 @@ from shapefile import (
from gisaf.database import db_session
from gisaf.config import conf
from gisaf.models.map_bases import MaplibreStyle
from gisaf.models.models_base import Model
from gisaf.models.metadata import gisaf_survey, gisaf_admin, survey
from gisaf.models.metadata import gisaf_survey, gisaf_admin, survey, raw_survey
from gisaf.models.misc import Qml
from gisaf.models.category import Category
from gisaf.models.to_migrate import InfoItem
@ -157,6 +158,7 @@ class SurveyModel(BaseSurveyModel):
# status: str = Field(sa_type=String(1))
get_gdf_with_related: ClassVar[bool] = False
category_name: ClassVar[str]
filtered_columns_on_map: ClassVar[list[str]] = [
'equip_id',
@ -219,9 +221,8 @@ class SurveyModel(BaseSurveyModel):
@classmethod
async def get_geojson(cls,
registry=None, simplify_tolerance=0, preserve_topology=False):
if registry is None:
from ..registry import registry
simplify_tolerance=0, preserve_topology=False):
from gisaf.registry import registry
## Fastest, but the id is in properties and needs the front end (eg Gisaf for mapbox)
## to move it at the feature level
@ -592,36 +593,35 @@ class GeoModelNoStatus(Model):
return zip_file
@classmethod
async def get_mapbox_style(cls):
async def get_maplibre_style(cls) -> MaplibreStyle | None:
"""
Get the mapbox style (paint, layout, attribution...)
"""
## If the model is from survey, it should have a category, which has a style
## Get from database
style = {}
if hasattr(cls, 'category'):
category = await Category.get_item_by_pk(pk=cls.category.name)
if category:
if category['mapbox_paint'] is not None:
style['paint'] = category['mapbox_paint']
if category['mapbox_layout'] is not None:
style['layout'] = category['mapbox_layout']
else:
category = None
qml = await Qml.get_item_by_pk(pk=cls.__name__)
if qml:
if qml['mapbox_paint']:
style['paint'] = qml['mapbox_paint']
if qml['mapbox_layout']:
style['layout'] = qml['mapbox_layout']
if cls.attribution is not None:
style['attribution'] = cls.attribution
style: MaplibreStyle | None
async with db_session() as session:
if hasattr(cls, 'category'):
category = await session.get(Category, cls.get_store_name())
if category:
style = MaplibreStyle(
layout=category.mapbox_layout,
paint=category.mapbox_paint,
)
else:
style = None
else:
qml = await session.get(Qml, cls.__name__)
if qml:
style = MaplibreStyle(
layout=qml.mapbox_layout,
paint=qml.mapbox_paint,
attribution=qml.attr,
)
else:
style = None
return style
@classmethod
async def get_features_attrs(cls, simplify_tolerance):
"""
@ -1094,10 +1094,13 @@ class RawSurveyBaseModel(BaseSurveyModel, GeoPointModelNoStatus):
"""
Abstract base class for category based raw survey point models
"""
# metadata: ClassVar[MetaData] = raw_survey
geom: Annotated[str, WKBElement] = Field(sa_type=Geometry('POINTZ', dimension=3,
srid=conf.geo.raw_survey.srid))
status: str = Field(sa_type=String(1))
__table_args__ = raw_survey.table_args
geom: Annotated[str, WKBElement] = Field(
sa_type=Geometry('POINTZ',
dimension=3,
srid=conf.geo.raw_survey.srid),
) # type: ignore
status: str = Field(sa_type=String(1)) # type: ignore
## store_name is set in category_models_maker.make_category_models
store_name: ClassVar[str | None] = None

66
src/gisaf/models/map_bases.py
View file

@ -28,30 +28,6 @@ class BaseStyle(Model, table=True):
return f'<models.BaseStyle {self.name:s}>'
class BaseMap(Model, table=True):
__table_args__ = gisaf_map.table_args
__tablename__: str = 'base_map' # type: ignore
class Admin:
menu = 'Other'
id: int | None = Field(primary_key=True, default=None)
name: str
layers: list['BaseMapLayer'] = Relationship(back_populates='base_map')
def __repr__(self) -> str:
return f'<models.BaseMap {self.name:s}>'
def __str__(self) -> str:
return self.name
@classmethod
def selectinload(cls) -> list[list['BaseMapLayer']]:
return [
cls.layers
]
class BaseMapLayer(Model, table=True):
__table_args__ = gisaf_map.table_args
__tablename__: str = 'base_map_layer' # type: ignore
@ -62,7 +38,7 @@ class BaseMapLayer(Model, table=True):
id: int | None = Field(primary_key=True, default=None)
base_map_id: int = Field(foreign_key=gisaf_map.table('base_map.id'),
index=True)
base_map: BaseMap = Relationship(back_populates='layers')
base_map: 'BaseMap' = Relationship() #back_populates='layers')
store: str = Field(sa_type=String(100)) # type: ignore
@classmethod
@ -78,8 +54,46 @@ class BaseMapLayer(Model, table=True):
return f"{self.store or '':s}"
class BaseMap(Model, table=True):
__table_args__ = gisaf_map.table_args
__tablename__: str = 'base_map' # type: ignore
class Admin:
menu = 'Other'
id: int | None = Field(primary_key=True, default=None)
name: str
# layers: list['BaseMapLayer'] = Relationship(
# back_populates='base_map',
# # link_model=BaseMapLayer
# )
def __repr__(self) -> str:
return f'<models.BaseMap {self.name:s}>'
def __str__(self) -> str:
return self.name
# @classmethod
# def selectinload(cls) -> list[list[BaseMapLayer]]:
# return [
# cls.layers
# ]
class BaseMapWithStores(BaseModel):
name: str
stores: list[str]
class MapInitData(BaseModel):
baseStyles: list[BaseStyle] = []
baseMaps: list[BaseMap] = []
baseMaps: list[BaseMapWithStores] = []
groups: list[CategoryGroup] = []
stores: list[Store] = []
class MaplibreStyle(BaseModel):
paint: dict[str, dict | list | float | int | str] | None = None
layout: dict[str, dict | list | float | int | str] | None = None
attribution: str | None = None

14
src/gisaf/models/misc.py
View file

@ -14,11 +14,11 @@ class NotADataframeError(Exception):
pass
class Qml(Model):
class Qml(Model, table=True):
"""
Model for storing qml (QGis style)
"""
model_config = ConfigDict(protected_namespaces=())
model_config = ConfigDict(protected_namespaces=()) # type: ignore
__table_args__ = gisaf_map.table_args
class Admin:
@ -26,11 +26,11 @@ class Qml(Model):
flask_admin_model_view = 'QmlModelView'
model_name: str | None = Field(default=None, primary_key=True)
qml: str
attr: str
style: str
mapbox_paint: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True))
mapbox_layout: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True))
qml: str | None = None
attr: str | None = None
style: str | None = None
mapbox_paint: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore
mapbox_layout: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore
def __repr__(self):
return '<models.Qml {self.model_name:s}>'.format(self=self)

8
src/gisaf/models/store.py
View file

@ -1,10 +1,4 @@
from typing import Any
from pydantic import BaseModel
from gisaf.models.geo_models_base import GeoModel, RawSurveyBaseModel, GeoPointSurveyModel
class MapLibreStyle(BaseModel):
...
class StoreNameOnly(BaseModel):
@ -12,6 +6,7 @@ class StoreNameOnly(BaseModel):
class Store(StoreNameOnly):
category: str | None = None
auto_import: bool
# base_gis_type: str
count: int | None = None
@ -39,7 +34,6 @@ class Store(StoreNameOnly):
#raw_model: GeoPointSurveyModel
#raw_model_store_name: str
status: str
store: str
style: str | None
symbol: str | None
title: str

8
src/gisaf/models/to_migrate.py
View file

@ -73,11 +73,3 @@ class FeatureInfo(BaseModel):
files: list[Attachment] = []
images: list[Attachment] = []
externalRecordUrl: str | None = None
class MapboxPaint(BaseModel):
...
class MapboxLayout(BaseModel):
...

15
src/gisaf/redis_tools.py
View file

@ -16,6 +16,7 @@ from redis import asyncio as aioredis
from gisaf.config import conf
# from gisaf.models.live import LiveModel
from gisaf.models.map_bases import MaplibreStyle
from gisaf.utils import (SHAPELY_TYPE_TO_MAPBOX_TYPE, DEFAULT_MAPBOX_LAYOUT,
DEFAULT_MAPBOX_PAINT, gisTypeSymbolMap)
from gisaf.registry import registry
@ -281,18 +282,14 @@ class Store:
registry.geom_live_defs[model_info['store']] = model_info
registry.update_live_layers()
async def get_mapbox_style(self, store_name):
async def get_maplibre_style(self, store_name) -> MaplibreStyle:
"""
Get the http headers (mapbox style) from the store name (layer_def)
"""
paint = await self.redis.get(self.get_mapbox_paint_channel(store_name))
layout = await self.redis.get(self.get_mapbox_layout_channel(store_name))
style = {}
if paint is not None:
style['paint'] = paint.decode()
if layout is not None:
style['layout'] = layout.decode()
return style
return MaplibreStyle(
paint=(await self.redis.get(self.get_mapbox_paint_channel(store_name))).decode(),
layout=(await self.redis.get(self.get_mapbox_layout_channel(store_name))).decode(),
)
async def get_layer_as_json(self, store_name):
"""

41
src/gisaf/registry.py
View file

@ -12,7 +12,7 @@ from pydantic import create_model
from sqlalchemy import text
from sqlalchemy.orm import selectinload, joinedload
from sqlalchemy.exc import NoResultFound
from sqlmodel import SQLModel, select, inspect, Relationship
from sqlmodel import SQLModel, col, select, inspect, Relationship
import pandas as pd
import numpy as np
@ -336,7 +336,7 @@ class ModelRegistry:
# for category in categories
# if self.raw_survey_models.get(category.table_name)}
async def get_model_id_params(self, model: SQLModel, id: int) -> FeatureInfo:
async def get_model_id_params(self, model: SQLModel, id: int) -> FeatureInfo | None:
"""
Return the parameters for this item (table name, id), displayed in info pane
"""
@ -389,8 +389,9 @@ class ModelRegistry:
"""
## Utility functions used with apply method (dataframes)
def fill_columns_from_custom_models(row) -> tuple[str, str, str, str, str, str, str]:
# model:
return (
row.model.__name__,
row.model.get_store_name(),
row.model.__name__,
row.model.description,
row.model.__table__.schema,
@ -429,7 +430,11 @@ class ModelRegistry:
self.categories = self.categories.merge(df_raw_models, left_on='store', right_index=True)
self.categories['custom'] = False
self.categories['is_db'] = True
self.categories.sort_index(inplace=True)
self.categories.reset_index(inplace=True)
self.categories.rename(columns={'name': 'category'}, inplace=True)
self.categories.set_index('store', inplace=True)
self.categories.sort_values('category')
# self.categories.sort_index(inplace=True)
# self.categories['name_letter'] = self.categories.index.str.slice(0, 1)
# self.categories['name_number'] = self.categories.index.str.slice(1).astype('int64')
# self.categories.sort_values(['name_letter', 'name_number'], inplace=True)
@ -467,6 +472,7 @@ class ModelRegistry:
self.custom_models['custom'] = True
self.custom_models['is_db'] = True
self.custom_models['raw_model_store_name'] = ''
self.custom_models['category'] = ''
self.custom_models['in_menu'] = self.custom_models.apply(
lambda row: getattr(row.model, 'in_menu', True),
axis=1
@ -512,6 +518,7 @@ class ModelRegistry:
self.custom_stores = self.custom_stores.loc[self.custom_stores.in_menu]
self.custom_stores['auto_import'] = False
self.custom_stores['is_line_work'] = False
self.custom_stores['category'] = ''
if len(self.custom_stores) > 0:
self.custom_stores['long_name'],\
@ -525,13 +532,18 @@ class ModelRegistry:
## Combine Misc (custom) and survey (auto) stores
## Retain only one status per category (defaultStatus, 'E'/existing by default)
self.stores = pd.concat([
self.categories[self.categories.status==conf.map.defaultStatus[0]].reset_index().set_index('store').sort_values('title'),
self.categories[self.categories.status==conf.map.defaultStatus[0]].sort_values('title'),
self.custom_models,
self.custom_stores
])#.drop(columns=['store_name'])
self.stores.drop(columns='name', inplace=True)
self.stores.index.name = 'name'
self.stores['in_menu'] = self.stores['in_menu'].astype(bool)
self.stores['status'].fillna('E', inplace=True)
self.categories.reset_index(inplace=True)
self.categories.set_index('category', inplace=True)
## Set in the stores dataframe some useful properties, from the model class
## Maybe at some point it makes sense to get away from class-based definitions
def fill_columns_from_model(row):
@ -576,18 +588,20 @@ class ModelRegistry:
## Add Misc and Live
self.primary_groups.append(CategoryGroup(
name='Misc',
long_name='Misc',
major=True,
long_name='Misc and old layers (not coming from our survey; '
'they will be organized, '
'eventually as the surveys get more complete)',
# description]='Misc and old layers (not coming from our survey; '
# 'they will be organized, '
# 'eventually as the surveys get more complete)',
categories=[],
))
self.primary_groups.append(CategoryGroup(
name='Live',
long_name='Live',
major=True,
long_name='Layers from data processing, sensors, etc, '
'and are updated automatically',
# long_name='Layers from data processing, sensors, etc, '
# 'and are updated automatically',
categories=[],
))
@ -671,7 +685,7 @@ class ModelRegistry:
df_live['auto_import'] = False
df_live['base_gis_type'] = df_live['gis_type']
df_live['custom'] = False
df_live['group'] = ''
df_live['group'] = 'Live'
df_live['in_menu'] = True
df_live['is_db'] = False
df_live['is_line_work'] = False
@ -681,8 +695,11 @@ class ModelRegistry:
df_live['minor_group_2'] = ''
df_live['status'] = 'E'
df_live['style'] = None
df_live['title'] = df_live['name']
df_live['category'] = ''
df_live.rename(columns={'name': 'title'}, inplace=True)
df_live.index.name = 'name'
registry.stores = pd.concat([registry.stores, df_live])
df_live.index.name = 'store'
for store, model_info in self.geom_live_defs.items():
## Add provided live layers in the stores df
# Create the pydantic model