philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Fix startup with no custom model

Browse source
This commit is contained in:
phil 2024-05-10 00:29:41 +02:00
parent 41e92fad57
commit 75471a4d6a
2 changed files with 298 additions and 244 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/gisaf/_version.py
View file

@ -1 +1 @@
__version__: str = '0.1.dev83+g77adcce.d20240508' __version__: str = '0.1.dev85+g41e92fa.d20240509'

456
src/gisaf/registry.py
View file

@ -1,6 +1,7 @@
""" """
Define the models for the ORM Define the models for the ORM
""" """
import logging import logging
import importlib import importlib
import pkgutil import pkgutil
@ -17,8 +18,14 @@ import pandas as pd
import numpy as np import numpy as np
from gisaf.config import conf from gisaf.config import conf
from gisaf.models import (misc, category as category_module, from gisaf.models import (
project, reconcile, map_bases, tags) misc,
category as category_module,
project,
reconcile,
map_bases,
tags,
)
from gisaf.models.geo_models_base import ( from gisaf.models.geo_models_base import (
LiveGeoModel, LiveGeoModel,
PlottableModel, PlottableModel,
@ -41,14 +48,16 @@ from gisaf.models.info import FeatureInfo, InfoCategory
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
category_model_mapper = { category_model_mapper = {
'Point': GeoPointSurveyModel, "Point": GeoPointSurveyModel,
'Line': GeoLineSurveyModel, "Line": GeoLineSurveyModel,
'Polygon': GeoPolygonSurveyModel, "Polygon": GeoPolygonSurveyModel,
} }
class NotInRegistry(Exception): class NotInRegistry(Exception):
pass pass
def import_submodules(package, recursive=True): def import_submodules(package, recursive=True):
"""Import all submodules of a module, recursively, including subpackages """Import all submodules of a module, recursively, including subpackages
@ -61,7 +70,7 @@ def import_submodules(package, recursive=True):
package = importlib.import_module(package) package = importlib.import_module(package)
results = {} results = {}
for loader, name, is_pkg in pkgutil.walk_packages(package.__path__): for loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
full_name = package.__name__ + '.' + name full_name = package.__name__ + "." + name
results[full_name] = importlib.import_module(full_name) results[full_name] = importlib.import_module(full_name)
if recursive and is_pkg: if recursive and is_pkg:
results.update(import_submodules(full_name)) results.update(import_submodules(full_name))
@ -74,6 +83,7 @@ class ModelRegistry:
Maintains registries for all kind of model types, eg. geom, data, values... Maintains registries for all kind of model types, eg. geom, data, values...
Provides tools to get the models from their names, table names, etc. Provides tools to get the models from their names, table names, etc.
""" """
stores: pd.DataFrame stores: pd.DataFrame
categories: pd.DataFrame categories: pd.DataFrame
primary_groups: list[CategoryGroup] primary_groups: list[CategoryGroup]
@ -110,7 +120,7 @@ class ModelRegistry:
Make (or refresh) the registry of models. Make (or refresh) the registry of models.
:return: :return:
""" """
logger.debug('make_registry') logger.debug("make_registry")
self.scan() self.scan()
await self.make_category_models() await self.make_category_models()
await self.build() await self.build()
@ -128,30 +138,33 @@ class ModelRegistry:
so that the models created are actually bound to the db connection so that the models created are actually bound to the db connection
:return: :return:
""" """
logger.debug('make_category_models') logger.debug("make_category_models")
async with db_session() as session: async with db_session() as session:
query = select(Category).order_by(Category.long_name).\ query = (
options(selectinload(Category.category_group)) # type: ignore select(Category)
.order_by(Category.long_name)
.options(selectinload(Category.category_group))
) # type: ignore
data = await session.exec(query) data = await session.exec(query)
categories: list[Category] = data.all() # type: ignore categories: list[Category] = data.all() # type: ignore
for category in categories: for category in categories:
## Several statuses can coexist for the same model, so ## Several statuses can coexist for the same model, so
## consider only the ones with the 'E' (existing) status ## consider only the ones with the 'E' (existing) status
## The other statuses are defined only for import (?) ## The other statuses are defined only for import (?)
if getattr(category, 'status', 'E') != 'E': if getattr(category, "status", "E") != "E":
continue continue
## Use pydantic create_model, supported by SQLModel ## Use pydantic create_model, supported by SQLModel
## See https://github.com/tiangolo/sqlmodel/issues/377 ## See https://github.com/tiangolo/sqlmodel/issues/377
store_name = survey.table(category.table_name) store_name = survey.table(category.table_name)
raw_store_name = raw_survey.table(f'RAW_{category.table_name}') raw_store_name = raw_survey.table(f"RAW_{category.table_name}")
raw_survey_field_definitions = { raw_survey_field_definitions = {
## FIXME: RawSurveyBaseModel.category should be a Category, not category.name ## FIXME: RawSurveyBaseModel.category should be a Category, not category.name
'category_name': (ClassVar[str], category.name), "category_name": (ClassVar[str], category.name),
## FIXME: Same for RawSurveyBaseModel.group ## FIXME: Same for RawSurveyBaseModel.group
'group_name': (ClassVar[str], category.category_group.name), "group_name": (ClassVar[str], category.category_group.name),
'viewable_role': (ClassVar[str], category.viewable_role), "viewable_role": (ClassVar[str], category.viewable_role),
'store_name': (ClassVar[str], raw_store_name), "store_name": (ClassVar[str], raw_store_name),
# 'icon': (str, ''), # 'icon': (str, ''),
} }
## Raw survey points ## Raw survey points
@ -160,31 +173,31 @@ class ModelRegistry:
__base__=RawSurveyBaseModel, __base__=RawSurveyBaseModel,
__model_name=category.raw_survey_table_name, __model_name=category.raw_survey_table_name,
__cls_kwargs__={ __cls_kwargs__={
'table': True, "table": True,
}, },
**raw_survey_field_definitions **raw_survey_field_definitions,
) )
except Exception as err: except Exception as err:
logger.exception(err) logger.exception(err)
logger.warning(err) logger.warning(err)
else: else:
logger.debug('Discovered {:s}'.format(category.raw_survey_table_name)) logger.debug("Discovered {:s}".format(category.raw_survey_table_name))
model_class = category_model_mapper.get(category.gis_type) model_class = category_model_mapper.get(category.gis_type)
## Final geometries ## Final geometries
try: try:
if model_class: if model_class:
survey_field_definitions = { survey_field_definitions = {
'category_name': (ClassVar[str], category.name), "category_name": (ClassVar[str], category.name),
'category': (ClassVar[Category], category), "category": (ClassVar[Category], category),
'group_name': (ClassVar[str], category.category_group.name), "group_name": (ClassVar[str], category.category_group.name),
'raw_store_name': (ClassVar[str], raw_store_name), "raw_store_name": (ClassVar[str], raw_store_name),
'viewable_role': (ClassVar[str], category.viewable_role), "viewable_role": (ClassVar[str], category.viewable_role),
'symbol': (ClassVar[str], category.symbol), "symbol": (ClassVar[str], category.symbol),
'equipment': (Equipment, Relationship()), "equipment": (Equipment, Relationship()),
'surveyor': (Surveyor, Relationship()), "surveyor": (Surveyor, Relationship()),
'accuracy': (Accuracy, Relationship()), "accuracy": (Accuracy, Relationship()),
'project': (Project, Relationship()), "project": (Project, Relationship()),
#'raw_model': (str, self.raw_survey_models.get(raw_store_name)), #'raw_model': (str, self.raw_survey_models.get(raw_store_name)),
# 'icon': (str, f'{survey.schema}-{category.table_name}'), # 'icon': (str, f'{survey.schema}-{category.table_name}'),
} }
@ -192,52 +205,66 @@ class ModelRegistry:
__base__=model_class, __base__=model_class,
__model_name=category.table_name, __model_name=category.table_name,
__cls_kwargs__={ __cls_kwargs__={
'table': True, "table": True,
}, },
**survey_field_definitions, **survey_field_definitions,
) )
except Exception as err: except Exception as err:
logger.warning(err) logger.warning(err)
else: else:
logger.debug('Discovered {:s}'.format(category.table_name)) logger.debug("Discovered {:s}".format(category.table_name))
logger.info('Discovered {:d} models'.format(len(categories))) logger.info("Discovered {:d} models".format(len(categories)))
def scan(self) -> None: def scan(self) -> None:
""" """
Scan all models defined explicitely (not the survey ones, Scan all models defined explicitely (not the survey ones,
which are defined by categories), and store them for reference. which are defined by categories), and store them for reference.
""" """
logger.debug('scan') logger.debug("scan")
## Scan the models defined in modules ## Scan the models defined in modules
for module_name, module in import_submodules(models).items(): for module_name, module in import_submodules(models).items():
if module_name.rsplit('.', 1)[-1] in ( if module_name.rsplit(".", 1)[-1] in (
'geo_models_base', "geo_models_base",
'models_base', "models_base",
): ):
continue continue
for name in dir(module): for name in dir(module):
obj = getattr(module, name) obj = getattr(module, name)
if hasattr(obj, '__module__') and obj.__module__.startswith(module.__name__)\ if (
and hasattr(obj, '__tablename__') and hasattr(obj, 'get_store_name'): hasattr(obj, "__module__")
and obj.__module__.startswith(module.__name__)
and hasattr(obj, "__tablename__")
and hasattr(obj, "get_store_name")
):
gis_type = self.add_model(obj) gis_type = self.add_model(obj)
logger.debug(f'Model {obj.get_store_name()} added in the registry from gisaf source tree as {gis_type}') logger.debug(
f"Model {obj.get_store_name()} added in the registry from gisaf source tree as {gis_type}"
)
## Scan the models defined in plugins (setuptools' entry points) ## Scan the models defined in plugins (setuptools' entry points)
for module_name, model in self.scan_entry_points(name='gisaf_extras.models').items(): for module_name, model in self.scan_entry_points(
name="gisaf_extras.models"
).items():
gis_type = self.add_model(model) gis_type = self.add_model(model)
logger.debug(f'Model {model.get_store_name()} added in the registry from {module_name} entry point as {gis_type}') logger.debug(
f"Model {model.get_store_name()} added in the registry from {module_name} entry point as {gis_type}"
)
for module_name, store in self.scan_entry_points(name='gisaf_extras.stores').items(): for module_name, store in self.scan_entry_points(
name="gisaf_extras.stores"
).items():
self.add_store(store) self.add_store(store)
logger.debug(f'Store {store} added in the registry from {module_name} gisaf_extras.stores entry point') logger.debug(
f"Store {store} added in the registry from {module_name} gisaf_extras.stores entry point"
)
## Add misc models ## Add misc models
for module in misc, category_module, project, reconcile, map_bases, tags: for module in misc, category_module, project, reconcile, map_bases, tags:
for name in dir(module): for name in dir(module):
obj = getattr(module, name) obj = getattr(module, name)
if hasattr(obj, '__module__') and hasattr(obj, '__tablename__'): if hasattr(obj, "__module__") and hasattr(obj, "__tablename__"):
self.misc[name] = obj self.misc[name] = obj
async def build(self) -> None: async def build(self) -> None:
@ -246,7 +273,7 @@ class ModelRegistry:
This should be executed after the discovery of surey models (categories) This should be executed after the discovery of surey models (categories)
and the scan of custom/module defined models. and the scan of custom/module defined models.
""" """
logger.debug('build') logger.debug("build")
## Combine all geom models (auto and custom) ## Combine all geom models (auto and custom)
self.geom = {**self.survey_models, **self.geom_custom} self.geom = {**self.survey_models, **self.geom_custom}
@ -268,13 +295,13 @@ class ModelRegistry:
self.make_menu() self.make_menu()
def populate_values_for_model(self): def populate_values_for_model(self):
''' """
Build a dict for quick access to the values from a model Build a dict for quick access to the values from a model
''' """
self.values_for_model = {} self.values_for_model = {}
for model_value in self.values.values(): for model_value in self.values.values():
for relationship in inspect(model_value).relationships: for relationship in inspect(model_value).relationships:
model = self.stores.loc[relationship.target.fullname, 'model'] model = self.stores.loc[relationship.target.fullname, "model"]
if model not in self.values_for_model: if model not in self.values_for_model:
self.values_for_model[model] = [] self.values_for_model[model] = []
self.values_for_model[model].append(model_value) self.values_for_model[model].append(model_value)
@ -292,23 +319,26 @@ class ModelRegistry:
logger.warning(err) logger.warning(err)
return named_objects return named_objects
def add_model(self, model) -> Literal['GeoModel', 'PlottableModel', 'Other model']: def add_model(self, model) -> Literal["GeoModel", "PlottableModel", "Other model"]:
""" """
Add the model to its proper dict for reference, return the type Add the model to its proper dict for reference, return the type
""" """
# if not hasattr(model, 'get_store_name'): # if not hasattr(model, 'get_store_name'):
# raise NotInRegistry() # raise NotInRegistry()
table_name = model.get_store_name() table_name = model.get_store_name()
if issubclass(model, GeoModel) and not \ if (
issubclass(model, RawSurveyBaseModel) and not model.hidden: issubclass(model, GeoModel)
and not issubclass(model, RawSurveyBaseModel)
and not model.hidden
):
self.geom_custom[table_name] = model self.geom_custom[table_name] = model
return 'GeoModel' return "GeoModel"
elif issubclass(model, PlottableModel): elif issubclass(model, PlottableModel):
self.values[table_name] = model self.values[table_name] = model
return 'PlottableModel' return "PlottableModel"
else: else:
self.other[table_name] = model self.other[table_name] = model
return 'Other model' return "Other model"
def add_store(self, store) -> None: def add_store(self, store) -> None:
self.geom_custom_store[store.name] = store self.geom_custom_store[store.name] = store
@ -320,7 +350,7 @@ class ModelRegistry:
""" """
self.menu = defaultdict(list) self.menu = defaultdict(list)
for name, model in self.stores.model.items(): for name, model in self.stores.model.items():
if hasattr(model, 'Admin'): if hasattr(model, "Admin"):
self.menu[model.Admin.menu].append(model) self.menu[model.Admin.menu].append(model)
# def get_raw_survey_model_mapping(self): # def get_raw_survey_model_mapping(self):
@ -343,8 +373,11 @@ class ModelRegistry:
if not model: if not model:
raise NotInRegistry raise NotInRegistry
async with db_session() as session: async with db_session() as session:
query = select(model).where(model.id == id).options( query = (
*(joinedload(jt) for jt in model.selectinload())) select(model)
.where(model.id == id)
.options(*(joinedload(jt) for jt in model.selectinload()))
)
result = await session.exec(query) result = await session.exec(query)
try: try:
item = result.one() item = result.one()
@ -355,20 +388,20 @@ class ModelRegistry:
raise NotInRegistry raise NotInRegistry
files, images = [], [] files, images = [], []
categorizedInfoItems: list[InfoCategory] | None categorizedInfoItems: list[InfoCategory] | None
if hasattr(item, 'get_categorized_info'): if hasattr(item, "get_categorized_info"):
categorizedInfoItems = await item.get_categorized_info() categorizedInfoItems = await item.get_categorized_info()
else: else:
categorizedInfoItems = None categorizedInfoItems = None
if hasattr(item, 'get_graph'): if hasattr(item, "get_graph"):
graph = item.get_graph() graph = item.get_graph()
else: else:
graph = None graph = None
if hasattr(item, 'Attachments'): if hasattr(item, "Attachments"):
if hasattr(item.Attachments, 'files'): if hasattr(item.Attachments, "files"):
files = await item.Attachments.files(item) files = await item.Attachments.files(item)
if hasattr(item.Attachments, 'images'): if hasattr(item.Attachments, "images"):
images = await item.Attachments.images(item) images = await item.Attachments.images(item)
if hasattr(item, 'get_external_record_url'): if hasattr(item, "get_external_record_url"):
externalRecordUrl = item.get_external_record_url() externalRecordUrl = item.get_external_record_url()
else: else:
externalRecordUrl = None externalRecordUrl = None
@ -391,11 +424,12 @@ class ModelRegistry:
Make registry for primary groups, categories and survey stores using Pandas dataframes. Make registry for primary groups, categories and survey stores using Pandas dataframes.
Used in GraphQl queries. Used in GraphQl queries.
""" """
## Utility functions used with apply method (dataframes) ## Utility functions used with apply method (dataframes)
def fill_columns_from_custom_models(row) -> tuple[str, str, str, str, str, str, str]: def fill_columns_from_custom_models(
# model: row,
) -> tuple[str, str, str, str, str, str]:
return ( return (
row.model.get_store_name(),
row.model.__name__, row.model.__name__,
row.model.description, row.model.description,
row.model.__table__.schema, row.model.__table__.schema,
@ -408,36 +442,42 @@ class ModelRegistry:
return ( return (
row.model.description, row.model.description,
row.model.description, row.model.description,
None ## Schema None, ## Schema
) )
def get_store_name(category) -> str: def get_store_name(category) -> str:
fragments = ['V', category.group, category.minor_group_1] fragments = ["V", category.group, category.minor_group_1]
if category.minor_group_2 != '----': if category.minor_group_2 != "----":
fragments.append(category.minor_group_2) fragments.append(category.minor_group_2)
return '.'.join([survey.name, '_'.join(fragments)]) return ".".join([survey.name, "_".join(fragments)])
self.categories = await Category.get_df() self.categories = await Category.get_df()
self.categories['title'] = self.categories.long_name.fillna(self.categories.description) self.categories["title"] = self.categories.long_name.fillna(
self.categories.description
)
self.categories['store'] = self.categories.apply(get_store_name, axis=1) self.categories["store"] = self.categories.apply(get_store_name, axis=1)
self.categories['count'] = pd.Series(dtype=pd.Int64Dtype()) self.categories["count"] = pd.Series(dtype=pd.Int64Dtype())
df_models = pd.DataFrame(self.geom.items(), df_models = pd.DataFrame(
columns=['store', 'model'] self.geom.items(), columns=["store", "model"]
).set_index('store') ).set_index("store")
df_raw_models = pd.DataFrame(self.raw_survey_models.items(), df_raw_models = pd.DataFrame(
columns=('store', 'raw_model') self.raw_survey_models.items(), columns=("store", "raw_model")
).set_index('store') ).set_index("store")
self.categories = self.categories.merge(df_models, left_on='store', right_index=True) self.categories = self.categories.merge(
self.categories = self.categories.merge(df_raw_models, left_on='store', right_index=True) df_models, left_on="store", right_index=True
self.categories['custom'] = False )
self.categories['is_db'] = True self.categories = self.categories.merge(
df_raw_models, left_on="store", right_index=True
)
self.categories["custom"] = False
self.categories["is_db"] = True
self.categories.reset_index(inplace=True) self.categories.reset_index(inplace=True)
self.categories.rename(columns={'name': 'category'}, inplace=True) self.categories.rename(columns={"name": "category"}, inplace=True)
self.categories.set_index('store', inplace=True) self.categories.set_index("store", inplace=True)
self.categories.sort_values('category') self.categories.sort_values("category")
# self.categories.sort_index(inplace=True) # self.categories.sort_index(inplace=True)
# self.categories['name_letter'] = self.categories.index.str.slice(0, 1) # self.categories['name_letter'] = self.categories.index.str.slice(0, 1)
# self.categories['name_number'] = self.categories.index.str.slice(1).astype('int64') # self.categories['name_number'] = self.categories.index.str.slice(1).astype('int64')
@ -455,50 +495,51 @@ class ModelRegistry:
# lambda row: row.raw_model.store_name, # lambda row: row.raw_model.store_name,
# axis=1 # axis=1
# ) # )
self.categories['is_line_work'] = self.categories.apply( self.categories["is_line_work"] = self.categories.apply(
lambda row: issubclass(row.model, LineWorkSurveyModel), lambda row: issubclass(row.model, LineWorkSurveyModel), axis=1
axis=1
) )
else: else:
self.categories['store_name'] = None self.categories["store_name"] = None
self.categories['raw_model_store_name'] = None self.categories["raw_model_store_name"] = None
self.categories['is_line_work'] = None self.categories["is_line_work"] = None
self.categories['raw_survey_model'] = None self.categories["raw_survey_model"] = None
## -------------------- ## --------------------
## Custom models (Misc) ## Custom models (Misc)
## -------------------- ## --------------------
self.custom_models = pd.DataFrame( self.custom_models = pd.DataFrame(
self.geom_custom.items(), self.geom_custom.items(), columns=["store", "model"]
columns=['store', 'model'] ).set_index("store")
).set_index('store') self.custom_models["group"] = "Misc"
self.custom_models['group'] = 'Misc' self.custom_models["custom"] = True
self.custom_models['custom'] = True self.custom_models["is_db"] = True
self.custom_models['is_db'] = True self.custom_models["raw_model_store_name"] = ""
self.custom_models['raw_model_store_name'] = '' self.custom_models["category"] = ""
self.custom_models['category'] = '' self.custom_models["in_menu"] = self.custom_models.apply(
self.custom_models['in_menu'] = self.custom_models.apply( lambda row: getattr(row.model, "in_menu", True), axis=1
lambda row: getattr(row.model, 'in_menu', True),
axis=1
) )
self.custom_models = self.custom_models.loc[self.custom_models.in_menu] self.custom_models = self.custom_models.loc[self.custom_models.in_menu]
self.custom_models['auto_import'] = False self.custom_models["auto_import"] = False
self.custom_models['is_line_work'] = False self.custom_models["is_line_work"] = False
self.custom_models['minor_group_1'] = None self.custom_models["minor_group_1"] = None
self.custom_models['minor_group_2'] = None self.custom_models["minor_group_2"] = None
if len(self.custom_models) > 0: if len(self.custom_models) > 0:
self.custom_models['name'],\ (
self.custom_models['long_name'],\ self.custom_models["long_name"],
self.custom_models['custom_description'],\ self.custom_models["custom_description"],
self.custom_models['db_schema'],\ self.custom_models["db_schema"],
self.custom_models['gis_type'],\ self.custom_models["gis_type"],
self.custom_models['style'],\ self.custom_models["style"],
self.custom_models['symbol'],\ self.custom_models["symbol"],
= zip(*self.custom_models.apply(fill_columns_from_custom_models, axis=1)) ) = zip(*self.custom_models.apply(fill_columns_from_custom_models, axis=1))
## Try to give a meaningful description, eg. including the source (db_schema) ## Try to give a meaningful description, eg. including the source (db_schema)
self.custom_models['description'] = self.custom_models['custom_description'].fillna(self.custom_models['long_name'] + '-' + self.custom_models['db_schema']) self.custom_models["description"] = self.custom_models[
self.custom_models['title'] = self.custom_models['long_name'] "custom_description"
].fillna(
self.custom_models["long_name"] + "-" + self.custom_models["db_schema"]
)
self.custom_models["title"] = self.custom_models["long_name"]
self.custom_models.fillna(np.nan, inplace=True) self.custom_models.fillna(np.nan, inplace=True)
self.custom_models.replace([np.nan], [None], inplace=True) self.custom_models.replace([np.nan], [None], inplace=True)
@ -506,47 +547,50 @@ class ModelRegistry:
## Custom stores (Community) ## Custom stores (Community)
## ------------------------- ## -------------------------
self.custom_stores = pd.DataFrame( self.custom_stores = pd.DataFrame(
self.geom_custom_store.items(), self.geom_custom_store.items(), columns=["store", "model"]
columns=['store', 'model'] ).set_index("store")
).set_index('store') self.custom_stores["group"] = "Community"
self.custom_stores['group'] = 'Community' self.custom_stores["custom"] = True
self.custom_stores['custom'] = True self.custom_stores["is_db"] = False
self.custom_stores['is_db'] = False
if len(self.custom_stores) == 0: if len(self.custom_stores) == 0:
self.custom_stores['in_menu'] = False self.custom_stores["in_menu"] = False
else: else:
self.custom_stores['in_menu'] = self.custom_stores.apply( self.custom_stores["in_menu"] = self.custom_stores.apply(
lambda row: getattr(row.model, 'in_menu', True), lambda row: getattr(row.model, "in_menu", True), axis=1
axis=1
) )
self.custom_stores = self.custom_stores.loc[self.custom_stores.in_menu] self.custom_stores = self.custom_stores.loc[self.custom_stores.in_menu]
self.custom_stores['auto_import'] = False self.custom_stores["auto_import"] = False
self.custom_stores['is_line_work'] = False self.custom_stores["is_line_work"] = False
self.custom_stores['category'] = '' self.custom_stores["category"] = ""
if len(self.custom_stores) > 0: if len(self.custom_stores) > 0:
self.custom_stores['long_name'],\ (
self.custom_stores['description'],\ self.custom_stores["long_name"],
self.custom_stores['db_schema'],\ self.custom_stores["description"],
= zip(*self.custom_stores.apply(fill_columns_from_custom_stores, axis=1)) self.custom_stores["db_schema"],
self.custom_stores['title'] = self.custom_stores['long_name'] ) = zip(*self.custom_stores.apply(fill_columns_from_custom_stores, axis=1))
self.custom_stores["title"] = self.custom_stores["long_name"]
self.custom_stores.fillna(np.nan, inplace=True) self.custom_stores.fillna(np.nan, inplace=True)
self.custom_stores.replace([np.nan], [None], inplace=True) self.custom_stores.replace([np.nan], [None], inplace=True)
## Combine Misc (custom) and survey (auto) stores ## Combine Misc (custom) and survey (auto) stores
## Retain only one status per category (defaultStatus, 'E'/existing by default) ## Retain only one status per category (defaultStatus, 'E'/existing by default)
self.stores = pd.concat([ self.stores = pd.concat(
[
self.custom_models, self.custom_models,
self.categories[self.categories.status==conf.map.defaultStatus[0]].sort_values('title'), self.categories[
self.custom_stores self.categories.status == conf.map.defaultStatus[0]
])#.drop(columns=['store_name']) ].sort_values("title"),
self.stores.drop(columns='name', inplace=True) self.custom_stores,
self.stores.index.name = 'name' ]
self.stores['in_menu'] = self.stores['in_menu'].astype(bool) ) # .drop(columns=['store_name'])
self.stores['status'].fillna('E', inplace=True) # self.stores.drop(columns='name', inplace=True)
self.stores.index.name = "name"
self.stores["in_menu"] = self.stores["in_menu"].astype(bool)
self.stores["status"].fillna("E", inplace=True)
self.categories.reset_index(inplace=True) self.categories.reset_index(inplace=True)
self.categories.set_index('category', inplace=True) self.categories.set_index("category", inplace=True)
## Set in the stores dataframe some useful properties, from the model class ## Set in the stores dataframe some useful properties, from the model class
## Maybe at some point it makes sense to get away from class-based definitions ## Maybe at some point it makes sense to get away from class-based definitions
@ -562,26 +606,29 @@ class ModelRegistry:
# self.stores['icon'],\ # self.stores['icon'],\
# self.stores['symbol'],\ # self.stores['symbol'],\
self.stores['mapbox_type_default'], \ (
self.stores['base_gis_type'], \ self.stores["mapbox_type_default"],
self.stores['z_index'], \ self.stores["base_gis_type"],
self.stores['attribution'] \ self.stores["z_index"],
= zip(*self.stores.apply(fill_columns_from_model, axis=1)) self.stores["attribution"],
) = zip(*self.stores.apply(fill_columns_from_model, axis=1))
self.stores['mapbox_type_custom'] = self.stores['mapbox_type_custom'].replace('', np.nan).fillna(np.nan) self.stores["mapbox_type_custom"] = (
self.stores['type'] = self.stores['mapbox_type_custom'].fillna( self.stores["mapbox_type_custom"].replace("", np.nan).fillna(np.nan)
self.stores['mapbox_type_default'] )
self.stores["type"] = self.stores["mapbox_type_custom"].fillna(
self.stores["mapbox_type_default"]
) )
self.stores['viewable_role'] = self.stores.apply( self.stores["viewable_role"] = self.stores.apply(
lambda row: getattr(row.model, 'viewable_role', None), lambda row: getattr(row.model, "viewable_role", None),
axis=1, axis=1,
) )
self.stores['viewable_role'].replace('', None, inplace=True) self.stores["viewable_role"].replace("", None, inplace=True)
# self.stores['gql_object_type'] = self.stores.apply(make_model_gql_object_type, axis=1) # self.stores['gql_object_type'] = self.stores.apply(make_model_gql_object_type, axis=1)
self.stores['is_live'] = False self.stores["is_live"] = False
self.stores['description'].fillna('', inplace=True) self.stores["description"].fillna("", inplace=True)
## Layer groups: Misc, survey's primary groups, Live ## Layer groups: Misc, survey's primary groups, Live
async with db_session() as session: async with db_session() as session:
@ -590,31 +637,38 @@ class ModelRegistry:
self.primary_groups = data.all() self.primary_groups = data.all()
## Add Misc and Live ## Add Misc and Live
self.primary_groups.insert(0, CategoryGroup( self.primary_groups.insert(
name='Misc', 0,
long_name='Misc', CategoryGroup(
name="Misc",
long_name="Misc",
major=True, major=True,
# description]='Misc and old layers (not coming from our survey; ' # description]='Misc and old layers (not coming from our survey; '
# 'they will be organized, ' # 'they will be organized, '
# 'eventually as the surveys get more complete)', # 'eventually as the surveys get more complete)',
categories=[], categories=[],
)) ),
)
self.primary_groups.append(CategoryGroup( self.primary_groups.append(
name='Live', CategoryGroup(
long_name='Live', name="Live",
long_name="Live",
major=True, major=True,
# long_name='Layers from data processing, sensors, etc, ' # long_name='Layers from data processing, sensors, etc, '
# 'and are updated automatically', # 'and are updated automatically',
categories=[], categories=[],
)) )
)
self.primary_groups.append(CategoryGroup( self.primary_groups.append(
name='Community', CategoryGroup(
name="Community",
major=True, major=True,
long_name='Layers from community', long_name="Layers from community",
categories=[], categories=[],
)) )
)
# def make_group(group): # def make_group(group):
# return GeomGroup( # return GeomGroup(
@ -629,7 +683,7 @@ class ModelRegistry:
""" """
Get information about the available stores Get information about the available stores
""" """
raise DeprecationWarning('get_stores was for graphql') raise DeprecationWarning("get_stores was for graphql")
async def update_stores_counts(self): async def update_stores_counts(self):
""" """
@ -639,14 +693,16 @@ class ModelRegistry:
# async with db.acquire(reuse=False) as connection: # async with db.acquire(reuse=False) as connection:
async with db_session() as session: async with db_session() as session:
rows = await session.exec(text(query)) rows = await session.exec(text(query))
all_tables_count = pd.DataFrame(rows, columns=['schema', 'table', 'count']) all_tables_count = pd.DataFrame(rows, columns=["schema", "table", "count"])
all_tables_count['store'] = all_tables_count['schema'] + '.' + all_tables_count['table'] all_tables_count["store"] = (
all_tables_count.set_index(['store'], inplace=True) all_tables_count["schema"] + "." + all_tables_count["table"]
)
all_tables_count.set_index(["store"], inplace=True)
## TODO: a DB VACUUM can be triggered if all counts are 0? ## TODO: a DB VACUUM can be triggered if all counts are 0?
## Update the count in registry's stores ## Update the count in registry's stores
self.stores.loc[:, 'count'] = all_tables_count['count'] self.stores.loc[:, "count"] = all_tables_count["count"]
# ## FIXME: count for custom stores # ## FIXME: count for custom stores
# store_df = self.stores.loc[(self.stores['count'] != 0) | (self.stores['is_live'])] # store_df = self.stores.loc[(self.stores['count'] != 0) | (self.stores['is_live'])]
# def set_count(row): # def set_count(row):
@ -663,14 +719,14 @@ class ModelRegistry:
""" """
## Remove existing live layers ## Remove existing live layers
self.geom_live = {} self.geom_live = {}
self.stores.drop(self.stores[self.stores.is_live == True].index, # noqa: E712 self.stores.drop(
inplace=True) self.stores[self.stores.is_live == True].index, # noqa: E712
df_live = pd.DataFrame.from_dict(self.geom_live_defs.values(), inplace=True,
orient='columns'
) )
df_live = pd.DataFrame.from_dict(self.geom_live_defs.values(), orient="columns")
if len(df_live) == 0: if len(df_live) == 0:
return return
df_live.set_index('store', inplace=True) df_live.set_index("store", inplace=True)
## Adjust column names ## Adjust column names
## and add columns, to make sure pandas dtypes are not changed when the ## and add columns, to make sure pandas dtypes are not changed when the
## dataframes are concat ## dataframes are concat
@ -685,37 +741,35 @@ class ModelRegistry:
# }, inplace=True # }, inplace=True
# ) # )
## Add columns ## Add columns
df_live['auto_import'] = False df_live["auto_import"] = False
df_live['base_gis_type'] = df_live['gis_type'] df_live["base_gis_type"] = df_live["gis_type"]
df_live['custom'] = False df_live["custom"] = False
df_live['group'] = 'Live' df_live["group"] = "Live"
df_live['in_menu'] = True df_live["in_menu"] = True
df_live['is_db'] = False df_live["is_db"] = False
df_live['is_line_work'] = False df_live["is_line_work"] = False
df_live['long_name'] = df_live['name'] df_live["long_name"] = df_live["name"]
df_live['minor_group_1'] = '' df_live["minor_group_1"] = ""
df_live['minor_group_2'] = '' df_live["minor_group_2"] = ""
df_live['status'] = 'E' df_live["status"] = "E"
df_live['style'] = None df_live["style"] = None
df_live['category'] = '' df_live["category"] = ""
df_live.rename(columns={'name': 'title'}, inplace=True) df_live.rename(columns={"name": "title"}, inplace=True)
df_live.index.name = 'name' df_live.index.name = "name"
registry.stores = pd.concat([registry.stores, df_live]) registry.stores = pd.concat([registry.stores, df_live])
df_live.index.name = 'store' df_live.index.name = "store"
for store, model_info in self.geom_live_defs.items(): for store, model_info in self.geom_live_defs.items():
## Add provided live layers in the stores df ## Add provided live layers in the stores df
# Create the pydantic model # Create the pydantic model
# NOTE: Unused at this point, but might be usedful # NOTE: Unused at this point, but might be usedful
field_definitions = { field_definitions = {
k: (ClassVar[v.__class__], v) k: (ClassVar[v.__class__], v) for k, v in model_info.items()
for k, v in model_info.items()
} }
self.geom_live[store] = create_model( self.geom_live[store] = create_model(
__model_name=store, __model_name=store, __base__=LiveGeoModel, **field_definitions
__base__= LiveGeoModel,
**field_definitions
) )
# Accessible as global # Accessible as global
registry: ModelRegistry = ModelRegistry() registry: ModelRegistry = ModelRegistry()