From 898197209ade3b9c0fe2b6b3ffc495acf2a01254 Mon Sep 17 00:00:00 2001 From: phil Date: Sun, 15 Dec 2024 01:50:23 +0100 Subject: [PATCH 1/2] Config: cleanup, use '__' as ENV nested delimiter; adjust CI test --- .forgejo/workflows/test.yaml | 2 +- src/gisaf/config.py | 120 ++++++++++++++++------------------- 2 files changed, 55 insertions(+), 67 deletions(-) diff --git a/.forgejo/workflows/test.yaml b/.forgejo/workflows/test.yaml index 926bd82..70956fc 100644 --- a/.forgejo/workflows/test.yaml +++ b/.forgejo/workflows/test.yaml @@ -31,4 +31,4 @@ jobs: run: uv pip install --python=$UV_PROJECT_ENVIRONMENT --no-deps . - name: Run tests (API call) - run: GISAF_DB_HOST=gisaf-database pytest -s tests/basic.py + run: GISAF__DB__HOST=gisaf-database pytest -s tests/basic.py diff --git a/src/gisaf/config.py b/src/gisaf/config.py index 7c68011..cc33eb6 100644 --- a/src/gisaf/config.py +++ b/src/gisaf/config.py @@ -3,27 +3,29 @@ import logging from pathlib import Path from typing import Any, Type, Tuple from yaml import safe_load +from importlib.metadata import version from xdg import BaseDirectory +from pydantic import BaseModel from pydantic_settings import ( BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict, + YamlConfigSettingsSource, ) -from pydantic.v1.utils import deep_update - -from importlib.metadata import version logger = logging.getLogger(__name__) ENV = environ.get("env", "prod") config_files = [ - Path(Path.cwd().root) / "etc" / "gisaf" / ENV, - Path(BaseDirectory.xdg_config_home) / "gisaf" / ENV, + Path(BaseDirectory.xdg_config_home) / "gisaf" / f"{ENV}.yaml", + Path(BaseDirectory.xdg_config_home) / "gisaf" / f"{ENV}.yml", + Path(Path.cwd().root) / "etc" / "gisaf" / f"{ENV}.yaml", + Path(Path.cwd().root) / "etc" / "gisaf" / f"{ENV}.yml", ] -class DashboardHome(BaseSettings): +class DashboardHome(BaseModel): title: str = "Gisaf - home/dashboards" content_file: Path = ( Path(Path.cwd().root) / "etc" / "gisaf" / "dashboard_home_content.html" @@ -33,7 +35,7 @@ class DashboardHome(BaseSettings): ) -class GisafConfig(BaseSettings): +class GisafConfig(BaseModel): title: str = "Gisaf" windowTitle: str = "Gisaf" debugLevel: str = "INFO" @@ -42,7 +44,7 @@ class GisafConfig(BaseSettings): use_pretty_errors: bool = False -class SpatialSysRef(BaseSettings): +class SpatialSysRef(BaseModel): author: str = "AVSM" ellps: str = "WGS84" k: int = 1 @@ -56,12 +58,12 @@ class SpatialSysRef(BaseSettings): y_0: float = 1328608.994 -class RawSurvey(BaseSettings): +class RawSurvey(BaseModel): spatial_sys_ref: SpatialSysRef = SpatialSysRef() srid: int = 910001 -class Geo(BaseSettings): +class Geo(BaseModel): raw_survey: RawSurvey = RawSurvey() simplify_geom_factor: int = 10000000 simplify_preserve_topology: bool = False @@ -69,17 +71,17 @@ class Geo(BaseSettings): srid_for_proj: int = 32644 -# class Flask(BaseSettings): +# class Flask(BaseModel): # secret_key: str # debug: int -class MQTT(BaseSettings): +class MQTT(BaseModel): broker: str = "localhost" port: int = 1883 -class GisafLive(BaseSettings): +class GisafLive(BaseModel): hostname: str = "localhost" port: int = 80 scheme: str = "http" @@ -87,24 +89,24 @@ class GisafLive(BaseSettings): mqtt: MQTT = MQTT() -class DefaultSurvey(BaseSettings): +class DefaultSurvey(BaseModel): surveyor_id: int = 1 equipment_id: int = 1 -class Survey(BaseSettings): +class Survey(BaseModel): db_schema_raw: str = "raw_survey" db_schema: str = "survey" default: DefaultSurvey = DefaultSurvey() -class Crypto(BaseSettings): +class Crypto(BaseModel): secret: str = "Gisaf big secret" algorithm: str = "HS256" expire: float = 21600 -class DB(BaseSettings): +class DB(BaseModel): # uri: str host: str = "localhost" port: int = 5432 @@ -124,21 +126,21 @@ class DB(BaseSettings): return f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}/{self.db}" -class Log(BaseSettings): +class Log(BaseModel): level: str = "WARNING" -class OGCAPILicense(BaseSettings): +class OGCAPILicense(BaseModel): name: str = "CC-BY 4.0 license" url: str = "https://creativecommons.org/licenses/by/4.0/" -class OGCAPIProvider(BaseSettings): +class OGCAPIProvider(BaseModel): name: str = "Organization Name" url: str = "https://pygeoapi.io" -class OGCAPIServerContact(BaseSettings): +class OGCAPIServerContact(BaseModel): name: str = "Lastname, Firstname" position: str = "Position Title" address: str = "Mailing Address" @@ -150,7 +152,7 @@ class OGCAPIServerContact(BaseSettings): url: str | None = None -class OGCAPIIdentification(BaseSettings): +class OGCAPIIdentification(BaseModel): title: str = "pygeoapi default instance" description: str = "pygeoapi provides an API to geospatial data" keywords: list[str] = ["geospatial", "data", "api"] @@ -159,26 +161,26 @@ class OGCAPIIdentification(BaseSettings): url: str = "http://example.org" -class OGCAPIMetadata(BaseSettings): +class OGCAPIMetadata(BaseModel): identification: OGCAPIIdentification = OGCAPIIdentification() license: OGCAPILicense = OGCAPILicense() provider: OGCAPIProvider = OGCAPIProvider() contact: OGCAPIServerContact = OGCAPIServerContact() -class ServerBind(BaseSettings): +class ServerBind(BaseModel): host: str = "0.0.0.0" port: int = 5000 -class OGCAPIServerMap(BaseSettings): +class OGCAPIServerMap(BaseModel): url: str = "https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png" attribution: str = ( """Wikimedia maps | Map data © OpenStreetMap contributors""" ) -class OGCAPIServer(BaseSettings): +class OGCAPIServer(BaseModel): bind: ServerBind = ServerBind() url: str = "https://example.org/ogcapi" mimetype: str = "application/json; charset=UTF-8" @@ -189,7 +191,7 @@ class OGCAPIServer(BaseSettings): map: OGCAPIServerMap = OGCAPIServerMap() -class OGCAPI(BaseSettings): +class OGCAPI(BaseModel): base_url: str = "http://example.org/ogcapi" bbox: list[float] = [-180, -90, 180, 90] log: Log = Log() @@ -197,7 +199,7 @@ class OGCAPI(BaseSettings): server: OGCAPIServer = OGCAPIServer() -class TileServer(BaseSettings): +class TileServer(BaseModel): baseDir: Path = Path(BaseDirectory.xdg_data_home) / "gisaf" / "mbtiles_files_dir" useRequestUrl: bool = False spriteBaseDir: Path = ( @@ -213,7 +215,7 @@ class TileServer(BaseSettings): self.spriteBaseDir.mkdir(parents=True, exist_ok=True) -class Map(BaseSettings): +class Map(BaseModel): tileServer: TileServer = TileServer() zoom: int = 14 pitch: int = 45 @@ -228,11 +230,11 @@ class Map(BaseSettings): tagKeys: list[str] = ["source"] -class Measures(BaseSettings): +class Measures(BaseModel): defaultStore: str | None = None -class BasketDefault(BaseSettings): +class BasketDefault(BaseModel): surveyor: str = "Default surveyor" equipment: str = "Default equipment" project: str = "Default project" @@ -240,40 +242,40 @@ class BasketDefault(BaseSettings): store: str | None = None -# class BasketOldDef(BaseSettings): +# class BasketOldDef(BaseModel): # base_dir: str -class Basket(BaseSettings): +class Basket(BaseModel): base_dir: str = "/var/local/gisaf/baskets" default: BasketDefault = BasketDefault() -class Plot(BaseSettings): +class Plot(BaseModel): maxDataSize: int = 10000 -class Dashboard(BaseSettings): +class Dashboard(BaseModel): base_source_url: str = "http://url.to.jupyter/lab/tree/" base_storage_dir: str = "/var/lib/share/gisaf/dashboard" base_storage_url: str = "/dashboard-attachment/" -class Widgets(BaseSettings): +class Widgets(BaseModel): footer: str = ( """Generated by Gisaf""" ) -class Admin(BaseSettings): +class Admin(BaseModel): basket: Basket = Basket() -class Attachments(BaseSettings): +class Attachments(BaseModel): base_dir: str = "/var/local/gisaf/attachments" -class Job(BaseSettings): +class Job(BaseModel): id: str func: str trigger: str @@ -281,7 +283,7 @@ class Job(BaseSettings): seconds: int | None = 0 -class Crs(BaseSettings): +class Crs(BaseModel): """ Handy definitions for crs-es """ @@ -295,7 +297,8 @@ class Crs(BaseSettings): class Config(BaseSettings): model_config = SettingsConfigDict( - # env_prefix='gisaf_', + env_prefix="GISAF__", + nested_model_default_partial_update=True, env_nested_delimiter="__", ) @@ -308,7 +311,15 @@ class Config(BaseSettings): dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, ) -> Tuple[PydanticBaseSettingsSource, ...]: - return env_settings, init_settings, file_secret_settings, config_file_settings # type: ignore + configs = [ + YamlConfigSettingsSource(settings_cls, yaml_file=cf) for cf in config_files + ] + return ( + env_settings, + init_settings, + file_secret_settings, + *configs, + ) # def __init__(self, **kwargs): # super().__init__(**kwargs) @@ -353,31 +364,8 @@ class Config(BaseSettings): ) -def config_file_settings() -> dict[str, Any]: - config: dict[str, Any] = {} - for p in config_files: - for suffix in {".yaml", ".yml"}: - path = p.with_suffix(suffix) - if not path.is_file(): - logger.info(f"No file found at `{path.resolve()}`") - continue - logger.debug(f"Reading config file `{path.resolve()}`") - if path.suffix in {".yaml", ".yml"}: - config = deep_update(config, load_yaml(path)) - else: - logger.info(f"Unknown config file extension `{path.suffix}`") - return config - - -def load_yaml(path: Path) -> dict[str, Any]: - with Path(path).open("r") as f: - config = safe_load(f) - if not isinstance(config, dict): - raise TypeError(f"Config file has no top-level mapping: {path}") - return config - - conf = Config() +breakpoint() # def set_app_config(app) -> None: # raw_configs = [] From abff3e0f1a90ed3c99859c9fd900a91f78c69e85 Mon Sep 17 00:00:00 2001 From: phil Date: Sun, 15 Dec 2024 01:53:03 +0100 Subject: [PATCH 2/2] Remove breakpoint --- src/gisaf/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gisaf/config.py b/src/gisaf/config.py index cc33eb6..ab771f2 100644 --- a/src/gisaf/config.py +++ b/src/gisaf/config.py @@ -365,7 +365,6 @@ class Config(BaseSettings): conf = Config() -breakpoint() # def set_app_config(app) -> None: # raw_configs = []