Config: cleanup, use '__' as ENV nested delimiter; adjust CI test

.forgejo/workflows/test.yaml

@@ -31,4 +31,4 @@ jobs:
         run: uv pip install --python=$UV_PROJECT_ENVIRONMENT --no-deps .
 
       - name: Run tests (API call)
-        run: GISAF_DB_HOST=gisaf-database pytest -s tests/basic.py
+        run: GISAF__DB__HOST=gisaf-database pytest -s tests/basic.py

src/gisaf/config.py

@@ -3,27 +3,29 @@ import logging
 from pathlib import Path
 from typing import Any, Type, Tuple
 from yaml import safe_load
+from importlib.metadata import version
 
 from xdg import BaseDirectory
+from pydantic import BaseModel
 from pydantic_settings import (
     BaseSettings,
     PydanticBaseSettingsSource,
     SettingsConfigDict,
+    YamlConfigSettingsSource,
 )
-from pydantic.v1.utils import deep_update
-
-from importlib.metadata import version
 
 logger = logging.getLogger(__name__)
 ENV = environ.get("env", "prod")
 
 config_files = [
-    Path(Path.cwd().root) / "etc" / "gisaf" / ENV,
+    Path(BaseDirectory.xdg_config_home) / "gisaf" / f"{ENV}.yaml",
-    Path(BaseDirectory.xdg_config_home) / "gisaf" / ENV,
+    Path(BaseDirectory.xdg_config_home) / "gisaf" / f"{ENV}.yml",
+    Path(Path.cwd().root) / "etc" / "gisaf" / f"{ENV}.yaml",
+    Path(Path.cwd().root) / "etc" / "gisaf" / f"{ENV}.yml",
 ]
 
 
-class DashboardHome(BaseSettings):
+class DashboardHome(BaseModel):
     title: str = "Gisaf - home/dashboards"
     content_file: Path = (
         Path(Path.cwd().root) / "etc" / "gisaf" / "dashboard_home_content.html"
@@ -33,7 +35,7 @@ class DashboardHome(BaseSettings):
     )
 
 
-class GisafConfig(BaseSettings):
+class GisafConfig(BaseModel):
     title: str = "Gisaf"
     windowTitle: str = "Gisaf"
     debugLevel: str = "INFO"
@@ -42,7 +44,7 @@ class GisafConfig(BaseSettings):
     use_pretty_errors: bool = False
 
 
-class SpatialSysRef(BaseSettings):
+class SpatialSysRef(BaseModel):
     author: str = "AVSM"
     ellps: str = "WGS84"
     k: int = 1
@@ -56,12 +58,12 @@ class SpatialSysRef(BaseSettings):
     y_0: float = 1328608.994
 
 
-class RawSurvey(BaseSettings):
+class RawSurvey(BaseModel):
     spatial_sys_ref: SpatialSysRef = SpatialSysRef()
     srid: int = 910001
 
 
-class Geo(BaseSettings):
+class Geo(BaseModel):
     raw_survey: RawSurvey = RawSurvey()
     simplify_geom_factor: int = 10000000
     simplify_preserve_topology: bool = False
@@ -69,17 +71,17 @@ class Geo(BaseSettings):
     srid_for_proj: int = 32644
 
 
-# class Flask(BaseSettings):
+# class Flask(BaseModel):
 #     secret_key: str
 #     debug: int
 
 
-class MQTT(BaseSettings):
+class MQTT(BaseModel):
     broker: str = "localhost"
     port: int = 1883
 
 
-class GisafLive(BaseSettings):
+class GisafLive(BaseModel):
     hostname: str = "localhost"
     port: int = 80
     scheme: str = "http"
@@ -87,24 +89,24 @@ class GisafLive(BaseSettings):
     mqtt: MQTT = MQTT()
 
 
-class DefaultSurvey(BaseSettings):
+class DefaultSurvey(BaseModel):
     surveyor_id: int = 1
     equipment_id: int = 1
 
 
-class Survey(BaseSettings):
+class Survey(BaseModel):
     db_schema_raw: str = "raw_survey"
     db_schema: str = "survey"
     default: DefaultSurvey = DefaultSurvey()
 
 
-class Crypto(BaseSettings):
+class Crypto(BaseModel):
     secret: str = "Gisaf big secret"
     algorithm: str = "HS256"
     expire: float = 21600
 
 
-class DB(BaseSettings):
+class DB(BaseModel):
     # uri: str
     host: str = "localhost"
     port: int = 5432
@@ -124,21 +126,21 @@ class DB(BaseSettings):
         return f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}/{self.db}"
 
 
-class Log(BaseSettings):
+class Log(BaseModel):
     level: str = "WARNING"
 
 
-class OGCAPILicense(BaseSettings):
+class OGCAPILicense(BaseModel):
     name: str = "CC-BY 4.0 license"
     url: str = "https://creativecommons.org/licenses/by/4.0/"
 
 
-class OGCAPIProvider(BaseSettings):
+class OGCAPIProvider(BaseModel):
     name: str = "Organization Name"
     url: str = "https://pygeoapi.io"
 
 
-class OGCAPIServerContact(BaseSettings):
+class OGCAPIServerContact(BaseModel):
     name: str = "Lastname, Firstname"
     position: str = "Position Title"
     address: str = "Mailing Address"
@@ -150,7 +152,7 @@ class OGCAPIServerContact(BaseSettings):
     url: str | None = None
 
 
-class OGCAPIIdentification(BaseSettings):
+class OGCAPIIdentification(BaseModel):
     title: str = "pygeoapi default instance"
     description: str = "pygeoapi provides an API to geospatial data"
     keywords: list[str] = ["geospatial", "data", "api"]
@@ -159,26 +161,26 @@ class OGCAPIIdentification(BaseSettings):
     url: str = "http://example.org"
 
 
-class OGCAPIMetadata(BaseSettings):
+class OGCAPIMetadata(BaseModel):
     identification: OGCAPIIdentification = OGCAPIIdentification()
     license: OGCAPILicense = OGCAPILicense()
     provider: OGCAPIProvider = OGCAPIProvider()
     contact: OGCAPIServerContact = OGCAPIServerContact()
 
 
-class ServerBind(BaseSettings):
+class ServerBind(BaseModel):
     host: str = "0.0.0.0"
     port: int = 5000
 
 
-class OGCAPIServerMap(BaseSettings):
+class OGCAPIServerMap(BaseModel):
     url: str = "https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png"
     attribution: str = (
         """<a href="https://wikimediafoundation.org/wiki/Maps_Terms_of_Use">Wikimedia maps</a> | Map data &copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>"""
     )
 
 
-class OGCAPIServer(BaseSettings):
+class OGCAPIServer(BaseModel):
     bind: ServerBind = ServerBind()
     url: str = "https://example.org/ogcapi"
     mimetype: str = "application/json; charset=UTF-8"
@@ -189,7 +191,7 @@ class OGCAPIServer(BaseSettings):
     map: OGCAPIServerMap = OGCAPIServerMap()
 
 
-class OGCAPI(BaseSettings):
+class OGCAPI(BaseModel):
     base_url: str = "http://example.org/ogcapi"
     bbox: list[float] = [-180, -90, 180, 90]
     log: Log = Log()
@@ -197,7 +199,7 @@ class OGCAPI(BaseSettings):
     server: OGCAPIServer = OGCAPIServer()
 
 
-class TileServer(BaseSettings):
+class TileServer(BaseModel):
     baseDir: Path = Path(BaseDirectory.xdg_data_home) / "gisaf" / "mbtiles_files_dir"
     useRequestUrl: bool = False
     spriteBaseDir: Path = (
@@ -213,7 +215,7 @@ class TileServer(BaseSettings):
         self.spriteBaseDir.mkdir(parents=True, exist_ok=True)
 
 
-class Map(BaseSettings):
+class Map(BaseModel):
     tileServer: TileServer = TileServer()
     zoom: int = 14
     pitch: int = 45
@@ -228,11 +230,11 @@ class Map(BaseSettings):
     tagKeys: list[str] = ["source"]
 
 
-class Measures(BaseSettings):
+class Measures(BaseModel):
     defaultStore: str | None = None
 
 
-class BasketDefault(BaseSettings):
+class BasketDefault(BaseModel):
     surveyor: str = "Default surveyor"
     equipment: str = "Default equipment"
     project: str = "Default project"
@@ -240,40 +242,40 @@ class BasketDefault(BaseSettings):
     store: str | None = None
 
 
-# class BasketOldDef(BaseSettings):
+# class BasketOldDef(BaseModel):
 #     base_dir: str
 
 
-class Basket(BaseSettings):
+class Basket(BaseModel):
     base_dir: str = "/var/local/gisaf/baskets"
     default: BasketDefault = BasketDefault()
 
 
-class Plot(BaseSettings):
+class Plot(BaseModel):
     maxDataSize: int = 10000
 
 
-class Dashboard(BaseSettings):
+class Dashboard(BaseModel):
     base_source_url: str = "http://url.to.jupyter/lab/tree/"
     base_storage_dir: str = "/var/lib/share/gisaf/dashboard"
     base_storage_url: str = "/dashboard-attachment/"
 
 
-class Widgets(BaseSettings):
+class Widgets(BaseModel):
     footer: str = (
         """Generated by <span class='link' onclick="window.open('https://redmine.auroville.org.in/projects/gisaf/')">Gisaf</span>"""
     )
 
 
-class Admin(BaseSettings):
+class Admin(BaseModel):
     basket: Basket = Basket()
 
 
-class Attachments(BaseSettings):
+class Attachments(BaseModel):
     base_dir: str = "/var/local/gisaf/attachments"
 
 
-class Job(BaseSettings):
+class Job(BaseModel):
     id: str
     func: str
     trigger: str
@@ -281,7 +283,7 @@ class Job(BaseSettings):
     seconds: int | None = 0
 
 
-class Crs(BaseSettings):
+class Crs(BaseModel):
     """
     Handy definitions for crs-es
     """
@@ -295,7 +297,8 @@ class Crs(BaseSettings):
 
 class Config(BaseSettings):
     model_config = SettingsConfigDict(
-        # env_prefix='gisaf_',
+        env_prefix="GISAF__",
+        nested_model_default_partial_update=True,
         env_nested_delimiter="__",
     )
 
@@ -308,7 +311,15 @@ class Config(BaseSettings):
         dotenv_settings: PydanticBaseSettingsSource,
         file_secret_settings: PydanticBaseSettingsSource,
     ) -> Tuple[PydanticBaseSettingsSource, ...]:
-        return env_settings, init_settings, file_secret_settings, config_file_settings  # type: ignore
+        configs = [
+            YamlConfigSettingsSource(settings_cls, yaml_file=cf) for cf in config_files
+        ]
+        return (
+            env_settings,
+            init_settings,
+            file_secret_settings,
+            *configs,
+        )
 
     # def __init__(self, **kwargs):
     #     super().__init__(**kwargs)
@@ -353,31 +364,8 @@ class Config(BaseSettings):
         )
 
 
-def config_file_settings() -> dict[str, Any]:
-    config: dict[str, Any] = {}
-    for p in config_files:
-        for suffix in {".yaml", ".yml"}:
-            path = p.with_suffix(suffix)
-            if not path.is_file():
-                logger.info(f"No file found at `{path.resolve()}`")
-                continue
-            logger.debug(f"Reading config file `{path.resolve()}`")
-            if path.suffix in {".yaml", ".yml"}:
-                config = deep_update(config, load_yaml(path))
-            else:
-                logger.info(f"Unknown config file extension `{path.suffix}`")
-    return config
-
-
-def load_yaml(path: Path) -> dict[str, Any]:
-    with Path(path).open("r") as f:
-        config = safe_load(f)
-    if not isinstance(config, dict):
-        raise TypeError(f"Config file has no top-level mapping: {path}")
-    return config
-
-
 conf = Config()
+breakpoint()
 
 # def set_app_config(app) -> None:
 #     raw_configs = []