Add missing dependencies

Add tile server
Config: add defults
Cosmetic refactorings

pdm.lock

@@ -5,7 +5,7 @@
 groups = ["default", "dev", "mqtt"]
 strategy = ["cross_platform"]
 lock_version = "4.4.1"
-content_hash = "sha256:d6bc84b5bf12fda8fd24858515794677046aca3dea340a40679d1276ae7a6ea9"
+content_hash = "sha256:75aa4cd0effa4fc41f312763423aa949a48e14e235b75bbe0a67fa762bc9660c"
 
 [[package]]
 name = "aiomqtt"
@@ -20,6 +20,16 @@ files = [
     {file = "aiomqtt-1.2.1.tar.gz", hash = "sha256:7582f4341f08ef7110dd9ab3a559454dc28ccda1eac502ff8f08a73b238ecede"},
 ]
 
+[[package]]
+name = "aiosqlite"
+version = "0.19.0"
+requires_python = ">=3.7"
+summary = "asyncio bridge to the standard sqlite3 module"
+files = [
+    {file = "aiosqlite-0.19.0-py3-none-any.whl", hash = "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96"},
+    {file = "aiosqlite-0.19.0.tar.gz", hash = "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d"},
+]
+
 [[package]]
 name = "annotated-types"
 version = "0.6.0"
@@ -308,17 +318,17 @@ files = [
 
 [[package]]
 name = "fastapi"
-version = "0.108.0"
+version = "0.109.2"
 requires_python = ">=3.8"
 summary = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
 dependencies = [
     "pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4",
-    "starlette<0.33.0,>=0.29.0",
+    "starlette<0.37.0,>=0.36.3",
     "typing-extensions>=4.8.0",
 ]
 files = [
-    {file = "fastapi-0.108.0-py3-none-any.whl", hash = "sha256:8c7bc6d315da963ee4cdb605557827071a9a7f95aeb8fcdd3bde48cdc8764dd7"},
+    {file = "fastapi-0.109.2-py3-none-any.whl", hash = "sha256:2c9bab24667293b501cad8dd388c05240c850b58ec5876ee3283c47d6e1e3a4d"},
-    {file = "fastapi-0.108.0.tar.gz", hash = "sha256:5056e504ac6395bf68493d71fcfc5352fdbd5fda6f88c21f6420d80d81163296"},
+    {file = "fastapi-0.109.2.tar.gz", hash = "sha256:f3817eac96fe4f65a2ebb4baa000f394e55f5fccdaf7f75250804bc58f354f73"},
 ]
 
 [[package]]
@@ -1162,15 +1172,15 @@ files = [
 
 [[package]]
 name = "starlette"
-version = "0.32.0.post1"
+version = "0.36.3"
 requires_python = ">=3.8"
 summary = "The little ASGI library that shines."
 dependencies = [
     "anyio<5,>=3.4.0",
 ]
 files = [
-    {file = "starlette-0.32.0.post1-py3-none-any.whl", hash = "sha256:cd0cb10ddb49313f609cedfac62c8c12e56c7314b66d89bb077ba228bada1b09"},
+    {file = "starlette-0.36.3-py3-none-any.whl", hash = "sha256:13d429aa93a61dc40bf503e8c801db1f1bca3dc706b10ef2434a36123568f044"},
-    {file = "starlette-0.32.0.post1.tar.gz", hash = "sha256:e54e2b7e2fb06dff9eac40133583f10dfa05913f5a85bf26f427c7a40a9a3d02"},
+    {file = "starlette-0.36.3.tar.gz", hash = "sha256:90a671733cfb35771d8cc605e0b679d23b992f8dcfad48cc60b38cb29aeb7080"},
 ]
 
 [[package]]
@@ -1202,6 +1212,15 @@ files = [
     {file = "types_pytz-2023.3.1.1-py3-none-any.whl", hash = "sha256:1999a123a3dc0e39a2ef6d19f3f8584211de9e6a77fe7a0259f04a524e90a5cf"},
 ]
 
+[[package]]
+name = "types-pyyaml"
+version = "6.0.12.12"
+summary = "Typing stubs for PyYAML"
+files = [
+    {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"},
+    {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"},
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.8.0"

pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 dependencies = [
     "apscheduler>=3.10.4",
     "asyncpg>=0.28.0",
-    "fastapi>=0.104.1",
+    "fastapi>=0.104.2",
     "geoalchemy2>=0.14.2",
     "geopandas>=0.14.0",
     "itsdangerous>=2.1.2",
@@ -26,6 +26,7 @@ dependencies = [
     "sqlmodel>=0.0.14",
     "uvicorn>=0.23.2",
     "websockets>=12.0",
+    "aiosqlite>=0.19.0",
 ]
 requires-python = ">=3.11,<4"
 readme = "README.md"
@@ -51,4 +52,5 @@ dev = [
     "pandas-stubs>=2.1.4.231218",
     "pretty-errors>=1.2.25",
     "types-psycopg2>=2.9.21.20",
+    "types-PyYAML>=6.0.12.12",
 ]

src/gisaf/_version.py

@@ -1 +1 @@
-__version__ = '2023.4.dev28+ge3ed311.d20240107'
+__version__ = '2023.4.dev33+g7e9e266.d20240210'

src/gisaf/admin.py

@@ -4,6 +4,7 @@ import logging
 
 from gisaf.live import live_server
 from gisaf.redis_tools import Store
+from gisaf.baskets import Basket
 
 logger = logging.getLogger('Gisaf admin manager')
 
@@ -13,6 +14,7 @@ class AdminManager:
     One instance only, handled by Gisaf's process.
     """
     store: Store
+    baskets: dict[str, Basket]
     async def setup_admin(self, app):
         """
         Create the default baskets, scan and create baskets

src/gisaf/api/map.py

@@ -1,15 +1,17 @@
 from collections import OrderedDict, defaultdict
 import logging
-from pathlib import Path
+from json import dumps
-from fastapi import Depends, FastAPI, HTTPException, status, responses
+
+from fastapi import FastAPI, Request, HTTPException, status, responses
 from sqlalchemy.orm import selectinload
+from sqlalchemy.exc import NoResultFound
 from sqlmodel import select
 
-from gisaf.config import conf
 from gisaf.models.map_bases import BaseMap, BaseMapLayer, BaseStyle, MapInitData
 from gisaf.registry import registry
 from gisaf.database import db_session
-from gisaf.models.authentication import User
+from gisaf.database import fastapi_db_session
+from gisaf.tiles import registry as tiles_registry
 
 logger = logging.getLogger(__name__)
 
@@ -21,20 +23,20 @@ async def get_base_styles():
     async with db_session() as session:
         query = select(BaseStyle.name)\
             .where(BaseStyle.enabled==True)\
-            .order_by(BaseStyle.id)
+            .order_by(BaseStyle.id)  # type: ignore # noqa: E712
         data = await session.exec(query)
         base_styles = data.all()
     ## TODO: tiles_registry
     logger.warning('TODO: tiles_registry')
     # base_styles.extend(tiles_registry.mbtiles.values())
-    return [BaseStyle(name=bs) for bs in base_styles]
+    return [BaseStyle(name=bs) for bs in base_styles] # type: ignore
 
 async def get_base_maps() -> list[BaseMap]:
     async with db_session() as session:
-        query1 = select(BaseMap).options(selectinload(BaseMap.layers))
+        query1 = select(BaseMap).options(selectinload(BaseMap.layers)) # type: ignore
         data1 = await session.exec(query1)
         base_maps = data1.all()
-        return base_maps
+        return base_maps # type: ignore
         base_map_dict = {bm.id: bm.name for bm in base_maps}
         query2 = select(BaseMapLayer).options(selectinload(BaseMapLayer.base_map))
         data2 = await session.exec(query2)
@@ -58,5 +60,27 @@ async def get_init_data() -> MapInitData:
         baseStyles=await get_base_styles(),
         baseMaps=await get_base_maps(),
         groups=registry.primary_groups,
-        stores=df.to_dict(orient='records')
+        stores=df.to_dict(orient='records') # type: ignore
     )
+
+@api.get('/base_style/{name}')
+async def get_base_style(request: Request, name: str,
+                         db_session: fastapi_db_session,
+                         ) -> BaseStyle:
+    data = await db_session.exec(select(BaseStyle).where(BaseStyle.name==name))
+    try:
+        base_style = data.one()
+    except NoResultFound:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+        # return BaseStyle(
+        #     name=name,
+        #     style=dumps({})
+        # )
+    if name in tiles_registry.mbtiles:
+        ## Try to get base_style from tiles_registry
+        tiles = tiles_registry.mbtiles['name']
+        style = dumps(await tiles.get_style(style_record=base_style,
+                                            request=request))
+    else:
+        style = base_style.style # type: ignore
+    return BaseStyle(name=name, style=style) # type: ignore

src/gisaf/application.py

@@ -8,6 +8,7 @@ from gisaf.api.geoapi import api as geoapi
 from gisaf.config import conf
 from gisaf.registry import registry
 from gisaf.redis_tools import setup_redis, setup_redis_cache, shutdown_redis
+from gisaf.tiles import registry as map_tile_registry
 from gisaf.live import setup_live
 
 logging.basicConfig(level=conf.gisaf.debugLevel)
@@ -20,8 +21,10 @@ async def lifespan(app: FastAPI):
     await setup_redis()
     await setup_redis_cache()
     await setup_live()
+    await map_tile_registry.setup()
     yield
     await shutdown_redis()
+    await map_tile_registry.shutdown()
 
 app = FastAPI(
     debug=False,

src/gisaf/config.py

@@ -23,81 +23,81 @@ config_files = [
 ]
 
 class DashboardHome(BaseSettings):
-    title: str
+    title: str = 'Gisaf - home/dashboards'
-    content_file: str
+    content_file: str = '/etc/gisaf/dashboard_home_content.html'
-    footer_file: str
+    footer_file: str = '/etc/gisaf/dashboard_home_footer.html'
 
 class GisafConfig(BaseSettings):
-    title: str
+    title: str = 'Gisaf'
-    windowTitle: str
+    windowTitle: str = 'Gisaf'
-    debugLevel: str
+    debugLevel: str = 'INFO'
-    dashboard_home: DashboardHome
+    dashboard_home: DashboardHome = DashboardHome()
     redirect: str = ''
     use_pretty_errors: bool = False
 
 class SpatialSysRef(BaseSettings):
-    author: str
+    author: str = 'AVSM'
-    ellps: str
+    ellps: str = 'WGS84'
-    k: int
+    k: int = 1
-    lat_0: float
+    lat_0: float = 12.01605433
-    lon_0: float
+    lon_0: float = 79.80998934
-    no_defs: bool
+    no_defs: bool = True
-    proj: str
+    proj: str = 'tmerc'
-    towgs84: str
+    towgs84: str = '0,0,0,0,0,0,0'
-    units: str
+    units: str = 'm'
-    x_0: float
+    x_0: float = 370455.630
-    y_0: float
+    y_0: float = 1328608.994
 
 class RawSurvey(BaseSettings):
-    spatial_sys_ref: SpatialSysRef
+    spatial_sys_ref: SpatialSysRef = SpatialSysRef()
-    srid: int
+    srid: int = 910001
 
 class Geo(BaseSettings):
-    raw_survey: RawSurvey
+    raw_survey: RawSurvey = RawSurvey()
-    simplify_geom_factor: int
+    simplify_geom_factor: int = 10000000
     simplify_preserve_topology: bool = False
-    srid: int
+    srid: int = 4326
-    srid_for_proj: int
+    srid_for_proj: int = 32644
 
-class Flask(BaseSettings):
+# class Flask(BaseSettings):
-    secret_key: str
+#     secret_key: str
-    debug: int
+#     debug: int
 
 class MQTT(BaseSettings):
     broker: str = 'localhost'
     port: int = 1883
 
 class GisafLive(BaseSettings):
-    hostname: str
+    hostname: str = 'localhost'
-    port: int
+    port: int = 80
-    scheme: str
+    scheme: str = 'http'
-    redis: str
+    redis: str = 'redis://localhost'
-    mqtt: MQTT
+    mqtt: MQTT = MQTT()
 
 class DefaultSurvey(BaseSettings):
-    surveyor_id: int
+    surveyor_id: int = 1
-    equipment_id: int
+    equipment_id: int = 1
 
 class Survey(BaseSettings):
-    model_config = ConfigDict(extra='ignore')
+    # model_config = ConfigDict(extra='ignore')
-    db_schema_raw: str
+    db_schema_raw: str = 'raw_survey'
-    db_schema: str
+    db_schema: str = 'survey'
-    default: DefaultSurvey
+    default: DefaultSurvey = DefaultSurvey()
 
 class Crypto(BaseSettings):
-    secret: str
+    secret: str = 'Gisaf big secret'
-    algorithm: str
+    algorithm: str = 'HS256'
-    expire: float
+    expire: float = 21600
 
 class DB(BaseSettings):
-    uri: str
+    # uri: str
-    host: str
+    host: str = 'localhost'
     port: int = 5432
-    user: str
+    user: str = 'gisaf'
-    db: str
+    db: str = 'gisaf'
-    password: str
+    password: str = 'secret'
-    debug: bool
+    debug: bool = False
-    info: bool
+    info: bool = True
     pool_size: int = 10
     max_overflow: int = 10
     echo: bool = False
@@ -110,119 +110,121 @@ class DB(BaseSettings):
 
 
 class Log(BaseSettings):
-    level: str
+    level: str = 'WARNING'
 
 class OGCAPILicense(BaseSettings):
-    name: str
+    name: str = 'CC-BY 4.0 license'
-    url: str
+    url: str = 'https://creativecommons.org/licenses/by/4.0/'
 
 class OGCAPIProvider(BaseSettings):
-    name: str
+    name: str = 'Organization Name'
-    url: str
+    url: str = 'https://pygeoapi.io'
 
 class OGCAPIServerContact(BaseSettings):
-    name: str
+    name: str = 'Lastname, Firstname'
-    address: str
+    position: str = 'Position Title'
-    city: str
+    address: str = 'Mailing Address'
-    stateorprovince: str
+    city: str = 'City'
-    postalcode: int
+    stateorprovince: str = 'Administrative Area'
-    country: str
+    postalcode: int = 0
-    email: str
+    country: str = 'Country'
+    email: str = 'you@example.org'
+    url: str | None = None
 
 class OGCAPIIdentification(BaseSettings):
-    title: str
+    title: str = 'pygeoapi default instance'
-    description: str
+    description: str = 'pygeoapi provides an API to geospatial data'
-    keywords: list[str]
+    keywords: list[str] = ['geospatial', 'data', 'api']
-    keywords_type: str
+    keywords_type: str = 'theme'
-    terms_of_service: str
+    terms_of_service: str = 'https://creativecommons.org/licenses/by/4.0/'
-    url: str
+    url: str = 'http://example.org'
 
 class OGCAPIMetadata(BaseSettings):
-    identification: OGCAPIIdentification
+    identification: OGCAPIIdentification = OGCAPIIdentification()
-    license: OGCAPILicense
+    license: OGCAPILicense = OGCAPILicense()
-    provider: OGCAPIProvider
+    provider: OGCAPIProvider = OGCAPIProvider()
-    contact: OGCAPIServerContact
+    contact: OGCAPIServerContact = OGCAPIServerContact()
 
 class ServerBind(BaseSettings):
-    host: str
+    host: str = '0.0.0.0'
-    port: int
+    port: int = 5000
 
 class OGCAPIServerMap(BaseSettings):
-    url: str
+    url: str = 'https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png'
-    attribution: str
+    attribution: str = '''<a href="https://wikimediafoundation.org/wiki/Maps_Terms_of_Use">Wikimedia maps</a> | Map data &copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'''
 
 class OGCAPIServer(BaseSettings):
-    bind: ServerBind
+    bind: ServerBind = ServerBind()
-    url: str
+    url: str = 'https://example.org/ogcapi'
-    mimetype: str
+    mimetype: str = 'application/json; charset=UTF-8'
-    encoding: str
+    encoding: str = 'utf-8'
-    language: str
+    language: str = 'en-US'
-    pretty_print: bool
+    pretty_print: bool = False
-    limit: int
+    limit: int = 1000
-    map: OGCAPIServerMap
+    map: OGCAPIServerMap = OGCAPIServerMap()
 
 class OGCAPI(BaseSettings):
-    base_url: str
+    base_url: str = 'http://example.org/ogcapi'
-    bbox: list[float]
+    bbox: list[float] = [-180, -90, 180, 90]
-    log: Log
+    log: Log = Log()
-    metadata: OGCAPIMetadata
+    metadata: OGCAPIMetadata = OGCAPIMetadata()
-    server: OGCAPIServer
+    server: OGCAPIServer = OGCAPIServer()
 
 class TileServer(BaseSettings):
-     baseDir: str
+     baseDir: str = '/path/to/mbtiles_files_dir'
      useRequestUrl: bool = False
-     spriteBaseDir: str
+     spriteBaseDir: str = '/path/to/mbtiles_sprites_dir'
-     spriteUrl: str
+     spriteUrl: str = '/tiles/sprite/sprite'
-     spriteBaseUrl: str
+     spriteBaseUrl: str = 'https://gisaf.example.org'
      openMapTilesKey: str | None = None
 
 class Map(BaseSettings):
-    tileServer: TileServer | None = None
+    tileServer: TileServer = TileServer()
-    zoom: int
+    zoom: int = 14
-    pitch: int
+    pitch: int = 45
-    lat: float
+    lat: float = 12.0000
-    lng: float
+    lng: float = 79.8106
-    bearing: float
+    bearing: float = 0
-    style: str
+    style: str = 'OSM (vector)'
-    opacity: float
+    opacity: float = 1
-    attribution: str
+    attribution: str = ''
-    status: list[str]
+    status: list[str] = ['E', 'F', 'D']
-    defaultStatus: list[str] # FIXME: should be str
+    defaultStatus: list[str] = ['E'] # FIXME: should be str
-    tagKeys: list[str]
+    tagKeys: list[str] = ['source']
 
 class Measures(BaseSettings):
-    defaultStore: str
+    defaultStore: str | None = None
 
 class BasketDefault(BaseSettings):
-    surveyor: str
+    surveyor: str = 'Default surveyor'
-    equipment: str
+    equipment: str = 'Default equipment'
-    project: str | None
+    project: str = 'Default project'
-    status: str
+    status: str = 'E'
-    store: str | None
+    store: str | None = None
 
-class BasketOldDef(BaseSettings):
+# class BasketOldDef(BaseSettings):
-    base_dir: str
+#     base_dir: str
 
 class Basket(BaseSettings):
-    base_dir: str
+    base_dir: str = '/var/local/gisaf/baskets'
-    default: BasketDefault
+    default: BasketDefault = BasketDefault()
 
 class Plot(BaseSettings):
-    maxDataSize: int
+    maxDataSize: int = 10000
 
 class Dashboard(BaseSettings):
-    base_source_url: str
+    base_source_url: str = 'http://url.to.jupyter/lab/tree/'
-    base_storage_dir: str
+    base_storage_dir: str = '/var/lib/share/gisaf/dashboard'
     base_storage_url: str = '/dashboard-attachment/'
 
 class Widgets(BaseSettings):
-    footer: str
+    footer: str = """Generated by <span class='link' onclick="window.open('https://redmine.auroville.org.in/projects/gisaf/')">Gisaf</span>"""
 
 class Admin(BaseSettings):
-    basket: Basket
+    basket: Basket = Basket()
 
 class Attachments(BaseSettings):
-    base_dir: str
+    base_dir: str = '/var/local/gisaf/attachments'
 
 class Job(BaseSettings):
     id: str
@@ -256,7 +258,7 @@ class Config(BaseSettings):
         dotenv_settings: PydanticBaseSettingsSource,
         file_secret_settings: PydanticBaseSettingsSource,
     ) -> Tuple[PydanticBaseSettingsSource, ...]:
-        return env_settings, init_settings, file_secret_settings, config_file_settings
+        return env_settings, init_settings, file_secret_settings, config_file_settings # type: ignore
 
     # def __init__(self, **kwargs):
     #     super().__init__(**kwargs)
@@ -268,29 +270,27 @@ class Config(BaseSettings):
     #         'web_mercator': 'epsg:3857',
     #     }
 
-    admin: Admin
+    admin: Admin = Admin()
-    attachments: Attachments
+    attachments: Attachments = Attachments()
-    basket: BasketOldDef
+    # basket: BasketOldDef = BasketOldDef()
     # crs: Crs
-    crypto: Crypto
+    crypto: Crypto = Crypto()
-    dashboard: Dashboard
+    dashboard: Dashboard = Dashboard()
-    db: DB
+    db: DB = DB()
-    flask: Flask
+    # flask: Flask
-    geo: Geo
+    geo: Geo = Geo()
-    gisaf: GisafConfig
+    gisaf: GisafConfig = GisafConfig()
-    gisaf_live: GisafLive
+    gisaf_live: GisafLive = GisafLive()
-    jobs: list[Job]
+    jobs: list[Job] = []
-    map: Map
+    map: Map = Map()
-    measures: Measures
+    measures: Measures = Measures()
-    ogcapi: OGCAPI
+    ogcapi: OGCAPI = OGCAPI()
-    plot: Plot
+    plot: Plot = Plot()
-    plugins: dict[str, dict[str, Any]]
+    plugins: dict[str, dict[str, Any]] = {}
-    survey: Survey
+    survey: Survey = Survey()
-    version: str
+    version: str = __version__
-    weather_station: dict[str, dict[str, Any]]
+    weather_station: dict[str, dict[str, Any]] = {}
-    widgets: Widgets
+    widgets: Widgets = Widgets()
-    #engine: AsyncEngine
-    #session_maker: sessionmaker
 
     @property
     def crs(self) -> Crs:
@@ -328,7 +328,7 @@ def load_yaml(path: Path) -> dict[str, Any]:
     return config
 
 
-conf = Config(version=__version__)
+conf = Config()
 
 # def set_app_config(app) -> None:
 #     raw_configs = []

src/gisaf/database.py

@@ -3,7 +3,7 @@ from typing import Annotated, Literal, Any
 from collections.abc import AsyncGenerator
 
 from sqlalchemy.ext.asyncio import create_async_engine
-from sqlalchemy.orm import joinedload, QueryableAttribute, InstrumentedAttribute
+from sqlalchemy.orm import joinedload, QueryableAttribute
 from sqlalchemy.sql.selectable import Select
 from sqlmodel import SQLModel, select
 from sqlmodel.ext.asyncio.session import AsyncSession
@@ -76,7 +76,7 @@ class BaseModel(SQLModel):
                 columns.add(*(col.name for col in cls.__table__.primary_key.columns))
                 query = select(*(getattr(cls, col) for col in columns))
             if where is not None:
-                query.append_whereclause(where)
+                query = query.where(where)
             ## Get the joined tables
             joined_tables = cls.selectinload()
             if with_related and len(joined_tables) > 0:

src/gisaf/models/geo_models_base.py

@@ -186,7 +186,7 @@ class SurveyModel(BaseSurveyModel):
 
     @declared_attr
     def __tablename__(cls) -> str:
-        return cls.__name__  # type: nocheck
+        return cls.__name__  # type: ignore
 
     async def get_survey_info(self):
         info = await super(SurveyModel, self).get_survey_info()

src/gisaf/models/map_bases.py

@@ -11,7 +11,7 @@ from gisaf.models.store import Store
 
 class BaseStyle(Model, table=True):
     __table_args__ = gisaf_map.table_args
-    __tablename__ = 'map_base_style'
+    __tablename__: str = 'map_base_style' # type: ignore
 
     class Admin:
         menu = 'Other'
@@ -19,18 +19,18 @@ class BaseStyle(Model, table=True):
 
     id: int | None = Field(primary_key=True, default=None)
     name: str
-    style: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True))
+    style: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore
-    mbtiles: str = Field(sa_type=String(50))
+    mbtiles: str = Field(sa_type=String(50)) # type: ignore
     static_tiles_url: str
     enabled: bool = True
 
     def __repr__(self):
-        return '<models.BaseStyle {self.name:s}>'.format(self=self)
+        return f'<models.BaseStyle {self.name:s}>'
 
 
 class BaseMap(Model, table=True):
     __table_args__ = gisaf_map.table_args
-    __tablename__ = 'base_map'
+    __tablename__: str = 'base_map' # type: ignore
 
     class Admin:
         menu = 'Other'
@@ -39,14 +39,14 @@ class BaseMap(Model, table=True):
     name: str
     layers: list['BaseMapLayer'] = Relationship(back_populates='base_map')
 
-    def __repr__(self):
+    def __repr__(self) -> str:
-        return '<models.BaseMap {self.name:s}>'.format(self=self)
+        return f'<models.BaseMap {self.name:s}>'
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self.name
 
     @classmethod
-    def selectinload(cls):
+    def selectinload(cls) -> list[list['BaseMapLayer']]:
         return [
             cls.layers
         ]
@@ -54,7 +54,7 @@ class BaseMap(Model, table=True):
 
 class BaseMapLayer(Model, table=True):
     __table_args__ = gisaf_map.table_args
-    __tablename__ = 'base_map_layer'
+    __tablename__: str = 'base_map_layer' # type: ignore
 
     class Admin:
         menu = 'Other'
@@ -63,7 +63,7 @@ class BaseMapLayer(Model, table=True):
     base_map_id: int = Field(foreign_key=gisaf_map.table('base_map.id'),
                              index=True)
     base_map: BaseMap = Relationship(back_populates='layers')
-    store: str = Field(sa_type=String(100))
+    store: str = Field(sa_type=String(100)) # type: ignore
 
     @classmethod
     def selectinload(cls):

src/gisaf/security.py

@@ -40,6 +40,17 @@ class TokenData(BaseModel):
 
 oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token", auto_error=False)
 
+credentials_exception = HTTPException(
+    status_code=status.HTTP_401_UNAUTHORIZED,
+    detail="Could not validate credentials",
+    headers={"WWW-Authenticate": "Bearer"},
+)
+
+expired_exception = HTTPException(
+    status_code=status.HTTP_401_UNAUTHORIZED,
+    detail="Token has expired",
+    headers={"WWW-Authenticate": "Bearer"},
+)
 
 def get_password_hash(password: str):
     return pwd_context.hash(password)
@@ -98,16 +109,6 @@ def verify_password(user: User, plain_password):
 
 async def get_current_user(
     token: str = Depends(oauth2_scheme)) -> UserRead | None:
-    credentials_exception = HTTPException(
-        status_code=status.HTTP_401_UNAUTHORIZED,
-        detail="Could not validate credentials",
-        headers={"WWW-Authenticate": "Bearer"},
-    )
-    expired_exception = HTTPException(
-        status_code=status.HTTP_401_UNAUTHORIZED,
-        detail="Token has expired",
-        headers={"WWW-Authenticate": "Bearer"},
-    )
     if token is None:
         return None
     try:

src/gisaf/tiles.py

@@ -0,0 +1,196 @@
+"""
+mbtile server
+
+Instructions (example):
+
+cd map  ## Matches tilesBaseDir in config
+
+curl -O http://download.geofabrik.de/asia/india/southern-zone-latest.osm.pbf
+
+tilemaker \
+    --config /usr/local/lib/gisaf_src/tilemaker_src/tilemaker/resources/config-openmaptiles.json \
+    --process /usr/local/lib/gisaf_src/tilemaker_src/tilemaker/resources/process-openmaptiles.lua \
+    --input southern-zone-latest.osm.pbf \
+    --output southern-zone-latest.mbtile
+
+
+----
+
+Get the style from https://github.com/openmaptiles, eg.
+curl -o osm-bright-full.json https://raw.githubusercontent.com/openmaptiles/osm-bright-gl-style/master/style.json
+## Minify json:
+python -c 'import json, sys;json.dump(json.load(sys.stdin), sys.stdout)'  < osm-bright-full.json > osm-bright.json
+
+And copy the style to the gisaf_map.map_base_style table,
+with the name matching the file name
+(TODO: this would need de-coupling the source mbtile and the style)
+
+----
+
+Get the sprites from openmaptiles:`
+
+cd tiles  ## Matches tilesSpriteBaseDir in config
+
+curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite.png'
+curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite.json'
+curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite@2x.png'
+curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite@2x.json'
+
+TODO: TO migrate - 3/1/2024: this was copied from legacy gisaf without change
+See: https://github.com/gis-ops/tutorials/blob/master/webservices/fastapi/fastapi_auth_vector_tiles.md
+"""
+
+from pathlib import Path
+from json import loads, dumps
+import logging
+
+from fastapi import FastAPI, Response, responses, HTTPException, status
+from fastapi.staticfiles import StaticFiles
+import aiosqlite
+
+from gisaf.config import conf
+
+logger = logging.getLogger('gisaf tile server')
+
+api = FastAPI(
+    default_response_class=responses.ORJSONResponse,
+)
+
+OSM_ATTRIBUTION = '<a href=\"http://www.openstreetmap.org/about/\" target=\"_blank\">&copy; OpenStreetMap contributors</a>'
+
+class MBTiles:
+    def __init__(self, file_path, style_name):
+        self.file_path = file_path
+        self.name = style_name
+        self.scheme = 'tms'
+        self.etag = f'W/"{hex(int(file_path.stat().st_mtime))[2:]}"'
+
+    async def connect(self):
+        self.db = await aiosqlite.connect(self.file_path)
+        self.metadata = {}
+        try:
+            async with self.db.execute('select name, value from metadata') as cursor:
+                async for row in cursor:
+                    self.metadata[row[0]] = row[1]
+        except aiosqlite.DatabaseError as err:
+            logger.warning(f'Cannot read {self.file_path}, will not be able to serve tiles (error: {err.args[0]})')
+
+        self.metadata['bounds'] = [float(v) for v in self.metadata['bounds'].split(',')]
+        self.metadata['maxzoom'] = int(self.metadata['maxzoom'])
+        self.metadata['minzoom'] = int(self.metadata['minzoom'])
+
+    async def get_style(self, style_record, request):
+        """
+        Generate on the fly the style
+        """
+        if conf.map.tileServer.useRequestUrl:
+            base_url = request.url.parent
+        else:
+            base_url = conf.map.tileServer.spriteBaseUrl
+        base_tiles_url = f"{base_url}/tiles/{self.name}"
+        scheme = self.scheme
+        ## TODO: avoid parse and serialize at every request
+        layers = loads(style_record['style'])['layers']
+        for layer in layers:
+            if 'source' in layer:
+                layer['source'] = 'gisafTiles'
+        resp = {
+            'basename': self.file_path.stem,
+            #'center': self.center,
+            'description': f'Extract of {self.file_path.stem} from OSM, powered by Gisaf',
+            'format': self.metadata['format'],
+            'id': f'gisaftiles_{self.name}',
+            'maskLevel': 5,
+            'name': self.name,
+            #'pixel_scale': 256,
+            #'planettime': '1499040000000',
+            'tilejson': '2.0.0',
+            'version': 8,
+            'glyphs': f"/assets/fonts/glyphs/{{fontstack}}/{{range}}.pbf",
+            'sprite': f"{base_url}{conf.map.tileServer.spriteUrl}",
+            'sources': {
+                'gisafTiles': {
+                    'type': 'vector',
+                    'tiles': [
+                        f'{base_tiles_url}/{{z}}/{{x}}/{{y}}.pbf',
+                    ],
+                    'maxzoom': self.metadata['maxzoom'],
+                    'minzoom': self.metadata['minzoom'],
+                    'bounds': self.metadata['bounds'],
+                    'scheme': scheme,
+                    'attribution': OSM_ATTRIBUTION,
+                    'version': self.metadata['version'],
+                }
+            },
+            'layers': layers,
+        }
+        return resp
+
+
+class MBTilesRegistry:
+    mbtiles: dict[str, MBTiles]
+    async def setup(self):
+        """
+        Read all mbtiles, construct styles
+        """
+        self.mbtiles = {}
+        for file_path in Path(conf.map.tileServer.baseDir).glob('*.mbtiles'):
+            mbtiles = MBTiles(file_path, file_path.stem)
+            self.mbtiles[file_path.stem] = mbtiles
+            await mbtiles.connect()
+
+    async def shutdown(self):
+        """
+        Tear down the connection to the mbtiles files
+        """
+        for mbtiles in self.mbtiles.values():
+            await mbtiles.db.close()
+
+
+@api.get('/{style_name}/{z}/{x}/{y}.pbf')
+async def get_tile(request, style_name: str, z:int, x: int, y: int,
+                   response: Response):
+    """
+    Return the specific tile
+    """
+    if style_name not in registry.mbtiles:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+    mbtiles = registry.mbtiles[style_name]
+
+    if request.headers.get('If-None-Match') == mbtiles.etag:
+        request.not_modified = True
+        return {}
+
+    response.headers['Content-Encoding'] = 'gzip'
+    response.headers['Content-Type'] = 'application/octet-stream'
+    request.response_etag = mbtiles.etag
+    async with mbtiles.db.execute('select tile_data from tiles where zoom_level=? and tile_column=? and tile_row=?',
+                           (z, x, y)) as cursor:
+        async for row in cursor:
+            return row[0]
+    raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+
+#@routes.get('/sprite/{name:\S+}')
+#async def get_sprite(request):
+
+
+@api.get('/{style_name}')
+async def get_style(request, style_name: str):
+    """
+    Return the base style.
+    Note that normal operations are processed through graphql (resolve_base_style)
+    :param request:
+    :return:
+    """
+    if style_name not in registry.mbtiles:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+    mbtiles = registry.mbtiles[style_name]
+    return await mbtiles.get_style()
+
+
+registry = MBTilesRegistry()
+
+api.mount("/sprite",
+          StaticFiles(directory=conf.map.tileServer.spriteBaseDir),
+          name="sprites")