From 5dacc908f2d2218ebb6139b8b20d0400fb9b3934 Mon Sep 17 00:00:00 2001 From: phil Date: Sat, 10 Feb 2024 19:26:38 +0530 Subject: [PATCH] Add missing dependencies Add tile server Config: add defults Cosmetic refactorings --- pdm.lock | 35 +++- pyproject.toml | 4 +- src/gisaf/_version.py | 2 +- src/gisaf/admin.py | 2 + src/gisaf/api/map.py | 42 +++- src/gisaf/application.py | 3 + src/gisaf/config.py | 286 ++++++++++++++-------------- src/gisaf/database.py | 4 +- src/gisaf/models/geo_models_base.py | 2 +- src/gisaf/models/map_bases.py | 22 +-- src/gisaf/security.py | 23 +-- src/gisaf/tiles.py | 196 +++++++++++++++++++ 12 files changed, 434 insertions(+), 187 deletions(-) create mode 100644 src/gisaf/tiles.py diff --git a/pdm.lock b/pdm.lock index f0c9f89..a39bfbe 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "mqtt"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:d6bc84b5bf12fda8fd24858515794677046aca3dea340a40679d1276ae7a6ea9" +content_hash = "sha256:75aa4cd0effa4fc41f312763423aa949a48e14e235b75bbe0a67fa762bc9660c" [[package]] name = "aiomqtt" @@ -20,6 +20,16 @@ files = [ {file = "aiomqtt-1.2.1.tar.gz", hash = "sha256:7582f4341f08ef7110dd9ab3a559454dc28ccda1eac502ff8f08a73b238ecede"}, ] +[[package]] +name = "aiosqlite" +version = "0.19.0" +requires_python = ">=3.7" +summary = "asyncio bridge to the standard sqlite3 module" +files = [ + {file = "aiosqlite-0.19.0-py3-none-any.whl", hash = "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96"}, + {file = "aiosqlite-0.19.0.tar.gz", hash = "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d"}, +] + [[package]] name = "annotated-types" version = "0.6.0" @@ -308,17 +318,17 @@ files = [ [[package]] name = "fastapi" -version = "0.108.0" +version = "0.109.2" requires_python = ">=3.8" summary = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" dependencies = [ "pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4", - "starlette<0.33.0,>=0.29.0", + "starlette<0.37.0,>=0.36.3", "typing-extensions>=4.8.0", ] files = [ - {file = "fastapi-0.108.0-py3-none-any.whl", hash = "sha256:8c7bc6d315da963ee4cdb605557827071a9a7f95aeb8fcdd3bde48cdc8764dd7"}, - {file = "fastapi-0.108.0.tar.gz", hash = "sha256:5056e504ac6395bf68493d71fcfc5352fdbd5fda6f88c21f6420d80d81163296"}, + {file = "fastapi-0.109.2-py3-none-any.whl", hash = "sha256:2c9bab24667293b501cad8dd388c05240c850b58ec5876ee3283c47d6e1e3a4d"}, + {file = "fastapi-0.109.2.tar.gz", hash = "sha256:f3817eac96fe4f65a2ebb4baa000f394e55f5fccdaf7f75250804bc58f354f73"}, ] [[package]] @@ -1162,15 +1172,15 @@ files = [ [[package]] name = "starlette" -version = "0.32.0.post1" +version = "0.36.3" requires_python = ">=3.8" summary = "The little ASGI library that shines." dependencies = [ "anyio<5,>=3.4.0", ] files = [ - {file = "starlette-0.32.0.post1-py3-none-any.whl", hash = "sha256:cd0cb10ddb49313f609cedfac62c8c12e56c7314b66d89bb077ba228bada1b09"}, - {file = "starlette-0.32.0.post1.tar.gz", hash = "sha256:e54e2b7e2fb06dff9eac40133583f10dfa05913f5a85bf26f427c7a40a9a3d02"}, + {file = "starlette-0.36.3-py3-none-any.whl", hash = "sha256:13d429aa93a61dc40bf503e8c801db1f1bca3dc706b10ef2434a36123568f044"}, + {file = "starlette-0.36.3.tar.gz", hash = "sha256:90a671733cfb35771d8cc605e0b679d23b992f8dcfad48cc60b38cb29aeb7080"}, ] [[package]] @@ -1202,6 +1212,15 @@ files = [ {file = "types_pytz-2023.3.1.1-py3-none-any.whl", hash = "sha256:1999a123a3dc0e39a2ef6d19f3f8584211de9e6a77fe7a0259f04a524e90a5cf"}, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.12" +summary = "Typing stubs for PyYAML" +files = [ + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, +] + [[package]] name = "typing-extensions" version = "4.8.0" diff --git a/pyproject.toml b/pyproject.toml index 8e47edf..bb0447e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [ dependencies = [ "apscheduler>=3.10.4", "asyncpg>=0.28.0", - "fastapi>=0.104.1", + "fastapi>=0.104.2", "geoalchemy2>=0.14.2", "geopandas>=0.14.0", "itsdangerous>=2.1.2", @@ -26,6 +26,7 @@ dependencies = [ "sqlmodel>=0.0.14", "uvicorn>=0.23.2", "websockets>=12.0", + "aiosqlite>=0.19.0", ] requires-python = ">=3.11,<4" readme = "README.md" @@ -51,4 +52,5 @@ dev = [ "pandas-stubs>=2.1.4.231218", "pretty-errors>=1.2.25", "types-psycopg2>=2.9.21.20", + "types-PyYAML>=6.0.12.12", ] diff --git a/src/gisaf/_version.py b/src/gisaf/_version.py index 1f6fcbc..84bd5f7 100644 --- a/src/gisaf/_version.py +++ b/src/gisaf/_version.py @@ -1 +1 @@ -__version__ = '2023.4.dev28+ge3ed311.d20240107' \ No newline at end of file +__version__ = '2023.4.dev33+g7e9e266.d20240210' \ No newline at end of file diff --git a/src/gisaf/admin.py b/src/gisaf/admin.py index 2d94742..1fde3d0 100644 --- a/src/gisaf/admin.py +++ b/src/gisaf/admin.py @@ -4,6 +4,7 @@ import logging from gisaf.live import live_server from gisaf.redis_tools import Store +from gisaf.baskets import Basket logger = logging.getLogger('Gisaf admin manager') @@ -13,6 +14,7 @@ class AdminManager: One instance only, handled by Gisaf's process. """ store: Store + baskets: dict[str, Basket] async def setup_admin(self, app): """ Create the default baskets, scan and create baskets diff --git a/src/gisaf/api/map.py b/src/gisaf/api/map.py index c2ea7a9..c3b1bff 100644 --- a/src/gisaf/api/map.py +++ b/src/gisaf/api/map.py @@ -1,15 +1,17 @@ from collections import OrderedDict, defaultdict import logging -from pathlib import Path -from fastapi import Depends, FastAPI, HTTPException, status, responses +from json import dumps + +from fastapi import FastAPI, Request, HTTPException, status, responses from sqlalchemy.orm import selectinload +from sqlalchemy.exc import NoResultFound from sqlmodel import select -from gisaf.config import conf from gisaf.models.map_bases import BaseMap, BaseMapLayer, BaseStyle, MapInitData from gisaf.registry import registry from gisaf.database import db_session -from gisaf.models.authentication import User +from gisaf.database import fastapi_db_session +from gisaf.tiles import registry as tiles_registry logger = logging.getLogger(__name__) @@ -21,20 +23,20 @@ async def get_base_styles(): async with db_session() as session: query = select(BaseStyle.name)\ .where(BaseStyle.enabled==True)\ - .order_by(BaseStyle.id) + .order_by(BaseStyle.id) # type: ignore # noqa: E712 data = await session.exec(query) base_styles = data.all() ## TODO: tiles_registry logger.warning('TODO: tiles_registry') # base_styles.extend(tiles_registry.mbtiles.values()) - return [BaseStyle(name=bs) for bs in base_styles] + return [BaseStyle(name=bs) for bs in base_styles] # type: ignore async def get_base_maps() -> list[BaseMap]: async with db_session() as session: - query1 = select(BaseMap).options(selectinload(BaseMap.layers)) + query1 = select(BaseMap).options(selectinload(BaseMap.layers)) # type: ignore data1 = await session.exec(query1) base_maps = data1.all() - return base_maps + return base_maps # type: ignore base_map_dict = {bm.id: bm.name for bm in base_maps} query2 = select(BaseMapLayer).options(selectinload(BaseMapLayer.base_map)) data2 = await session.exec(query2) @@ -58,5 +60,27 @@ async def get_init_data() -> MapInitData: baseStyles=await get_base_styles(), baseMaps=await get_base_maps(), groups=registry.primary_groups, - stores=df.to_dict(orient='records') + stores=df.to_dict(orient='records') # type: ignore ) + +@api.get('/base_style/{name}') +async def get_base_style(request: Request, name: str, + db_session: fastapi_db_session, + ) -> BaseStyle: + data = await db_session.exec(select(BaseStyle).where(BaseStyle.name==name)) + try: + base_style = data.one() + except NoResultFound: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + # return BaseStyle( + # name=name, + # style=dumps({}) + # ) + if name in tiles_registry.mbtiles: + ## Try to get base_style from tiles_registry + tiles = tiles_registry.mbtiles['name'] + style = dumps(await tiles.get_style(style_record=base_style, + request=request)) + else: + style = base_style.style # type: ignore + return BaseStyle(name=name, style=style) # type: ignore diff --git a/src/gisaf/application.py b/src/gisaf/application.py index c0daa2e..1238f09 100644 --- a/src/gisaf/application.py +++ b/src/gisaf/application.py @@ -8,6 +8,7 @@ from gisaf.api.geoapi import api as geoapi from gisaf.config import conf from gisaf.registry import registry from gisaf.redis_tools import setup_redis, setup_redis_cache, shutdown_redis +from gisaf.tiles import registry as map_tile_registry from gisaf.live import setup_live logging.basicConfig(level=conf.gisaf.debugLevel) @@ -20,8 +21,10 @@ async def lifespan(app: FastAPI): await setup_redis() await setup_redis_cache() await setup_live() + await map_tile_registry.setup() yield await shutdown_redis() + await map_tile_registry.shutdown() app = FastAPI( debug=False, diff --git a/src/gisaf/config.py b/src/gisaf/config.py index 8fd2089..a0220f6 100644 --- a/src/gisaf/config.py +++ b/src/gisaf/config.py @@ -23,81 +23,81 @@ config_files = [ ] class DashboardHome(BaseSettings): - title: str - content_file: str - footer_file: str + title: str = 'Gisaf - home/dashboards' + content_file: str = '/etc/gisaf/dashboard_home_content.html' + footer_file: str = '/etc/gisaf/dashboard_home_footer.html' class GisafConfig(BaseSettings): - title: str - windowTitle: str - debugLevel: str - dashboard_home: DashboardHome + title: str = 'Gisaf' + windowTitle: str = 'Gisaf' + debugLevel: str = 'INFO' + dashboard_home: DashboardHome = DashboardHome() redirect: str = '' use_pretty_errors: bool = False class SpatialSysRef(BaseSettings): - author: str - ellps: str - k: int - lat_0: float - lon_0: float - no_defs: bool - proj: str - towgs84: str - units: str - x_0: float - y_0: float + author: str = 'AVSM' + ellps: str = 'WGS84' + k: int = 1 + lat_0: float = 12.01605433 + lon_0: float = 79.80998934 + no_defs: bool = True + proj: str = 'tmerc' + towgs84: str = '0,0,0,0,0,0,0' + units: str = 'm' + x_0: float = 370455.630 + y_0: float = 1328608.994 class RawSurvey(BaseSettings): - spatial_sys_ref: SpatialSysRef - srid: int + spatial_sys_ref: SpatialSysRef = SpatialSysRef() + srid: int = 910001 class Geo(BaseSettings): - raw_survey: RawSurvey - simplify_geom_factor: int + raw_survey: RawSurvey = RawSurvey() + simplify_geom_factor: int = 10000000 simplify_preserve_topology: bool = False - srid: int - srid_for_proj: int + srid: int = 4326 + srid_for_proj: int = 32644 -class Flask(BaseSettings): - secret_key: str - debug: int +# class Flask(BaseSettings): +# secret_key: str +# debug: int class MQTT(BaseSettings): broker: str = 'localhost' port: int = 1883 class GisafLive(BaseSettings): - hostname: str - port: int - scheme: str - redis: str - mqtt: MQTT + hostname: str = 'localhost' + port: int = 80 + scheme: str = 'http' + redis: str = 'redis://localhost' + mqtt: MQTT = MQTT() class DefaultSurvey(BaseSettings): - surveyor_id: int - equipment_id: int + surveyor_id: int = 1 + equipment_id: int = 1 class Survey(BaseSettings): - model_config = ConfigDict(extra='ignore') - db_schema_raw: str - db_schema: str - default: DefaultSurvey + # model_config = ConfigDict(extra='ignore') + db_schema_raw: str = 'raw_survey' + db_schema: str = 'survey' + default: DefaultSurvey = DefaultSurvey() class Crypto(BaseSettings): - secret: str - algorithm: str - expire: float + secret: str = 'Gisaf big secret' + algorithm: str = 'HS256' + expire: float = 21600 class DB(BaseSettings): - uri: str - host: str + # uri: str + host: str = 'localhost' port: int = 5432 - user: str - db: str - password: str - debug: bool - info: bool + user: str = 'gisaf' + db: str = 'gisaf' + password: str = 'secret' + debug: bool = False + info: bool = True pool_size: int = 10 max_overflow: int = 10 echo: bool = False @@ -110,119 +110,121 @@ class DB(BaseSettings): class Log(BaseSettings): - level: str + level: str = 'WARNING' class OGCAPILicense(BaseSettings): - name: str - url: str + name: str = 'CC-BY 4.0 license' + url: str = 'https://creativecommons.org/licenses/by/4.0/' class OGCAPIProvider(BaseSettings): - name: str - url: str + name: str = 'Organization Name' + url: str = 'https://pygeoapi.io' class OGCAPIServerContact(BaseSettings): - name: str - address: str - city: str - stateorprovince: str - postalcode: int - country: str - email: str + name: str = 'Lastname, Firstname' + position: str = 'Position Title' + address: str = 'Mailing Address' + city: str = 'City' + stateorprovince: str = 'Administrative Area' + postalcode: int = 0 + country: str = 'Country' + email: str = 'you@example.org' + url: str | None = None class OGCAPIIdentification(BaseSettings): - title: str - description: str - keywords: list[str] - keywords_type: str - terms_of_service: str - url: str + title: str = 'pygeoapi default instance' + description: str = 'pygeoapi provides an API to geospatial data' + keywords: list[str] = ['geospatial', 'data', 'api'] + keywords_type: str = 'theme' + terms_of_service: str = 'https://creativecommons.org/licenses/by/4.0/' + url: str = 'http://example.org' class OGCAPIMetadata(BaseSettings): - identification: OGCAPIIdentification - license: OGCAPILicense - provider: OGCAPIProvider - contact: OGCAPIServerContact + identification: OGCAPIIdentification = OGCAPIIdentification() + license: OGCAPILicense = OGCAPILicense() + provider: OGCAPIProvider = OGCAPIProvider() + contact: OGCAPIServerContact = OGCAPIServerContact() class ServerBind(BaseSettings): - host: str - port: int + host: str = '0.0.0.0' + port: int = 5000 class OGCAPIServerMap(BaseSettings): - url: str - attribution: str + url: str = 'https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png' + attribution: str = '''Wikimedia maps | Map data © OpenStreetMap contributors''' class OGCAPIServer(BaseSettings): - bind: ServerBind - url: str - mimetype: str - encoding: str - language: str - pretty_print: bool - limit: int - map: OGCAPIServerMap + bind: ServerBind = ServerBind() + url: str = 'https://example.org/ogcapi' + mimetype: str = 'application/json; charset=UTF-8' + encoding: str = 'utf-8' + language: str = 'en-US' + pretty_print: bool = False + limit: int = 1000 + map: OGCAPIServerMap = OGCAPIServerMap() class OGCAPI(BaseSettings): - base_url: str - bbox: list[float] - log: Log - metadata: OGCAPIMetadata - server: OGCAPIServer + base_url: str = 'http://example.org/ogcapi' + bbox: list[float] = [-180, -90, 180, 90] + log: Log = Log() + metadata: OGCAPIMetadata = OGCAPIMetadata() + server: OGCAPIServer = OGCAPIServer() class TileServer(BaseSettings): - baseDir: str + baseDir: str = '/path/to/mbtiles_files_dir' useRequestUrl: bool = False - spriteBaseDir: str - spriteUrl: str - spriteBaseUrl: str + spriteBaseDir: str = '/path/to/mbtiles_sprites_dir' + spriteUrl: str = '/tiles/sprite/sprite' + spriteBaseUrl: str = 'https://gisaf.example.org' openMapTilesKey: str | None = None class Map(BaseSettings): - tileServer: TileServer | None = None - zoom: int - pitch: int - lat: float - lng: float - bearing: float - style: str - opacity: float - attribution: str - status: list[str] - defaultStatus: list[str] # FIXME: should be str - tagKeys: list[str] + tileServer: TileServer = TileServer() + zoom: int = 14 + pitch: int = 45 + lat: float = 12.0000 + lng: float = 79.8106 + bearing: float = 0 + style: str = 'OSM (vector)' + opacity: float = 1 + attribution: str = '' + status: list[str] = ['E', 'F', 'D'] + defaultStatus: list[str] = ['E'] # FIXME: should be str + tagKeys: list[str] = ['source'] class Measures(BaseSettings): - defaultStore: str + defaultStore: str | None = None class BasketDefault(BaseSettings): - surveyor: str - equipment: str - project: str | None - status: str - store: str | None + surveyor: str = 'Default surveyor' + equipment: str = 'Default equipment' + project: str = 'Default project' + status: str = 'E' + store: str | None = None -class BasketOldDef(BaseSettings): - base_dir: str +# class BasketOldDef(BaseSettings): +# base_dir: str class Basket(BaseSettings): - base_dir: str - default: BasketDefault + base_dir: str = '/var/local/gisaf/baskets' + default: BasketDefault = BasketDefault() class Plot(BaseSettings): - maxDataSize: int + maxDataSize: int = 10000 class Dashboard(BaseSettings): - base_source_url: str - base_storage_dir: str + base_source_url: str = 'http://url.to.jupyter/lab/tree/' + base_storage_dir: str = '/var/lib/share/gisaf/dashboard' base_storage_url: str = '/dashboard-attachment/' class Widgets(BaseSettings): - footer: str + footer: str = """Generated by Gisaf""" class Admin(BaseSettings): - basket: Basket + basket: Basket = Basket() class Attachments(BaseSettings): - base_dir: str + base_dir: str = '/var/local/gisaf/attachments' class Job(BaseSettings): id: str @@ -256,7 +258,7 @@ class Config(BaseSettings): dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, ) -> Tuple[PydanticBaseSettingsSource, ...]: - return env_settings, init_settings, file_secret_settings, config_file_settings + return env_settings, init_settings, file_secret_settings, config_file_settings # type: ignore # def __init__(self, **kwargs): # super().__init__(**kwargs) @@ -268,29 +270,27 @@ class Config(BaseSettings): # 'web_mercator': 'epsg:3857', # } - admin: Admin - attachments: Attachments - basket: BasketOldDef + admin: Admin = Admin() + attachments: Attachments = Attachments() + # basket: BasketOldDef = BasketOldDef() # crs: Crs - crypto: Crypto - dashboard: Dashboard - db: DB - flask: Flask - geo: Geo - gisaf: GisafConfig - gisaf_live: GisafLive - jobs: list[Job] - map: Map - measures: Measures - ogcapi: OGCAPI - plot: Plot - plugins: dict[str, dict[str, Any]] - survey: Survey - version: str - weather_station: dict[str, dict[str, Any]] - widgets: Widgets - #engine: AsyncEngine - #session_maker: sessionmaker + crypto: Crypto = Crypto() + dashboard: Dashboard = Dashboard() + db: DB = DB() + # flask: Flask + geo: Geo = Geo() + gisaf: GisafConfig = GisafConfig() + gisaf_live: GisafLive = GisafLive() + jobs: list[Job] = [] + map: Map = Map() + measures: Measures = Measures() + ogcapi: OGCAPI = OGCAPI() + plot: Plot = Plot() + plugins: dict[str, dict[str, Any]] = {} + survey: Survey = Survey() + version: str = __version__ + weather_station: dict[str, dict[str, Any]] = {} + widgets: Widgets = Widgets() @property def crs(self) -> Crs: @@ -328,7 +328,7 @@ def load_yaml(path: Path) -> dict[str, Any]: return config -conf = Config(version=__version__) +conf = Config() # def set_app_config(app) -> None: # raw_configs = [] diff --git a/src/gisaf/database.py b/src/gisaf/database.py index de0ceb8..3b64d5d 100644 --- a/src/gisaf/database.py +++ b/src/gisaf/database.py @@ -3,7 +3,7 @@ from typing import Annotated, Literal, Any from collections.abc import AsyncGenerator from sqlalchemy.ext.asyncio import create_async_engine -from sqlalchemy.orm import joinedload, QueryableAttribute, InstrumentedAttribute +from sqlalchemy.orm import joinedload, QueryableAttribute from sqlalchemy.sql.selectable import Select from sqlmodel import SQLModel, select from sqlmodel.ext.asyncio.session import AsyncSession @@ -76,7 +76,7 @@ class BaseModel(SQLModel): columns.add(*(col.name for col in cls.__table__.primary_key.columns)) query = select(*(getattr(cls, col) for col in columns)) if where is not None: - query.append_whereclause(where) + query = query.where(where) ## Get the joined tables joined_tables = cls.selectinload() if with_related and len(joined_tables) > 0: diff --git a/src/gisaf/models/geo_models_base.py b/src/gisaf/models/geo_models_base.py index f25e2f9..e825fb9 100644 --- a/src/gisaf/models/geo_models_base.py +++ b/src/gisaf/models/geo_models_base.py @@ -186,7 +186,7 @@ class SurveyModel(BaseSurveyModel): @declared_attr def __tablename__(cls) -> str: - return cls.__name__ # type: nocheck + return cls.__name__ # type: ignore async def get_survey_info(self): info = await super(SurveyModel, self).get_survey_info() diff --git a/src/gisaf/models/map_bases.py b/src/gisaf/models/map_bases.py index 3c3e760..8472344 100644 --- a/src/gisaf/models/map_bases.py +++ b/src/gisaf/models/map_bases.py @@ -11,7 +11,7 @@ from gisaf.models.store import Store class BaseStyle(Model, table=True): __table_args__ = gisaf_map.table_args - __tablename__ = 'map_base_style' + __tablename__: str = 'map_base_style' # type: ignore class Admin: menu = 'Other' @@ -19,18 +19,18 @@ class BaseStyle(Model, table=True): id: int | None = Field(primary_key=True, default=None) name: str - style: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) - mbtiles: str = Field(sa_type=String(50)) + style: dict[str, Any] | None = Field(sa_type=JSON(none_as_null=True)) # type: ignore + mbtiles: str = Field(sa_type=String(50)) # type: ignore static_tiles_url: str enabled: bool = True def __repr__(self): - return ''.format(self=self) + return f'' class BaseMap(Model, table=True): __table_args__ = gisaf_map.table_args - __tablename__ = 'base_map' + __tablename__: str = 'base_map' # type: ignore class Admin: menu = 'Other' @@ -39,14 +39,14 @@ class BaseMap(Model, table=True): name: str layers: list['BaseMapLayer'] = Relationship(back_populates='base_map') - def __repr__(self): - return ''.format(self=self) + def __repr__(self) -> str: + return f'' - def __str__(self): + def __str__(self) -> str: return self.name @classmethod - def selectinload(cls): + def selectinload(cls) -> list[list['BaseMapLayer']]: return [ cls.layers ] @@ -54,7 +54,7 @@ class BaseMap(Model, table=True): class BaseMapLayer(Model, table=True): __table_args__ = gisaf_map.table_args - __tablename__ = 'base_map_layer' + __tablename__: str = 'base_map_layer' # type: ignore class Admin: menu = 'Other' @@ -63,7 +63,7 @@ class BaseMapLayer(Model, table=True): base_map_id: int = Field(foreign_key=gisaf_map.table('base_map.id'), index=True) base_map: BaseMap = Relationship(back_populates='layers') - store: str = Field(sa_type=String(100)) + store: str = Field(sa_type=String(100)) # type: ignore @classmethod def selectinload(cls): diff --git a/src/gisaf/security.py b/src/gisaf/security.py index 79011c7..a03943d 100644 --- a/src/gisaf/security.py +++ b/src/gisaf/security.py @@ -40,6 +40,17 @@ class TokenData(BaseModel): oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token", auto_error=False) +credentials_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate credentials", + headers={"WWW-Authenticate": "Bearer"}, +) + +expired_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Token has expired", + headers={"WWW-Authenticate": "Bearer"}, +) def get_password_hash(password: str): return pwd_context.hash(password) @@ -97,17 +108,7 @@ def verify_password(user: User, plain_password): async def get_current_user( - token: str = Depends(oauth2_scheme)) -> UserRead | None: - credentials_exception = HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Could not validate credentials", - headers={"WWW-Authenticate": "Bearer"}, - ) - expired_exception = HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Token has expired", - headers={"WWW-Authenticate": "Bearer"}, - ) + token: str = Depends(oauth2_scheme)) -> UserRead | None: if token is None: return None try: diff --git a/src/gisaf/tiles.py b/src/gisaf/tiles.py new file mode 100644 index 0000000..081bd06 --- /dev/null +++ b/src/gisaf/tiles.py @@ -0,0 +1,196 @@ +""" +mbtile server + +Instructions (example): + +cd map ## Matches tilesBaseDir in config + +curl -O http://download.geofabrik.de/asia/india/southern-zone-latest.osm.pbf + +tilemaker \ + --config /usr/local/lib/gisaf_src/tilemaker_src/tilemaker/resources/config-openmaptiles.json \ + --process /usr/local/lib/gisaf_src/tilemaker_src/tilemaker/resources/process-openmaptiles.lua \ + --input southern-zone-latest.osm.pbf \ + --output southern-zone-latest.mbtile + + +---- + +Get the style from https://github.com/openmaptiles, eg. +curl -o osm-bright-full.json https://raw.githubusercontent.com/openmaptiles/osm-bright-gl-style/master/style.json +## Minify json: +python -c 'import json, sys;json.dump(json.load(sys.stdin), sys.stdout)' < osm-bright-full.json > osm-bright.json + +And copy the style to the gisaf_map.map_base_style table, +with the name matching the file name +(TODO: this would need de-coupling the source mbtile and the style) + +---- + +Get the sprites from openmaptiles:` + +cd tiles ## Matches tilesSpriteBaseDir in config + +curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite.png' +curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite.json' +curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite@2x.png' +curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite@2x.json' + +TODO: TO migrate - 3/1/2024: this was copied from legacy gisaf without change +See: https://github.com/gis-ops/tutorials/blob/master/webservices/fastapi/fastapi_auth_vector_tiles.md +""" + +from pathlib import Path +from json import loads, dumps +import logging + +from fastapi import FastAPI, Response, responses, HTTPException, status +from fastapi.staticfiles import StaticFiles +import aiosqlite + +from gisaf.config import conf + +logger = logging.getLogger('gisaf tile server') + +api = FastAPI( + default_response_class=responses.ORJSONResponse, +) + +OSM_ATTRIBUTION = '© OpenStreetMap contributors' + +class MBTiles: + def __init__(self, file_path, style_name): + self.file_path = file_path + self.name = style_name + self.scheme = 'tms' + self.etag = f'W/"{hex(int(file_path.stat().st_mtime))[2:]}"' + + async def connect(self): + self.db = await aiosqlite.connect(self.file_path) + self.metadata = {} + try: + async with self.db.execute('select name, value from metadata') as cursor: + async for row in cursor: + self.metadata[row[0]] = row[1] + except aiosqlite.DatabaseError as err: + logger.warning(f'Cannot read {self.file_path}, will not be able to serve tiles (error: {err.args[0]})') + + self.metadata['bounds'] = [float(v) for v in self.metadata['bounds'].split(',')] + self.metadata['maxzoom'] = int(self.metadata['maxzoom']) + self.metadata['minzoom'] = int(self.metadata['minzoom']) + + async def get_style(self, style_record, request): + """ + Generate on the fly the style + """ + if conf.map.tileServer.useRequestUrl: + base_url = request.url.parent + else: + base_url = conf.map.tileServer.spriteBaseUrl + base_tiles_url = f"{base_url}/tiles/{self.name}" + scheme = self.scheme + ## TODO: avoid parse and serialize at every request + layers = loads(style_record['style'])['layers'] + for layer in layers: + if 'source' in layer: + layer['source'] = 'gisafTiles' + resp = { + 'basename': self.file_path.stem, + #'center': self.center, + 'description': f'Extract of {self.file_path.stem} from OSM, powered by Gisaf', + 'format': self.metadata['format'], + 'id': f'gisaftiles_{self.name}', + 'maskLevel': 5, + 'name': self.name, + #'pixel_scale': 256, + #'planettime': '1499040000000', + 'tilejson': '2.0.0', + 'version': 8, + 'glyphs': f"/assets/fonts/glyphs/{{fontstack}}/{{range}}.pbf", + 'sprite': f"{base_url}{conf.map.tileServer.spriteUrl}", + 'sources': { + 'gisafTiles': { + 'type': 'vector', + 'tiles': [ + f'{base_tiles_url}/{{z}}/{{x}}/{{y}}.pbf', + ], + 'maxzoom': self.metadata['maxzoom'], + 'minzoom': self.metadata['minzoom'], + 'bounds': self.metadata['bounds'], + 'scheme': scheme, + 'attribution': OSM_ATTRIBUTION, + 'version': self.metadata['version'], + } + }, + 'layers': layers, + } + return resp + + +class MBTilesRegistry: + mbtiles: dict[str, MBTiles] + async def setup(self): + """ + Read all mbtiles, construct styles + """ + self.mbtiles = {} + for file_path in Path(conf.map.tileServer.baseDir).glob('*.mbtiles'): + mbtiles = MBTiles(file_path, file_path.stem) + self.mbtiles[file_path.stem] = mbtiles + await mbtiles.connect() + + async def shutdown(self): + """ + Tear down the connection to the mbtiles files + """ + for mbtiles in self.mbtiles.values(): + await mbtiles.db.close() + + +@api.get('/{style_name}/{z}/{x}/{y}.pbf') +async def get_tile(request, style_name: str, z:int, x: int, y: int, + response: Response): + """ + Return the specific tile + """ + if style_name not in registry.mbtiles: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + mbtiles = registry.mbtiles[style_name] + + if request.headers.get('If-None-Match') == mbtiles.etag: + request.not_modified = True + return {} + + response.headers['Content-Encoding'] = 'gzip' + response.headers['Content-Type'] = 'application/octet-stream' + request.response_etag = mbtiles.etag + async with mbtiles.db.execute('select tile_data from tiles where zoom_level=? and tile_column=? and tile_row=?', + (z, x, y)) as cursor: + async for row in cursor: + return row[0] + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + + +#@routes.get('/sprite/{name:\S+}') +#async def get_sprite(request): + + +@api.get('/{style_name}') +async def get_style(request, style_name: str): + """ + Return the base style. + Note that normal operations are processed through graphql (resolve_base_style) + :param request: + :return: + """ + if style_name not in registry.mbtiles: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + mbtiles = registry.mbtiles[style_name] + return await mbtiles.get_style() + + +registry = MBTilesRegistry() + +api.mount("/sprite", + StaticFiles(directory=conf.map.tileServer.spriteBaseDir), + name="sprites")