""" mbtile server Instructions (example): cd map ## Matches tilesBaseDir in config curl http://download.geofabrik.de/asia/india/southern-zone-latest.osm.pbf -o osm.pbf TILEMAKER_SRC=/home/phil/gisaf_misc/tilemaker # Or, for fish set TILEMAKER_SRC /home/phil/gisaf_misc/tilemaker cp $TILEMAKER_SRC/resources/config-openmaptiles.json . cp $TILEMAKER_SRC/resources/process-openmaptiles.lua . ## Edit config-openmaptiles.json, eg add in "settings": # "bounding_box":[79.76777,11.96541,79.86909,12.04497] vi config-openmaptiles.json ## Generate mbtile database: tilemaker \ --config config-openmaptiles.json \ --process process-openmaptiles.lua \ --input osm.pbf \ --output osm.mbtiles ## Generate static tiles files mkdir osm tilemaker \ --config config-openmaptiles.json \ --process process-openmaptiles.lua \ --input osm.pbf \ --output osm ---- Get the style from https://github.com/openmaptiles, eg. curl -o osm-bright-full.json https://raw.githubusercontent.com/openmaptiles/osm-bright-gl-style/master/style.json ## Minify json: python -c 'import json, sys;json.dump(json.load(sys.stdin), sys.stdout)' < osm-bright-full.json > osm-bright.json ---- Get the sprites from openmaptiles: cd tiles ## Matches tilesSpriteBaseDir in config curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite.png' curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite.json' curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite@2x.png' curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite@2x.json' """ # noqa: E501 import logging import tarfile from pathlib import Path from json import loads, dumps from io import BytesIO from fastapi import FastAPI, Response, HTTPException, Request from fastapi.staticfiles import StaticFiles import aiosqlite from treetrail.config import conf from treetrail.models import BaseMapStyles from treetrail.utils import mkdir logger = logging.getLogger('treetrail tile server') tiles_app = FastAPI() def get_tiles_tar_path(style): ## FIXME: use conf return Path(__file__).parent.parent/f'treetrail-app/src/data/tiles/{style}.tar' OSM_ATTRIBUTION = '' \ '© OpenStreetMap contributors' class MBTiles: def __init__(self, file_path, style_name): self.file_path = file_path self.name = style_name self.scheme = 'tms' self.etag = f'W/"{hex(int(file_path.stat().st_mtime))[2:]}"' self.style_layers: list[dict] ## FIXME: use conf try: with open(Path(__file__).parent.parent / 'treetrail-app' / 'src' / 'assets' / 'map' / 'style.json') as f: style = loads(f.read()) self.style_layers = style['layers'] except FileNotFoundError: self.style_layers = [] for layer in self.style_layers: if 'source' in layer: layer['source'] = 'treeTrailTiles' async def connect(self): self.db = await aiosqlite.connect(self.file_path) self.metadata = {} try: async with self.db.execute('select name, value from metadata') as cursor: async for row in cursor: self.metadata[row[0]] = row[1] except aiosqlite.DatabaseError as err: logger.warning(f'Cannot read {self.file_path}, will not be able' f' to serve tiles (error: {err.args[0]})') ## Fix types if 'bounds' in self.metadata: self.metadata['bounds'] = [float(v) for v in self.metadata['bounds'].split(',')] self.metadata['maxzoom'] = int(self.metadata['maxzoom']) self.metadata['minzoom'] = int(self.metadata['minzoom']) logger.info(f'Serving tiles in {self.file_path}') async def get_style(self, request: Request): """ Generate on the fly the style """ if conf.tiles.useRequestUrl: base_url = str(request.base_url).removesuffix("/") else: base_url = conf.tiles.spriteBaseUrl base_tiles_url = f"{base_url}/tiles/{self.name}" scheme = self.scheme resp = { 'basename': self.file_path.stem, #'center': self.center, 'description': f'Extract of {self.file_path.stem} from OSM by Gisaf', 'format': self.metadata['format'], 'id': f'gisaftiles_{self.name}', 'maskLevel': 5, 'name': self.name, #'pixel_scale': 256, #'planettime': '1499040000000', 'tilejson': '2.0.0', 'version': 8, 'glyphs': "/assets/fonts/glyphs/{fontstack}/{range}.pbf", 'sprite': f"{conf.tiles.spriteUrl}", 'sources': { 'treeTrailTiles': { 'type': 'vector', 'tiles': [ f'{base_tiles_url}/{{z}}/{{x}}/{{y}}.pbf', ], 'maxzoom': self.metadata['maxzoom'], 'minzoom': self.metadata['minzoom'], 'bounds': self.metadata['bounds'], 'scheme': scheme, 'attribution': OSM_ATTRIBUTION, 'version': self.metadata['version'], } }, 'layers': self.style_layers, } return resp async def get_tile(self, z, x, y): async with self.db.execute( 'select tile_data from tiles where zoom_level=? ' \ 'and tile_column=? and tile_row=?', (z, x, y)) as cursor: async for row in cursor: return row[0] async def get_all_tiles_tar(self, style, request): s = 0 n = 0 buf = BytesIO() with tarfile.open(fileobj=buf, mode='w') as tar: ## Add tiles async with self.db.execute('select zoom_level, tile_column, ' \ 'tile_row, tile_data from tiles') as cursor: async for row in cursor: z, x, y, tile = row tar_info = tarfile.TarInfo() tar_info.path = f'{style}/{z}/{x}/{y}.pbf' tar_info.size = len(tile) tar.addfile(tar_info, BytesIO(tile)) logger.debug(f'Added {style}/{z}/{x}/{y} ({len(tile)})') n += 1 s += len(tile) logger.info(f'Added {n} files ({s} bytes)') ## Add style tar_info = tarfile.TarInfo() tar_info.path = f'style/{style}' style_definition = await self.get_style(request) style_data = dumps(style_definition, check_circular=False).encode('utf-8') tar_info.size = len(style_data) tar.addfile(tar_info, BytesIO(style_data)) ## Add sprites ex. /tiles/sprite/sprite.json and /tiles/sprite/sprite.png tar.add(conf.tiles.spriteBaseDir, 'sprite') ## Extract buf.seek(0) ## XXX: Could write to file: #file_path = get_tiles_tar_path(style) return buf.read() class MBTilesRegistry: mbtiles: dict[str, MBTiles] async def setup(self, app): """ Read all mbtiles, construct styles """ self.mbtiles = {} for file_path in Path(conf.tiles.baseDir).glob('*.mbtiles'): mbtiles = MBTiles(file_path, file_path.stem) self.mbtiles[file_path.stem] = mbtiles await mbtiles.connect() async def shutdown(self, app): """ Tear down the connection to the mbtiles files """ for mbtiles in self.mbtiles.values(): await mbtiles.db.close() gzip_headers = { 'Content-Encoding': 'gzip', 'Content-Type': 'application/octet-stream', } tar_headers = { 'Content-Type': 'application/x-tar', } @tiles_app.get('/styles') async def get_styles() -> BaseMapStyles: """Styles for the map background. There are 2 types: - found on the embedded tiles server, that can be used offline - external providers, defined in the config with a simple url """ return BaseMapStyles( external=conf.mapStyles, embedded=list(registry.mbtiles.keys()) ) @tiles_app.get('/{style_name}/{z}/{x}/{y}.pbf') async def get_tile(style_name:str, z:int, x:int, y:int): """ Return the specific tile """ ## TODO: implement etag #if request.headers.get('If-None-Match') == mbtiles.etag: # request.not_modified = True # return web.Response(body=None) #request.response_etag = mbtiles.etag if style_name not in registry.mbtiles: raise HTTPException(status_code=404) mbtiles = registry.mbtiles[style_name] try: tile = await mbtiles.get_tile(z, x, y) except Exception as err: logger.info(f'Cannot get tile {z}, {x}, {y}') logger.exception(err) raise HTTPException(status_code=404) else: return Response(content=tile, media_type="application/json", headers=gzip_headers) @tiles_app.get('/{style_name}/all.tar') async def get_tiles_tar(style_name, request: Request): """ Get a tar file with all the tiles. Typically, used to feed into the browser's cache for offline use. """ mbtiles: MBTiles = registry.mbtiles[style_name] tar = await mbtiles.get_all_tiles_tar(style_name, request) return Response(content=tar, media_type="application/x-tar", headers=tar_headers) #@tiles_app.get('/sprite/{name:\S+}') #async def get_sprite(request): @tiles_app.get('/style/{style_name}') async def get_style(style_name: str, request: Request): """ Return the base style. """ if style_name not in registry.mbtiles: raise HTTPException(status_code=404) mbtiles = registry.mbtiles[style_name] return await mbtiles.get_style(request) registry = MBTilesRegistry() tiles_app.mount("/sprite", StaticFiles(directory=mkdir(conf.tiles.spriteBaseDir)), name="tiles_sprites") tiles_app.mount('/osm', StaticFiles(directory=mkdir(conf.tiles.osmBaseDir)), name='tiles_osm')