Initial commit
This commit is contained in:
commit
f4cf78603a
25 changed files with 2895 additions and 0 deletions
307
src/treetrail/tiles.py
Normal file
307
src/treetrail/tiles.py
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
"""
|
||||
mbtile server
|
||||
|
||||
Instructions (example):
|
||||
|
||||
cd map ## Matches tilesBaseDir in config
|
||||
|
||||
curl http://download.geofabrik.de/asia/india/southern-zone-latest.osm.pbf -o osm.pbf
|
||||
|
||||
TILEMAKER_SRC=/home/phil/gisaf_misc/tilemaker
|
||||
# Or, for fish
|
||||
set TILEMAKER_SRC /home/phil/gisaf_misc/tilemaker
|
||||
cp $TILEMAKER_SRC/resources/config-openmaptiles.json .
|
||||
cp $TILEMAKER_SRC/resources/process-openmaptiles.lua .
|
||||
|
||||
## Edit config-openmaptiles.json, eg add in "settings":
|
||||
# "bounding_box":[79.76777,11.96541,79.86909,12.04497]
|
||||
vi config-openmaptiles.json
|
||||
|
||||
## Generate mbtile database:
|
||||
tilemaker \
|
||||
--config config-openmaptiles.json \
|
||||
--process process-openmaptiles.lua \
|
||||
--input osm.pbf \
|
||||
--output osm.mbtiles
|
||||
|
||||
|
||||
## Generate static tiles files
|
||||
mkdir osm
|
||||
tilemaker \
|
||||
--config config-openmaptiles.json \
|
||||
--process process-openmaptiles.lua \
|
||||
--input osm.pbf \
|
||||
--output osm
|
||||
|
||||
----
|
||||
|
||||
Get the style from https://github.com/openmaptiles, eg.
|
||||
curl -o osm-bright-full.json https://raw.githubusercontent.com/openmaptiles/osm-bright-gl-style/master/style.json
|
||||
## Minify json:
|
||||
python -c 'import json, sys;json.dump(json.load(sys.stdin), sys.stdout)' < osm-bright-full.json > osm-bright.json
|
||||
|
||||
----
|
||||
|
||||
Get the sprites from openmaptiles:
|
||||
|
||||
cd tiles ## Matches tilesSpriteBaseDir in config
|
||||
|
||||
curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite.png'
|
||||
curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite.json'
|
||||
curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite@2x.png'
|
||||
curl -O 'https://openmaptiles.github.io/osm-bright-gl-style/sprite@2x.json'
|
||||
|
||||
""" # noqa: E501
|
||||
|
||||
import logging
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
from json import loads, dumps
|
||||
from io import BytesIO
|
||||
|
||||
from fastapi import FastAPI, Response, HTTPException, Request
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
import aiosqlite
|
||||
|
||||
from treetrail.config import conf
|
||||
from treetrail.models import BaseMapStyles
|
||||
from treetrail.utils import mkdir
|
||||
|
||||
logger = logging.getLogger('treetrail tile server')
|
||||
|
||||
tiles_app = FastAPI()
|
||||
|
||||
def get_tiles_tar_path(style):
|
||||
## FIXME: use conf
|
||||
return Path(__file__).parent.parent/f'treetrail-app/src/data/tiles/{style}.tar'
|
||||
|
||||
|
||||
OSM_ATTRIBUTION = '<a href=\"http://www.openstreetmap.org/about/" target="_blank">' \
|
||||
'© OpenStreetMap contributors</a>'
|
||||
|
||||
|
||||
class MBTiles:
|
||||
def __init__(self, file_path, style_name):
|
||||
self.file_path = file_path
|
||||
self.name = style_name
|
||||
self.scheme = 'tms'
|
||||
self.etag = f'W/"{hex(int(file_path.stat().st_mtime))[2:]}"'
|
||||
self.style_layers: list[dict]
|
||||
## FIXME: use conf
|
||||
try:
|
||||
with open(Path(__file__).parent.parent / 'treetrail-app' / 'src' /
|
||||
'assets' / 'map' / 'style.json') as f:
|
||||
style = loads(f.read())
|
||||
self.style_layers = style['layers']
|
||||
except FileNotFoundError:
|
||||
self.style_layers = []
|
||||
for layer in self.style_layers:
|
||||
if 'source' in layer:
|
||||
layer['source'] = 'treeTrailTiles'
|
||||
|
||||
async def connect(self):
|
||||
self.db = await aiosqlite.connect(self.file_path)
|
||||
self.metadata = {}
|
||||
try:
|
||||
async with self.db.execute('select name, value from metadata') as cursor:
|
||||
async for row in cursor:
|
||||
self.metadata[row[0]] = row[1]
|
||||
except aiosqlite.DatabaseError as err:
|
||||
logger.warning(f'Cannot read {self.file_path}, will not be able'
|
||||
f' to serve tiles (error: {err.args[0]})')
|
||||
|
||||
## Fix types
|
||||
if 'bounds' in self.metadata:
|
||||
self.metadata['bounds'] = [float(v)
|
||||
for v in self.metadata['bounds'].split(',')]
|
||||
self.metadata['maxzoom'] = int(self.metadata['maxzoom'])
|
||||
self.metadata['minzoom'] = int(self.metadata['minzoom'])
|
||||
logger.info(f'Serving tiles in {self.file_path}')
|
||||
|
||||
async def get_style(self, request: Request):
|
||||
"""
|
||||
Generate on the fly the style
|
||||
"""
|
||||
if conf.tiles.useRequestUrl:
|
||||
base_url = str(request.base_url).removesuffix("/")
|
||||
else:
|
||||
base_url = conf.tiles.spriteBaseUrl
|
||||
base_tiles_url = f"{base_url}/tiles/{self.name}"
|
||||
scheme = self.scheme
|
||||
resp = {
|
||||
'basename': self.file_path.stem,
|
||||
#'center': self.center,
|
||||
'description': f'Extract of {self.file_path.stem} from OSM by Gisaf',
|
||||
'format': self.metadata['format'],
|
||||
'id': f'gisaftiles_{self.name}',
|
||||
'maskLevel': 5,
|
||||
'name': self.name,
|
||||
#'pixel_scale': 256,
|
||||
#'planettime': '1499040000000',
|
||||
'tilejson': '2.0.0',
|
||||
'version': 8,
|
||||
'glyphs': "/assets/fonts/glyphs/{fontstack}/{range}.pbf",
|
||||
'sprite': f"{base_url}{conf.tiles.spriteUrl}",
|
||||
'sources': {
|
||||
'treeTrailTiles': {
|
||||
'type': 'vector',
|
||||
'tiles': [
|
||||
f'{base_tiles_url}/{{z}}/{{x}}/{{y}}.pbf',
|
||||
],
|
||||
'maxzoom': self.metadata['maxzoom'],
|
||||
'minzoom': self.metadata['minzoom'],
|
||||
'bounds': self.metadata['bounds'],
|
||||
'scheme': scheme,
|
||||
'attribution': OSM_ATTRIBUTION,
|
||||
'version': self.metadata['version'],
|
||||
}
|
||||
},
|
||||
'layers': self.style_layers,
|
||||
}
|
||||
return resp
|
||||
|
||||
async def get_tile(self, z, x, y):
|
||||
async with self.db.execute(
|
||||
'select tile_data from tiles where zoom_level=? ' \
|
||||
'and tile_column=? and tile_row=?', (z, x, y)) as cursor:
|
||||
async for row in cursor:
|
||||
return row[0]
|
||||
|
||||
async def get_all_tiles_tar(self, style, request):
|
||||
s = 0
|
||||
n = 0
|
||||
buf = BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode='w') as tar:
|
||||
## Add tiles
|
||||
async with self.db.execute('select zoom_level, tile_column, ' \
|
||||
'tile_row, tile_data from tiles') as cursor:
|
||||
async for row in cursor:
|
||||
z, x, y, tile = row
|
||||
tar_info = tarfile.TarInfo()
|
||||
tar_info.path = f'{style}/{z}/{x}/{y}.pbf'
|
||||
tar_info.size = len(tile)
|
||||
tar.addfile(tar_info, BytesIO(tile))
|
||||
logger.debug(f'Added {style}/{z}/{x}/{y} ({len(tile)})')
|
||||
n += 1
|
||||
s += len(tile)
|
||||
logger.info(f'Added {n} files ({s} bytes)')
|
||||
|
||||
## Add style
|
||||
tar_info = tarfile.TarInfo()
|
||||
tar_info.path = f'style/{style}'
|
||||
style_definition = await self.get_style(request)
|
||||
style_data = dumps(style_definition, check_circular=False).encode('utf-8')
|
||||
tar_info.size = len(style_data)
|
||||
tar.addfile(tar_info, BytesIO(style_data))
|
||||
|
||||
## Add sprites ex. /tiles/sprite/sprite.json and /tiles/sprite/sprite.png
|
||||
tar.add(conf.tiles.spriteBaseDir, 'sprite')
|
||||
|
||||
## Extract
|
||||
buf.seek(0)
|
||||
## XXX: Could write to file:
|
||||
#file_path = get_tiles_tar_path(style)
|
||||
return buf.read()
|
||||
|
||||
class MBTilesRegistry:
|
||||
mbtiles: dict[str, MBTiles]
|
||||
async def setup(self, app):
|
||||
"""
|
||||
Read all mbtiles, construct styles
|
||||
"""
|
||||
self.mbtiles = {}
|
||||
for file_path in Path(conf.tiles.baseDir).glob('*.mbtiles'):
|
||||
mbtiles = MBTiles(file_path, file_path.stem)
|
||||
self.mbtiles[file_path.stem] = mbtiles
|
||||
await mbtiles.connect()
|
||||
|
||||
async def shutdown(self, app):
|
||||
"""
|
||||
Tear down the connection to the mbtiles files
|
||||
"""
|
||||
for mbtiles in self.mbtiles.values():
|
||||
await mbtiles.db.close()
|
||||
|
||||
|
||||
gzip_headers = {
|
||||
'Content-Encoding': 'gzip',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
}
|
||||
|
||||
tar_headers = {
|
||||
'Content-Type': 'application/x-tar',
|
||||
}
|
||||
|
||||
|
||||
@tiles_app.get('/styles')
|
||||
async def get_styles() -> BaseMapStyles:
|
||||
"""Styles for the map background. There are 2 types:
|
||||
- found on the embedded tiles server, that can be used offline
|
||||
- external providers, defined in the config with a simple url
|
||||
"""
|
||||
return BaseMapStyles(
|
||||
external=conf.mapStyles,
|
||||
embedded=list(registry.mbtiles.keys())
|
||||
)
|
||||
|
||||
|
||||
@tiles_app.get('/{style_name}/{z}/{x}/{y}.pbf')
|
||||
async def get_tile(style_name:str, z:int, x:int, y:int):
|
||||
"""
|
||||
Return the specific tile
|
||||
"""
|
||||
## TODO: implement etag
|
||||
#if request.headers.get('If-None-Match') == mbtiles.etag:
|
||||
# request.not_modified = True
|
||||
# return web.Response(body=None)
|
||||
#request.response_etag = mbtiles.etag
|
||||
|
||||
if style_name not in registry.mbtiles:
|
||||
raise HTTPException(status_code=404)
|
||||
mbtiles = registry.mbtiles[style_name]
|
||||
try:
|
||||
tile = await mbtiles.get_tile(z, x, y)
|
||||
except Exception as err:
|
||||
logger.info(f'Cannot get tile {z}, {x}, {y}')
|
||||
logger.exception(err)
|
||||
raise HTTPException(status_code=404)
|
||||
else:
|
||||
return Response(content=tile,
|
||||
media_type="application/json",
|
||||
headers=gzip_headers)
|
||||
|
||||
|
||||
@tiles_app.get('/{style_name}/all.tar')
|
||||
async def get_tiles_tar(style_name, request: Request):
|
||||
"""
|
||||
Get a tar file with all the tiles. Typically, used to feed into
|
||||
the browser's cache for offline use.
|
||||
"""
|
||||
mbtiles: MBTiles = registry.mbtiles[style_name]
|
||||
tar = await mbtiles.get_all_tiles_tar(style_name, request)
|
||||
return Response(content=tar, media_type="application/x-tar", headers=tar_headers)
|
||||
|
||||
#@tiles_app.get('/sprite/{name:\S+}')
|
||||
#async def get_sprite(request):
|
||||
|
||||
|
||||
@tiles_app.get('/style/{style_name}')
|
||||
async def get_style(style_name: str, request: Request):
|
||||
"""
|
||||
Return the base style.
|
||||
"""
|
||||
if style_name not in registry.mbtiles:
|
||||
raise HTTPException(status_code=404)
|
||||
mbtiles = registry.mbtiles[style_name]
|
||||
|
||||
return await mbtiles.get_style(request)
|
||||
|
||||
|
||||
registry = MBTilesRegistry()
|
||||
|
||||
tiles_app.mount("/sprite",
|
||||
StaticFiles(directory=mkdir(conf.tiles.spriteBaseDir)),
|
||||
name="tiles_sprites")
|
||||
tiles_app.mount('/osm',
|
||||
StaticFiles(directory=mkdir(conf.tiles.osmBaseDir)),
|
||||
name='tiles_osm')
|
||||
Loading…
Add table
Add a link
Reference in a new issue