diff --git a/init-server.py b/init-server.py index 01f0375..228d886 100755 --- a/init-server.py +++ b/init-server.py @@ -101,6 +101,7 @@ def prepare_http_host(c): for file in [ 'downloader.py', 'mounter.py', + 'metadata_to_tilejson.py', ]: put( c, @@ -122,6 +123,18 @@ def prepare_http_host(c): def debug_tmp(c): + for file in [ + 'downloader.py', + 'mounter.py', + 'metadata_to_tilejson.py', + ]: + put( + c, + SCRIPTS_DIR / 'http_host' / file, + HTTP_HOST_BIN, + permissions='755', + ) + for file in ['nginx_template.conf', 'nginx_sync.py']: put( c, @@ -130,6 +143,9 @@ def debug_tmp(c): create_parent_dir=True, ) + c.sudo('chown -R ofm:ofm /data/ofm/http_host') + c.sudo('chown -R nginx:nginx /data/ofm/http_host/logs_nginx') + @click.command() @click.argument('hostname') @@ -137,12 +153,11 @@ def debug_tmp(c): @click.option('--user', help='SSH user (if not in .ssh/config)') @click.option('--tile-gen', is_flag=True, help='Install tile-gen task') @click.option('--http-host', is_flag=True, help='Install http-host task') -@click.option('--reboot', 'do_reboot', is_flag=True, help='Reboot after installation') @click.option('--debug', is_flag=True) @click.option( '--skip-shared', is_flag=True, help='Skip the shared installtion step (useful for development)' ) -def main(hostname, user, port, tile_gen, http_host, skip_shared, do_reboot, debug): +def main(hostname, user, port, tile_gen, http_host, skip_shared, debug): if not debug and not click.confirm(f'Run script on {hostname}?'): return @@ -182,9 +197,6 @@ def main(hostname, user, port, tile_gen, http_host, skip_shared, do_reboot, debu if http_host: prepare_http_host(c) - if do_reboot: - reboot(c) - if __name__ == '__main__': main() diff --git a/scripts/http_host/.gitignore b/scripts/http_host/.gitignore new file mode 100644 index 0000000..a6c57f5 --- /dev/null +++ b/scripts/http_host/.gitignore @@ -0,0 +1 @@ +*.json diff --git a/scripts/http_host/metadata_to_tilejson.py b/scripts/http_host/metadata_to_tilejson.py new file mode 100755 index 0000000..bacb66e --- /dev/null +++ b/scripts/http_host/metadata_to_tilejson.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import json +from pathlib import Path +from pprint import pprint + +import click + + +@click.command() +@click.argument( + 'metadata_path', type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path) +) +@click.argument('tilejson_path', type=click.Path(path_type=Path)) +@click.argument('url_prefix') +@click.option('--minify', is_flag=True, help='Minify the generated JSON') +def cli(metadata_path: Path, tilejson_path: Path, url_prefix: str, minify: bool): + """ + Takes a MBTiles metadata.json and generates a TileJSON 3.0.0 file + + URL_PREFIX: Base URL to use as a prefix for tiles in the generated TileJSON. + + Reference: https://github.com/mapbox/tilejson-spec/tree/master/3.0.0 + """ + + tilejson = dict(tilejson='3.0.0') + + with open(metadata_path) as fp: + metadata = json.load(fp) + + metadata_json_key = json.loads(metadata.pop('json')) + + tilejson['tiles'] = [url_prefix.rstrip('/') + '/{z}/{x}/{y}.pbf'] + + '' + tilejson['vector_layers'] = metadata_json_key.pop('vector_layers') + assert not metadata_json_key # check that no more keys left + + tilejson['attribution'] = metadata.pop('attribution') + tilejson['bounds'] = [float(n) for n in metadata.pop('bounds').split(',')] + tilejson['center'] = [float(n) for n in metadata.pop('center').split(',')] + tilejson['center'][2] = 1 + + tilejson['description'] = metadata.pop('description') + + tilejson['maxzoom'] = int(metadata.pop('maxzoom')) + tilejson['minzoom'] = int(metadata.pop('minzoom')) + + tilejson['name'] = metadata.pop('name') + tilejson['version'] = metadata.pop('version') + + with open(tilejson_path, 'w') as fp: + if minify: + json.dump(tilejson, fp, ensure_ascii=False, separators=(',', ':')) + else: + json.dump(tilejson, fp, ensure_ascii=False, indent=2) + + +if __name__ == '__main__': + cli() diff --git a/scripts/http_host/nginx_sync/nginx_sync.py b/scripts/http_host/nginx_sync/nginx_sync.py index dd0d066..380c852 100755 --- a/scripts/http_host/nginx_sync/nginx_sync.py +++ b/scripts/http_host/nginx_sync/nginx_sync.py @@ -7,6 +7,9 @@ from pathlib import Path import click +RUNS_DIR = Path('/data/ofm/http_host/runs') + + @click.command() def cli(): if not Path('/etc/fstab').exists(): @@ -30,9 +33,42 @@ def cli(): area, version = subdir.name.split('-') + run_dir = RUNS_DIR / area / version + if not run_dir.is_dir(): + print(f"{run_dir} doesn't exists, skipping") + continue + + tilejson_path = run_dir / 'tilejson-tiles-org.json' + + metadata_path = subdir / 'metadata.json' + if not metadata_path.is_file(): + print(f"{metadata_path} doesn't exists, skipping") + continue + + url_prefix = f'https://tiles.openfreemap.org/{area}/{version}/tiles//' + + subprocess.run( + [ + sys.executable, + Path(__file__).parent.parent / 'metadata_to_tilejson.py', + '--minify', + metadata_path, + tilejson_path, + url_prefix, + ], + check=True, + ) + version_str = rf""" - location /{area}/{version}/ {{ # trailing hash important - alias {subdir}/; # trailing hash important + location /{area}/{version} {{ # no trailing hash + alias {tilejson_path}; # no trailing hash + + add_header Cache-Control public; + expires 10y; + }} + + location /{area}/{version}/ {{ # trailing hash + alias {subdir}/tiles/; # trailing hash try_files $uri @empty; add_header Cache-Control public; @@ -45,7 +81,7 @@ def cli(): if not help_text: help_text = ( '\ntest with:\n' - f'curl -H "Host: ofm" -I http://localhost/{area}/{version}/tiles/14/8529/5975.pbf' + f'curl -H "Host: ofm" -I http://localhost/{area}/{version}/14/8529/5975.pbf' ) nginx_template = nginx_template.replace('___LOCATION_BLOCKS___', location_block_str) diff --git a/scripts/http_host/nginx_sync/nginx_template.conf b/scripts/http_host/nginx_sync/nginx_template.conf index 6b065ec..cd4a2c4 100644 --- a/scripts/http_host/nginx_sync/nginx_template.conf +++ b/scripts/http_host/nginx_sync/nginx_template.conf @@ -1,5 +1,5 @@ server { - server_name ofm tiles.openfreemaps.org; + server_name ofm tiles.openfreemap.org; # disabling access log by default # access_log /data/ofm/http_host/logs_nginx/nginx-access.log access_json buffer=32k;