From 9059c27707eaad0af025a28507438d972a19626f Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Wed, 3 Jan 2024 20:48:03 +0100 Subject: [PATCH] host_manager --- init-server.py | 12 ++-- scripts/http_host/host_manager.py | 9 ++- scripts/http_host/http_host_lib/__init__.py | 3 +- .../http_host_lib/download_tileset.py | 2 +- scripts/http_host/http_host_lib/nginx.py | 69 +++++++++++-------- .../http_host_lib/set_tileset_versions.py | 4 +- ssh_lib/pkg_base.py | 2 +- 7 files changed, 59 insertions(+), 42 deletions(-) diff --git a/init-server.py b/init-server.py index ddbfbb2..6f31558 100755 --- a/init-server.py +++ b/init-server.py @@ -88,7 +88,7 @@ def prepare_tile_gen(c): c.sudo('chown ofm:ofm -R /data/ofm/tile_gen/bin') -def prepare_http_host(c): +def prepare_http_host(c, skip_cron: bool): nginx(c) certbot(c) c1000k(c) @@ -107,7 +107,8 @@ def prepare_http_host(c): c.sudo('/data/ofm/venv/bin/pip install -e /data/ofm/http_host/bin') # always last - put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') + if not skip_cron: + put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') def upload_https_host_files(c): @@ -131,7 +132,7 @@ def upload_certificates(c): def debug_tmp(c): upload_https_host_files(c) - put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') + # put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') @click.command() @@ -144,7 +145,8 @@ def debug_tmp(c): @click.option( '--skip-shared', is_flag=True, help='Skip the shared installtion step (useful for development)' ) -def main(hostname, user, port, tile_gen, http_host, skip_shared, debug): +@click.option('--skip-cron', is_flag=True, help='Skip the cronjob (useful for development)') +def main(hostname, user, port, tile_gen, http_host, skip_shared, skip_cron, debug): if not debug and not click.confirm(f'Run script on {hostname}?'): return @@ -182,7 +184,7 @@ def main(hostname, user, port, tile_gen, http_host, skip_shared, debug): prepare_tile_gen(c) if http_host: - prepare_http_host(c) + prepare_http_host(c, skip_cron=skip_cron) if __name__ == '__main__': diff --git a/scripts/http_host/host_manager.py b/scripts/http_host/host_manager.py index ca89d77..dc07f9b 100755 --- a/scripts/http_host/host_manager.py +++ b/scripts/http_host/host_manager.py @@ -19,12 +19,12 @@ from http_host_lib.utils import assert_linux, assert_single_process, assert_sudo def cli(): """ Manages OpenFreeMap HTTP hosts, including:\n - - Deploying the correct versions of tilesets\n - Downloading assets\n - Downloading tilesets\n - Mounting directories\n - Updating nginx config\n - - Running the sync cron task every minute + - Setting the latest versions of tilesets\n + - Running the sync cron task (called every minute) """ @@ -170,11 +170,10 @@ def sync(ctx): print('running sync') print(datetime.datetime.now(tz=datetime.timezone.utc)) - assert_single_process() - download_done = False download_done += ctx.invoke(download_tileset, area='monaco') - # download_done += ctx.invoke(download_tileset, area='planet') + download_done += ctx.invoke(download_tileset, area='planet') + if download_done: ctx.invoke(mount) diff --git a/scripts/http_host/http_host_lib/__init__.py b/scripts/http_host/http_host_lib/__init__.py index c6db8bf..63f4722 100644 --- a/scripts/http_host/http_host_lib/__init__.py +++ b/scripts/http_host/http_host_lib/__init__.py @@ -6,4 +6,5 @@ TEMPLATES_DIR = Path(__file__).parent / 'templates' DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets') -MNT_DIR = Path('/mnt/ofm/') +MNT_DIR = Path('/mnt/ofm') +OFM_CONFIG_DIR = Path('/data/ofm/config') diff --git a/scripts/http_host/http_host_lib/download_tileset.py b/scripts/http_host/http_host_lib/download_tileset.py index d81f37c..8f153ed 100644 --- a/scripts/http_host/http_host_lib/download_tileset.py +++ b/scripts/http_host/http_host_lib/download_tileset.py @@ -13,7 +13,7 @@ def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> boo returns True if downloaded something """ - click.echo(f'downloading area: {area}, version: {version}') + click.echo(f'downloading {area} {version}') version_dir = runs_dir / area / version btrfs_file = version_dir / 'tiles.btrfs' diff --git a/scripts/http_host/http_host_lib/nginx.py b/scripts/http_host/http_host_lib/nginx.py index 7b1d151..311574c 100644 --- a/scripts/http_host/http_host_lib/nginx.py +++ b/scripts/http_host/http_host_lib/nginx.py @@ -2,7 +2,7 @@ import subprocess import sys from pathlib import Path -from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR, TEMPLATES_DIR +from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR, OFM_CONFIG_DIR, TEMPLATES_DIR def write_nginx_config(): @@ -15,9 +15,7 @@ def write_nginx_config(): for subdir in MNT_DIR.iterdir(): if not subdir.is_dir(): continue - area, version = subdir.name.split('-') - location_str += create_version_location(area, version, subdir) if not curl_text: @@ -27,8 +25,7 @@ def write_nginx_config(): f'curl -I https://tiles.openfreemap.org/{area}/{version}/14/8529/5975.pbf' ) - for area in ['monaco', 'planet']: - location_str += create_latest_location(area) + location_str += create_latest_locations() nginx_template = nginx_template.replace('___LOCATION_BLOCKS___', location_str) @@ -71,7 +68,43 @@ def create_version_location(area: str, version: str, subdir: Path) -> str: # TODO # target 10y return f""" - location /{area}/{version} {{ # no trailing hash + location /{area}/{version} {{ # no trailing hash + alias {tilejson_path}; # no trailing hash + default_type application/json; + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; + expires 1d; + }} + + location /{area}/{version}/ {{ # trailing hash + alias {subdir}/tiles/; # trailing hash + try_files $uri @empty; + + add_header Content-Encoding gzip; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; + expires 1d; # target 10y + }} + """ + + +def create_latest_locations() -> str: + location_str = '' + + local_version_files = OFM_CONFIG_DIR.glob('tileset_version_*.txt') + for file in local_version_files: + area = file.stem.split('_')[-1] + with open(file) as fp: + version = fp.read().strip() + print(f' setting latest version for {area}: {version}') + + run_dir = DEFAULT_RUNS_DIR / area / version + tilejson_path = run_dir / 'tilejson-tiles-org.json' + assert tilejson_path.exists() + + location_str += f""" + location /{area} {{ # no trailing hash alias {tilejson_path}; # no trailing hash default_type application/json; @@ -79,26 +112,6 @@ def create_version_location(area: str, version: str, subdir: Path) -> str: add_header Cache-Control public; expires 1d; }} + """ - location /{area}/{version}/ {{ # trailing hash - alias {subdir}/tiles/; # trailing hash - try_files $uri @empty; - - add_header Content-Encoding gzip; - add_header 'Access-Control-Allow-Origin' '*' always; - add_header Cache-Control public; - expires 1d; # target 10y - }} - """ - - -def create_latest_location(area: str) -> str: - local_version_file = Path(f'/data/ofm/config/deployed_tiles_{area}.txt') - - if not local_version_file.exists(): - return '' - - with open(local_version_file) as fp: - version_str = fp.read().strip() - - print(version_str) + return location_str diff --git a/scripts/http_host/http_host_lib/set_tileset_versions.py b/scripts/http_host/http_host_lib/set_tileset_versions.py index 73eb9c4..cbaf886 100644 --- a/scripts/http_host/http_host_lib/set_tileset_versions.py +++ b/scripts/http_host/http_host_lib/set_tileset_versions.py @@ -2,6 +2,8 @@ from pathlib import Path import requests +from http_host_lib import OFM_CONFIG_DIR + def set_tileset_versions(): need_nginx_sync = False @@ -12,7 +14,7 @@ def set_tileset_versions(): remote_version = r.text.strip() print(f' remote version for {area}: {remote_version}') - local_version_file = Path(f'/data/ofm/config/deployed_tiles_{area}.txt') + local_version_file = OFM_CONFIG_DIR / f'tileset_version_{area}.txt' if not local_version_file.exists(): local_version_start = None diff --git a/ssh_lib/pkg_base.py b/ssh_lib/pkg_base.py index 3414f64..98c07be 100644 --- a/ssh_lib/pkg_base.py +++ b/ssh_lib/pkg_base.py @@ -18,7 +18,7 @@ def pkg_base(c): 'unzip', 'wget', 'psmisc', - 'util-linux' + 'util-linux', # 'btrfs-progs', #