From 5ad3987c269d46582aeffd70c248362f633cb889 Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Wed, 3 Jan 2024 19:36:52 +0100 Subject: [PATCH] host_manager --- init-server.py | 2 +- scripts/http_host/cron.d/ofm_http_host | 2 +- scripts/http_host/host_manager.py | 14 +-- scripts/http_host/http_host_lib/nginx.py | 110 ++++++++++-------- ...loy_tileset.py => set_tileset_versions.py} | 2 +- 5 files changed, 69 insertions(+), 61 deletions(-) rename scripts/http_host/http_host_lib/{deploy_tileset.py => set_tileset_versions.py} (97%) diff --git a/init-server.py b/init-server.py index 5416bec..475f19a 100755 --- a/init-server.py +++ b/init-server.py @@ -107,7 +107,7 @@ def prepare_http_host(c): c.sudo('/data/ofm/venv/bin/pip install -e /data/ofm/http_host/bin') # always last - # put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') + put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') def upload_https_host_files(c): diff --git a/scripts/http_host/cron.d/ofm_http_host b/scripts/http_host/cron.d/ofm_http_host index 4c7cca7..442734a 100644 --- a/scripts/http_host/cron.d/ofm_http_host +++ b/scripts/http_host/cron.d/ofm_http_host @@ -1,5 +1,5 @@ PYTHON=/data/ofm/venv/bin/python BIN=/data/ofm/http_host/bin -# every minute download_asset.py +# every minute sync * * * * * ofm sudo $PYTHON -u $BIN/host_manager.py sync >> /data/ofm/http_host/logs/host_manager_sync.log 2>&1 diff --git a/scripts/http_host/host_manager.py b/scripts/http_host/host_manager.py index 624ed81..fe90d43 100755 --- a/scripts/http_host/host_manager.py +++ b/scripts/http_host/host_manager.py @@ -7,11 +7,11 @@ from pathlib import Path import click import requests from http_host_lib import DEFAULT_ASSETS_DIR, DEFAULT_RUNS_DIR, MNT_DIR -from http_host_lib.deploy_tileset import deploy_tileset from http_host_lib.download_fonts import download_fonts from http_host_lib.download_tileset import download_and_extract_tileset from http_host_lib.mount import clean_up_mounts, create_fstab from http_host_lib.nginx import write_nginx_config +from http_host_lib.set_tileset_versions import set_tileset_versions from http_host_lib.utils import assert_linux, assert_single_process, assert_sudo @@ -114,13 +114,13 @@ def mount(): @cli.command() -def deploy_tileset_version(): +def set_latest_versions(): """ - Deploys the latest tileset version specified by + Sets the latest version of the tilesets to the version specified by https://assets.openfreemap.com/versions/deployed_planet.txt - 1. Check if the given version is present on the disk and is mounted - 2. Write to a version file + 1. Checks if the given version is present on the disk and is mounted + 2. Writes to a version file """ assert_linux() @@ -129,7 +129,7 @@ def deploy_tileset_version(): if not MNT_DIR.exists(): sys.exit('mount needs to be run first') - return deploy_tileset() + return set_tileset_versions() @cli.command() @@ -166,7 +166,7 @@ def sync(ctx): ctx.invoke(download_assets) - deploy_done = ctx.invoke(deploy_tileset_version) + deploy_done = ctx.invoke(set_latest_versions) if download_done or deploy_done: ctx.invoke(nginx_sync) diff --git a/scripts/http_host/http_host_lib/nginx.py b/scripts/http_host/http_host_lib/nginx.py index 43c7e05..ef16b10 100644 --- a/scripts/http_host/http_host_lib/nginx.py +++ b/scripts/http_host/http_host_lib/nginx.py @@ -9,7 +9,7 @@ def write_nginx_config(): with open(TEMPLATES_DIR / 'nginx_cf.conf') as fp: nginx_template = fp.read() - location_block_str = '' + location_str = '' curl_text = '' for subdir in MNT_DIR.iterdir(): @@ -18,55 +18,7 @@ def write_nginx_config(): area, version = subdir.name.split('-') - run_dir = DEFAULT_RUNS_DIR / area / version - if not run_dir.is_dir(): - print(f" {run_dir} doesn't exists, skipping") - continue - - tilejson_path = run_dir / 'tilejson-tiles-org.json' - - metadata_path = subdir / 'metadata.json' - if not metadata_path.is_file(): - print(f" {metadata_path} doesn't exists, skipping") - continue - - url_prefix = f'https://tiles.openfreemap.org/{area}/{version}' - - subprocess.run( - [ - sys.executable, - Path(__file__).parent.parent / 'metadata_to_tilejson.py', - '--minify', - metadata_path, - tilejson_path, - url_prefix, - ], - check=True, - ) - - # TODO # target 10y - version_str = f""" - location /{area}/{version} {{ # no trailing hash - alias {tilejson_path}; # no trailing hash - default_type application/json; - - add_header 'Access-Control-Allow-Origin' '*' always; - add_header Cache-Control public; - expires 1d; - }} - - location /{area}/{version}/ {{ # trailing hash - alias {subdir}/tiles/; # trailing hash - try_files $uri @empty; - - add_header Content-Encoding gzip; - add_header 'Access-Control-Allow-Origin' '*' always; - add_header Cache-Control public; - expires 1d; # target 10y - }} - """ - - location_block_str += version_str + location_str += create_version_location(area, version, subdir) if not curl_text: curl_text = ( @@ -75,7 +27,9 @@ def write_nginx_config(): f'curl -I https://tiles.openfreemap.org/{area}/{version}/14/8529/5975.pbf' ) - nginx_template = nginx_template.replace('___LOCATION_BLOCKS___', location_block_str) + location_str += create_deployed_location(area, version, subdir) + + nginx_template = nginx_template.replace('___LOCATION_BLOCKS___', location_str) with open('/data/nginx/sites/ofm-tiles-org.conf', 'w') as fp: fp.write(nginx_template) @@ -85,3 +39,57 @@ def write_nginx_config(): subprocess.run(['systemctl', 'reload', 'nginx'], check=True) print(curl_text) + + +def create_version_location(area: str, version: str, subdir: Path): + run_dir = DEFAULT_RUNS_DIR / area / version + if not run_dir.is_dir(): + print(f" {run_dir} doesn't exists, skipping") + return '' + + tilejson_path = run_dir / 'tilejson-tiles-org.json' + + metadata_path = subdir / 'metadata.json' + if not metadata_path.is_file(): + print(f" {metadata_path} doesn't exists, skipping") + return '' + + url_prefix = f'https://tiles.openfreemap.org/{area}/{version}' + + subprocess.run( + [ + sys.executable, + Path(__file__).parent.parent / 'metadata_to_tilejson.py', + '--minify', + metadata_path, + tilejson_path, + url_prefix, + ], + check=True, + ) + + # TODO # target 10y + return f""" + location /{area}/{version} {{ # no trailing hash + alias {tilejson_path}; # no trailing hash + default_type application/json; + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; + expires 1d; + }} + + location /{area}/{version}/ {{ # trailing hash + alias {subdir}/tiles/; # trailing hash + try_files $uri @empty; + + add_header Content-Encoding gzip; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; + expires 1d; # target 10y + }} + """ + + +def create_deployed_location(area: str, version: str, subdir: Path): + pass diff --git a/scripts/http_host/http_host_lib/deploy_tileset.py b/scripts/http_host/http_host_lib/set_tileset_versions.py similarity index 97% rename from scripts/http_host/http_host_lib/deploy_tileset.py rename to scripts/http_host/http_host_lib/set_tileset_versions.py index 889bb5c..73eb9c4 100644 --- a/scripts/http_host/http_host_lib/deploy_tileset.py +++ b/scripts/http_host/http_host_lib/set_tileset_versions.py @@ -3,7 +3,7 @@ from pathlib import Path import requests -def deploy_tileset(): +def set_tileset_versions(): need_nginx_sync = False for area in ['planet', 'monaco']: