host_manager

init-server.py

@@ -88,7 +88,7 @@ def prepare_tile_gen(c):
     c.sudo('chown ofm:ofm -R /data/ofm/tile_gen/bin')
 
 
-def prepare_http_host(c):
+def prepare_http_host(c, skip_cron: bool):
     nginx(c)
     certbot(c)
     c1000k(c)
@@ -107,7 +107,8 @@ def prepare_http_host(c):
     c.sudo('/data/ofm/venv/bin/pip install -e /data/ofm/http_host/bin')
 
     # always last
-    put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
+    if not skip_cron:
+        put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
 
 
 def upload_https_host_files(c):
@@ -131,7 +132,7 @@ def upload_certificates(c):
 
 def debug_tmp(c):
     upload_https_host_files(c)
-    put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
+    # put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
 
 
 @click.command()
@@ -144,7 +145,8 @@ def debug_tmp(c):
 @click.option(
     '--skip-shared', is_flag=True, help='Skip the shared installtion step (useful for development)'
 )
-def main(hostname, user, port, tile_gen, http_host, skip_shared, debug):
+@click.option('--skip-cron', is_flag=True, help='Skip the cronjob (useful for development)')
+def main(hostname, user, port, tile_gen, http_host, skip_shared, skip_cron, debug):
     if not debug and not click.confirm(f'Run script on {hostname}?'):
         return
 
@@ -182,7 +184,7 @@ def main(hostname, user, port, tile_gen, http_host, skip_shared, debug):
         prepare_tile_gen(c)
 
     if http_host:
-        prepare_http_host(c)
+        prepare_http_host(c, skip_cron=skip_cron)
 
 
 if __name__ == '__main__':

scripts/http_host/host_manager.py

@@ -19,12 +19,12 @@ from http_host_lib.utils import assert_linux, assert_single_process, assert_sudo
 def cli():
     """
     Manages OpenFreeMap HTTP hosts, including:\n
-    - Deploying the correct versions of tilesets\n
     - Downloading assets\n
     - Downloading tilesets\n
     - Mounting directories\n
     - Updating nginx config\n
-    - Running the sync cron task every minute
+    - Setting the latest versions of tilesets\n
+    - Running the sync cron task (called every minute)
     """
 
 
@@ -170,11 +170,10 @@ def sync(ctx):
     print('running sync')
     print(datetime.datetime.now(tz=datetime.timezone.utc))
 
-    assert_single_process()
-
     download_done = False
     download_done += ctx.invoke(download_tileset, area='monaco')
-    # download_done += ctx.invoke(download_tileset, area='planet')
+    download_done += ctx.invoke(download_tileset, area='planet')
+
     if download_done:
         ctx.invoke(mount)
 

scripts/http_host/http_host_lib/__init__.py

@@ -6,4 +6,5 @@ TEMPLATES_DIR = Path(__file__).parent / 'templates'
 DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
 DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets')
 
-MNT_DIR = Path('/mnt/ofm/')
+MNT_DIR = Path('/mnt/ofm')
+OFM_CONFIG_DIR = Path('/data/ofm/config')

scripts/http_host/http_host_lib/download_tileset.py

@@ -13,7 +13,7 @@ def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> boo
     returns True if downloaded something
     """
 
-    click.echo(f'downloading area: {area}, version: {version}')
+    click.echo(f'downloading {area} {version}')
 
     version_dir = runs_dir / area / version
     btrfs_file = version_dir / 'tiles.btrfs'

scripts/http_host/http_host_lib/nginx.py

@@ -2,7 +2,7 @@ import subprocess
 import sys
 from pathlib import Path
 
-from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR, TEMPLATES_DIR
+from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR, OFM_CONFIG_DIR, TEMPLATES_DIR
 
 
 def write_nginx_config():
@@ -15,9 +15,7 @@ def write_nginx_config():
     for subdir in MNT_DIR.iterdir():
         if not subdir.is_dir():
             continue
-
         area, version = subdir.name.split('-')
-
         location_str += create_version_location(area, version, subdir)
 
         if not curl_text:
@@ -27,8 +25,7 @@ def write_nginx_config():
                 f'curl -I https://tiles.openfreemap.org/{area}/{version}/14/8529/5975.pbf'
             )
 
-    for area in ['monaco', 'planet']:
-        location_str += create_latest_location(area)
+    location_str += create_latest_locations()
 
     nginx_template = nginx_template.replace('___LOCATION_BLOCKS___', location_str)
 
@@ -71,7 +68,43 @@ def create_version_location(area: str, version: str, subdir: Path) -> str:
 
     # TODO # target 10y
     return f"""
-        location /{area}/{version} {{    # no trailing hash
+    location /{area}/{version} {{    # no trailing hash
+        alias {tilejson_path};       # no trailing hash
+        default_type application/json;
+
+        add_header 'Access-Control-Allow-Origin' '*' always;
+        add_header Cache-Control public;
+        expires 1d;
+    }}
+
+    location /{area}/{version}/ {{    # trailing hash
+        alias {subdir}/tiles/;        # trailing hash
+        try_files $uri @empty;
+
+        add_header Content-Encoding gzip;
+        add_header 'Access-Control-Allow-Origin' '*' always;
+        add_header Cache-Control public;
+        expires 1d;  # target 10y
+    }}
+    """
+
+
+def create_latest_locations() -> str:
+    location_str = ''
+
+    local_version_files = OFM_CONFIG_DIR.glob('tileset_version_*.txt')
+    for file in local_version_files:
+        area = file.stem.split('_')[-1]
+        with open(file) as fp:
+            version = fp.read().strip()
+        print(f'  setting latest version for {area}: {version}')
+
+        run_dir = DEFAULT_RUNS_DIR / area / version
+        tilejson_path = run_dir / 'tilejson-tiles-org.json'
+        assert tilejson_path.exists()
+
+        location_str += f"""
+        location /{area} {{          # no trailing hash
             alias {tilejson_path};       # no trailing hash
             default_type application/json;
 
@@ -79,26 +112,6 @@ def create_version_location(area: str, version: str, subdir: Path) -> str:
             add_header Cache-Control public;
             expires 1d;
         }}
+        """
 
-        location /{area}/{version}/ {{    # trailing hash
-            alias {subdir}/tiles/;        # trailing hash
-            try_files $uri @empty;
-
-            add_header Content-Encoding gzip;
-            add_header 'Access-Control-Allow-Origin' '*' always;
-            add_header Cache-Control public;
-            expires 1d;  # target 10y
-        }}
-    """
-
-
-def create_latest_location(area: str) -> str:
-    local_version_file = Path(f'/data/ofm/config/deployed_tiles_{area}.txt')
-
-    if not local_version_file.exists():
-        return ''
-
-    with open(local_version_file) as fp:
-        version_str = fp.read().strip()
-
-    print(version_str)
+    return location_str

scripts/http_host/http_host_lib/set_tileset_versions.py

@@ -2,6 +2,8 @@ from pathlib import Path
 
 import requests
 
+from http_host_lib import OFM_CONFIG_DIR
+
 
 def set_tileset_versions():
     need_nginx_sync = False
@@ -12,7 +14,7 @@ def set_tileset_versions():
         remote_version = r.text.strip()
         print(f'  remote version for {area}: {remote_version}')
 
-        local_version_file = Path(f'/data/ofm/config/deployed_tiles_{area}.txt')
+        local_version_file = OFM_CONFIG_DIR / f'tileset_version_{area}.txt'
 
         if not local_version_file.exists():
             local_version_start = None

ssh_lib/pkg_base.py

@@ -18,7 +18,7 @@ def pkg_base(c):
         'unzip',
         'wget',
         'psmisc',
-        'util-linux'
+        'util-linux',
         #
         'btrfs-progs',
         #