versions

modules/http_host/http_host.py

@@ -1,8 +1,5 @@
 #!/usr/bin/env python3
 
-import datetime
-import sys
-
 import click
 from http_host_lib.assets import (
     download_assets,
@@ -11,11 +8,9 @@ from http_host_lib.btrfs import (
     download_area_version,
     get_versions_for_area,
 )
-from http_host_lib.config import config
 from http_host_lib.mount import auto_mount_unmount
-from http_host_lib.nginx import write_nginx_config
+from http_host_lib.sync import full_sync
-from http_host_lib.set_tileset_versions import set_tileset_versions
+from http_host_lib.versions import sync_version_files
-from http_host_lib.utils import assert_linux, assert_sudo
 
 
 @click.group()
@@ -25,9 +20,8 @@ def cli():
     - Downloading btrfs images\n
     - Downloading assets\n
     - Mounting directories\n
-    - Updating nginx config\n
     - Getting the deployed versions of tilesets\n
-    - Running the sync cron task (called every minute)
+    - Running the sync cron task (called every minute with http-host-autoupdate)
     """
 
 
@@ -43,7 +37,7 @@ def download_btrfs(area: str, version: str):
     Use --version=1 to list all available versions
     """
 
-    return download_area_version(area, version)
+    download_area_version(area, version)
 
 
 @cli.command(name='download-assets')
@@ -65,75 +59,25 @@ def mount():
     auto_mount_unmount()
 
 
-@cli.command()
+@cli.command(name='sync-version-files')
-def set_latest_versions():
+def sync_version_files_():
     """
-    Sets the latest version of the tilesets to the version specified by
+    Syncs the version files from remote to local.
-    https://assets.openfreemap.com/versions/deployed_planet.txt
+    Remove versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
-
-    1. Checks if the given version is present on the disk and is mounted
-    2. Writes to a version file
     """
 
-    print('running set_latest_versions')
+    sync_version_files()
-
-    assert_linux()
-    assert_sudo()
-
-    if not config.mnt_dir.exists():
-        sys.exit('  mount needs to be run first')
-
-    return set_tileset_versions()
-
-
-@cli.command()
-def nginx_sync():
-    """
-    Syncs the nginx config to the state of the system
-    """
-
-    print('running nginx_sync')
-
-    assert_linux()
-    assert_sudo()
-
-    if not config.mnt_dir.exists():
-        sys.exit('  mount needs to be run first')
-
-    write_nginx_config()
 
 
 @cli.command()
 @click.option('--force', is_flag=True, help='Force nginx sync run')
-@click.pass_context
+def sync(force):
-def sync(ctx, force):
     """
     Runs the sync task, normally called by cron every minute
     On a new server this also takes care of everything, no need to run anything manually.
     """
 
-    print('---')
+    full_sync(force)
-    print('running sync')
-    print(datetime.datetime.now(tz=datetime.timezone.utc))
-
-    assert_linux()
-    assert_sudo()
-
-    download_done = False
-    download_done += ctx.invoke(download_btrfs, area='monaco')
-
-    if not config.host_config.get('skip_planet'):
-        download_done += ctx.invoke(download_btrfs, area='planet')
-
-    if download_done:
-        ctx.invoke(mount)
-
-    ctx.invoke(download_assets)
-
-    deploy_done = ctx.invoke(set_latest_versions)
-
-    if download_done or deploy_done or force:
-        ctx.invoke(nginx_sync)
 
 
 @cli.command()

modules/http_host/http_host_lib/btrfs.py

@@ -8,7 +8,7 @@ from http_host_lib.config import config
 from http_host_lib.utils import download_file_aria2, get_remote_file_size
 
 
-def download_area_version(area: str, version: str):
+def download_area_version(area: str, version: str) -> bool:
     """
     Downloads and uncompresses tiles.btrfs files from the btrfs bucket
     """

modules/http_host/http_host_lib/config.py

@@ -15,6 +15,7 @@ class Configuration:
 
     mnt_dir = Path('/mnt/ofm')
     ofm_config_dir = Path('/data/ofm/config')
+    deployed_versions_dir = ofm_config_dir / 'deployed_versions'
 
     certs_dir = Path('/data/nginx/certs')
     nginx_confs = Path(__file__).parent / 'nginx_confs'

modules/http_host/http_host_lib/nginx.py

@@ -8,6 +8,9 @@ from http_host_lib.utils import python_venv_executable
 
 
 def write_nginx_config():
+    if not config.mnt_dir.exists():
+        sys.exit('  mount needs to be run first')
+
     curl_text_mix = ''
 
     domain_le = config.host_config['domain_le']

modules/http_host/http_host_lib/set_tileset_versions.py

@@ -1,45 +0,0 @@
-from pathlib import Path
-
-import requests
-
-from http_host_lib.config import config
-
-
-def set_tileset_versions():
-    need_nginx_sync = False
-
-    for area in ['planet', 'monaco']:
-        r = requests.get(f'https://assets.openfreemap.com/versions/deployed_{area}.txt', timeout=30)
-        r.raise_for_status()
-        remote_version = r.text.strip()
-        print(f'  remote version for {area}: {remote_version}')
-
-        local_version_file = config.ofm_config_dir / f'tileset_version_{area}.txt'
-
-        if not local_version_file.exists():
-            local_version_start = None
-        else:
-            with open(local_version_file) as fp:
-                local_version_start = fp.read()
-
-        if not remote_version:
-            print('    remote version not specified')
-            if local_version_start is not None:
-                local_version_file.unlink()
-                need_nginx_sync = True
-            continue
-
-        mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
-        if not mnt_file.exists():
-            print('    local version does not exist')
-            if local_version_start is not None:
-                local_version_file.unlink()
-                need_nginx_sync = True
-            continue
-
-        if remote_version != local_version_start:
-            with open(local_version_file, 'w') as fp:
-                fp.write(remote_version)
-            need_nginx_sync = True
-
-    return need_nginx_sync

modules/http_host/http_host_lib/sync.py

@@ -0,0 +1,39 @@
+from datetime import datetime, timezone
+
+from http_host_lib.assets import download_assets
+from http_host_lib.btrfs import download_area_version
+from http_host_lib.config import config
+from http_host_lib.mount import auto_mount_unmount
+from http_host_lib.nginx import write_nginx_config
+from http_host_lib.utils import assert_linux, assert_sudo
+from http_host_lib.versions import sync_version_files
+
+
+def full_sync(force=False):
+    """
+    Runs the sync task, normally called by cron every minute
+    On a new server this also takes care of everything, no need to run anything manually.
+    """
+
+    print('---')
+    print('running full_sync')
+    print(datetime.now(tz=timezone.utc))
+
+    assert_linux()
+    assert_sudo()
+
+    download_done = False
+    download_done += download_area_version(area='monaco', version='latest')
+
+    if not config.host_config.get('skip_planet'):
+        download_done += download_area_version(area='planet', version='latest')
+
+    if download_done or force:
+        auto_mount_unmount()
+
+    download_assets()
+
+    versions_changed = sync_version_files()
+
+    if download_done or versions_changed or force:
+        write_nginx_config()

modules/http_host/http_host_lib/versions.py

@@ -0,0 +1,52 @@
+import sys
+from pathlib import Path
+
+import requests
+
+from http_host_lib.config import config
+from http_host_lib.utils import assert_linux, assert_sudo
+
+
+def sync_version_files() -> bool:
+    """
+    Syncs the version files from remote to local.
+    Remove versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
+    """
+
+    print('Syncing local version files')
+
+    assert_linux()
+    assert_sudo()
+
+    if not config.mnt_dir.exists():
+        sys.exit('  mount needs to be run first')
+
+    need_nginx_sync = False
+
+    for area in config.areas:
+        r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
+        r.raise_for_status()
+        remote_version = r.text.strip()
+        assert remote_version
+        print(f'  remote version for {area}: {remote_version}')
+
+        local_version_file = config.deployed_versions_dir / f'{area}.txt'
+
+        try:
+            local_version_old = local_version_file.read_text()
+        except Exception:
+            local_version_old = None
+
+        mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
+        if not mnt_file.exists():
+            print('    local version does not exist')
+            if local_version_old is not None:
+                local_version_file.unlink()
+                need_nginx_sync = True
+            continue
+
+        if remote_version != local_version_old:
+            local_version_file.write_text(remote_version)
+            need_nginx_sync = True
+
+    return need_nginx_sync

ssh_lib/tasks.py

@@ -4,9 +4,9 @@ import sys
 from ssh_lib import (
     CONFIG_DIR,
     HTTP_HOST_BIN,
+    MODULES_DIR,
     OFM_DIR,
     REMOTE_CONFIG,
-    MODULES_DIR,
     TILE_GEN_BIN,
     VENV_BIN,
     dotenv_val,
@@ -138,6 +138,7 @@ def run_http_host_sync(c):
 
 
 def upload_http_host_files(c):
+    c.sudo(f'rm -rf {HTTP_HOST_BIN}')
     c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
 
     put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')