versions

modules/http_host/http_host_lib/btrfs.py

@@ -8,7 +8,7 @@ from http_host_lib.config import config
 from http_host_lib.utils import download_file_aria2, get_remote_file_size
 
 
-def download_area_version(area: str, version: str):
+def download_area_version(area: str, version: str) -> bool:
     """
     Downloads and uncompresses tiles.btrfs files from the btrfs bucket
     """

modules/http_host/http_host_lib/config.py

@@ -15,6 +15,7 @@ class Configuration:
 
     mnt_dir = Path('/mnt/ofm')
     ofm_config_dir = Path('/data/ofm/config')
+    deployed_versions_dir = ofm_config_dir / 'deployed_versions'
 
     certs_dir = Path('/data/nginx/certs')
     nginx_confs = Path(__file__).parent / 'nginx_confs'

modules/http_host/http_host_lib/nginx.py

@@ -8,6 +8,9 @@ from http_host_lib.utils import python_venv_executable
 
 
 def write_nginx_config():
+    if not config.mnt_dir.exists():
+        sys.exit('  mount needs to be run first')
+
     curl_text_mix = ''
 
     domain_le = config.host_config['domain_le']

modules/http_host/http_host_lib/set_tileset_versions.py

@@ -1,45 +0,0 @@
-from pathlib import Path
-
-import requests
-
-from http_host_lib.config import config
-
-
-def set_tileset_versions():
-    need_nginx_sync = False
-
-    for area in ['planet', 'monaco']:
-        r = requests.get(f'https://assets.openfreemap.com/versions/deployed_{area}.txt', timeout=30)
-        r.raise_for_status()
-        remote_version = r.text.strip()
-        print(f'  remote version for {area}: {remote_version}')
-
-        local_version_file = config.ofm_config_dir / f'tileset_version_{area}.txt'
-
-        if not local_version_file.exists():
-            local_version_start = None
-        else:
-            with open(local_version_file) as fp:
-                local_version_start = fp.read()
-
-        if not remote_version:
-            print('    remote version not specified')
-            if local_version_start is not None:
-                local_version_file.unlink()
-                need_nginx_sync = True
-            continue
-
-        mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
-        if not mnt_file.exists():
-            print('    local version does not exist')
-            if local_version_start is not None:
-                local_version_file.unlink()
-                need_nginx_sync = True
-            continue
-
-        if remote_version != local_version_start:
-            with open(local_version_file, 'w') as fp:
-                fp.write(remote_version)
-            need_nginx_sync = True
-
-    return need_nginx_sync

modules/http_host/http_host_lib/sync.py

@@ -0,0 +1,39 @@
+from datetime import datetime, timezone
+
+from http_host_lib.assets import download_assets
+from http_host_lib.btrfs import download_area_version
+from http_host_lib.config import config
+from http_host_lib.mount import auto_mount_unmount
+from http_host_lib.nginx import write_nginx_config
+from http_host_lib.utils import assert_linux, assert_sudo
+from http_host_lib.versions import sync_version_files
+
+
+def full_sync(force=False):
+    """
+    Runs the sync task, normally called by cron every minute
+    On a new server this also takes care of everything, no need to run anything manually.
+    """
+
+    print('---')
+    print('running full_sync')
+    print(datetime.now(tz=timezone.utc))
+
+    assert_linux()
+    assert_sudo()
+
+    download_done = False
+    download_done += download_area_version(area='monaco', version='latest')
+
+    if not config.host_config.get('skip_planet'):
+        download_done += download_area_version(area='planet', version='latest')
+
+    if download_done or force:
+        auto_mount_unmount()
+
+    download_assets()
+
+    versions_changed = sync_version_files()
+
+    if download_done or versions_changed or force:
+        write_nginx_config()

modules/http_host/http_host_lib/versions.py

@@ -0,0 +1,52 @@
+import sys
+from pathlib import Path
+
+import requests
+
+from http_host_lib.config import config
+from http_host_lib.utils import assert_linux, assert_sudo
+
+
+def sync_version_files() -> bool:
+    """
+    Syncs the version files from remote to local.
+    Remove versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
+    """
+
+    print('Syncing local version files')
+
+    assert_linux()
+    assert_sudo()
+
+    if not config.mnt_dir.exists():
+        sys.exit('  mount needs to be run first')
+
+    need_nginx_sync = False
+
+    for area in config.areas:
+        r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
+        r.raise_for_status()
+        remote_version = r.text.strip()
+        assert remote_version
+        print(f'  remote version for {area}: {remote_version}')
+
+        local_version_file = config.deployed_versions_dir / f'{area}.txt'
+
+        try:
+            local_version_old = local_version_file.read_text()
+        except Exception:
+            local_version_old = None
+
+        mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
+        if not mnt_file.exists():
+            print('    local version does not exist')
+            if local_version_old is not None:
+                local_version_file.unlink()
+                need_nginx_sync = True
+            continue
+
+        if remote_version != local_version_old:
+            local_version_file.write_text(remote_version)
+            need_nginx_sync = True
+
+    return need_nginx_sync