refactor, auto_clean

modules/http_host/cron.d/ofm_http_host

@@ -1,4 +1,4 @@
 # every minute sync, locking so that only one process can run at a time
-* * * * * ofm     /usr/bin/flock -n /tmp/hostmanager.lockfile -c 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync >> /data/ofm/http_host/logs/host_manager_sync.log 2>&1'
+* * * * * ofm     /usr/bin/flock -n /tmp/http_host.lockfile -c 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/http_host.py sync >> /data/ofm/http_host/logs/http_host_sync.log 2>&1'
 
 

modules/http_host/http_host.py

@@ -9,8 +9,9 @@ from http_host_lib.btrfs import (
     get_versions_for_area,
 )
 from http_host_lib.mount import auto_mount_unmount
-from http_host_lib.sync import full_sync
-from http_host_lib.versions import sync_version_files
+from http_host_lib.nginx import write_nginx_config
+from http_host_lib.sync import auto_clean_btrfs, full_sync
+from http_host_lib.versions import fetch_version_files
 
 
 @click.group()
@@ -19,8 +20,8 @@ def cli():
     Manages OpenFreeMap HTTP hosts, including:\n
     - Downloading btrfs images\n
     - Downloading assets\n
-    - Mounting directories\n
-    - Getting the deployed versions of tilesets\n
+    - Mounting downloaded btrfs images\n
+    - Fetches version files\n
     - Running the sync cron task (called every minute with http-host-autoupdate)
     """
 
@@ -59,14 +60,32 @@ def mount():
     auto_mount_unmount()
 
 
-@cli.command(name='sync-version-files')
-def sync_version_files_():
+@cli.command(name='fetch-versions')
+def fetch_version_files_():
     """
-    Syncs the version files from remote to local.
-    Remove versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
+    Fetches the version files from remote to local.
+    Remote versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
     """
 
-    sync_version_files()
+    fetch_version_files()
+
+
+@cli.command()
+def auto_clean():
+    """
+    Cleans the old btrfs images
+    """
+
+    auto_clean_btrfs()
+
+
+@cli.command()
+def nginx_config():
+    """
+    Writes the nginx config files and reloads nginx
+    """
+
+    write_nginx_config()
 
 
 @cli.command()

modules/http_host/http_host_lib/btrfs.py

@@ -18,8 +18,18 @@ def download_area_version(area: str, version: str) -> bool:
 
     versions = get_versions_for_area(area)
 
+    # latest version
     if version == 'latest':
         selected_version = versions[-1]
+
+    # deployed version
+    elif version == 'deployed':
+        try:
+            selected_version = (config.deployed_versions_dir / f'{area}.txt').read_text().strip()
+        except Exception:
+            return False
+
+    # specific version
     else:
         if version not in versions:
             available_versions_str = '\n'.join(versions)
@@ -44,7 +54,7 @@ def download_and_extract_btrfs(area: str, version: str) -> bool:
     returns True if download successful, False if skipped
     """
 
-    print(f'downloading and extracting btrfs for: {area} {version}')
+    print(f'downloading btrfs: {area} {version}')
 
     version_dir = config.runs_dir / area / version
     btrfs_file = version_dir / 'tiles.btrfs'
@@ -62,11 +72,13 @@ def download_and_extract_btrfs(area: str, version: str) -> bool:
     disk_free = shutil.disk_usage(temp_dir).free
     file_size = get_remote_file_size(url)
     if not file_size:
-        raise ValueError('Cannot get remote file size')
+        print(f'Cannot get remote file size for {url}')
+        return False
 
     needed_space = file_size * 3
     if disk_free < needed_space:
-        raise ValueError(f'Not enough disk space. Needed: {needed_space}, free space: {disk_free}')
+        print(f'Not enough disk space. Needed: {needed_space}, free space: {disk_free}')
+        return False
 
     target_file = temp_dir / 'tiles.btrfs.gz'
     download_file_aria2(url, target_file)

modules/http_host/http_host_lib/nginx.py

@@ -200,16 +200,27 @@ def create_version_location(
 def create_latest_locations(*, local: str, domain: str) -> str:
     location_str = ''
 
-    local_version_files = config.ofm_config_dir.glob('tileset_version_*.txt')
+    local_version_files = config.deployed_versions_dir.glob('*.txt')
+
     for file in local_version_files:
-        area = file.stem.split('_')[-1]
+        area = file.stem
         with open(file) as fp:
             version = fp.read().strip()
-        print(f'  setting latest version for {area}: {version}')
 
+        print(f'  linking latest version for {area}: {version}')
+
+        # checking runs dir
         run_dir = config.runs_dir / area / version
         tilejson_path = run_dir / f'tilejson-{local}.json'
-        assert tilejson_path.is_file()
+        if not tilejson_path.is_file():
+            print(f'    error with latest: {tilejson_path} does not exist')
+            continue
+
+        # checking mnt dir
+        mnt_file = Path(f'/mnt/ofm/{area}-{version}/metadata.json')
+        if not mnt_file.is_file():
+            print(f'    error with latest: {mnt_file} does not exist')
+            continue
 
         location_str += f"""
         location = /{area} {{          # no trailing slash

modules/http_host/http_host_lib/sync.py

@@ -1,3 +1,4 @@
+import shutil
 from datetime import datetime, timezone
 
 from http_host_lib.assets import download_assets
@@ -6,7 +7,7 @@ from http_host_lib.config import config
 from http_host_lib.mount import auto_mount_unmount
 from http_host_lib.nginx import write_nginx_config
 from http_host_lib.utils import assert_linux, assert_sudo
-from http_host_lib.versions import sync_version_files
+from http_host_lib.versions import fetch_version_files
 
 
 def full_sync(force=False):
@@ -22,18 +23,70 @@ def full_sync(force=False):
     assert_linux()
     assert_sudo()
 
-    download_done = False
-    download_done += download_area_version(area='monaco', version='latest')
+    # start
 
-    if not config.host_config.get('skip_planet'):
-        download_done += download_area_version(area='planet', version='latest')
-
-    if download_done or force:
-        auto_mount_unmount()
+    versions_changed = fetch_version_files()
 
     download_assets()
 
-    versions_changed = sync_version_files()
+    btrfs_downloaded = False
 
-    if download_done or versions_changed or force:
+    # download latest and deployed monaco
+    btrfs_downloaded += download_area_version(area='monaco', version='latest')
+    btrfs_downloaded += download_area_version(area='monaco', version='deployed')
+
+    # download latest and deployed planet
+    if not config.host_config.get('skip_planet'):
+        btrfs_downloaded += download_area_version(area='planet', version='latest')
+        btrfs_downloaded += download_area_version(area='planet', version='deployed')
+
+    if btrfs_downloaded or versions_changed or force:
+        auto_clean_btrfs()
+        auto_mount_unmount()
         write_nginx_config()
+
+
+def auto_clean_btrfs():
+    """
+    Clean old btrfs runs
+
+    For each area we keep max two versions:
+    1. The newest one available locally
+    2. The one currently deployed, specified in /data/ofm/config/deployed_versions
+    3. If there is no deployed version, then we include the second newest one
+    """
+
+    for area in config.areas:
+        area_dir = config.runs_dir / area
+        if not area_dir.is_dir():
+            continue
+
+        local_versions = sorted([i.name for i in area_dir.iterdir()])
+
+        versions_to_keep = set()
+
+        # add newest version
+        if local_versions:
+            versions_to_keep.add(local_versions[-1])
+
+        # add deployed version
+        try:
+            deployed_version_file = config.deployed_versions_dir / f'{area}.txt'
+            deployed_version = deployed_version_file.read_text().strip()
+            if (config.runs_dir / area / deployed_version).exists():
+                versions_to_keep.add(deployed_version)
+        except Exception:
+            pass
+
+        # if still only one version, we include the second newest one
+        if len(versions_to_keep) == 1 and len(local_versions) >= 2:
+            versions_to_keep.add(local_versions[-2])
+
+        print(f'  keeping versions for {area}: {sorted(versions_to_keep)}')
+
+        versions_to_remove = set(local_versions).difference(versions_to_keep)
+
+        for version in versions_to_remove:
+            print(f'  removing version for {area}: {version}')
+            version_dir = config.runs_dir / area / version
+            shutil.rmtree(version_dir)

modules/http_host/http_host_lib/versions.py

@@ -1,16 +1,13 @@
-import sys
-from pathlib import Path
-
 import requests
 
 from http_host_lib.config import config
 from http_host_lib.utils import assert_linux, assert_sudo
 
 
-def sync_version_files() -> bool:
+def fetch_version_files() -> bool:
     """
     Syncs the version files from remote to local.
-    Remove versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
+    Remote versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
     """
 
     print('Syncing local version files')
@@ -18,9 +15,6 @@ def sync_version_files() -> bool:
     assert_linux()
     assert_sudo()
 
-    if not config.mnt_dir.exists():
-        sys.exit('  mount needs to be run first')
-
     need_nginx_sync = False
 
     for area in config.areas:
@@ -37,15 +31,8 @@ def sync_version_files() -> bool:
         except Exception:
             local_version_old = None
 
-        mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
-        if not mnt_file.exists():
-            print('    local version does not exist')
-            if local_version_old is not None:
-                local_version_file.unlink()
-                need_nginx_sync = True
-            continue
-
         if remote_version != local_version_old:
+            config.deployed_versions_dir.mkdir(exist_ok=True, parents=True)
             local_version_file.write_text(remote_version)
             need_nginx_sync = True