config, set_version

modules/http_host/http_host_lib/btrfs.py

@@ -2,9 +2,8 @@ import shutil
 import subprocess
 import sys
 
-import requests
-
 from http_host_lib.config import config
+from http_host_lib.shared import get_versions_for_area
 from http_host_lib.utils import download_file_aria2, get_remote_file_size
 
 
@@ -41,14 +40,6 @@ def download_area_version(area: str, version: str) -> bool:
     return download_and_extract_btrfs(area, selected_version)
 
 
-def get_versions_for_area(area: str) -> list:
-    r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30)
-    r.raise_for_status()
-
-    versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')]
-    return sorted(versions)
-
-
 def download_and_extract_btrfs(area: str, version: str) -> bool:
     """
     returns True if download successful, False if skipped

modules/http_host/http_host_lib/shared.py

@@ -0,0 +1 @@
+../../tile_gen/tile_gen_lib/shared.py

modules/loadbalancer/loadbalancer.py

@@ -111,22 +111,6 @@ def run_area(c, area):
     return results
 
 
-def check_host(domain, host_ip, area, version):
-    # check TileJSON first
-    url = f'https://{domain}/{area}'
-    tilejson_str = pycurl_get(url, domain, host_ip)
-    tilejson = json.loads(tilejson_str)
-    tiles_url = tilejson['tiles'][0]
-    version_in_tilejson = tiles_url.split('/')[4]
-    assert version_in_tilejson == version
-
-    # check actual vector tile
-    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
-    assert pycurl_status(url, domain, host_ip) == 200
-
-    # check style
-    url = f'https://{domain}/styles/bright'
-    assert pycurl_status(url, domain, host_ip) == 200
 
 
 def get_target_version(area):

modules/loadbalancer/loadbalancer_lib/shared.py

@@ -0,0 +1 @@
+../../tile_gen/tile_gen_lib/shared.py

modules/set_version/setup.py

@@ -1,17 +0,0 @@
-from setuptools import find_packages, setup
-
-
-requirements = [
-    'click',
-    'requests',
-    'pycurl',
-    'python-dotenv',
-    'questionary',
-]
-
-
-setup(
-    python_requires='>=3.10',
-    install_requires=requirements,
-    packages=find_packages(),
-)

modules/set_version/setversion.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python3
-
-import subprocess
-
-import click
-import questionary
-from setversion_lib import RCLONE_BIN, RCLONE_CONF
-
-
-@click.group()
-def cli():
-    """
-    Sets deployed reference versions
-    """
-
-
-@cli.command()
-@click.argument('area', required=True)
-def interactive(area):
-    versions = get_available_versions(area)[::-1]
-
-    choices = [questionary.Choice(title=r, value=i) for i, r in enumerate(versions)]
-    answer = questionary.select(f'Select version for: {area}', choices=choices).ask()
-
-    selected = versions[answer]
-
-    set_version(area, selected)
-
-
-def get_available_versions(area):
-    p = subprocess.run(
-        [
-            RCLONE_BIN,
-            'cat',
-            f'remote:ofm-{area}/dirs.txt',
-        ],
-        env=dict(RCLONE_CONFIG=RCLONE_CONF),
-        check=True,
-        capture_output=True,
-        text=True,
-    )
-    versions = [l.strip() for l in p.stdout.strip().splitlines()]
-    versions.sort()
-
-    return versions
-
-
-def set_version(area, version):
-    subprocess.run(
-        [
-            RCLONE_BIN,
-            'rcat',
-            f'remote:ofm-assets/versions/deployed_{area}.txt',
-        ],
-        env=dict(RCLONE_CONFIG=RCLONE_CONF),
-        check=True,
-        input=version.encode(),
-    )
-
-
-if __name__ == '__main__':
-    cli()

modules/set_version/setversion_lib/__init__.py

@@ -1,16 +0,0 @@
-from pathlib import Path
-
-
-if Path('/data/ofm/config').exists():
-    OFM_CONFIG_DIR = Path('/data/ofm/config')
-else:
-    OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
-
-assert OFM_CONFIG_DIR.exists()
-
-RCLONE_CONF = OFM_CONFIG_DIR / 'rclone.conf'
-
-if Path('/opt/homebrew/bin/rclone').exists():
-    RCLONE_BIN = '/opt/homebrew/bin/rclone'
-else:
-    RCLONE_BIN = 'rclone'

modules/tile_gen/tile_gen.py

@@ -4,6 +4,7 @@ import click
 from tile_gen_lib.btrfs import make_btrfs
 from tile_gen_lib.planetiler import run_planetiler
 from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
+from tile_gen_lib.set_version import check_and_set_version
 
 
 @click.group()
@@ -48,5 +49,18 @@ def make_indexes():
         make_indexes_for_bucket(bucket)
 
 
+@cli.command()
+@click.argument('area', required=True)
+@click.option(
+    '--version', default='latest', help='Optional version string, like "20231227_043106_pt"'
+)
+def set_version(area, version):
+    """
+    Set versions for a given area
+    """
+
+    check_and_set_version(area, version)
+
+
 if __name__ == '__main__':
     cli()

modules/tile_gen/tile_gen_lib/set_version.py

@@ -0,0 +1,31 @@
+import subprocess
+
+from http_host_lib.config import config
+
+from tile_gen_lib.shared import get_versions_for_area
+
+
+def check_all_hosts(area, version):
+    pass
+
+
+def check_and_set_version(area, version):
+    if version == 'latest':
+        versions = get_versions_for_area(area)
+        version = versions[-1]
+
+    if not check_all_hosts(area, version):
+        return
+
+
+def set_version(area, version):
+    subprocess.run(
+        [
+            'rclone',
+            'rcat',
+            f'remote:ofm-assets/deployed_versions/{area}.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=version.strip().encode(),
+    )

modules/tile_gen/tile_gen_lib/shared.py

@@ -1,7 +1,45 @@
+import json
 from io import BytesIO
 from pathlib import Path
 
 import pycurl
+import requests
+
+
+def get_versions_for_area(area: str) -> list:
+    r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30)
+    r.raise_for_status()
+
+    versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')]
+    return sorted(versions)
+
+
+def check_host_version(domain, host_ip, area, version):
+    # check actual vector tile
+    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
+    assert pycurl_status(url, domain, host_ip) == 200
+
+    # check style
+    url = f'https://{domain}/styles/bright'
+    assert pycurl_status(url, domain, host_ip) == 200
+
+
+def check_host_latest(domain, host_ip, area, version):
+    # check TileJSON first
+    url = f'https://{domain}/{area}'
+    tilejson_str = pycurl_get(url, domain, host_ip)
+    tilejson = json.loads(tilejson_str)
+    tiles_url = tilejson['tiles'][0]
+    version_in_tilejson = tiles_url.split('/')[4]
+    assert version_in_tilejson == version
+
+    # check actual vector tile
+    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
+    assert pycurl_status(url, domain, host_ip) == 200
+
+    # check style
+    url = f'https://{domain}/styles/bright'
+    assert pycurl_status(url, domain, host_ip) == 200
 
 
 def pycurl_status(url, domain, host_ip):