config, set_version

init-server.py

@@ -11,12 +11,11 @@ from ssh_lib.tasks import (
     run_http_host_sync,
     setup_ledns_writer,
     setup_loadbalancer,
-    upload_http_host_config,
+    upload_config_json,
     upload_http_host_files,
 )
 from ssh_lib.utils import (
     put,
-    put_dir,
 )
 
 
@@ -65,7 +64,7 @@ def http_host_static(hostname, user, port):
     c = get_connection(hostname, user, port)
 
     prepare_shared(c)
-    upload_http_host_config(c)
+    upload_config_json(c)
 
     prepare_http_host(c)
 
@@ -83,8 +82,8 @@ def http_host_autoupdate(hostname, user, port):
     c.sudo('rm -f /etc/cron.d/ofm_http_host')
 
     prepare_shared(c)
-    upload_http_host_config(c)
 
+    upload_config_json(c)
     prepare_http_host(c)
 
     run_http_host_sync(c)
@@ -132,8 +131,7 @@ def loadbalancer(hostname, user, port):
 def debug(hostname, user, port):
     c = get_connection(hostname, user, port)
 
-    upload_http_host_files(c)
-    # run_http_host_sync(c)
+    upload_config_json(c)
 
 
 if __name__ == '__main__':

modules/http_host/http_host_lib/btrfs.py

@@ -2,9 +2,8 @@ import shutil
 import subprocess
 import sys
 
-import requests
-
 from http_host_lib.config import config
+from http_host_lib.shared import get_versions_for_area
 from http_host_lib.utils import download_file_aria2, get_remote_file_size
 
 
@@ -41,14 +40,6 @@ def download_area_version(area: str, version: str) -> bool:
     return download_and_extract_btrfs(area, selected_version)
 
 
-def get_versions_for_area(area: str) -> list:
-    r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30)
-    r.raise_for_status()
-
-    versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')]
-    return sorted(versions)
-
-
 def download_and_extract_btrfs(area: str, version: str) -> bool:
     """
     returns True if download successful, False if skipped

modules/http_host/http_host_lib/shared.py

@@ -0,0 +1 @@
+../../tile_gen/tile_gen_lib/shared.py

modules/loadbalancer/loadbalancer.py

@@ -111,22 +111,6 @@ def run_area(c, area):
     return results
 
 
-def check_host(domain, host_ip, area, version):
-    # check TileJSON first
-    url = f'https://{domain}/{area}'
-    tilejson_str = pycurl_get(url, domain, host_ip)
-    tilejson = json.loads(tilejson_str)
-    tiles_url = tilejson['tiles'][0]
-    version_in_tilejson = tiles_url.split('/')[4]
-    assert version_in_tilejson == version
-
-    # check actual vector tile
-    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
-    assert pycurl_status(url, domain, host_ip) == 200
-
-    # check style
-    url = f'https://{domain}/styles/bright'
-    assert pycurl_status(url, domain, host_ip) == 200
 
 
 def get_target_version(area):

modules/loadbalancer/loadbalancer_lib/shared.py

@@ -0,0 +1 @@
+../../tile_gen/tile_gen_lib/shared.py

modules/set_version/setup.py

@@ -1,17 +0,0 @@
-from setuptools import find_packages, setup
-
-
-requirements = [
-    'click',
-    'requests',
-    'pycurl',
-    'python-dotenv',
-    'questionary',
-]
-
-
-setup(
-    python_requires='>=3.10',
-    install_requires=requirements,
-    packages=find_packages(),
-)

modules/set_version/setversion.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python3
-
-import subprocess
-
-import click
-import questionary
-from setversion_lib import RCLONE_BIN, RCLONE_CONF
-
-
-@click.group()
-def cli():
-    """
-    Sets deployed reference versions
-    """
-
-
-@cli.command()
-@click.argument('area', required=True)
-def interactive(area):
-    versions = get_available_versions(area)[::-1]
-
-    choices = [questionary.Choice(title=r, value=i) for i, r in enumerate(versions)]
-    answer = questionary.select(f'Select version for: {area}', choices=choices).ask()
-
-    selected = versions[answer]
-
-    set_version(area, selected)
-
-
-def get_available_versions(area):
-    p = subprocess.run(
-        [
-            RCLONE_BIN,
-            'cat',
-            f'remote:ofm-{area}/dirs.txt',
-        ],
-        env=dict(RCLONE_CONFIG=RCLONE_CONF),
-        check=True,
-        capture_output=True,
-        text=True,
-    )
-    versions = [l.strip() for l in p.stdout.strip().splitlines()]
-    versions.sort()
-
-    return versions
-
-
-def set_version(area, version):
-    subprocess.run(
-        [
-            RCLONE_BIN,
-            'rcat',
-            f'remote:ofm-assets/versions/deployed_{area}.txt',
-        ],
-        env=dict(RCLONE_CONFIG=RCLONE_CONF),
-        check=True,
-        input=version.encode(),
-    )
-
-
-if __name__ == '__main__':
-    cli()

modules/set_version/setversion_lib/__init__.py

@@ -1,16 +0,0 @@
-from pathlib import Path
-
-
-if Path('/data/ofm/config').exists():
-    OFM_CONFIG_DIR = Path('/data/ofm/config')
-else:
-    OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
-
-assert OFM_CONFIG_DIR.exists()
-
-RCLONE_CONF = OFM_CONFIG_DIR / 'rclone.conf'
-
-if Path('/opt/homebrew/bin/rclone').exists():
-    RCLONE_BIN = '/opt/homebrew/bin/rclone'
-else:
-    RCLONE_BIN = 'rclone'

modules/tile_gen/tile_gen.py

@@ -4,6 +4,7 @@ import click
 from tile_gen_lib.btrfs import make_btrfs
 from tile_gen_lib.planetiler import run_planetiler
 from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
+from tile_gen_lib.set_version import check_and_set_version
 
 
 @click.group()
@@ -48,5 +49,18 @@ def make_indexes():
         make_indexes_for_bucket(bucket)
 
 
+@cli.command()
+@click.argument('area', required=True)
+@click.option(
+    '--version', default='latest', help='Optional version string, like "20231227_043106_pt"'
+)
+def set_version(area, version):
+    """
+    Set versions for a given area
+    """
+
+    check_and_set_version(area, version)
+
+
 if __name__ == '__main__':
     cli()

modules/tile_gen/tile_gen_lib/set_version.py

@@ -0,0 +1,31 @@
+import subprocess
+
+from http_host_lib.config import config
+
+from tile_gen_lib.shared import get_versions_for_area
+
+
+def check_all_hosts(area, version):
+    pass
+
+
+def check_and_set_version(area, version):
+    if version == 'latest':
+        versions = get_versions_for_area(area)
+        version = versions[-1]
+
+    if not check_all_hosts(area, version):
+        return
+
+
+def set_version(area, version):
+    subprocess.run(
+        [
+            'rclone',
+            'rcat',
+            f'remote:ofm-assets/deployed_versions/{area}.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=version.strip().encode(),
+    )

modules/tile_gen/tile_gen_lib/shared.py

@@ -1,7 +1,45 @@
+import json
 from io import BytesIO
 from pathlib import Path
 
 import pycurl
+import requests
+
+
+def get_versions_for_area(area: str) -> list:
+    r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30)
+    r.raise_for_status()
+
+    versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')]
+    return sorted(versions)
+
+
+def check_host_version(domain, host_ip, area, version):
+    # check actual vector tile
+    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
+    assert pycurl_status(url, domain, host_ip) == 200
+
+    # check style
+    url = f'https://{domain}/styles/bright'
+    assert pycurl_status(url, domain, host_ip) == 200
+
+
+def check_host_latest(domain, host_ip, area, version):
+    # check TileJSON first
+    url = f'https://{domain}/{area}'
+    tilejson_str = pycurl_get(url, domain, host_ip)
+    tilejson = json.loads(tilejson_str)
+    tiles_url = tilejson['tiles'][0]
+    version_in_tilejson = tiles_url.split('/')[4]
+    assert version_in_tilejson == version
+
+    # check actual vector tile
+    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
+    assert pycurl_status(url, domain, host_ip) == 200
+
+    # check style
+    url = f'https://{domain}/styles/bright'
+    assert pycurl_status(url, domain, host_ip) == 200
 
 
 def pycurl_status(url, domain, host_ip):

prepare-virtualenv.sh

@@ -14,7 +14,6 @@ pip install -e .
 pip install -e modules/http_host
 pip install -e modules/tile_gen
 pip install -e modules/loadbalancer
-pip install -e modules/set_version
 
 
 

ssh_lib/tasks.py

@@ -78,40 +78,6 @@ def prepare_tile_gen(c):
     put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')
 
 
-def upload_http_host_config(c):
-    domain_le = dotenv_val('DOMAIN_LE').lower()
-    domain_ledns = dotenv_val('DOMAIN_LEDNS').lower()
-    skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
-    le_email = dotenv_val('LE_EMAIL').lower()
-
-    if not (domain_le or domain_ledns):
-        sys.exit('Please specify DOMAIN_LE or DOMAIN_LEDNS in config/.env')
-
-    if domain_le and not le_email:
-        sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE')
-
-    host_config = {
-        'domain_le': domain_le,
-        'domain_ledns': domain_ledns,
-        'skip_planet': skip_planet,
-        'le_email': le_email,
-    }
-
-    host_config_str = json.dumps(host_config, indent=2, ensure_ascii=False)
-    print(host_config_str)
-    put_str(c, '/data/ofm/config/http_host.json', host_config_str)
-
-    if domain_ledns:
-        assert (CONFIG_DIR / 'rclone.conf').exists()
-        put(
-            c,
-            CONFIG_DIR / 'rclone.conf',
-            f'{REMOTE_CONFIG}/rclone.conf',
-            permissions=400,
-        )
-        put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_ledns_reader', '/etc/cron.d/')
-
-
 def prepare_http_host(c):
     kernel_tweaks_ofm(c)
 
@@ -128,6 +94,16 @@ def prepare_http_host(c):
 
     upload_http_host_files(c)
 
+    if dotenv_val('DOMAIN_LEDNS'):
+        assert (CONFIG_DIR / 'rclone.conf').exists()
+        put(
+            c,
+            CONFIG_DIR / 'rclone.conf',
+            f'{REMOTE_CONFIG}/rclone.conf',
+            permissions=400,
+        )
+        put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_ledns_reader', '/etc/cron.d/')
+
     c.sudo(f'{VENV_BIN}/pip install -e {HTTP_HOST_BIN} --use-pep517')
 
 
@@ -219,21 +195,38 @@ def setup_ledns_writer(c):
     )
 
 
-def setup_loadbalancer(c):
+def upload_config_json(c):
+    domain_le = dotenv_val('DOMAIN_LE').lower()
     domain_ledns = dotenv_val('DOMAIN_LEDNS').lower()
+    skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
+    le_email = dotenv_val('LE_EMAIL').lower()
+
+    if not (domain_le or domain_ledns):
+        sys.exit('Please specify DOMAIN_LE or DOMAIN_LEDNS in config/.env')
+
+    if domain_le and not le_email:
+        sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE')
+
     http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()]
     assert (CONFIG_DIR / 'cloudflare.ini').exists()
 
     config = {
+        'domain_le': domain_le,
         'domain_ledns': domain_ledns,
+        'le_email': le_email,
+        'skip_planet': skip_planet,
         'http_host_list': http_host_list,
         'telegram_token': dotenv_val('TELEGRAM_TOKEN'),
         'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'),
     }
 
     config_str = json.dumps(config, indent=2, ensure_ascii=False)
-    # print(config_str)
-    put_str(c, f'{REMOTE_CONFIG}/loadbalancer.json', config_str)
+    print(config_str)
+    put_str(c, f'{REMOTE_CONFIG}/config.json', config_str)
+
+
+def setup_loadbalancer(c):
+    upload_config_json(c)
 
     put(
         c,