work

README.md

@@ -190,8 +190,6 @@ See [dev setup docs](docs/dev_setup.md).
 Updated Planetiler version to latest
 Updated OpenJDK to 24 via Temurin repo
 
-
-
 ##### v0.8
 
 Lot of self-hosting related fixes.

init-server.py

@@ -121,11 +121,12 @@ def http_host_sync(hostname, user, port, noninteractive):
     run_http_host_sync(c)
 
 
-@cli.command()
-@common_options
-def debug(hostname, user, port, noninteractive):
-    c = get_connection(hostname, user, port)
-    run_http_host_sync(c)
+#
+# @cli.command()
+# @common_options
+# def debug(hostname, user, port, noninteractive):
+#     c = get_connection(hostname, user, port)
+#     run_http_host_sync(c)
 
 
 if __name__ == '__main__':

modules/http_host/http_host_lib/nginx.py

@@ -320,15 +320,3 @@ def create_latest_locations(*, local: str, domain: str) -> str:
         """
 
     return location_str
-
-
-def write_roundrobin_reader_script(domain_roundrobin):
-    script = f"""
-#!/usr/bin/env bash
-export RCLONE_CONFIG=/data/ofm/config/rclone.conf
-rclone copyto -v "remote:ofm-private/roundrobin/{domain_roundrobin}/ofm_roundrobin.cert" /data/nginx/certs/ofm_roundrobin.cert
-rclone copyto -v "remote:ofm-private/roundrobin/{domain_roundrobin}/ofm_roundrobin.key" /data/nginx/certs/ofm_roundrobin.key
-    """.strip()
-
-    with open(config.http_host_bin / 'roundrobin_reader.sh', 'w') as fp:
-        fp.write(script)

modules/http_host/http_host_lib/shared.py

@@ -1 +0,0 @@
-../../tile_gen/tile_gen_lib/shared.py

modules/tile_gen/cron.d/ofm_tile_gen

@@ -5,8 +5,8 @@ LOG_DIR=/data/ofm/tile_gen/logs
 # every day at 23:10, make a monaco run
 10 23 * * * ofm    $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
 
-# debug monaco run, every minute
-#*/1 * * * * ofm    $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
+# debug monaco run, normally disabled, enable to run every minute
+#* * * * * ofm    $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
 
 # every minute, set monaco to latest
 * * * * * ofm     $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1

modules/tile_gen/setup.py

@@ -3,7 +3,6 @@ from setuptools import find_packages, setup
 
 requirements = [
     'click',
-    'pycurl',
     'requests',
 ]
 

modules/tile_gen/tile_gen.py

@@ -3,9 +3,12 @@ from datetime import datetime, timezone
 
 import click
 from tile_gen_lib.btrfs import make_btrfs
+from tile_gen_lib.get_version_shared import (
+    get_deployed_version,
+    get_versions_for_area,
+)
 from tile_gen_lib.planetiler import run_planetiler
-from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
-from tile_gen_lib.set_version import check_and_set_version
+from tile_gen_lib.rclone import make_indexes_for_bucket, set_version_on_bucket, upload_area
 
 
 now = datetime.now(timezone.utc)
@@ -71,7 +74,22 @@ def set_version(area, version):
 
     print(f'---\n{now}\nStarting set-version {area}')
 
-    check_and_set_version(area, version)
+    if version == 'latest':
+        versions = get_versions_for_area(area)
+        if not versions:
+            print(f'  No versions found for {area}')
+            return
+
+        version = versions[-1]
+        print(f'  Latest version on bucket: {area} {version}')
+
+    try:
+        if get_deployed_version(area)['version'] == version:
+            return
+    except Exception:
+        pass
+
+    set_version_on_bucket(area, version)
 
 
 if __name__ == '__main__':

modules/tile_gen/tile_gen_lib/config.py

@@ -1,4 +1,3 @@
-import json
 import subprocess
 from pathlib import Path
 
@@ -22,8 +21,6 @@ class Configuration:
         repo_root = Path(__file__).parent.parent.parent.parent
         ofm_config_dir = repo_root / 'config'
 
-    ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
-
     rclone_config = ofm_config_dir / 'rclone.conf'
     rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip()
 

modules/tile_gen/tile_gen_lib/get_version_shared.py

@@ -0,0 +1,48 @@
+"""
+This file is shared / symlinked between tile_gen_lib and http_host_lib
+"""
+
+from datetime import datetime, timezone
+
+import requests
+
+
+def get_versions_for_area(area: str) -> list:
+    """
+    Download the files.txt and check for the runs with the "done" file present
+    """
+    r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30)
+    r.raise_for_status()
+
+    versions = []
+
+    files = r.text.splitlines()
+    for f in files:
+        if not f.startswith(f'areas/{area}/'):
+            continue
+        if not f.endswith('/done'):
+            continue
+        version_str = f.split('/')[2]
+        versions.append(version_str)
+
+    return sorted(versions)
+
+
+def get_deployed_version(area: str) -> dict:
+    r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
+    r.raise_for_status()
+    version = r.text.strip()
+
+    last_modified_str = r.headers.get('Last-Modified')
+    last_modified = parse_http_last_modified(last_modified_str)
+
+    return dict(
+        version=version,
+        last_modified=last_modified,
+    )
+
+
+def parse_http_last_modified(date_string) -> datetime:
+    parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT')
+    parsed_date = parsed_date.replace(tzinfo=timezone.utc)
+    return parsed_date

modules/tile_gen/tile_gen_lib/rclone.py

@@ -132,3 +132,17 @@ def make_indexes_for_bucket(bucket):
         check=True,
         input=index_str.encode(),
     )
+
+
+def set_version_on_bucket(area, version):
+    print(f'setting version: {area} {version}')
+    subprocess.run(
+        [
+            config.rclone_bin,
+            'rcat',
+            f'remote:ofm-assets/deployed_versions/{area}.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=version.strip().encode(),
+    )

modules/tile_gen/tile_gen_lib/set_version.py

@@ -1,56 +0,0 @@
-import subprocess
-
-from .config import config
-from .shared import check_host_version, get_deployed_version, get_versions_for_area
-
-
-def check_and_set_version(area, version):
-    if version == 'latest':
-        versions = get_versions_for_area(area)
-        if not versions:
-            print(f'  No versions found for {area}')
-            return
-
-        version = versions[-1]
-        print(f'  Latest version on bucket: {area} {version}')
-
-    if not check_all_hosts(area, version):
-        return
-
-    try:
-        if get_deployed_version(area)['version'] == version:
-            return
-    except Exception:
-        pass
-
-    set_version(area, version)
-
-
-def set_version(area, version):
-    print(f'setting version: {area} {version}')
-    subprocess.run(
-        [
-            config.rclone_bin,
-            'rcat',
-            f'remote:ofm-assets/deployed_versions/{area}.txt',
-        ],
-        env=dict(RCLONE_CONFIG=config.rclone_config),
-        check=True,
-        input=version.strip().encode(),
-    )
-
-
-def check_all_hosts(area, version) -> bool:
-    oc = config.ofm_config
-
-    domain = oc['domain_roundrobin'] or oc['domain_direct']
-    print(f'Using domain: {domain}')
-
-    try:
-        for host_ip in oc['http_host_list']:
-            print(f'Checking {area} {version} on host {host_ip}')
-            check_host_version(domain, host_ip, area, version)
-        return True
-    except Exception:
-        print('Error, version not available')
-        return False

modules/tile_gen/tile_gen_lib/shared.py

@@ -1,134 +0,0 @@
-import json
-from datetime import datetime, timezone
-from io import BytesIO
-from pathlib import Path
-
-import pycurl
-import requests
-
-
-def get_versions_for_area(area: str) -> list:
-    """
-    Download the files.txt and check for the runs with the "done" file present
-    """
-    r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30)
-    r.raise_for_status()
-
-    versions = []
-
-    files = r.text.splitlines()
-    for f in files:
-        if not f.startswith(f'areas/{area}/'):
-            continue
-        if not f.endswith('/done'):
-            continue
-        version_str = f.split('/')[2]
-        versions.append(version_str)
-
-    return sorted(versions)
-
-
-def get_deployed_version(area: str) -> dict:
-    r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
-    r.raise_for_status()
-    version = r.text.strip()
-
-    last_modified_str = r.headers.get('Last-Modified')
-    last_modified = parse_http_last_modified(last_modified_str)
-
-    return dict(
-        version=version,
-        last_modified=last_modified,
-    )
-
-
-def parse_http_last_modified(date_string) -> datetime:
-    parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT')
-    parsed_date = parsed_date.replace(tzinfo=timezone.utc)
-    return parsed_date
-
-
-def check_host_version(domain, host_ip, area, version):
-    # check versioned TileJSON
-    check_tilejson(f'https://{domain}/{area}/{version}', domain, host_ip, version)
-
-    # check actual vector tile
-    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
-    assert pycurl_status(url, domain, host_ip) == 200
-
-
-def check_host_latest(domain, host_ip, area, version):
-    # check latest TileJSON
-    check_tilejson(f'https://{domain}/{area}', domain, host_ip, version)
-
-    # check versioned TileJSON
-    check_tilejson(f'https://{domain}/{area}/{version}', domain, host_ip, version)
-
-    # check actual vector tile
-    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
-    assert pycurl_status(url, domain, host_ip) == 200
-
-    # check style
-    url = f'https://{domain}/styles/bright'
-    assert pycurl_status(url, domain, host_ip) == 200
-
-
-def check_tilejson(url, domain, host_ip, version):
-    tilejson_str = pycurl_get(url, domain, host_ip)
-    tilejson = json.loads(tilejson_str)
-    tiles_url = tilejson['tiles'][0]
-    version_in_tilejson = tiles_url.split('/')[4]
-    assert version_in_tilejson == version
-
-
-# pycurl
-
-
-def pycurl_status(url, domain, host_ip):
-    """
-    Uses pycurl to make a HTTPS HEAD request using custom resolving,
-    checks if the status code is 200
-    """
-
-    c = pycurl.Curl()
-    c.setopt(c.URL, url)
-
-    # linux needs CA certs specified manually
-    if Path('/etc/ssl/certs/ca-certificates.crt').exists():
-        c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
-
-    c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
-    c.setopt(c.NOBODY, True)
-    c.setopt(c.TIMEOUT, 5)
-    c.perform()
-    status_code = c.getinfo(c.RESPONSE_CODE)
-    c.close()
-
-    return status_code
-
-
-def pycurl_get(url, domain, host_ip):
-    """
-    Uses pycurl to make a HTTPS GET request using custom resolving,
-    checks if the status code is 200, and returns the content.
-    """
-
-    buffer = BytesIO()
-    c = pycurl.Curl()
-    c.setopt(c.URL, url)
-
-    # linux needs CA certs specified manually
-    if Path('/etc/ssl/certs/ca-certificates.crt').exists():
-        c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
-
-    c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
-    c.setopt(c.WRITEDATA, buffer)
-    c.setopt(c.TIMEOUT, 5)
-    c.perform()
-    status_code = c.getinfo(c.RESPONSE_CODE)
-    c.close()
-
-    if status_code != 200:
-        raise ValueError(f'status code: {status_code}')
-
-    return buffer.getvalue().decode('utf8')

ssh_lib/tasks.py

@@ -33,8 +33,6 @@ def prepare_shared(c):
     c.sudo(f'chown ofm:ofm {REMOTE_CONFIG}')
     c.sudo(f'chown ofm:ofm {OFM_DIR}')
 
-    upload_config_json(c)
-
     prepare_venv(c)
 
 
@@ -86,6 +84,8 @@ def prepare_http_host(c):
     kernel_somaxconn65k(c)
     kernel_limits1m(c)
 
+    upload_config_json(c)
+
     nginx(c)
     certbot(c)