diff --git a/README.md b/README.md index 53a5a2c..e694fe9 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,6 @@ See [dev setup docs](docs/dev_setup.md). Updated Planetiler version to latest Updated OpenJDK to 24 via Temurin repo - - ##### v0.8 Lot of self-hosting related fixes. diff --git a/init-server.py b/init-server.py index e9c6965..3b4ce9d 100755 --- a/init-server.py +++ b/init-server.py @@ -121,11 +121,12 @@ def http_host_sync(hostname, user, port, noninteractive): run_http_host_sync(c) -@cli.command() -@common_options -def debug(hostname, user, port, noninteractive): - c = get_connection(hostname, user, port) - run_http_host_sync(c) +# +# @cli.command() +# @common_options +# def debug(hostname, user, port, noninteractive): +# c = get_connection(hostname, user, port) +# run_http_host_sync(c) if __name__ == '__main__': diff --git a/modules/http_host/http_host_lib/nginx.py b/modules/http_host/http_host_lib/nginx.py index dc7dfaf..3acee7b 100644 --- a/modules/http_host/http_host_lib/nginx.py +++ b/modules/http_host/http_host_lib/nginx.py @@ -320,15 +320,3 @@ def create_latest_locations(*, local: str, domain: str) -> str: """ return location_str - - -def write_roundrobin_reader_script(domain_roundrobin): - script = f""" -#!/usr/bin/env bash -export RCLONE_CONFIG=/data/ofm/config/rclone.conf -rclone copyto -v "remote:ofm-private/roundrobin/{domain_roundrobin}/ofm_roundrobin.cert" /data/nginx/certs/ofm_roundrobin.cert -rclone copyto -v "remote:ofm-private/roundrobin/{domain_roundrobin}/ofm_roundrobin.key" /data/nginx/certs/ofm_roundrobin.key - """.strip() - - with open(config.http_host_bin / 'roundrobin_reader.sh', 'w') as fp: - fp.write(script) diff --git a/modules/http_host/http_host_lib/shared.py b/modules/http_host/http_host_lib/shared.py deleted file mode 120000 index 37f60fb..0000000 --- a/modules/http_host/http_host_lib/shared.py +++ /dev/null @@ -1 +0,0 @@ -../../tile_gen/tile_gen_lib/shared.py \ No newline at end of file diff --git a/modules/tile_gen/cron.d/ofm_tile_gen b/modules/tile_gen/cron.d/ofm_tile_gen index df95cbf..786ffe0 100644 --- a/modules/tile_gen/cron.d/ofm_tile_gen +++ b/modules/tile_gen/cron.d/ofm_tile_gen @@ -5,8 +5,8 @@ LOG_DIR=/data/ofm/tile_gen/logs # every day at 23:10, make a monaco run 10 23 * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1 -# debug monaco run, every minute -#*/1 * * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1 +# debug monaco run, normally disabled, enable to run every minute +#* * * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1 # every minute, set monaco to latest * * * * * ofm $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1 diff --git a/modules/tile_gen/setup.py b/modules/tile_gen/setup.py index b0aa452..16d6b17 100644 --- a/modules/tile_gen/setup.py +++ b/modules/tile_gen/setup.py @@ -3,7 +3,6 @@ from setuptools import find_packages, setup requirements = [ 'click', - 'pycurl', 'requests', ] diff --git a/modules/tile_gen/tile_gen.py b/modules/tile_gen/tile_gen.py index 917590c..59f8ea0 100755 --- a/modules/tile_gen/tile_gen.py +++ b/modules/tile_gen/tile_gen.py @@ -3,9 +3,12 @@ from datetime import datetime, timezone import click from tile_gen_lib.btrfs import make_btrfs +from tile_gen_lib.get_version_shared import ( + get_deployed_version, + get_versions_for_area, +) from tile_gen_lib.planetiler import run_planetiler -from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area -from tile_gen_lib.set_version import check_and_set_version +from tile_gen_lib.rclone import make_indexes_for_bucket, set_version_on_bucket, upload_area now = datetime.now(timezone.utc) @@ -71,7 +74,22 @@ def set_version(area, version): print(f'---\n{now}\nStarting set-version {area}') - check_and_set_version(area, version) + if version == 'latest': + versions = get_versions_for_area(area) + if not versions: + print(f' No versions found for {area}') + return + + version = versions[-1] + print(f' Latest version on bucket: {area} {version}') + + try: + if get_deployed_version(area)['version'] == version: + return + except Exception: + pass + + set_version_on_bucket(area, version) if __name__ == '__main__': diff --git a/modules/tile_gen/tile_gen_lib/config.py b/modules/tile_gen/tile_gen_lib/config.py index e64eebe..2124859 100644 --- a/modules/tile_gen/tile_gen_lib/config.py +++ b/modules/tile_gen/tile_gen_lib/config.py @@ -1,4 +1,3 @@ -import json import subprocess from pathlib import Path @@ -22,8 +21,6 @@ class Configuration: repo_root = Path(__file__).parent.parent.parent.parent ofm_config_dir = repo_root / 'config' - ofm_config = json.loads((ofm_config_dir / 'config.json').read_text()) - rclone_config = ofm_config_dir / 'rclone.conf' rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip() diff --git a/modules/tile_gen/tile_gen_lib/get_version_shared.py b/modules/tile_gen/tile_gen_lib/get_version_shared.py new file mode 100644 index 0000000..5e8cefd --- /dev/null +++ b/modules/tile_gen/tile_gen_lib/get_version_shared.py @@ -0,0 +1,48 @@ +""" +This file is shared / symlinked between tile_gen_lib and http_host_lib +""" + +from datetime import datetime, timezone + +import requests + + +def get_versions_for_area(area: str) -> list: + """ + Download the files.txt and check for the runs with the "done" file present + """ + r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30) + r.raise_for_status() + + versions = [] + + files = r.text.splitlines() + for f in files: + if not f.startswith(f'areas/{area}/'): + continue + if not f.endswith('/done'): + continue + version_str = f.split('/')[2] + versions.append(version_str) + + return sorted(versions) + + +def get_deployed_version(area: str) -> dict: + r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30) + r.raise_for_status() + version = r.text.strip() + + last_modified_str = r.headers.get('Last-Modified') + last_modified = parse_http_last_modified(last_modified_str) + + return dict( + version=version, + last_modified=last_modified, + ) + + +def parse_http_last_modified(date_string) -> datetime: + parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT') + parsed_date = parsed_date.replace(tzinfo=timezone.utc) + return parsed_date diff --git a/modules/tile_gen/tile_gen_lib/rclone.py b/modules/tile_gen/tile_gen_lib/rclone.py index 2d6c50c..76ae275 100644 --- a/modules/tile_gen/tile_gen_lib/rclone.py +++ b/modules/tile_gen/tile_gen_lib/rclone.py @@ -132,3 +132,17 @@ def make_indexes_for_bucket(bucket): check=True, input=index_str.encode(), ) + + +def set_version_on_bucket(area, version): + print(f'setting version: {area} {version}') + subprocess.run( + [ + config.rclone_bin, + 'rcat', + f'remote:ofm-assets/deployed_versions/{area}.txt', + ], + env=dict(RCLONE_CONFIG=config.rclone_config), + check=True, + input=version.strip().encode(), + ) diff --git a/modules/tile_gen/tile_gen_lib/set_version.py b/modules/tile_gen/tile_gen_lib/set_version.py deleted file mode 100644 index 0629c64..0000000 --- a/modules/tile_gen/tile_gen_lib/set_version.py +++ /dev/null @@ -1,56 +0,0 @@ -import subprocess - -from .config import config -from .shared import check_host_version, get_deployed_version, get_versions_for_area - - -def check_and_set_version(area, version): - if version == 'latest': - versions = get_versions_for_area(area) - if not versions: - print(f' No versions found for {area}') - return - - version = versions[-1] - print(f' Latest version on bucket: {area} {version}') - - if not check_all_hosts(area, version): - return - - try: - if get_deployed_version(area)['version'] == version: - return - except Exception: - pass - - set_version(area, version) - - -def set_version(area, version): - print(f'setting version: {area} {version}') - subprocess.run( - [ - config.rclone_bin, - 'rcat', - f'remote:ofm-assets/deployed_versions/{area}.txt', - ], - env=dict(RCLONE_CONFIG=config.rclone_config), - check=True, - input=version.strip().encode(), - ) - - -def check_all_hosts(area, version) -> bool: - oc = config.ofm_config - - domain = oc['domain_roundrobin'] or oc['domain_direct'] - print(f'Using domain: {domain}') - - try: - for host_ip in oc['http_host_list']: - print(f'Checking {area} {version} on host {host_ip}') - check_host_version(domain, host_ip, area, version) - return True - except Exception: - print('Error, version not available') - return False diff --git a/modules/tile_gen/tile_gen_lib/shared.py b/modules/tile_gen/tile_gen_lib/shared.py deleted file mode 100644 index c2f05f8..0000000 --- a/modules/tile_gen/tile_gen_lib/shared.py +++ /dev/null @@ -1,134 +0,0 @@ -import json -from datetime import datetime, timezone -from io import BytesIO -from pathlib import Path - -import pycurl -import requests - - -def get_versions_for_area(area: str) -> list: - """ - Download the files.txt and check for the runs with the "done" file present - """ - r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30) - r.raise_for_status() - - versions = [] - - files = r.text.splitlines() - for f in files: - if not f.startswith(f'areas/{area}/'): - continue - if not f.endswith('/done'): - continue - version_str = f.split('/')[2] - versions.append(version_str) - - return sorted(versions) - - -def get_deployed_version(area: str) -> dict: - r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30) - r.raise_for_status() - version = r.text.strip() - - last_modified_str = r.headers.get('Last-Modified') - last_modified = parse_http_last_modified(last_modified_str) - - return dict( - version=version, - last_modified=last_modified, - ) - - -def parse_http_last_modified(date_string) -> datetime: - parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT') - parsed_date = parsed_date.replace(tzinfo=timezone.utc) - return parsed_date - - -def check_host_version(domain, host_ip, area, version): - # check versioned TileJSON - check_tilejson(f'https://{domain}/{area}/{version}', domain, host_ip, version) - - # check actual vector tile - url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf' - assert pycurl_status(url, domain, host_ip) == 200 - - -def check_host_latest(domain, host_ip, area, version): - # check latest TileJSON - check_tilejson(f'https://{domain}/{area}', domain, host_ip, version) - - # check versioned TileJSON - check_tilejson(f'https://{domain}/{area}/{version}', domain, host_ip, version) - - # check actual vector tile - url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf' - assert pycurl_status(url, domain, host_ip) == 200 - - # check style - url = f'https://{domain}/styles/bright' - assert pycurl_status(url, domain, host_ip) == 200 - - -def check_tilejson(url, domain, host_ip, version): - tilejson_str = pycurl_get(url, domain, host_ip) - tilejson = json.loads(tilejson_str) - tiles_url = tilejson['tiles'][0] - version_in_tilejson = tiles_url.split('/')[4] - assert version_in_tilejson == version - - -# pycurl - - -def pycurl_status(url, domain, host_ip): - """ - Uses pycurl to make a HTTPS HEAD request using custom resolving, - checks if the status code is 200 - """ - - c = pycurl.Curl() - c.setopt(c.URL, url) - - # linux needs CA certs specified manually - if Path('/etc/ssl/certs/ca-certificates.crt').exists(): - c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt') - - c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}']) - c.setopt(c.NOBODY, True) - c.setopt(c.TIMEOUT, 5) - c.perform() - status_code = c.getinfo(c.RESPONSE_CODE) - c.close() - - return status_code - - -def pycurl_get(url, domain, host_ip): - """ - Uses pycurl to make a HTTPS GET request using custom resolving, - checks if the status code is 200, and returns the content. - """ - - buffer = BytesIO() - c = pycurl.Curl() - c.setopt(c.URL, url) - - # linux needs CA certs specified manually - if Path('/etc/ssl/certs/ca-certificates.crt').exists(): - c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt') - - c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}']) - c.setopt(c.WRITEDATA, buffer) - c.setopt(c.TIMEOUT, 5) - c.perform() - status_code = c.getinfo(c.RESPONSE_CODE) - c.close() - - if status_code != 200: - raise ValueError(f'status code: {status_code}') - - return buffer.getvalue().decode('utf8') diff --git a/ssh_lib/tasks.py b/ssh_lib/tasks.py index e50fb6c..23a20bd 100644 --- a/ssh_lib/tasks.py +++ b/ssh_lib/tasks.py @@ -33,8 +33,6 @@ def prepare_shared(c): c.sudo(f'chown ofm:ofm {REMOTE_CONFIG}') c.sudo(f'chown ofm:ofm {OFM_DIR}') - upload_config_json(c) - prepare_venv(c) @@ -86,6 +84,8 @@ def prepare_http_host(c): kernel_somaxconn65k(c) kernel_limits1m(c) + upload_config_json(c) + nginx(c) certbot(c)