diff --git a/init-server.py b/init-server.py index 0b0edc2..251fd94 100755 --- a/init-server.py +++ b/init-server.py @@ -11,12 +11,11 @@ from ssh_lib.tasks import ( run_http_host_sync, setup_ledns_writer, setup_loadbalancer, - upload_http_host_config, + upload_config_json, upload_http_host_files, ) from ssh_lib.utils import ( put, - put_dir, ) @@ -65,7 +64,7 @@ def http_host_static(hostname, user, port): c = get_connection(hostname, user, port) prepare_shared(c) - upload_http_host_config(c) + upload_config_json(c) prepare_http_host(c) @@ -83,8 +82,8 @@ def http_host_autoupdate(hostname, user, port): c.sudo('rm -f /etc/cron.d/ofm_http_host') prepare_shared(c) - upload_http_host_config(c) + upload_config_json(c) prepare_http_host(c) run_http_host_sync(c) @@ -132,8 +131,7 @@ def loadbalancer(hostname, user, port): def debug(hostname, user, port): c = get_connection(hostname, user, port) - upload_http_host_files(c) - # run_http_host_sync(c) + upload_config_json(c) if __name__ == '__main__': diff --git a/modules/http_host/http_host_lib/btrfs.py b/modules/http_host/http_host_lib/btrfs.py index 4fedb72..4f6e6bf 100644 --- a/modules/http_host/http_host_lib/btrfs.py +++ b/modules/http_host/http_host_lib/btrfs.py @@ -2,9 +2,8 @@ import shutil import subprocess import sys -import requests - from http_host_lib.config import config +from http_host_lib.shared import get_versions_for_area from http_host_lib.utils import download_file_aria2, get_remote_file_size @@ -41,14 +40,6 @@ def download_area_version(area: str, version: str) -> bool: return download_and_extract_btrfs(area, selected_version) -def get_versions_for_area(area: str) -> list: - r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30) - r.raise_for_status() - - versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')] - return sorted(versions) - - def download_and_extract_btrfs(area: str, version: str) -> bool: """ returns True if download successful, False if skipped diff --git a/modules/http_host/http_host_lib/shared.py b/modules/http_host/http_host_lib/shared.py new file mode 120000 index 0000000..37f60fb --- /dev/null +++ b/modules/http_host/http_host_lib/shared.py @@ -0,0 +1 @@ +../../tile_gen/tile_gen_lib/shared.py \ No newline at end of file diff --git a/modules/loadbalancer/loadbalancer.py b/modules/loadbalancer/loadbalancer.py index bf92d0c..5bbe551 100755 --- a/modules/loadbalancer/loadbalancer.py +++ b/modules/loadbalancer/loadbalancer.py @@ -111,22 +111,6 @@ def run_area(c, area): return results -def check_host(domain, host_ip, area, version): - # check TileJSON first - url = f'https://{domain}/{area}' - tilejson_str = pycurl_get(url, domain, host_ip) - tilejson = json.loads(tilejson_str) - tiles_url = tilejson['tiles'][0] - version_in_tilejson = tiles_url.split('/')[4] - assert version_in_tilejson == version - - # check actual vector tile - url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf' - assert pycurl_status(url, domain, host_ip) == 200 - - # check style - url = f'https://{domain}/styles/bright' - assert pycurl_status(url, domain, host_ip) == 200 def get_target_version(area): diff --git a/modules/loadbalancer/loadbalancer_lib/shared.py b/modules/loadbalancer/loadbalancer_lib/shared.py new file mode 120000 index 0000000..37f60fb --- /dev/null +++ b/modules/loadbalancer/loadbalancer_lib/shared.py @@ -0,0 +1 @@ +../../tile_gen/tile_gen_lib/shared.py \ No newline at end of file diff --git a/modules/set_version/setup.py b/modules/set_version/setup.py deleted file mode 100644 index 66d3b58..0000000 --- a/modules/set_version/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import find_packages, setup - - -requirements = [ - 'click', - 'requests', - 'pycurl', - 'python-dotenv', - 'questionary', -] - - -setup( - python_requires='>=3.10', - install_requires=requirements, - packages=find_packages(), -) diff --git a/modules/set_version/setversion.py b/modules/set_version/setversion.py deleted file mode 100755 index 3352c5c..0000000 --- a/modules/set_version/setversion.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess - -import click -import questionary -from setversion_lib import RCLONE_BIN, RCLONE_CONF - - -@click.group() -def cli(): - """ - Sets deployed reference versions - """ - - -@cli.command() -@click.argument('area', required=True) -def interactive(area): - versions = get_available_versions(area)[::-1] - - choices = [questionary.Choice(title=r, value=i) for i, r in enumerate(versions)] - answer = questionary.select(f'Select version for: {area}', choices=choices).ask() - - selected = versions[answer] - - set_version(area, selected) - - -def get_available_versions(area): - p = subprocess.run( - [ - RCLONE_BIN, - 'cat', - f'remote:ofm-{area}/dirs.txt', - ], - env=dict(RCLONE_CONFIG=RCLONE_CONF), - check=True, - capture_output=True, - text=True, - ) - versions = [l.strip() for l in p.stdout.strip().splitlines()] - versions.sort() - - return versions - - -def set_version(area, version): - subprocess.run( - [ - RCLONE_BIN, - 'rcat', - f'remote:ofm-assets/versions/deployed_{area}.txt', - ], - env=dict(RCLONE_CONFIG=RCLONE_CONF), - check=True, - input=version.encode(), - ) - - -if __name__ == '__main__': - cli() diff --git a/modules/set_version/setversion_lib/__init__.py b/modules/set_version/setversion_lib/__init__.py deleted file mode 100644 index 77672e6..0000000 --- a/modules/set_version/setversion_lib/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from pathlib import Path - - -if Path('/data/ofm/config').exists(): - OFM_CONFIG_DIR = Path('/data/ofm/config') -else: - OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config' - -assert OFM_CONFIG_DIR.exists() - -RCLONE_CONF = OFM_CONFIG_DIR / 'rclone.conf' - -if Path('/opt/homebrew/bin/rclone').exists(): - RCLONE_BIN = '/opt/homebrew/bin/rclone' -else: - RCLONE_BIN = 'rclone' diff --git a/modules/tile_gen/tile_gen.py b/modules/tile_gen/tile_gen.py index ae373d3..1b8f315 100755 --- a/modules/tile_gen/tile_gen.py +++ b/modules/tile_gen/tile_gen.py @@ -4,6 +4,7 @@ import click from tile_gen_lib.btrfs import make_btrfs from tile_gen_lib.planetiler import run_planetiler from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area +from tile_gen_lib.set_version import check_and_set_version @click.group() @@ -48,5 +49,18 @@ def make_indexes(): make_indexes_for_bucket(bucket) +@cli.command() +@click.argument('area', required=True) +@click.option( + '--version', default='latest', help='Optional version string, like "20231227_043106_pt"' +) +def set_version(area, version): + """ + Set versions for a given area + """ + + check_and_set_version(area, version) + + if __name__ == '__main__': cli() diff --git a/modules/tile_gen/tile_gen_lib/host_check.py b/modules/tile_gen/tile_gen_lib/host_check.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/tile_gen/tile_gen_lib/set_version.py b/modules/tile_gen/tile_gen_lib/set_version.py new file mode 100644 index 0000000..596f133 --- /dev/null +++ b/modules/tile_gen/tile_gen_lib/set_version.py @@ -0,0 +1,31 @@ +import subprocess + +from http_host_lib.config import config + +from tile_gen_lib.shared import get_versions_for_area + + +def check_all_hosts(area, version): + pass + + +def check_and_set_version(area, version): + if version == 'latest': + versions = get_versions_for_area(area) + version = versions[-1] + + if not check_all_hosts(area, version): + return + + +def set_version(area, version): + subprocess.run( + [ + 'rclone', + 'rcat', + f'remote:ofm-assets/deployed_versions/{area}.txt', + ], + env=dict(RCLONE_CONFIG=config.rclone_config), + check=True, + input=version.strip().encode(), + ) diff --git a/modules/loadbalancer/loadbalancer_lib/curl.py b/modules/tile_gen/tile_gen_lib/shared.py similarity index 53% rename from modules/loadbalancer/loadbalancer_lib/curl.py rename to modules/tile_gen/tile_gen_lib/shared.py index 2a1a11c..0bccf78 100644 --- a/modules/loadbalancer/loadbalancer_lib/curl.py +++ b/modules/tile_gen/tile_gen_lib/shared.py @@ -1,7 +1,45 @@ +import json from io import BytesIO from pathlib import Path import pycurl +import requests + + +def get_versions_for_area(area: str) -> list: + r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30) + r.raise_for_status() + + versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')] + return sorted(versions) + + +def check_host_version(domain, host_ip, area, version): + # check actual vector tile + url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf' + assert pycurl_status(url, domain, host_ip) == 200 + + # check style + url = f'https://{domain}/styles/bright' + assert pycurl_status(url, domain, host_ip) == 200 + + +def check_host_latest(domain, host_ip, area, version): + # check TileJSON first + url = f'https://{domain}/{area}' + tilejson_str = pycurl_get(url, domain, host_ip) + tilejson = json.loads(tilejson_str) + tiles_url = tilejson['tiles'][0] + version_in_tilejson = tiles_url.split('/')[4] + assert version_in_tilejson == version + + # check actual vector tile + url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf' + assert pycurl_status(url, domain, host_ip) == 200 + + # check style + url = f'https://{domain}/styles/bright' + assert pycurl_status(url, domain, host_ip) == 200 def pycurl_status(url, domain, host_ip): diff --git a/prepare-virtualenv.sh b/prepare-virtualenv.sh index 14c33e5..9bf7631 100755 --- a/prepare-virtualenv.sh +++ b/prepare-virtualenv.sh @@ -14,7 +14,6 @@ pip install -e . pip install -e modules/http_host pip install -e modules/tile_gen pip install -e modules/loadbalancer -pip install -e modules/set_version diff --git a/ssh_lib/tasks.py b/ssh_lib/tasks.py index c774d55..9be5d9a 100644 --- a/ssh_lib/tasks.py +++ b/ssh_lib/tasks.py @@ -78,40 +78,6 @@ def prepare_tile_gen(c): put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/') -def upload_http_host_config(c): - domain_le = dotenv_val('DOMAIN_LE').lower() - domain_ledns = dotenv_val('DOMAIN_LEDNS').lower() - skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true' - le_email = dotenv_val('LE_EMAIL').lower() - - if not (domain_le or domain_ledns): - sys.exit('Please specify DOMAIN_LE or DOMAIN_LEDNS in config/.env') - - if domain_le and not le_email: - sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE') - - host_config = { - 'domain_le': domain_le, - 'domain_ledns': domain_ledns, - 'skip_planet': skip_planet, - 'le_email': le_email, - } - - host_config_str = json.dumps(host_config, indent=2, ensure_ascii=False) - print(host_config_str) - put_str(c, '/data/ofm/config/http_host.json', host_config_str) - - if domain_ledns: - assert (CONFIG_DIR / 'rclone.conf').exists() - put( - c, - CONFIG_DIR / 'rclone.conf', - f'{REMOTE_CONFIG}/rclone.conf', - permissions=400, - ) - put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_ledns_reader', '/etc/cron.d/') - - def prepare_http_host(c): kernel_tweaks_ofm(c) @@ -128,6 +94,16 @@ def prepare_http_host(c): upload_http_host_files(c) + if dotenv_val('DOMAIN_LEDNS'): + assert (CONFIG_DIR / 'rclone.conf').exists() + put( + c, + CONFIG_DIR / 'rclone.conf', + f'{REMOTE_CONFIG}/rclone.conf', + permissions=400, + ) + put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_ledns_reader', '/etc/cron.d/') + c.sudo(f'{VENV_BIN}/pip install -e {HTTP_HOST_BIN} --use-pep517') @@ -219,21 +195,38 @@ def setup_ledns_writer(c): ) -def setup_loadbalancer(c): +def upload_config_json(c): + domain_le = dotenv_val('DOMAIN_LE').lower() domain_ledns = dotenv_val('DOMAIN_LEDNS').lower() + skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true' + le_email = dotenv_val('LE_EMAIL').lower() + + if not (domain_le or domain_ledns): + sys.exit('Please specify DOMAIN_LE or DOMAIN_LEDNS in config/.env') + + if domain_le and not le_email: + sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE') + http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()] assert (CONFIG_DIR / 'cloudflare.ini').exists() config = { + 'domain_le': domain_le, 'domain_ledns': domain_ledns, + 'le_email': le_email, + 'skip_planet': skip_planet, 'http_host_list': http_host_list, 'telegram_token': dotenv_val('TELEGRAM_TOKEN'), 'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'), } config_str = json.dumps(config, indent=2, ensure_ascii=False) - # print(config_str) - put_str(c, f'{REMOTE_CONFIG}/loadbalancer.json', config_str) + print(config_str) + put_str(c, f'{REMOTE_CONFIG}/config.json', config_str) + + +def setup_loadbalancer(c): + upload_config_json(c) put( c,