From eb2d82d764a1551fc56c6ca1ff4e5ea93464a7b8 Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Tue, 10 Sep 2024 01:46:44 +0200 Subject: [PATCH] fix auto-update --- init-server.py | 6 +++--- modules/http_host/http_host.py | 1 - modules/http_host/http_host_lib/config.py | 15 ++++++++------- modules/http_host/http_host_lib/nginx.py | 19 +++++++++---------- modules/http_host/http_host_lib/sync.py | 2 +- modules/http_host/setup.py | 1 + modules/tile_gen/cron.d/ofm_tile_gen | 17 ++++++++++------- modules/tile_gen/tile_gen_lib/config.py | 3 --- ssh_lib/tasks.py | 5 +++-- 9 files changed, 35 insertions(+), 34 deletions(-) diff --git a/init-server.py b/init-server.py index 251fd94..0ff8fa4 100755 --- a/init-server.py +++ b/init-server.py @@ -12,7 +12,6 @@ from ssh_lib.tasks import ( setup_ledns_writer, setup_loadbalancer, upload_config_json, - upload_http_host_files, ) from ssh_lib.utils import ( put, @@ -93,14 +92,15 @@ def http_host_autoupdate(hostname, user, port): @cli.command() @common_options -def tile_gen(hostname, user, port): +@click.option('--cron', is_flag=True, help='Enable cron task') +def tile_gen(hostname, user, port, cron): if not click.confirm(f'Run script on {hostname}?'): return c = get_connection(hostname, user, port) prepare_shared(c) - prepare_tile_gen(c) + prepare_tile_gen(c, enable_cron=cron) @cli.command() diff --git a/modules/http_host/http_host.py b/modules/http_host/http_host.py index 5cee87b..e05a03c 100755 --- a/modules/http_host/http_host.py +++ b/modules/http_host/http_host.py @@ -106,5 +106,4 @@ def debug(): if __name__ == '__main__': - # print(config.host_config) cli() diff --git a/modules/http_host/http_host_lib/config.py b/modules/http_host/http_host_lib/config.py index 397ede3..bca1e0a 100644 --- a/modules/http_host/http_host_lib/config.py +++ b/modules/http_host/http_host_lib/config.py @@ -1,4 +1,5 @@ import json +import subprocess from pathlib import Path @@ -14,17 +15,17 @@ class Configuration: assets_dir = http_host_dir / 'assets' mnt_dir = Path('/mnt/ofm') - ofm_config_dir = Path('/data/ofm/config') - deployed_versions_dir = ofm_config_dir / 'deployed_versions' certs_dir = Path('/data/nginx/certs') nginx_confs = Path(__file__).parent / 'nginx_confs' - try: - with open(ofm_config_dir / 'http_host.json') as fp: - host_config = json.load(fp) - except Exception: - host_config = {} + ofm_config_dir = Path('/data/ofm/config') + deployed_versions_dir = ofm_config_dir / 'deployed_versions' + + ofm_config = json.loads((ofm_config_dir / 'config.json').read_text()) + + rclone_config = ofm_config_dir / 'rclone.conf' + rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip() config = Configuration() diff --git a/modules/http_host/http_host_lib/nginx.py b/modules/http_host/http_host_lib/nginx.py index 7014a25..5c627ea 100644 --- a/modules/http_host/http_host_lib/nginx.py +++ b/modules/http_host/http_host_lib/nginx.py @@ -13,8 +13,8 @@ def write_nginx_config(): curl_text_mix = '' - domain_le = config.host_config['domain_le'] - domain_ledns = config.host_config['domain_ledns'] + domain_le = config.ofm_config['domain_le'] + domain_ledns = config.ofm_config['domain_ledns'] # remove old configs and certs for file in Path('/data/nginx/sites').glob('ofm_*.conf'): @@ -25,7 +25,7 @@ def write_nginx_config(): # processing Round Robin DNS config if domain_ledns: - if not (config.ofm_config_dir / 'rclone.conf').is_file(): + if not config.rclone_config.is_file(): sys.exit('rclone.conf missing') # download the ledns certificate from bucket using rclone @@ -64,7 +64,7 @@ def write_nginx_config(): '--webroot-path=/data/nginx/acme-challenges', '--noninteractive', '-m', - config.host_config['le_email'], + config.ofm_config['le_email'], '--agree-tos', '--cert-name=ofm_le', # '--staging', @@ -125,12 +125,11 @@ def create_location_blocks(*, local, domain): area=area, version=version, subdir=subdir, local=local, domain=domain ) - if not curl_text: - curl_text = ( - '\ntest with:\n' - f'curl -H "Host: __LOCAL__" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n' - f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf' - ) + curl_text += ( + '\ntest with:\n' + f'curl -H "Host: __LOCAL__" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n' + f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf' + ) location_str += create_latest_locations(local=local, domain=domain) diff --git a/modules/http_host/http_host_lib/sync.py b/modules/http_host/http_host_lib/sync.py index 4122207..a2bfd1b 100644 --- a/modules/http_host/http_host_lib/sync.py +++ b/modules/http_host/http_host_lib/sync.py @@ -36,7 +36,7 @@ def full_sync(force=False): btrfs_downloaded += download_area_version(area='monaco', version='deployed') # download latest and deployed planet - if not config.host_config.get('skip_planet'): + if not config.ofm_config.get('skip_planet'): btrfs_downloaded += download_area_version(area='planet', version='latest') btrfs_downloaded += download_area_version(area='planet', version='deployed') diff --git a/modules/http_host/setup.py b/modules/http_host/setup.py index 16d6b17..b0aa452 100644 --- a/modules/http_host/setup.py +++ b/modules/http_host/setup.py @@ -3,6 +3,7 @@ from setuptools import find_packages, setup requirements = [ 'click', + 'pycurl', 'requests', ] diff --git a/modules/tile_gen/cron.d/ofm_tile_gen b/modules/tile_gen/cron.d/ofm_tile_gen index fd0ee06..2839a53 100644 --- a/modules/tile_gen/cron.d/ofm_tile_gen +++ b/modules/tile_gen/cron.d/ofm_tile_gen @@ -1,15 +1,18 @@ +# Define common variables +CMD="sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py" +LOG_DIR=/data/ofm/tile_gen/logs + # every hour, make a monaco run -10 * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-tiles monaco --upload >> /data/ofm/tile_gen/logs/monaco-make-tiles.log 2>&1 +10 * * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1 # every minute, set monaco to latest -* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py tile_gen.py set-version monaco >> /data/ofm/tile_gen/logs/monaco-set-version.log 2>&1 +* * * * * ofm $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1 # every Tuesday morning at 00:10, make a planet run -10 0 * * 2 ofm sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-tiles planet --upload >> /data/ofm/tile_gen/logs/planet-make-tiles.log 2>&1 - - +10 0 * * 2 ofm $CMD make-tiles planet --upload >> $LOG_DIR/planet-make-tiles.log 2>&1 +# every Saturday at 8:00, set planet to latest +0 9 * * 6 ofm $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1 # once per minute, create indexes -* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-indexes >> /data/ofm/tile_gen/logs/make-indexes-cron.log 2>&1 - +* * * * * ofm $CMD make-indexes >> $LOG_DIR/make-indexes-cron.log 2>&1 diff --git a/modules/tile_gen/tile_gen_lib/config.py b/modules/tile_gen/tile_gen_lib/config.py index 09704c1..cfa870d 100644 --- a/modules/tile_gen/tile_gen_lib/config.py +++ b/modules/tile_gen/tile_gen_lib/config.py @@ -21,14 +21,11 @@ class Configuration: ofm_config_dir = Path('/data/ofm/config') else: repo_root = Path(__file__).parent.parent.parent.parent - ofm_config_dir = repo_root / 'config' ofm_config = json.loads((ofm_config_dir / 'config.json').read_text()) - pprint(ofm_config) rclone_config = ofm_config_dir / 'rclone.conf' - rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip() diff --git a/ssh_lib/tasks.py b/ssh_lib/tasks.py index 2089699..cf42863 100644 --- a/ssh_lib/tasks.py +++ b/ssh_lib/tasks.py @@ -47,7 +47,7 @@ def prepare_venv(c): sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh') -def prepare_tile_gen(c): +def prepare_tile_gen(c, *, enable_cron): c.sudo('rm -f /etc/cron.d/ofm_tile_gen') install_planetiler(c) @@ -76,7 +76,8 @@ def prepare_tile_gen(c): c.sudo('chown ofm:ofm /data/ofm/tile_gen/{,*}') c.sudo(f'chown ofm:ofm -R {TILE_GEN_BIN}') - put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/') + if enable_cron: + put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/') def prepare_http_host(c):