From d753c8738aec38b17f53daf8fb7aaeb581dc81f7 Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Thu, 29 Aug 2024 18:40:32 +0200 Subject: [PATCH] versions --- modules/http_host/http_host.py | 78 +++---------------- modules/http_host/http_host_lib/btrfs.py | 2 +- modules/http_host/http_host_lib/config.py | 1 + modules/http_host/http_host_lib/nginx.py | 3 + .../http_host_lib/set_tileset_versions.py | 45 ----------- modules/http_host/http_host_lib/sync.py | 39 ++++++++++ modules/http_host/http_host_lib/versions.py | 52 +++++++++++++ ssh_lib/tasks.py | 3 +- 8 files changed, 109 insertions(+), 114 deletions(-) delete mode 100644 modules/http_host/http_host_lib/set_tileset_versions.py create mode 100644 modules/http_host/http_host_lib/sync.py create mode 100644 modules/http_host/http_host_lib/versions.py diff --git a/modules/http_host/http_host.py b/modules/http_host/http_host.py index acd7449..ff7b361 100755 --- a/modules/http_host/http_host.py +++ b/modules/http_host/http_host.py @@ -1,8 +1,5 @@ #!/usr/bin/env python3 -import datetime -import sys - import click from http_host_lib.assets import ( download_assets, @@ -11,11 +8,9 @@ from http_host_lib.btrfs import ( download_area_version, get_versions_for_area, ) -from http_host_lib.config import config from http_host_lib.mount import auto_mount_unmount -from http_host_lib.nginx import write_nginx_config -from http_host_lib.set_tileset_versions import set_tileset_versions -from http_host_lib.utils import assert_linux, assert_sudo +from http_host_lib.sync import full_sync +from http_host_lib.versions import sync_version_files @click.group() @@ -25,9 +20,8 @@ def cli(): - Downloading btrfs images\n - Downloading assets\n - Mounting directories\n - - Updating nginx config\n - Getting the deployed versions of tilesets\n - - Running the sync cron task (called every minute) + - Running the sync cron task (called every minute with http-host-autoupdate) """ @@ -43,7 +37,7 @@ def download_btrfs(area: str, version: str): Use --version=1 to list all available versions """ - return download_area_version(area, version) + download_area_version(area, version) @cli.command(name='download-assets') @@ -65,75 +59,25 @@ def mount(): auto_mount_unmount() -@cli.command() -def set_latest_versions(): +@cli.command(name='sync-version-files') +def sync_version_files_(): """ - Sets the latest version of the tilesets to the version specified by - https://assets.openfreemap.com/versions/deployed_planet.txt - - 1. Checks if the given version is present on the disk and is mounted - 2. Writes to a version file + Syncs the version files from remote to local. + Remove versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt """ - print('running set_latest_versions') - - assert_linux() - assert_sudo() - - if not config.mnt_dir.exists(): - sys.exit(' mount needs to be run first') - - return set_tileset_versions() - - -@cli.command() -def nginx_sync(): - """ - Syncs the nginx config to the state of the system - """ - - print('running nginx_sync') - - assert_linux() - assert_sudo() - - if not config.mnt_dir.exists(): - sys.exit(' mount needs to be run first') - - write_nginx_config() + sync_version_files() @cli.command() @click.option('--force', is_flag=True, help='Force nginx sync run') -@click.pass_context -def sync(ctx, force): +def sync(force): """ Runs the sync task, normally called by cron every minute On a new server this also takes care of everything, no need to run anything manually. """ - print('---') - print('running sync') - print(datetime.datetime.now(tz=datetime.timezone.utc)) - - assert_linux() - assert_sudo() - - download_done = False - download_done += ctx.invoke(download_btrfs, area='monaco') - - if not config.host_config.get('skip_planet'): - download_done += ctx.invoke(download_btrfs, area='planet') - - if download_done: - ctx.invoke(mount) - - ctx.invoke(download_assets) - - deploy_done = ctx.invoke(set_latest_versions) - - if download_done or deploy_done or force: - ctx.invoke(nginx_sync) + full_sync(force) @cli.command() diff --git a/modules/http_host/http_host_lib/btrfs.py b/modules/http_host/http_host_lib/btrfs.py index 4c325f9..b713534 100644 --- a/modules/http_host/http_host_lib/btrfs.py +++ b/modules/http_host/http_host_lib/btrfs.py @@ -8,7 +8,7 @@ from http_host_lib.config import config from http_host_lib.utils import download_file_aria2, get_remote_file_size -def download_area_version(area: str, version: str): +def download_area_version(area: str, version: str) -> bool: """ Downloads and uncompresses tiles.btrfs files from the btrfs bucket """ diff --git a/modules/http_host/http_host_lib/config.py b/modules/http_host/http_host_lib/config.py index 8739f17..397ede3 100644 --- a/modules/http_host/http_host_lib/config.py +++ b/modules/http_host/http_host_lib/config.py @@ -15,6 +15,7 @@ class Configuration: mnt_dir = Path('/mnt/ofm') ofm_config_dir = Path('/data/ofm/config') + deployed_versions_dir = ofm_config_dir / 'deployed_versions' certs_dir = Path('/data/nginx/certs') nginx_confs = Path(__file__).parent / 'nginx_confs' diff --git a/modules/http_host/http_host_lib/nginx.py b/modules/http_host/http_host_lib/nginx.py index c814342..e4c8cb0 100644 --- a/modules/http_host/http_host_lib/nginx.py +++ b/modules/http_host/http_host_lib/nginx.py @@ -8,6 +8,9 @@ from http_host_lib.utils import python_venv_executable def write_nginx_config(): + if not config.mnt_dir.exists(): + sys.exit(' mount needs to be run first') + curl_text_mix = '' domain_le = config.host_config['domain_le'] diff --git a/modules/http_host/http_host_lib/set_tileset_versions.py b/modules/http_host/http_host_lib/set_tileset_versions.py deleted file mode 100644 index 75e7cc5..0000000 --- a/modules/http_host/http_host_lib/set_tileset_versions.py +++ /dev/null @@ -1,45 +0,0 @@ -from pathlib import Path - -import requests - -from http_host_lib.config import config - - -def set_tileset_versions(): - need_nginx_sync = False - - for area in ['planet', 'monaco']: - r = requests.get(f'https://assets.openfreemap.com/versions/deployed_{area}.txt', timeout=30) - r.raise_for_status() - remote_version = r.text.strip() - print(f' remote version for {area}: {remote_version}') - - local_version_file = config.ofm_config_dir / f'tileset_version_{area}.txt' - - if not local_version_file.exists(): - local_version_start = None - else: - with open(local_version_file) as fp: - local_version_start = fp.read() - - if not remote_version: - print(' remote version not specified') - if local_version_start is not None: - local_version_file.unlink() - need_nginx_sync = True - continue - - mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json') - if not mnt_file.exists(): - print(' local version does not exist') - if local_version_start is not None: - local_version_file.unlink() - need_nginx_sync = True - continue - - if remote_version != local_version_start: - with open(local_version_file, 'w') as fp: - fp.write(remote_version) - need_nginx_sync = True - - return need_nginx_sync diff --git a/modules/http_host/http_host_lib/sync.py b/modules/http_host/http_host_lib/sync.py new file mode 100644 index 0000000..02d5f8c --- /dev/null +++ b/modules/http_host/http_host_lib/sync.py @@ -0,0 +1,39 @@ +from datetime import datetime, timezone + +from http_host_lib.assets import download_assets +from http_host_lib.btrfs import download_area_version +from http_host_lib.config import config +from http_host_lib.mount import auto_mount_unmount +from http_host_lib.nginx import write_nginx_config +from http_host_lib.utils import assert_linux, assert_sudo +from http_host_lib.versions import sync_version_files + + +def full_sync(force=False): + """ + Runs the sync task, normally called by cron every minute + On a new server this also takes care of everything, no need to run anything manually. + """ + + print('---') + print('running full_sync') + print(datetime.now(tz=timezone.utc)) + + assert_linux() + assert_sudo() + + download_done = False + download_done += download_area_version(area='monaco', version='latest') + + if not config.host_config.get('skip_planet'): + download_done += download_area_version(area='planet', version='latest') + + if download_done or force: + auto_mount_unmount() + + download_assets() + + versions_changed = sync_version_files() + + if download_done or versions_changed or force: + write_nginx_config() diff --git a/modules/http_host/http_host_lib/versions.py b/modules/http_host/http_host_lib/versions.py new file mode 100644 index 0000000..01905de --- /dev/null +++ b/modules/http_host/http_host_lib/versions.py @@ -0,0 +1,52 @@ +import sys +from pathlib import Path + +import requests + +from http_host_lib.config import config +from http_host_lib.utils import assert_linux, assert_sudo + + +def sync_version_files() -> bool: + """ + Syncs the version files from remote to local. + Remove versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt + """ + + print('Syncing local version files') + + assert_linux() + assert_sudo() + + if not config.mnt_dir.exists(): + sys.exit(' mount needs to be run first') + + need_nginx_sync = False + + for area in config.areas: + r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30) + r.raise_for_status() + remote_version = r.text.strip() + assert remote_version + print(f' remote version for {area}: {remote_version}') + + local_version_file = config.deployed_versions_dir / f'{area}.txt' + + try: + local_version_old = local_version_file.read_text() + except Exception: + local_version_old = None + + mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json') + if not mnt_file.exists(): + print(' local version does not exist') + if local_version_old is not None: + local_version_file.unlink() + need_nginx_sync = True + continue + + if remote_version != local_version_old: + local_version_file.write_text(remote_version) + need_nginx_sync = True + + return need_nginx_sync diff --git a/ssh_lib/tasks.py b/ssh_lib/tasks.py index 9221379..939ab32 100644 --- a/ssh_lib/tasks.py +++ b/ssh_lib/tasks.py @@ -4,9 +4,9 @@ import sys from ssh_lib import ( CONFIG_DIR, HTTP_HOST_BIN, + MODULES_DIR, OFM_DIR, REMOTE_CONFIG, - MODULES_DIR, TILE_GEN_BIN, VENV_BIN, dotenv_val, @@ -138,6 +138,7 @@ def run_http_host_sync(c): def upload_http_host_files(c): + c.sudo(f'rm -rf {HTTP_HOST_BIN}') c.sudo(f'mkdir -p {HTTP_HOST_BIN}') put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')