From 3654d523d2b3383214d09feb221aba54d9fdec3f Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Wed, 3 Jan 2024 20:07:22 +0100 Subject: [PATCH] cron fixes --- scripts/http_host/cron.d/ofm_http_host | 3 +- scripts/http_host/host_manager.py | 16 ++++- .../http_host_lib/download_tileset.py | 2 +- scripts/http_host/http_host_lib/mount.py | 58 +++++++++---------- scripts/http_host/http_host_lib/utils.py | 2 +- 5 files changed, 47 insertions(+), 34 deletions(-) diff --git a/scripts/http_host/cron.d/ofm_http_host b/scripts/http_host/cron.d/ofm_http_host index dcda3db..fc5afa1 100644 --- a/scripts/http_host/cron.d/ofm_http_host +++ b/scripts/http_host/cron.d/ofm_http_host @@ -1,5 +1,4 @@ # every minute sync, locking so that only one process can run at a time -* * * * * ofm /usr/bin/flock -n /tmp/hostmanager.lockfile -c \ - 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync >> /data/ofm/http_host/logs/host_manager_sync.log 2>&1' +* * * * * ofm /usr/bin/flock -n /tmp/hostmanager.lockfile -c 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync >> /data/ofm/http_host/logs/host_manager_sync.log 2>&1' diff --git a/scripts/http_host/host_manager.py b/scripts/http_host/host_manager.py index fe90d43..ca89d77 100755 --- a/scripts/http_host/host_manager.py +++ b/scripts/http_host/host_manager.py @@ -43,6 +43,8 @@ def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Pat Version can also be specified. """ + print('running download_tileset') + if area not in {'planet', 'monaco'}: sys.exit('Please specify area: "planet" or "monaco"') @@ -83,6 +85,8 @@ def download_assets(assets_dir: Path): Downloads and extracts assets """ + print('running download_assets') + if not assets_dir: assets_dir = DEFAULT_ASSETS_DIR @@ -98,6 +102,9 @@ def mount(): Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically. When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way. """ + + print('running mount') + assert_linux() assert_sudo() @@ -123,6 +130,8 @@ def set_latest_versions(): 2. Writes to a version file """ + print('running set_latest_versions') + assert_linux() assert_sudo() @@ -138,6 +147,8 @@ def nginx_sync(): Syncs the nginx config to the state of the system """ + print('running nginx_sync') + assert_linux() assert_sudo() @@ -154,13 +165,16 @@ def sync(ctx): Runs the sync task, normally called by cron every minute On a new server this also takes care of everything, no need to run anything manually. """ + + print('---') + print('running sync') print(datetime.datetime.now(tz=datetime.timezone.utc)) assert_single_process() download_done = False download_done += ctx.invoke(download_tileset, area='monaco') - download_done += ctx.invoke(download_tileset, area='planet') + # download_done += ctx.invoke(download_tileset, area='planet') if download_done: ctx.invoke(mount) diff --git a/scripts/http_host/http_host_lib/download_tileset.py b/scripts/http_host/http_host_lib/download_tileset.py index a7b2d59..d81f37c 100644 --- a/scripts/http_host/http_host_lib/download_tileset.py +++ b/scripts/http_host/http_host_lib/download_tileset.py @@ -18,7 +18,7 @@ def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> boo version_dir = runs_dir / area / version btrfs_file = version_dir / 'tiles.btrfs' if btrfs_file.exists(): - print(' file exists, skipping download') + print(' file exists, skipping download') return False temp_dir = runs_dir / '_tmp' diff --git a/scripts/http_host/http_host_lib/mount.py b/scripts/http_host/http_host_lib/mount.py index dca53b3..0607bbb 100644 --- a/scripts/http_host/http_host_lib/mount.py +++ b/scripts/http_host/http_host_lib/mount.py @@ -4,11 +4,39 @@ from pathlib import Path from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR +def create_fstab(): + fstab_new = [] + + for area in ['planet', 'monaco']: + area_dir = (DEFAULT_RUNS_DIR / area).resolve() + if not area_dir.exists(): + continue + + versions = sorted(area_dir.iterdir()) + for version in versions: + version_str = version.name + btrfs_file = area_dir / version_str / 'tiles.btrfs' + if not btrfs_file.is_file(): + continue + + mnt_folder = MNT_DIR / f'{area}-{version_str}' + mnt_folder.mkdir(exist_ok=True, parents=True) + + fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n') + print(f' created fstab entry for {btrfs_file} -> {mnt_folder}') + + with open('/etc/fstab') as fp: + fstab_orig = [l for l in fp.readlines() if f'{MNT_DIR}/' not in l] + + with open('/etc/fstab', 'w') as fp: + fp.writelines(fstab_orig + fstab_new) + + def clean_up_mounts(mnt_dir): if not mnt_dir.exists(): return - print('Cleaning up mounts') + print(' cleaning up mounts') # handle deleted files p = subprocess.run(['mount'], capture_output=True, text=True, check=True) @@ -32,31 +60,3 @@ def clean_up_mounts(mnt_dir): print(f' removing old mount {subdir}') subprocess.run(['umount', subdir], check=True) subdir.rmdir() - - -def create_fstab(): - fstab_new = [] - - for area in ['planet', 'monaco']: - area_dir = (DEFAULT_RUNS_DIR / area).resolve() - if not area_dir.exists(): - continue - - versions = sorted(area_dir.iterdir()) - for version in versions: - version_str = version.name - btrfs_file = area_dir / version_str / 'tiles.btrfs' - if not btrfs_file.is_file(): - continue - - mnt_folder = MNT_DIR / f'{area}-{version_str}' - mnt_folder.mkdir(exist_ok=True, parents=True) - - fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n') - print(f'Created fstab entry for {btrfs_file} -> {mnt_folder}') - - with open('/etc/fstab') as fp: - fstab_orig = [l for l in fp.readlines() if f'{MNT_DIR}/' not in l] - - with open('/etc/fstab', 'w') as fp: - fp.writelines(fstab_orig + fstab_new) diff --git a/scripts/http_host/http_host_lib/utils.py b/scripts/http_host/http_host_lib/utils.py index cb6d987..2042ce5 100644 --- a/scripts/http_host/http_host_lib/utils.py +++ b/scripts/http_host/http_host_lib/utils.py @@ -39,7 +39,7 @@ def get_remote_file_size(url: str) -> int | None: def download_file_aria2(url: str, local_file: Path): - print(f'Downloading: {url} into {local_file}') + print(f' downloading: {url} into {local_file}') subprocess.run( [