diff --git a/scripts/http_host/.gitignore b/scripts/http_host/.gitignore deleted file mode 100644 index 5f718dc..0000000 --- a/scripts/http_host/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.json - -_assets diff --git a/scripts/http_host/host_manager.py b/scripts/http_host/host_manager.py index f6323e2..2a29bd3 100755 --- a/scripts/http_host/host_manager.py +++ b/scripts/http_host/host_manager.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 +import datetime +import subprocess import sys from pathlib import Path import click import requests +from http_host_lib import DEFAULT_ASSETS_DIR, DEFAULT_RUNS_DIR, MNT_DIR from http_host_lib.download_fonts import download_fonts from http_host_lib.download_tileset import download_and_extract_tileset - - -DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') -DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets') +from http_host_lib.mount import clean_up_mounts, create_fstab +from http_host_lib.utils import assert_linux, assert_single_process, assert_sudo @click.group() @@ -88,5 +89,42 @@ def download_assets(assets_dir: Path): download_fonts(assets_dir) +@cli.command() +def mount(): + """ + Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically. + When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way. + """ + assert_linux() + assert_sudo() + + if not DEFAULT_RUNS_DIR.exists(): + sys.exit('download_tileset needs to be run first') + + clean_up_mounts(MNT_DIR) + create_fstab() + + print('Running mount -a') + subprocess.run(['mount', '-a'], check=True) + + clean_up_mounts(MNT_DIR) + + +@cli.command() +@click.pass_context +def sync(ctx): + """ + Runs the sync task, normally called by cron every minute + """ + print(datetime.datetime.now(tz=datetime.timezone.utc)) + + ctx.invoke(download_tileset, area='monaco') + # ctx.invoke(download_tileset, area='planet') + ctx.invoke(download_assets) + ctx.invoke(mount) + + if __name__ == '__main__': + # TODO + assert_single_process() cli() diff --git a/scripts/http_host/http_host_lib/__init__.py b/scripts/http_host/http_host_lib/__init__.py index e69de29..4f817ad 100644 --- a/scripts/http_host/http_host_lib/__init__.py +++ b/scripts/http_host/http_host_lib/__init__.py @@ -0,0 +1,7 @@ +from pathlib import Path + + +DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') +DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets') + +MNT_DIR = Path('/mnt/ofm/') diff --git a/scripts/http_host/mounter.py b/scripts/http_host/http_host_lib/mount.py old mode 100755 new mode 100644 similarity index 52% rename from scripts/http_host/mounter.py rename to scripts/http_host/http_host_lib/mount.py index 575506f..dca53b3 --- a/scripts/http_host/mounter.py +++ b/scripts/http_host/http_host_lib/mount.py @@ -1,37 +1,44 @@ -#!/usr/bin/env python3 -import datetime -import os import subprocess -import sys from pathlib import Path -import click +from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR -@click.command() -def cli(): - """ - Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically. - When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way. - """ +def clean_up_mounts(mnt_dir): + if not mnt_dir.exists(): + return - print(datetime.datetime.now(tz=datetime.timezone.utc)) + print('Cleaning up mounts') - if not Path('/etc/fstab').exists(): - sys.exit('Needs to be run on Linux') + # handle deleted files + p = subprocess.run(['mount'], capture_output=True, text=True, check=True) + lines = [l for l in p.stdout.splitlines() if f'{mnt_dir}/' in l and '(deleted)' in l] - if os.geteuid() != 0: - sys.exit('Needs sudo') + for l in lines: + mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0]) + print(f' removing deleted mount {mnt_path}') + assert mnt_path.exists() + subprocess.run(['umount', mnt_path], check=True) + mnt_path.rmdir() - if not Path('/data/ofm/http_host/runs').exists(): - sys.exit('download_tiles.py needs to be run first') + # clean all mounts not in current fstab + with open('/etc/fstab') as fp: + fstab_str = fp.read() - clean_up_mounts() + for subdir in mnt_dir.iterdir(): + if f'{subdir} ' in fstab_str: + continue + print(f' removing old mount {subdir}') + subprocess.run(['umount', subdir], check=True) + subdir.rmdir() + + +def create_fstab(): fstab_new = [] for area in ['planet', 'monaco']: - area_dir = (Path('/data/ofm/http_host/runs') / area).resolve() + area_dir = (DEFAULT_RUNS_DIR / area).resolve() if not area_dir.exists(): continue @@ -42,56 +49,14 @@ def cli(): if not btrfs_file.is_file(): continue - mnt_folder = Path('/mnt/ofm') / f'{area}-{version_str}' + mnt_folder = MNT_DIR / f'{area}-{version_str}' mnt_folder.mkdir(exist_ok=True, parents=True) fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n') print(f'Created fstab entry for {btrfs_file} -> {mnt_folder}') with open('/etc/fstab') as fp: - fstab_orig = [l for l in fp.readlines() if '/mnt/ofm/' not in l] + fstab_orig = [l for l in fp.readlines() if f'{MNT_DIR}/' not in l] with open('/etc/fstab', 'w') as fp: fp.writelines(fstab_orig + fstab_new) - - print('Running mount -a') - subprocess.run(['mount', '-a'], check=True) - - clean_up_mounts() - print('DONE') - - print('\n\n\n') - - -def clean_up_mounts(): - mnt_dir = Path('/mnt/ofm') - if not mnt_dir.exists(): - return - - print('Cleaning up mounts') - - with open('/etc/fstab') as fp: - fstab_str = fp.read() - - # handle deleted files - p = subprocess.run(['mount'], capture_output=True, text=True, check=True) - lines = [l for l in p.stdout.splitlines() if '/mnt/ofm/' in l and '(deleted)' in l] - for l in lines: - mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0]) - print(f'removing deleted mount {mnt_path}') - assert mnt_path.exists() - subprocess.run(['umount', mnt_path], check=True) - mnt_path.rmdir() - - # clean all mounts not in current fstab - for subdir in mnt_dir.iterdir(): - if f'{subdir} ' in fstab_str: - continue - - print(f'removing old mount {subdir}') - subprocess.run(['umount', subdir], check=True) - subdir.rmdir() - - -if __name__ == '__main__': - cli() diff --git a/scripts/http_host/http_host_lib/utils.py b/scripts/http_host/http_host_lib/utils.py index 9075ad1..7eefc7c 100644 --- a/scripts/http_host/http_host_lib/utils.py +++ b/scripts/http_host/http_host_lib/utils.py @@ -16,6 +16,11 @@ def assert_linux(): sys.exit('Needs to be run on Linux') +# TODO +def assert_single_process(): + pass + + def download_if_size_differs(url: str, local_file: Path) -> bool: if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url): download_file_aria2(url, local_file) diff --git a/ssh_lib/utils.py b/ssh_lib/utils.py index 2e5bfb3..d7d74f7 100644 --- a/ssh_lib/utils.py +++ b/ssh_lib/utils.py @@ -52,6 +52,7 @@ def put_dir( set_permission(c, remote_dir, permissions=dir_permissions, owner=owner, group=group) for file in files: + print(f'uploading {remote_dir}/{file.name}') put(c, file, f'{remote_dir}/{file.name}', file_permissions, owner, group)