host_manager work

scripts/http_host/.gitignore

@@ -1,3 +0,0 @@
-*.json
-
-_assets

scripts/http_host/host_manager.py

@@ -1,15 +1,16 @@
 #!/usr/bin/env python3
+import datetime
+import subprocess
 import sys
 from pathlib import Path
 
 import click
 import requests
+from http_host_lib import DEFAULT_ASSETS_DIR, DEFAULT_RUNS_DIR, MNT_DIR
 from http_host_lib.download_fonts import download_fonts
 from http_host_lib.download_tileset import download_and_extract_tileset
-
+from http_host_lib.mount import clean_up_mounts, create_fstab
-
+from http_host_lib.utils import assert_linux, assert_single_process, assert_sudo
-DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
-DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets')
 
 
 @click.group()
@@ -88,5 +89,42 @@ def download_assets(assets_dir: Path):
     download_fonts(assets_dir)
 
 
+@cli.command()
+def mount():
+    """
+    Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
+    When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
+    """
+    assert_linux()
+    assert_sudo()
+
+    if not DEFAULT_RUNS_DIR.exists():
+        sys.exit('download_tileset needs to be run first')
+
+    clean_up_mounts(MNT_DIR)
+    create_fstab()
+
+    print('Running mount -a')
+    subprocess.run(['mount', '-a'], check=True)
+
+    clean_up_mounts(MNT_DIR)
+
+
+@cli.command()
+@click.pass_context
+def sync(ctx):
+    """
+    Runs the sync task, normally called by cron every minute
+    """
+    print(datetime.datetime.now(tz=datetime.timezone.utc))
+
+    ctx.invoke(download_tileset, area='monaco')
+    # ctx.invoke(download_tileset, area='planet')
+    ctx.invoke(download_assets)
+    ctx.invoke(mount)
+
+
 if __name__ == '__main__':
+    # TODO
+    assert_single_process()
     cli()

scripts/http_host/http_host_lib/__init__.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+
+DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
+DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets')
+
+MNT_DIR = Path('/mnt/ofm/')

scripts/http_host/http_host_lib/mount.py

@@ -1,37 +1,44 @@
-#!/usr/bin/env python3
-import datetime
-import os
 import subprocess
-import sys
 from pathlib import Path
 
-import click
+from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR
 
 
-@click.command()
+def clean_up_mounts(mnt_dir):
-def cli():
+    if not mnt_dir.exists():
-    """
+        return
-    Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
-    When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
-    """
 
-    print(datetime.datetime.now(tz=datetime.timezone.utc))
+    print('Cleaning up mounts')
 
-    if not Path('/etc/fstab').exists():
+    # handle deleted files
-        sys.exit('Needs to be run on Linux')
+    p = subprocess.run(['mount'], capture_output=True, text=True, check=True)
+    lines = [l for l in p.stdout.splitlines() if f'{mnt_dir}/' in l and '(deleted)' in l]
 
-    if os.geteuid() != 0:
+    for l in lines:
-        sys.exit('Needs sudo')
+        mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0])
+        print(f'  removing deleted mount {mnt_path}')
+        assert mnt_path.exists()
+        subprocess.run(['umount', mnt_path], check=True)
+        mnt_path.rmdir()
 
-    if not Path('/data/ofm/http_host/runs').exists():
+    # clean all mounts not in current fstab
-        sys.exit('download_tiles.py needs to be run first')
+    with open('/etc/fstab') as fp:
+        fstab_str = fp.read()
 
-    clean_up_mounts()
+    for subdir in mnt_dir.iterdir():
+        if f'{subdir} ' in fstab_str:
+            continue
 
+        print(f'  removing old mount {subdir}')
+        subprocess.run(['umount', subdir], check=True)
+        subdir.rmdir()
+
+
+def create_fstab():
     fstab_new = []
 
     for area in ['planet', 'monaco']:
-        area_dir = (Path('/data/ofm/http_host/runs') / area).resolve()
+        area_dir = (DEFAULT_RUNS_DIR / area).resolve()
         if not area_dir.exists():
             continue
 
@@ -42,56 +49,14 @@ def cli():
             if not btrfs_file.is_file():
                 continue
 
-            mnt_folder = Path('/mnt/ofm') / f'{area}-{version_str}'
+            mnt_folder = MNT_DIR / f'{area}-{version_str}'
             mnt_folder.mkdir(exist_ok=True, parents=True)
 
             fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n')
             print(f'Created fstab entry for {btrfs_file} -> {mnt_folder}')
 
     with open('/etc/fstab') as fp:
-        fstab_orig = [l for l in fp.readlines() if '/mnt/ofm/' not in l]
+        fstab_orig = [l for l in fp.readlines() if f'{MNT_DIR}/' not in l]
 
     with open('/etc/fstab', 'w') as fp:
         fp.writelines(fstab_orig + fstab_new)
-
-    print('Running mount -a')
-    subprocess.run(['mount', '-a'], check=True)
-
-    clean_up_mounts()
-    print('DONE')
-
-    print('\n\n\n')
-
-
-def clean_up_mounts():
-    mnt_dir = Path('/mnt/ofm')
-    if not mnt_dir.exists():
-        return
-
-    print('Cleaning up mounts')
-
-    with open('/etc/fstab') as fp:
-        fstab_str = fp.read()
-
-    # handle deleted files
-    p = subprocess.run(['mount'], capture_output=True, text=True, check=True)
-    lines = [l for l in p.stdout.splitlines() if '/mnt/ofm/' in l and '(deleted)' in l]
-    for l in lines:
-        mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0])
-        print(f'removing deleted mount {mnt_path}')
-        assert mnt_path.exists()
-        subprocess.run(['umount', mnt_path], check=True)
-        mnt_path.rmdir()
-
-    # clean all mounts not in current fstab
-    for subdir in mnt_dir.iterdir():
-        if f'{subdir} ' in fstab_str:
-            continue
-
-        print(f'removing old mount {subdir}')
-        subprocess.run(['umount', subdir], check=True)
-        subdir.rmdir()
-
-
-if __name__ == '__main__':
-    cli()

scripts/http_host/http_host_lib/utils.py

@@ -16,6 +16,11 @@ def assert_linux():
         sys.exit('Needs to be run on Linux')
 
 
+# TODO
+def assert_single_process():
+    pass
+
+
 def download_if_size_differs(url: str, local_file: Path) -> bool:
     if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url):
         download_file_aria2(url, local_file)

ssh_lib/utils.py

@@ -52,6 +52,7 @@ def put_dir(
     set_permission(c, remote_dir, permissions=dir_permissions, owner=owner, group=group)
 
     for file in files:
+        print(f'uploading {remote_dir}/{file.name}')
         put(c, file, f'{remote_dir}/{file.name}', file_permissions, owner, group)