host_manager

prepare-virtualenv.sh

@@ -11,4 +11,4 @@ source venv/bin/activate
 pip install -U pip wheel setuptools
 pip install -e .
 
-
+pip install -e scripts/http_host

scripts/http_host/download_assets.py

@@ -1,112 +0,0 @@
-#!/usr/bin/env python3
-import datetime
-import shutil
-import subprocess
-import sys
-from pathlib import Path
-
-import click
-import requests
-
-
-DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets')
-
-
-@click.command()
-@click.option(
-    '--assets-dir',
-    help='Specify assets directory',
-    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
-)
-def cli(assets_dir):
-    """
-    Downloads and extracts assets
-    """
-
-    print(datetime.datetime.now(tz=datetime.timezone.utc))
-
-    if not assets_dir:
-        assets_dir = DEFAULT_ASSETS_DIR
-
-    if not assets_dir.parent.exists():
-        sys.exit("asset dir's parent doesn't exist")
-
-    download_fonts(assets_dir)
-
-    print('\n\n\n')
-
-
-def download_fonts(assets_dir):
-    """
-    Download and extract font assets if their file differ.
-    Making updates atomic, with extract to temp + move instead of extracting in place.
-    """
-
-    fonts_dir = assets_dir / 'fonts'
-    fonts_dir.mkdir(exist_ok=True, parents=True)
-
-    fonts_temp = assets_dir / 'fonts_temp'
-
-    for font in ['ml', 'omt', 'pm']:
-        url = f'https://assets.openfreemap.com/fonts/{font}.tgz'
-        local_file = fonts_dir / f'{font}.tgz'
-        if not download_if_size_differs(url, local_file):
-            continue
-
-        shutil.rmtree(fonts_temp, ignore_errors=True)
-        fonts_temp.mkdir()
-
-        subprocess.run(
-            ['tar', '-xzf', local_file, '-C', fonts_temp],
-            check=True,
-        )
-
-        target_dir = fonts_dir / font
-        target_dir_renamed = fonts_dir / f'{font}.bak'
-        temp_dir = fonts_temp / font
-
-        if target_dir.exists():
-            target_dir.rename(target_dir_renamed)
-        temp_dir.rename(target_dir)
-
-        shutil.rmtree(target_dir_renamed, ignore_errors=True)
-
-    shutil.rmtree(fonts_temp, ignore_errors=True)
-
-
-def download_if_size_differs(url: str, local_file: Path):
-    if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url):
-        download_file(url, local_file)
-        return True
-
-    return False
-
-
-def get_remote_file_size(url: str):
-    r = requests.head(url)
-    size = r.headers.get('Content-Length')
-    return int(size) if size else None
-
-
-def download_file(url, local_file):
-    click.echo(f'Downloading: {url} into {local_file}')
-
-    subprocess.run(
-        [
-            'aria2c',
-            '--split=8',
-            '--max-connection-per-server=8',
-            '--file-allocation=none',
-            '--min-split-size=1M',
-            '-d',
-            local_file.parent,
-            '-o',
-            local_file.name,
-            url,
-        ],
-        check=True,
-    )
-
-
-if __name__ == '__main__':
-    cli()

scripts/http_host/download_tiles.py

@@ -1,119 +0,0 @@
-#!/usr/bin/env python3
-import datetime
-import shutil
-import subprocess
-import sys
-from pathlib import Path
-
-import click
-import requests
-
-
-DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
-
-
-@click.command()
-@click.argument('area', required=False)
-@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"')
-@click.option(
-    '--runs-dir',
-    help='Specify /runs directory',
-    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
-)
-@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
-@click.option('--run-mounter', is_flag=True, help='Run mounter.py after download is complete')
-def cli(area: str, version: str, list_versions: bool, runs_dir: Path, run_mounter: bool):
-    """
-    Downloads and extracts the latest tiles.btrfs file from the public bucket.
-    Specific version can also be specified.
-    """
-
-    print(datetime.datetime.now(tz=datetime.timezone.utc))
-
-    if area not in {'planet', 'monaco'}:
-        sys.exit('Please specify are: "planet" or "monaco"')
-
-    r = requests.get(f'https://{area}.openfreemap.com/dirs.txt')
-    r.raise_for_status()
-
-    versions = sorted(r.text.splitlines())
-
-    all_versions_str = '\n'.join(versions)
-    if list_versions:
-        print(all_versions_str)
-        return
-
-    if version == 'latest':
-        selected_version = versions[-1]
-    else:
-        if version not in versions:
-            sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}')
-        selected_version = version
-
-    if not runs_dir and not Path('/data/ofm').exists():
-        sys.exit('Please specify a runs dir with --runs-dir')
-
-    changed = download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR)
-
-    if changed and run_mounter:
-        print('running mounter.py')
-        subprocess.run(
-            [sys.executable, Path(__file__).parent / 'mounter.py'],
-            check=True,
-        )
-
-        print('running nginx_sync.py')
-        subprocess.run(
-            [sys.executable, Path(__file__).parent / 'nginx_sync' / 'nginx_sync.py'],
-            check=True,
-        )
-
-    print('\n\n\n')
-
-
-def download(area: str, version: str, runs_dir: Path) -> bool:
-    click.echo(f'Downloading: area: {area}, version: {version}')
-
-    version_dir = runs_dir / area / version
-    btrfs_file = version_dir / 'tiles.btrfs'
-    if btrfs_file.exists():
-        print('File exists, skipping download')
-        return False
-
-    temp_dir = runs_dir / '_tmp'
-    if temp_dir.exists():
-        sys.exit(f'{temp_dir} dir exists, please delete it first')
-
-    temp_dir.mkdir(parents=True)
-
-    url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz'
-    print(url)
-
-    subprocess.run(
-        [
-            'aria2c',
-            '--split=8',
-            '--max-connection-per-server=8',
-            '--file-allocation=none',
-            '--dir',
-            temp_dir,
-            url,
-        ],
-        check=True,
-    )
-
-    subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
-    btrfs_src = temp_dir / 'tiles.btrfs'
-
-    shutil.rmtree(version_dir, ignore_errors=True)
-    version_dir.mkdir(parents=True)
-
-    btrfs_src.rename(btrfs_file)
-
-    shutil.rmtree(temp_dir)
-
-    return True
-
-
-if __name__ == '__main__':
-    cli()

scripts/http_host/host_manager.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+import click
+import requests
+from http_host_lib.download_fonts import download_fonts
+from http_host_lib.download_tileset import download_and_extract_tileset
+
+
+DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
+DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets')
+
+
+@click.group()
+def cli():
+    """
+    Manages OpenFreeMap HTTP hosts, including:\n
+    - Downloading tilesets\n
+    - Downloading assets\n
+    - Deploying the correct versions of tilesets\n
+    - Mounting directories\n
+    - Updating nginx config\n
+    """
+
+
+@cli.command()
+@click.argument('area', required=False)
+@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"')
+@click.option(
+    '--runs-dir',
+    help='Specify runs directory',
+    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
+)
+@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
+def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Path):
+    """
+    Downloads and extracts the latest tiles.btrfs file from the public bucket.
+    Version can also be specified.
+    """
+
+    if area not in {'planet', 'monaco'}:
+        sys.exit('Please specify area: "planet" or "monaco"')
+
+    r = requests.get(f'https://{area}.openfreemap.com/dirs.txt')
+    r.raise_for_status()
+
+    versions = sorted(r.text.splitlines())
+
+    all_versions_str = '\n'.join(versions)
+    if list_versions:
+        print(all_versions_str)
+        return
+
+    if version == 'latest':
+        selected_version = versions[-1]
+    else:
+        if version not in versions:
+            sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}')
+        selected_version = version
+
+    if not runs_dir:
+        runs_dir = DEFAULT_RUNS_DIR
+
+    if not runs_dir.parent.exists():
+        sys.exit("run dir's parent doesn't exist")
+
+    download_and_extract_tileset(area, selected_version, runs_dir)
+
+
+@cli.command()
+@click.option(
+    '--assets-dir',
+    help='Specify assets directory',
+    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
+)
+def download_assets(assets_dir: Path):
+    """
+    Downloads and extracts assets
+    """
+
+    if not assets_dir:
+        assets_dir = DEFAULT_ASSETS_DIR
+
+    if not assets_dir.parent.exists():
+        sys.exit("asset dir's parent doesn't exist")
+
+    download_fonts(assets_dir)
+
+
+if __name__ == '__main__':
+    cli()

scripts/http_host/http_host_lib/download_fonts.py

@@ -0,0 +1,43 @@
+import shutil
+import subprocess
+from pathlib import Path
+
+from http_host_lib.utils import download_if_size_differs
+
+
+def download_fonts(assets_dir: Path):
+    """
+    Download and extract font assets if their file size differ.
+    Making updates atomic, with extraction to a temp dest + rename
+    """
+
+    fonts_dir = assets_dir / 'fonts'
+    fonts_dir.mkdir(exist_ok=True, parents=True)
+
+    fonts_temp = assets_dir / 'fonts_temp'
+
+    for font in ['ml', 'omt', 'pm']:
+        url = f'https://assets.openfreemap.com/fonts/{font}.tgz'
+        local_file = fonts_dir / f'{font}.tgz'
+        if not download_if_size_differs(url, local_file):
+            continue
+
+        shutil.rmtree(fonts_temp, ignore_errors=True)
+        fonts_temp.mkdir()
+
+        subprocess.run(
+            ['tar', '-xzf', local_file, '-C', fonts_temp],
+            check=True,
+        )
+
+        target_dir = fonts_dir / font
+        target_dir_renamed = fonts_dir / f'{font}.bak'
+        temp_dir = fonts_temp / font
+
+        if target_dir.exists():
+            target_dir.rename(target_dir_renamed)
+        temp_dir.rename(target_dir)
+
+        shutil.rmtree(target_dir_renamed, ignore_errors=True)
+
+    shutil.rmtree(fonts_temp, ignore_errors=True)

scripts/http_host/http_host_lib/download_tileset.py

@@ -0,0 +1,42 @@
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+from http_host_lib.utils import download_file_aria2
+
+
+def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> bool:
+    """
+    returns True if downloaded something
+    """
+
+    click.echo(f'Downloading: area: {area}, version: {version}')
+
+    version_dir = runs_dir / area / version
+    btrfs_file = version_dir / 'tiles.btrfs'
+    if btrfs_file.exists():
+        print('File exists, skipping download')
+        return False
+
+    temp_dir = runs_dir / '_tmp'
+    if temp_dir.exists():
+        sys.exit(f'{temp_dir} dir exists, avoiding parallel run')
+
+    temp_dir.mkdir(parents=True)
+
+    url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz'
+    target_file = temp_dir / 'tiles.btrfs.gz'
+    download_file_aria2(url, target_file)
+
+    subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
+    btrfs_src = temp_dir / 'tiles.btrfs'
+
+    shutil.rmtree(version_dir, ignore_errors=True)
+    version_dir.mkdir(parents=True)
+
+    btrfs_src.rename(btrfs_file)
+
+    shutil.rmtree(temp_dir)
+    return True

scripts/http_host/http_host_lib/utils.py

@@ -0,0 +1,50 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import requests
+
+
+def assert_sudo():
+    if os.geteuid() != 0:
+        sys.exit('Needs sudo')
+
+
+def assert_linux():
+    if not Path('/etc/fstab').exists():
+        sys.exit('Needs to be run on Linux')
+
+
+def download_if_size_differs(url: str, local_file: Path) -> bool:
+    if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url):
+        download_file_aria2(url, local_file)
+        return True
+
+    return False
+
+
+def get_remote_file_size(url: str) -> int | None:
+    r = requests.head(url)
+    size = r.headers.get('Content-Length')
+    return int(size) if size else None
+
+
+def download_file_aria2(url: str, local_file: Path):
+    print(f'Downloading: {url} into {local_file}')
+
+    subprocess.run(
+        [
+            'aria2c',
+            '--split=8',
+            '--max-connection-per-server=8',
+            '--file-allocation=none',
+            '--min-split-size=1M',
+            '-d',
+            local_file.parent,
+            '-o',
+            local_file.name,
+            url,
+        ],
+        check=True,
+    )

scripts/http_host/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+
+setup(
+    python_requires='>=3.10',
+    packages=['http_host_lib'],
+)

setup.py

@@ -11,7 +11,6 @@ requirements = [
 ]
 
 setup(
-    name='openfreemap',
     python_requires='>=3.10',
     install_requires=requirements,
     packages=['ssh_lib'],