From ac9803ee69ea204d1f292c0a7f28c86e24bb9386 Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Tue, 2 Jan 2024 23:01:17 +0100 Subject: [PATCH] host_manager --- prepare-virtualenv.sh | 2 +- scripts/http_host/download_assets.py | 112 ----------------- scripts/http_host/download_tiles.py | 119 ------------------ scripts/http_host/host_manager.py | 92 ++++++++++++++ scripts/http_host/http_host_lib/__init__.py | 0 .../http_host/http_host_lib/download_fonts.py | 43 +++++++ .../http_host_lib/download_tileset.py | 42 +++++++ scripts/http_host/http_host_lib/utils.py | 50 ++++++++ scripts/http_host/setup.py | 7 ++ setup.py | 1 - 10 files changed, 235 insertions(+), 233 deletions(-) delete mode 100755 scripts/http_host/download_assets.py delete mode 100755 scripts/http_host/download_tiles.py create mode 100755 scripts/http_host/host_manager.py create mode 100644 scripts/http_host/http_host_lib/__init__.py create mode 100644 scripts/http_host/http_host_lib/download_fonts.py create mode 100644 scripts/http_host/http_host_lib/download_tileset.py create mode 100644 scripts/http_host/http_host_lib/utils.py create mode 100644 scripts/http_host/setup.py diff --git a/prepare-virtualenv.sh b/prepare-virtualenv.sh index e663ff1..4ff80d5 100755 --- a/prepare-virtualenv.sh +++ b/prepare-virtualenv.sh @@ -11,4 +11,4 @@ source venv/bin/activate pip install -U pip wheel setuptools pip install -e . - +pip install -e scripts/http_host diff --git a/scripts/http_host/download_assets.py b/scripts/http_host/download_assets.py deleted file mode 100755 index 43dbb21..0000000 --- a/scripts/http_host/download_assets.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -import datetime -import shutil -import subprocess -import sys -from pathlib import Path - -import click -import requests - - -DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets') - - -@click.command() -@click.option( - '--assets-dir', - help='Specify assets directory', - type=click.Path(dir_okay=True, file_okay=False, path_type=Path), -) -def cli(assets_dir): - """ - Downloads and extracts assets - """ - - print(datetime.datetime.now(tz=datetime.timezone.utc)) - - if not assets_dir: - assets_dir = DEFAULT_ASSETS_DIR - - if not assets_dir.parent.exists(): - sys.exit("asset dir's parent doesn't exist") - - download_fonts(assets_dir) - - print('\n\n\n') - - -def download_fonts(assets_dir): - """ - Download and extract font assets if their file differ. - Making updates atomic, with extract to temp + move instead of extracting in place. - """ - - fonts_dir = assets_dir / 'fonts' - fonts_dir.mkdir(exist_ok=True, parents=True) - - fonts_temp = assets_dir / 'fonts_temp' - - for font in ['ml', 'omt', 'pm']: - url = f'https://assets.openfreemap.com/fonts/{font}.tgz' - local_file = fonts_dir / f'{font}.tgz' - if not download_if_size_differs(url, local_file): - continue - - shutil.rmtree(fonts_temp, ignore_errors=True) - fonts_temp.mkdir() - - subprocess.run( - ['tar', '-xzf', local_file, '-C', fonts_temp], - check=True, - ) - - target_dir = fonts_dir / font - target_dir_renamed = fonts_dir / f'{font}.bak' - temp_dir = fonts_temp / font - - if target_dir.exists(): - target_dir.rename(target_dir_renamed) - temp_dir.rename(target_dir) - - shutil.rmtree(target_dir_renamed, ignore_errors=True) - - shutil.rmtree(fonts_temp, ignore_errors=True) - - -def download_if_size_differs(url: str, local_file: Path): - if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url): - download_file(url, local_file) - return True - - return False - - -def get_remote_file_size(url: str): - r = requests.head(url) - size = r.headers.get('Content-Length') - return int(size) if size else None - - -def download_file(url, local_file): - click.echo(f'Downloading: {url} into {local_file}') - - subprocess.run( - [ - 'aria2c', - '--split=8', - '--max-connection-per-server=8', - '--file-allocation=none', - '--min-split-size=1M', - '-d', - local_file.parent, - '-o', - local_file.name, - url, - ], - check=True, - ) - - -if __name__ == '__main__': - cli() diff --git a/scripts/http_host/download_tiles.py b/scripts/http_host/download_tiles.py deleted file mode 100755 index f072886..0000000 --- a/scripts/http_host/download_tiles.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -import datetime -import shutil -import subprocess -import sys -from pathlib import Path - -import click -import requests - - -DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') - - -@click.command() -@click.argument('area', required=False) -@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"') -@click.option( - '--runs-dir', - help='Specify /runs directory', - type=click.Path(dir_okay=True, file_okay=False, path_type=Path), -) -@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate') -@click.option('--run-mounter', is_flag=True, help='Run mounter.py after download is complete') -def cli(area: str, version: str, list_versions: bool, runs_dir: Path, run_mounter: bool): - """ - Downloads and extracts the latest tiles.btrfs file from the public bucket. - Specific version can also be specified. - """ - - print(datetime.datetime.now(tz=datetime.timezone.utc)) - - if area not in {'planet', 'monaco'}: - sys.exit('Please specify are: "planet" or "monaco"') - - r = requests.get(f'https://{area}.openfreemap.com/dirs.txt') - r.raise_for_status() - - versions = sorted(r.text.splitlines()) - - all_versions_str = '\n'.join(versions) - if list_versions: - print(all_versions_str) - return - - if version == 'latest': - selected_version = versions[-1] - else: - if version not in versions: - sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}') - selected_version = version - - if not runs_dir and not Path('/data/ofm').exists(): - sys.exit('Please specify a runs dir with --runs-dir') - - changed = download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR) - - if changed and run_mounter: - print('running mounter.py') - subprocess.run( - [sys.executable, Path(__file__).parent / 'mounter.py'], - check=True, - ) - - print('running nginx_sync.py') - subprocess.run( - [sys.executable, Path(__file__).parent / 'nginx_sync' / 'nginx_sync.py'], - check=True, - ) - - print('\n\n\n') - - -def download(area: str, version: str, runs_dir: Path) -> bool: - click.echo(f'Downloading: area: {area}, version: {version}') - - version_dir = runs_dir / area / version - btrfs_file = version_dir / 'tiles.btrfs' - if btrfs_file.exists(): - print('File exists, skipping download') - return False - - temp_dir = runs_dir / '_tmp' - if temp_dir.exists(): - sys.exit(f'{temp_dir} dir exists, please delete it first') - - temp_dir.mkdir(parents=True) - - url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz' - print(url) - - subprocess.run( - [ - 'aria2c', - '--split=8', - '--max-connection-per-server=8', - '--file-allocation=none', - '--dir', - temp_dir, - url, - ], - check=True, - ) - - subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True) - btrfs_src = temp_dir / 'tiles.btrfs' - - shutil.rmtree(version_dir, ignore_errors=True) - version_dir.mkdir(parents=True) - - btrfs_src.rename(btrfs_file) - - shutil.rmtree(temp_dir) - - return True - - -if __name__ == '__main__': - cli() diff --git a/scripts/http_host/host_manager.py b/scripts/http_host/host_manager.py new file mode 100755 index 0000000..f6323e2 --- /dev/null +++ b/scripts/http_host/host_manager.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +import click +import requests +from http_host_lib.download_fonts import download_fonts +from http_host_lib.download_tileset import download_and_extract_tileset + + +DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') +DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets') + + +@click.group() +def cli(): + """ + Manages OpenFreeMap HTTP hosts, including:\n + - Downloading tilesets\n + - Downloading assets\n + - Deploying the correct versions of tilesets\n + - Mounting directories\n + - Updating nginx config\n + """ + + +@cli.command() +@click.argument('area', required=False) +@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"') +@click.option( + '--runs-dir', + help='Specify runs directory', + type=click.Path(dir_okay=True, file_okay=False, path_type=Path), +) +@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate') +def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Path): + """ + Downloads and extracts the latest tiles.btrfs file from the public bucket. + Version can also be specified. + """ + + if area not in {'planet', 'monaco'}: + sys.exit('Please specify area: "planet" or "monaco"') + + r = requests.get(f'https://{area}.openfreemap.com/dirs.txt') + r.raise_for_status() + + versions = sorted(r.text.splitlines()) + + all_versions_str = '\n'.join(versions) + if list_versions: + print(all_versions_str) + return + + if version == 'latest': + selected_version = versions[-1] + else: + if version not in versions: + sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}') + selected_version = version + + if not runs_dir: + runs_dir = DEFAULT_RUNS_DIR + + if not runs_dir.parent.exists(): + sys.exit("run dir's parent doesn't exist") + + download_and_extract_tileset(area, selected_version, runs_dir) + + +@cli.command() +@click.option( + '--assets-dir', + help='Specify assets directory', + type=click.Path(dir_okay=True, file_okay=False, path_type=Path), +) +def download_assets(assets_dir: Path): + """ + Downloads and extracts assets + """ + + if not assets_dir: + assets_dir = DEFAULT_ASSETS_DIR + + if not assets_dir.parent.exists(): + sys.exit("asset dir's parent doesn't exist") + + download_fonts(assets_dir) + + +if __name__ == '__main__': + cli() diff --git a/scripts/http_host/http_host_lib/__init__.py b/scripts/http_host/http_host_lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/http_host/http_host_lib/download_fonts.py b/scripts/http_host/http_host_lib/download_fonts.py new file mode 100644 index 0000000..5de908a --- /dev/null +++ b/scripts/http_host/http_host_lib/download_fonts.py @@ -0,0 +1,43 @@ +import shutil +import subprocess +from pathlib import Path + +from http_host_lib.utils import download_if_size_differs + + +def download_fonts(assets_dir: Path): + """ + Download and extract font assets if their file size differ. + Making updates atomic, with extraction to a temp dest + rename + """ + + fonts_dir = assets_dir / 'fonts' + fonts_dir.mkdir(exist_ok=True, parents=True) + + fonts_temp = assets_dir / 'fonts_temp' + + for font in ['ml', 'omt', 'pm']: + url = f'https://assets.openfreemap.com/fonts/{font}.tgz' + local_file = fonts_dir / f'{font}.tgz' + if not download_if_size_differs(url, local_file): + continue + + shutil.rmtree(fonts_temp, ignore_errors=True) + fonts_temp.mkdir() + + subprocess.run( + ['tar', '-xzf', local_file, '-C', fonts_temp], + check=True, + ) + + target_dir = fonts_dir / font + target_dir_renamed = fonts_dir / f'{font}.bak' + temp_dir = fonts_temp / font + + if target_dir.exists(): + target_dir.rename(target_dir_renamed) + temp_dir.rename(target_dir) + + shutil.rmtree(target_dir_renamed, ignore_errors=True) + + shutil.rmtree(fonts_temp, ignore_errors=True) diff --git a/scripts/http_host/http_host_lib/download_tileset.py b/scripts/http_host/http_host_lib/download_tileset.py new file mode 100644 index 0000000..1430677 --- /dev/null +++ b/scripts/http_host/http_host_lib/download_tileset.py @@ -0,0 +1,42 @@ +import shutil +import subprocess +import sys +from pathlib import Path + +import click +from http_host_lib.utils import download_file_aria2 + + +def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> bool: + """ + returns True if downloaded something + """ + + click.echo(f'Downloading: area: {area}, version: {version}') + + version_dir = runs_dir / area / version + btrfs_file = version_dir / 'tiles.btrfs' + if btrfs_file.exists(): + print('File exists, skipping download') + return False + + temp_dir = runs_dir / '_tmp' + if temp_dir.exists(): + sys.exit(f'{temp_dir} dir exists, avoiding parallel run') + + temp_dir.mkdir(parents=True) + + url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz' + target_file = temp_dir / 'tiles.btrfs.gz' + download_file_aria2(url, target_file) + + subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True) + btrfs_src = temp_dir / 'tiles.btrfs' + + shutil.rmtree(version_dir, ignore_errors=True) + version_dir.mkdir(parents=True) + + btrfs_src.rename(btrfs_file) + + shutil.rmtree(temp_dir) + return True diff --git a/scripts/http_host/http_host_lib/utils.py b/scripts/http_host/http_host_lib/utils.py new file mode 100644 index 0000000..9075ad1 --- /dev/null +++ b/scripts/http_host/http_host_lib/utils.py @@ -0,0 +1,50 @@ +import os +import subprocess +import sys +from pathlib import Path + +import requests + + +def assert_sudo(): + if os.geteuid() != 0: + sys.exit('Needs sudo') + + +def assert_linux(): + if not Path('/etc/fstab').exists(): + sys.exit('Needs to be run on Linux') + + +def download_if_size_differs(url: str, local_file: Path) -> bool: + if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url): + download_file_aria2(url, local_file) + return True + + return False + + +def get_remote_file_size(url: str) -> int | None: + r = requests.head(url) + size = r.headers.get('Content-Length') + return int(size) if size else None + + +def download_file_aria2(url: str, local_file: Path): + print(f'Downloading: {url} into {local_file}') + + subprocess.run( + [ + 'aria2c', + '--split=8', + '--max-connection-per-server=8', + '--file-allocation=none', + '--min-split-size=1M', + '-d', + local_file.parent, + '-o', + local_file.name, + url, + ], + check=True, + ) diff --git a/scripts/http_host/setup.py b/scripts/http_host/setup.py new file mode 100644 index 0000000..870df9c --- /dev/null +++ b/scripts/http_host/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + + +setup( + python_requires='>=3.10', + packages=['http_host_lib'], +) diff --git a/setup.py b/setup.py index ef74b36..eabd88b 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,6 @@ requirements = [ ] setup( - name='openfreemap', python_requires='>=3.10', install_requires=requirements, packages=['ssh_lib'],