diff --git a/.gitignore b/.gitignore index e59c050..ecbdef4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,10 +10,11 @@ rclone.conf .DS_Store node_modules +tmp +temp + -/venv /.idea -/temp -/rclone-ofm.conf -/scripts/tile_gen/extract_mbtiles/out +venv + diff --git a/config/rclone.conf.sample b/config/rclone.conf.sample index 188c608..a87ee7e 100644 --- a/config/rclone.conf.sample +++ b/config/rclone.conf.sample @@ -4,4 +4,4 @@ provider = Cloudflare access_key_id = xxx secret_access_key = xxx endpoint = https://xxx.r2.cloudflarestorage.com - +no_check_bucket = true diff --git a/init-server.py b/init-server.py index 75788c0..bce1d36 100755 --- a/init-server.py +++ b/init-server.py @@ -6,7 +6,14 @@ from dotenv import dotenv_values from fabric import Config, Connection from ssh_lib.benchmark import c1000k -from ssh_lib.config import CONFIG_DIR, OFM_DIR, REMOTE_CONFIG, SCRIPTS_DIR, TILE_GEN_BIN +from ssh_lib.config import ( + CONFIG_DIR, + HTTP_HOST_BIN, + OFM_DIR, + REMOTE_CONFIG, + SCRIPTS_DIR, + TILE_GEN_BIN, +) from ssh_lib.kernel import set_cpu_governor, setup_kernel_settings from ssh_lib.nginx import certbot, nginx from ssh_lib.pkg_base import pkg_base, pkg_upgrade @@ -91,9 +98,23 @@ def prepare_tile_gen(c): def prepare_http_host(c): - nginx(c) - certbot(c) - c1000k(c) + # nginx(c) + # certbot(c) + # c1000k(c) + + prepare_venv(c) + + c.sudo(f'mkdir -p {HTTP_HOST_BIN}') + + for file in [ + 'downloader.py', + ]: + put( + c, + SCRIPTS_DIR / 'http_host' / file, + HTTP_HOST_BIN, + permissions='755', + ) def debug_tmp(c): diff --git a/scripts/http_host/downloader.py b/scripts/http_host/downloader.py new file mode 100755 index 0000000..592c8ef --- /dev/null +++ b/scripts/http_host/downloader.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +import shutil +import subprocess +import sys +from pathlib import Path + +import click +import requests + + +DEFAULT_RUNS_DIR = '/data/ofm/http_host/runs' + + +@click.command() +@click.option('--area', default='planet', help='The area to process') +@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"') +@click.option( + '--runs-dir', + help='Specify /runs directory', + type=click.Path(dir_okay=True, file_okay=False, path_type=Path), +) +@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate') +def cli(area: str, version: str, list_versions: bool, runs_dir: Path): + if area not in {'planet', 'monaco'}: + sys.exit('Area must be planet or monaco') + + r = requests.get(f'https://{area}.openfreemap.com/dirs.txt') + r.raise_for_status() + + versions = sorted(r.text.splitlines()) + + all_versions_str = '\n'.join(versions) + if list_versions: + print(all_versions_str) + return + + if version == 'latest': + selected_version = versions[-1] + else: + if version not in versions: + sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}') + selected_version = version + + download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR) + + +def download(area: str, version: str, runs_dir: Path): + click.echo(f'Downloading: area: {area}, version: {version}') + + version_dir = runs_dir / version + btrfs_file = version_dir / 'tiles.btrfs' + if btrfs_file.exists(): + print('File exists, skipping download') + return + + temp_dir = runs_dir / '_tmp' + shutil.rmtree(temp_dir, ignore_errors=True) + temp_dir.mkdir(parents=True) + + gzip_file = temp_dir / 'tiles.btrfs.gz' + + url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz' + print(url) + + subprocess.run( + [ + 'aria2c', + '--split=8', + '--max-connection-per-server=8', + '--file-allocation=none', + '-o', + gzip_file, + url, + ], + check=True, + ) + + subprocess.run(['unpigz', gzip_file]) + btrfs_src = temp_dir / 'tiles.btrfs' + + version_dir.mkdir() + btrfs_src.rename(btrfs_file) + + shutil.rmtree(temp_dir) + + +if __name__ == '__main__': + cli() diff --git a/scripts/http_host/downloader/downloader.py b/scripts/http_host/downloader/downloader.py deleted file mode 100755 index 2b9f930..0000000 --- a/scripts/http_host/downloader/downloader.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - -import click - - -@click.command() -@click.option('--area', default='planet', help='The area to process') -@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"') -def cli(area, version): - click.echo(f'Area: {area}, version: {version}') - - -if __name__ == '__main__': - cli() diff --git a/scripts/prepare-virtualenv.sh b/scripts/prepare-virtualenv.sh index f8a51ef..8e060fe 100755 --- a/scripts/prepare-virtualenv.sh +++ b/scripts/prepare-virtualenv.sh @@ -11,6 +11,6 @@ venv/bin/pip -V venv/bin/pip install -U pip wheel setuptools -venv/bin/pip install click +venv/bin/pip install click requests diff --git a/setup.py b/setup.py index eba2aa5..13d57d0 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ requirements = [ 'python-dotenv', 'click', 'nginxfmt', + 'requests', ] setup( diff --git a/ssh_lib/config.py b/ssh_lib/config.py index b0bb975..cd84eee 100644 --- a/ssh_lib/config.py +++ b/ssh_lib/config.py @@ -11,3 +11,4 @@ ASSETS_DIR = Path(__file__).parent / 'assets' OFM_DIR = '/data/ofm' REMOTE_CONFIG = '/data/ofm/config' TILE_GEN_BIN = '/data/ofm/tile_gen/bin' +HTTP_HOST_BIN = '/data/ofm/http_host/bin'