diff --git a/init-server.py b/init-server.py index c1e287e..e175e49 100755 --- a/init-server.py +++ b/init-server.py @@ -131,7 +131,7 @@ def debug(hostname, user, port): c = get_connection(hostname, user, port) upload_http_host_files(c) - run_http_host_sync(c) + # run_http_host_sync(c) # upload_http_host_config(c) # upload_http_host_files(c) diff --git a/scripts/http_host/host_manager.py b/scripts/http_host/http_host.py similarity index 73% rename from scripts/http_host/host_manager.py rename to scripts/http_host/http_host.py index 64315ae..2d33ffe 100755 --- a/scripts/http_host/host_manager.py +++ b/scripts/http_host/http_host.py @@ -11,7 +11,11 @@ from http_host_lib.assets import ( download_and_extract_asset_tar_gz, download_sprites, ) -from http_host_lib.btrfs import download_and_extract_tileset +from http_host_lib.btrfs import ( + download_and_extract_btrfs, + download_area_version, + get_versions_for_area, +) from http_host_lib.config import config from http_host_lib.mount import clean_up_mounts, create_fstab from http_host_lib.nginx import write_nginx_config @@ -27,7 +31,7 @@ def cli(): - Downloading tilesets\n - Mounting directories\n - Updating nginx config\n - - Setting the latest versions of tilesets\n + - Getting the deployed versions of tilesets\n - Running the sync cron task (called every minute) """ @@ -35,47 +39,13 @@ def cli(): @cli.command() @click.argument('area', required=False) @click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"') -@click.option( - '--runs-dir', - help='Specify runs directory', - type=click.Path(dir_okay=True, file_okay=False, path_type=Path), -) -@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate') -def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Path): +def download_btrfs(area: str, version: str): """ - Downloads and extracts the latest tiles.btrfs file from the public bucket. - Version can also be specified. + Downloads and extracts tiles.btrfs file from the btrfs bucket + Version can be "latest" or specified """ - print('running download_tileset') - - if area not in {'planet', 'monaco'}: - sys.exit(' please specify area: "planet" or "monaco"') - - r = requests.get(f'https://{area}.openfreemap.com/dirs.txt', timeout=30) - r.raise_for_status() - - versions = sorted(r.text.splitlines()) - - all_versions_str = '\n'.join(versions) - if list_versions: - print(all_versions_str) - return - - if version == 'latest': - selected_version = versions[-1] - else: - if version not in versions: - sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}') - selected_version = version - - if not runs_dir: - runs_dir = config.runs_dir - - if not runs_dir.parent.exists(): - sys.exit("runs dir's parent doesn't exist") - - return download_and_extract_tileset(area, selected_version, runs_dir) + return download_area_version(area, version) @cli.command() @@ -199,6 +169,12 @@ def sync(ctx, force): ctx.invoke(nginx_sync) +@cli.command() +def debug(): + versions = get_versions_for_area('monaco') + print(versions) + + if __name__ == '__main__': # print(config.host_config) cli() diff --git a/scripts/http_host/http_host_lib/btrfs.py b/scripts/http_host/http_host_lib/btrfs.py index aabaed1..b2ec98f 100644 --- a/scripts/http_host/http_host_lib/btrfs.py +++ b/scripts/http_host/http_host_lib/btrfs.py @@ -1,30 +1,60 @@ import shutil import subprocess -from pathlib import Path +import sys -import click +import requests +from http_host_lib.config import config from http_host_lib.utils import download_file_aria2, get_remote_file_size -def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> bool: +def download_area_version(area: str, version: str): + print('running download_btrfs') + + if area not in config.areas: + sys.exit(f' please specify area: {config.areas}') + + versions = get_versions_for_area(area) + + if version == 'latest': + selected_version = versions[-1] + else: + if version not in versions: + available_versions_str = '\n'.join(versions) + sys.exit( + f'Requested version is not available.\nAvailable versions for {area}:\n{available_versions_str}' + ) + selected_version = version + + return download_and_extract_btrfs(area, selected_version) + + +def get_versions_for_area(area: str) -> list: + r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30) + r.raise_for_status() + + versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')] + return sorted(versions) + + +def download_and_extract_btrfs(area: str, version: str) -> bool: """ - returns True if downloaded something + returns True if download successful, False if skipped """ - click.echo(f'downloading {area} {version}') + print(f'downloading {area} {version}') - version_dir = runs_dir / area / version + version_dir = config.runs_dir / area / version btrfs_file = version_dir / 'tiles.btrfs' if btrfs_file.exists(): print(' file exists, skipping download') return False - temp_dir = runs_dir / '_tmp' + temp_dir = config.runs_dir / '_tmp' shutil.rmtree(temp_dir, ignore_errors=True) temp_dir.mkdir(parents=True) - url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz' + url = f'https://btrfs.openfreemap.com/areas/{area}/{version}/tiles.btrfs.gz' # check disk space disk_free = shutil.disk_usage(temp_dir).free diff --git a/scripts/http_host/http_host_lib/config.py b/scripts/http_host/http_host_lib/config.py index 00616cb..8739f17 100644 --- a/scripts/http_host/http_host_lib/config.py +++ b/scripts/http_host/http_host_lib/config.py @@ -3,6 +3,8 @@ from pathlib import Path class Configuration: + areas = ['planet', 'monaco'] + http_host_dir = Path('/data/ofm/http_host') http_host_bin = http_host_dir / 'bin' diff --git a/scripts/tile_gen/tile_gen_lib/config.py b/scripts/tile_gen/tile_gen_lib/config.py index a23c05c..8a2b5e9 100644 --- a/scripts/tile_gen/tile_gen_lib/config.py +++ b/scripts/tile_gen/tile_gen_lib/config.py @@ -2,6 +2,8 @@ from pathlib import Path class Configuration: + areas = ['planet', 'monaco'] + tile_gen_dir = Path('/data/ofm/tile_gen') tile_gen_bin = tile_gen_dir / 'bin' @@ -15,7 +17,5 @@ class Configuration: ofm_config_dir = Path('/data/ofm/config') rclone_config = ofm_config_dir / 'rclone.conf' - areas = ['planet', 'monaco'] - config = Configuration()