refactor

init-server.py

@@ -131,7 +131,7 @@ def debug(hostname, user, port):
     c = get_connection(hostname, user, port)
 
     upload_http_host_files(c)
-    run_http_host_sync(c)
+    # run_http_host_sync(c)
 
     # upload_http_host_config(c)
     # upload_http_host_files(c)

scripts/http_host/http_host.py

@@ -11,7 +11,11 @@ from http_host_lib.assets import (
     download_and_extract_asset_tar_gz,
     download_sprites,
 )
-from http_host_lib.btrfs import download_and_extract_tileset
+from http_host_lib.btrfs import (
+    download_and_extract_btrfs,
+    download_area_version,
+    get_versions_for_area,
+)
 from http_host_lib.config import config
 from http_host_lib.mount import clean_up_mounts, create_fstab
 from http_host_lib.nginx import write_nginx_config
@@ -27,7 +31,7 @@ def cli():
     - Downloading tilesets\n
     - Mounting directories\n
     - Updating nginx config\n
-    - Setting the latest versions of tilesets\n
+    - Getting the deployed versions of tilesets\n
     - Running the sync cron task (called every minute)
     """
 
@@ -35,47 +39,13 @@ def cli():
 @cli.command()
 @click.argument('area', required=False)
 @click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"')
-@click.option(
+def download_btrfs(area: str, version: str):
-    '--runs-dir',
-    help='Specify runs directory',
-    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
-)
-@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
-def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Path):
     """
-    Downloads and extracts the latest tiles.btrfs file from the public bucket.
+    Downloads and extracts tiles.btrfs file from the btrfs bucket
-    Version can also be specified.
+    Version can be "latest" or specified
     """
 
-    print('running download_tileset')
+    return download_area_version(area, version)
-
-    if area not in {'planet', 'monaco'}:
-        sys.exit('  please specify area: "planet" or "monaco"')
-
-    r = requests.get(f'https://{area}.openfreemap.com/dirs.txt', timeout=30)
-    r.raise_for_status()
-
-    versions = sorted(r.text.splitlines())
-
-    all_versions_str = '\n'.join(versions)
-    if list_versions:
-        print(all_versions_str)
-        return
-
-    if version == 'latest':
-        selected_version = versions[-1]
-    else:
-        if version not in versions:
-            sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}')
-        selected_version = version
-
-    if not runs_dir:
-        runs_dir = config.runs_dir
-
-    if not runs_dir.parent.exists():
-        sys.exit("runs dir's parent doesn't exist")
-
-    return download_and_extract_tileset(area, selected_version, runs_dir)
 
 
 @cli.command()
@@ -199,6 +169,12 @@ def sync(ctx, force):
         ctx.invoke(nginx_sync)
 
 
+@cli.command()
+def debug():
+    versions = get_versions_for_area('monaco')
+    print(versions)
+
+
 if __name__ == '__main__':
     # print(config.host_config)
     cli()

scripts/http_host/http_host_lib/btrfs.py

@@ -1,30 +1,60 @@
 import shutil
 import subprocess
-from pathlib import Path
+import sys
 
-import click
+import requests
 
+from http_host_lib.config import config
 from http_host_lib.utils import download_file_aria2, get_remote_file_size
 
 
-def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> bool:
+def download_area_version(area: str, version: str):
+    print('running download_btrfs')
+
+    if area not in config.areas:
+        sys.exit(f'  please specify area: {config.areas}')
+
+    versions = get_versions_for_area(area)
+
+    if version == 'latest':
+        selected_version = versions[-1]
+    else:
+        if version not in versions:
+            available_versions_str = '\n'.join(versions)
+            sys.exit(
+                f'Requested version is not available.\nAvailable versions for {area}:\n{available_versions_str}'
+            )
+        selected_version = version
+
+    return download_and_extract_btrfs(area, selected_version)
+
+
+def get_versions_for_area(area: str) -> list:
+    r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30)
+    r.raise_for_status()
+
+    versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')]
+    return sorted(versions)
+
+
+def download_and_extract_btrfs(area: str, version: str) -> bool:
     """
-    returns True if downloaded something
+    returns True if download successful, False if skipped
     """
 
-    click.echo(f'downloading {area} {version}')
+    print(f'downloading {area} {version}')
 
-    version_dir = runs_dir / area / version
+    version_dir = config.runs_dir / area / version
     btrfs_file = version_dir / 'tiles.btrfs'
     if btrfs_file.exists():
         print('  file exists, skipping download')
         return False
 
-    temp_dir = runs_dir / '_tmp'
+    temp_dir = config.runs_dir / '_tmp'
     shutil.rmtree(temp_dir, ignore_errors=True)
     temp_dir.mkdir(parents=True)
 
-    url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz'
+    url = f'https://btrfs.openfreemap.com/areas/{area}/{version}/tiles.btrfs.gz'
 
     # check disk space
     disk_free = shutil.disk_usage(temp_dir).free

scripts/http_host/http_host_lib/config.py

@@ -3,6 +3,8 @@ from pathlib import Path
 
 
 class Configuration:
+    areas = ['planet', 'monaco']
+
     http_host_dir = Path('/data/ofm/http_host')
 
     http_host_bin = http_host_dir / 'bin'

scripts/tile_gen/tile_gen_lib/config.py

@@ -2,6 +2,8 @@ from pathlib import Path
 
 
 class Configuration:
+    areas = ['planet', 'monaco']
+
     tile_gen_dir = Path('/data/ofm/tile_gen')
 
     tile_gen_bin = tile_gen_dir / 'bin'
@@ -15,7 +17,5 @@ class Configuration:
     ofm_config_dir = Path('/data/ofm/config')
     rclone_config = ofm_config_dir / 'rclone.conf'
 
-    areas = ['planet', 'monaco']
-
 
 config = Configuration()