asset downloading simplified

config/.env.sample

@@ -7,5 +7,5 @@ DOMAIN_DIRECT=direct.openfreemap.org
 # Domain via CloudFlare, using origin certificates
 DOMAIN_CF=tiles.openfreemap.org
 
-# Set to true to skip the full planet download. Useful for testing.
+# Skip the full planet download, useful for testing (true/false)
 SKIP_PLANET=false

scripts/http_host/host_manager.py

@@ -9,6 +9,7 @@ import click
 import requests
 from http_host_lib import DEFAULT_ASSETS_DIR, DEFAULT_RUNS_DIR, MNT_DIR
 from http_host_lib.download_assets import (
+    download_and_extract_asset_tar_gz,
     download_fonts,
     download_natural_earth,
     download_sprites,
@@ -99,11 +100,11 @@ def download_assets(assets_dir: Path):
     if not assets_dir.parent.exists():
         sys.exit("asset dir's parent doesn't exist")
 
-    download_fonts(assets_dir)
+    download_and_extract_asset_tar_gz(assets_dir, 'fonts')
-    download_styles(assets_dir)
+    download_and_extract_asset_tar_gz(assets_dir, 'styles')
-    download_sprites(assets_dir)
+    download_and_extract_asset_tar_gz(assets_dir, 'natural_earth')
 
-    download_natural_earth(assets_dir)
+    download_sprites(assets_dir)
 
 
 @cli.command()

scripts/http_host/http_host_lib/download_assets.py

@@ -7,59 +7,36 @@ import requests
 from http_host_lib.utils import download_file_aria2, download_if_size_differs
 
 
-def download_fonts(assets_dir: Path):
+def download_and_extract_asset_tar_gz(assets_dir, asset_kind):
     """
-    Download and extract font assets if their file size differ.
+    Download and extract asset.tgz if the file size differ or not available locally
     """
 
-    fonts_dir = assets_dir / 'fonts'
+    print(f'Downloading asset {asset_kind}')
-    fonts_dir.mkdir(exist_ok=True, parents=True)
 
-    url = 'https://assets.openfreemap.com/fonts/ofm.tar.gz'
+    asset_dir = assets_dir / asset_kind
-    local_file = fonts_dir / 'ofm.tar.gz'
+    asset_dir.mkdir(exist_ok=True, parents=True)
+
+    url = f'https://assets.openfreemap.com/{asset_kind}/ofm.tar.gz'
+    local_file = asset_dir / 'ofm.tar.gz'
     if not download_if_size_differs(url, local_file):
         return
 
-    ofm_dir = fonts_dir / 'ofm'
+    ofm_dir = asset_dir / 'ofm'
-    ofm_dir_bak = fonts_dir / 'ofm.bak'
+    ofm_dir_bak = asset_dir / 'ofm.bak'
     shutil.rmtree(ofm_dir_bak, ignore_errors=True)
     if ofm_dir.exists():
         ofm_dir.rename(ofm_dir_bak)
 
     subprocess.run(
-        ['tar', '-xzf', local_file, '-C', fonts_dir],
+        ['tar', '-xzf', local_file, '-C', asset_dir],
-        check=True,
-    )
-
-
-def download_styles(assets_dir: Path):
-    """
-    Download and extract style assets if their file size differ.
-    """
-
-    styles_dir = assets_dir / 'styles'
-    styles_dir.mkdir(exist_ok=True, parents=True)
-
-    url = 'https://assets.openfreemap.com/styles/ofm.tar.gz'
-    local_file = styles_dir / 'ofm.tar.gz'
-    if not download_if_size_differs(url, local_file):
-        return
-
-    ofm_dir = styles_dir / 'ofm'
-    ofm_dir_bak = styles_dir / 'ofm.bak'
-    shutil.rmtree(ofm_dir_bak, ignore_errors=True)
-    if ofm_dir.exists():
-        ofm_dir.rename(ofm_dir_bak)
-
-    subprocess.run(
-        ['tar', '-xzf', local_file, '-C', styles_dir],
         check=True,
     )
 
 
 def download_sprites(assets_dir: Path):
     """
-    Download and extract sprites if a version is not available locally
+    Sprites are special assets, as we have to keep the old versions indefinitely
     """
 
     sprites_dir = assets_dir / 'sprites'
@@ -85,26 +62,3 @@ def download_sprites(assets_dir: Path):
             check=True,
         )
         local_file.unlink()
-
-
-def download_natural_earth(assets_dir: Path):
-    ne_dir = assets_dir / 'natural_earth'
-
-    if (ne_dir / 'tiles' / 'natural_earth_2_shaded_relief.raster' / '0' / '0' / '0.png').exists():
-        return
-
-    ne_dir.mkdir(exist_ok=True, parents=True)
-
-    subprocess.run(
-        [
-            'git',
-            'clone',
-            '--depth=1',
-            '-b',
-            'gh-pages',
-            'https://github.com/lukasmartinelli/naturalearthtiles.git',
-            ne_dir,
-        ]
-    )
-
-    shutil.rmtree(ne_dir / '.git')