cron

init-server.py

@@ -107,10 +107,11 @@ def upload_https_host_files(c):
     c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
 
     for file in [
+        'deploy_tiles_version.py',
         'download_assets.py',
         'download_tiles.py',
-        'mounter.py',
         'metadata_to_tilejson.py',
+        'mounter.py',
     ]:
         put(
             c,
@@ -154,7 +155,17 @@ def upload_certificates(c):
 
 
 def debug_tmp(c):
-    upload_https_host_files(c)
+    # upload_https_host_files(c)
+
+    for file in [
+        'deploy_tiles_version.py',
+    ]:
+        put(
+            c,
+            SCRIPTS_DIR / 'http_host' / file,
+            HTTP_HOST_BIN,
+            permissions='755',
+        )
 
 
 @click.command()

scripts/http_host/cron.d/ofm_http_host

@@ -1,4 +1,9 @@
 PYTHON=/data/ofm/venv/bin/python
+BIN=/data/ofm/http_host/bin
 
 # every minute download_asset.py
-* * * * *     root     $PYTHON /data/ofm/http_host/bin/download_assets.py > /data/ofm/http_host/logs/download_assets.log 2>&1
+* * * * * root  $PYTHON $BIN/download_assets.py >> /data/ofm/http_host/logs/download_assets.log 2>&1
+* * * * * root  $PYTHON $BIN/download_tiles.py planet --run-mounter >> /data/ofm/http_host/logs/download_tiles_planet.log 2>&1
+* * * * * root  $PYTHON $BIN/download_tiles.py monaco --run-mounter >> /data/ofm/http_host/logs/download_tiles_monaco.log 2>&1
+* * * * * root  $PYTHON $BIN/deploy_tiles_version.py >> /data/ofm/http_host/logs/deploy_tiles_version.log 2>&1
+

scripts/http_host/deploy_tiles_version.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+import datetime
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+import requests
+
+
+@click.command()
+def cli():
+    """
+    Deploys the version of tiles specified by https://assets.openfreemap.com/versions/deployed_tiles_planet.txt
+
+    1. Checking if the given version is present on the disk and mounted
+    2. Writing to a version file
+    3. Calling nginx_sync to update the /planet location block
+    """
+
+    print(datetime.datetime.now(tz=datetime.timezone.utc))
+
+    if not Path('/etc/fstab').exists():
+        sys.exit('Needs to be run on Linux')
+
+    if os.geteuid() != 0:
+        sys.exit('Needs sudo')
+
+    if not Path('/mnt/ofm').exists():
+        sys.exit('mounter.py needs to be run first')
+
+    need_nginx_sync = False
+
+    for area in ['planet', 'monaco']:
+        r = requests.get(f'https://assets.openfreemap.com/versions/deployed_tiles_{area}.txt')
+        r.raise_for_status()
+        version_str = r.text.strip()
+        print(f'remote version for {area}: {version_str}')
+
+        local_version_file = Path(f'/data/ofm/config/deployed_tiles_{area}.txt')
+
+        if not local_version_file.exists():
+            local_version_start = None
+        else:
+            with open(local_version_file) as fp:
+                local_version_start = fp.read()
+
+        if not version_str:
+            print('  remote version not specified')
+            if local_version_start is not None:
+                local_version_file.unlink()
+                need_nginx_sync = True
+            continue
+
+        mnt_file = Path(f'/mnt/ofm/{area}-{version_str}/metadata.json')
+        if not mnt_file.exists():
+            print('  local version does not exist')
+            if local_version_start is not None:
+                local_version_file.unlink()
+                need_nginx_sync = True
+            continue
+
+        if version_str != local_version_start:
+            with open(local_version_file, 'w') as fp:
+                fp.write(version_str)
+            need_nginx_sync = True
+
+    if need_nginx_sync:
+        print('running nginx_sync.py')
+
+        subprocess.run(
+            [sys.executable, Path(__file__).parent / 'nginx_sync' / 'nginx_sync.py'],
+            check=True,
+        )
+
+    print('\n\n\n')
+
+
+if __name__ == '__main__':
+    cli()

scripts/http_host/download_assets.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import datetime
 import shutil
 import subprocess
 import sys
@@ -22,6 +23,8 @@ def cli(assets_dir):
     Downloads and extracts assets
     """
 
+    print(datetime.datetime.now(tz=datetime.timezone.utc))
+
     if not assets_dir:
         assets_dir = DEFAULT_ASSETS_DIR
 
@@ -30,6 +33,8 @@ def cli(assets_dir):
 
     download_fonts(assets_dir)
 
+    print('\n\n\n')
+
 
 def download_fonts(assets_dir):
     """

scripts/http_host/download_tiles.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import datetime
 import shutil
 import subprocess
 import sys
@@ -20,12 +21,15 @@ DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
     type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
 )
 @click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
-def cli(area: str, version: str, list_versions: bool, runs_dir: Path):
+@click.option('--run-mounter', is_flag=True, help='Run mounter.py after download is complete')
+def cli(area: str, version: str, list_versions: bool, runs_dir: Path, run_mounter: bool):
     """
     Downloads and extracts the latest tiles.btrfs file from the public bucket.
     Specific version can also be specified.
     """
 
+    print(datetime.datetime.now(tz=datetime.timezone.utc))
+
     if area not in {'planet', 'monaco'}:
         sys.exit('Please specify are: "planet" or "monaco"')
 
@@ -49,17 +53,27 @@ def cli(area: str, version: str, list_versions: bool, runs_dir: Path):
     if not runs_dir and not Path('/data/ofm').exists():
         sys.exit('Please specify a runs dir with --runs-dir')
 
-    download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR)
+    changed = download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR)
+
+    if changed and run_mounter:
+        print('running mounter.py')
+
+        subprocess.run(
+            [sys.executable, Path(__file__).parent / 'mounter.py'],
+            check=True,
+        )
+
+    print('\n\n\n')
 
 
-def download(area: str, version: str, runs_dir: Path):
+def download(area: str, version: str, runs_dir: Path) -> bool:
     click.echo(f'Downloading: area: {area}, version: {version}')
 
     version_dir = runs_dir / area / version
     btrfs_file = version_dir / 'tiles.btrfs'
     if btrfs_file.exists():
         print('File exists, skipping download')
-        return
+        return False
 
     temp_dir = runs_dir / '_tmp'
     if temp_dir.exists():
@@ -93,6 +107,8 @@ def download(area: str, version: str, runs_dir: Path):
 
     shutil.rmtree(temp_dir)
 
+    return True
+
 
 if __name__ == '__main__':
     cli()

scripts/http_host/metadata_to_tilejson.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 import json
 from pathlib import Path
-from pprint import pprint
 
 import click
 

scripts/http_host/mounter.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import datetime
 import os
 import subprocess
 import sys
@@ -14,6 +15,8 @@ def cli():
     When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
     """
 
+    print(datetime.datetime.now(tz=datetime.timezone.utc))
+
     if not Path('/etc/fstab').exists():
         sys.exit('Needs to be run on Linux')
 
@@ -57,6 +60,8 @@ def cli():
     clean_up_mounts()
     print('DONE')
 
+    print('\n\n\n')
+
 
 def clean_up_mounts():
     mnt_dir = Path('/mnt/ofm')