diff --git a/init-server.py b/init-server.py index 0f04cf5..d19fdd8 100755 --- a/init-server.py +++ b/init-server.py @@ -107,10 +107,11 @@ def upload_https_host_files(c): c.sudo(f'mkdir -p {HTTP_HOST_BIN}') for file in [ + 'deploy_tiles_version.py', 'download_assets.py', 'download_tiles.py', - 'mounter.py', 'metadata_to_tilejson.py', + 'mounter.py', ]: put( c, @@ -154,7 +155,17 @@ def upload_certificates(c): def debug_tmp(c): - upload_https_host_files(c) + # upload_https_host_files(c) + + for file in [ + 'deploy_tiles_version.py', + ]: + put( + c, + SCRIPTS_DIR / 'http_host' / file, + HTTP_HOST_BIN, + permissions='755', + ) @click.command() diff --git a/scripts/http_host/cron.d/ofm_http_host b/scripts/http_host/cron.d/ofm_http_host index 554a2c6..a5fcf2c 100644 --- a/scripts/http_host/cron.d/ofm_http_host +++ b/scripts/http_host/cron.d/ofm_http_host @@ -1,4 +1,9 @@ PYTHON=/data/ofm/venv/bin/python +BIN=/data/ofm/http_host/bin # every minute download_asset.py -* * * * * root $PYTHON /data/ofm/http_host/bin/download_assets.py > /data/ofm/http_host/logs/download_assets.log 2>&1 +* * * * * root $PYTHON $BIN/download_assets.py >> /data/ofm/http_host/logs/download_assets.log 2>&1 +* * * * * root $PYTHON $BIN/download_tiles.py planet --run-mounter >> /data/ofm/http_host/logs/download_tiles_planet.log 2>&1 +* * * * * root $PYTHON $BIN/download_tiles.py monaco --run-mounter >> /data/ofm/http_host/logs/download_tiles_monaco.log 2>&1 +* * * * * root $PYTHON $BIN/deploy_tiles_version.py >> /data/ofm/http_host/logs/deploy_tiles_version.log 2>&1 + diff --git a/scripts/http_host/deploy_tiles_version.py b/scripts/http_host/deploy_tiles_version.py new file mode 100755 index 0000000..12b09d8 --- /dev/null +++ b/scripts/http_host/deploy_tiles_version.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +import datetime +import os +import subprocess +import sys +from pathlib import Path + +import click +import requests + + +@click.command() +def cli(): + """ + Deploys the version of tiles specified by https://assets.openfreemap.com/versions/deployed_tiles_planet.txt + + 1. Checking if the given version is present on the disk and mounted + 2. Writing to a version file + 3. Calling nginx_sync to update the /planet location block + """ + + print(datetime.datetime.now(tz=datetime.timezone.utc)) + + if not Path('/etc/fstab').exists(): + sys.exit('Needs to be run on Linux') + + if os.geteuid() != 0: + sys.exit('Needs sudo') + + if not Path('/mnt/ofm').exists(): + sys.exit('mounter.py needs to be run first') + + need_nginx_sync = False + + for area in ['planet', 'monaco']: + r = requests.get(f'https://assets.openfreemap.com/versions/deployed_tiles_{area}.txt') + r.raise_for_status() + version_str = r.text.strip() + print(f'remote version for {area}: {version_str}') + + local_version_file = Path(f'/data/ofm/config/deployed_tiles_{area}.txt') + + if not local_version_file.exists(): + local_version_start = None + else: + with open(local_version_file) as fp: + local_version_start = fp.read() + + if not version_str: + print(' remote version not specified') + if local_version_start is not None: + local_version_file.unlink() + need_nginx_sync = True + continue + + mnt_file = Path(f'/mnt/ofm/{area}-{version_str}/metadata.json') + if not mnt_file.exists(): + print(' local version does not exist') + if local_version_start is not None: + local_version_file.unlink() + need_nginx_sync = True + continue + + if version_str != local_version_start: + with open(local_version_file, 'w') as fp: + fp.write(version_str) + need_nginx_sync = True + + if need_nginx_sync: + print('running nginx_sync.py') + + subprocess.run( + [sys.executable, Path(__file__).parent / 'nginx_sync' / 'nginx_sync.py'], + check=True, + ) + + print('\n\n\n') + + +if __name__ == '__main__': + cli() diff --git a/scripts/http_host/download_assets.py b/scripts/http_host/download_assets.py index 446dadb..43dbb21 100755 --- a/scripts/http_host/download_assets.py +++ b/scripts/http_host/download_assets.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import datetime import shutil import subprocess import sys @@ -22,6 +23,8 @@ def cli(assets_dir): Downloads and extracts assets """ + print(datetime.datetime.now(tz=datetime.timezone.utc)) + if not assets_dir: assets_dir = DEFAULT_ASSETS_DIR @@ -30,6 +33,8 @@ def cli(assets_dir): download_fonts(assets_dir) + print('\n\n\n') + def download_fonts(assets_dir): """ diff --git a/scripts/http_host/download_tiles.py b/scripts/http_host/download_tiles.py index 14bdb5b..af22828 100755 --- a/scripts/http_host/download_tiles.py +++ b/scripts/http_host/download_tiles.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import datetime import shutil import subprocess import sys @@ -20,12 +21,15 @@ DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') type=click.Path(dir_okay=True, file_okay=False, path_type=Path), ) @click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate') -def cli(area: str, version: str, list_versions: bool, runs_dir: Path): +@click.option('--run-mounter', is_flag=True, help='Run mounter.py after download is complete') +def cli(area: str, version: str, list_versions: bool, runs_dir: Path, run_mounter: bool): """ Downloads and extracts the latest tiles.btrfs file from the public bucket. Specific version can also be specified. """ + print(datetime.datetime.now(tz=datetime.timezone.utc)) + if area not in {'planet', 'monaco'}: sys.exit('Please specify are: "planet" or "monaco"') @@ -49,17 +53,27 @@ def cli(area: str, version: str, list_versions: bool, runs_dir: Path): if not runs_dir and not Path('/data/ofm').exists(): sys.exit('Please specify a runs dir with --runs-dir') - download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR) + changed = download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR) + + if changed and run_mounter: + print('running mounter.py') + + subprocess.run( + [sys.executable, Path(__file__).parent / 'mounter.py'], + check=True, + ) + + print('\n\n\n') -def download(area: str, version: str, runs_dir: Path): +def download(area: str, version: str, runs_dir: Path) -> bool: click.echo(f'Downloading: area: {area}, version: {version}') version_dir = runs_dir / area / version btrfs_file = version_dir / 'tiles.btrfs' if btrfs_file.exists(): print('File exists, skipping download') - return + return False temp_dir = runs_dir / '_tmp' if temp_dir.exists(): @@ -93,6 +107,8 @@ def download(area: str, version: str, runs_dir: Path): shutil.rmtree(temp_dir) + return True + if __name__ == '__main__': cli() diff --git a/scripts/http_host/metadata_to_tilejson.py b/scripts/http_host/metadata_to_tilejson.py index bacb66e..caa0177 100755 --- a/scripts/http_host/metadata_to_tilejson.py +++ b/scripts/http_host/metadata_to_tilejson.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import json from pathlib import Path -from pprint import pprint import click diff --git a/scripts/http_host/mounter.py b/scripts/http_host/mounter.py index c82cbf6..4ce973c 100755 --- a/scripts/http_host/mounter.py +++ b/scripts/http_host/mounter.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import datetime import os import subprocess import sys @@ -14,6 +15,8 @@ def cli(): When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way. """ + print(datetime.datetime.now(tz=datetime.timezone.utc)) + if not Path('/etc/fstab').exists(): sys.exit('Needs to be run on Linux') @@ -57,6 +60,8 @@ def cli(): clean_up_mounts() print('DONE') + print('\n\n\n') + def clean_up_mounts(): mnt_dir = Path('/mnt/ofm')