diff --git a/init-server.py b/init-server.py index d46c4ad..2ffd51f 100755 --- a/init-server.py +++ b/init-server.py @@ -73,7 +73,6 @@ def prepare_tile_gen(c): c, SCRIPTS_DIR / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py', f'{TILE_GEN_BIN}/extract_mbtiles/extract_mbtiles.py', - permissions='755', create_parent_dir=True, ) @@ -81,7 +80,6 @@ def prepare_tile_gen(c): c, SCRIPTS_DIR / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py', f'{TILE_GEN_BIN}/shrink_btrfs/shrink_btrfs.py', - permissions='755', create_parent_dir=True, ) @@ -105,6 +103,7 @@ def prepare_http_host(c): prepare_venv(c) + c.sudo('mkdir -p /data/ofm/http_host/logs_nginx') c.sudo(f'mkdir -p {HTTP_HOST_BIN}') for file in [ @@ -118,33 +117,25 @@ def prepare_http_host(c): permissions='755', ) + for file in ['nginx_site.conf', 'nginx_sync.py']: + put( + c, + SCRIPTS_DIR / 'http_host' / 'nginx_sync' / file, + f'{HTTP_HOST_BIN}/nginx_sync/{file}', + create_parent_dir=True, + ) + c.sudo('chown -R ofm:ofm /data/ofm/http_host') + c.sudo('chown -R nginx:nginx /data/ofm/http_host/logs_nginx') def debug_tmp(c): - # for file in [ - # 'extract_btrfs.sh', - # 'planetiler_monaco.sh', - # 'planetiler_planet.sh', - # 'cloudflare_index.sh', - # 'cloudflare_upload.sh', - # ]: - # put( - # c, - # SCRIPTS_DIR / 'tile_gen' / file, - # TILE_GEN_BIN, - # permissions='755', - # ) - - for file in [ - 'downloader.py', - 'mounter.py', - ]: + for file in ['nginx_site.conf', 'nginx_sync.py']: put( c, - SCRIPTS_DIR / 'http_host' / file, - HTTP_HOST_BIN, - permissions='755', + SCRIPTS_DIR / 'http_host' / 'nginx_sync' / file, + f'{HTTP_HOST_BIN}/nginx_sync/{file}', + create_parent_dir=True, ) diff --git a/scripts/http_host/downloader.py b/scripts/http_host/downloader.py index 626049e..14bdb5b 100755 --- a/scripts/http_host/downloader.py +++ b/scripts/http_host/downloader.py @@ -21,6 +21,11 @@ DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') ) @click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate') def cli(area: str, version: str, list_versions: bool, runs_dir: Path): + """ + Downloads and extracts the latest tiles.btrfs file from the public bucket. + Specific version can also be specified. + """ + if area not in {'planet', 'monaco'}: sys.exit('Please specify are: "planet" or "monaco"') diff --git a/scripts/http_host/mounter.py b/scripts/http_host/mounter.py index 8b941a8..a64d659 100755 --- a/scripts/http_host/mounter.py +++ b/scripts/http_host/mounter.py @@ -7,34 +7,28 @@ from pathlib import Path import click -DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') - - @click.command() -@click.option( - '--runs-dir', - help='Specify /runs directory', - type=click.Path(dir_okay=True, file_okay=False, path_type=Path), -) -def cli(runs_dir: Path): - if os.geteuid() != 0: - sys.exit('Needs sudo') - - if not runs_dir and not Path('/data/ofm').exists(): - sys.exit('Please specify a runs dir with --runs-dir') +def cli(): + """ + Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically. + When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way. + """ if not Path('/etc/fstab').exists(): sys.exit('Needs to be run on Linux') - if not runs_dir: - runs_dir = DEFAULT_RUNS_DIR + if os.geteuid() != 0: + sys.exit('Needs sudo') + + if not Path('/data/ofm/http_host/runs').exists(): + sys.exit('downloader.py needs to be run first') clean_up_mounts() fstab_new = [] for area in ['planet', 'monaco']: - area_dir = (runs_dir / area).resolve() + area_dir = (Path('/data/ofm/http_host/runs') / area).resolve() if not area_dir.exists(): continue diff --git a/scripts/http_host/nginx_site.conf b/scripts/http_host/nginx_site.conf deleted file mode 100644 index ef28bc9..0000000 --- a/scripts/http_host/nginx_site.conf +++ /dev/null @@ -1,21 +0,0 @@ -server { - server_name ofm tiles.openfreemaps.org; - # test with - # curl -H "Host: ofm" http://localhost/planet/20231208_091355/tiles/11/637/1141.pbf - - #access_log /data/ofm/logs/nginx-access.log access_json; - access_log off; - error_log /data/ofm/logs/nginx-error.log; - - # trailing / important - location /planet/20231208_091355/ { - alias /data/ofm/runs/planet_20231208_091355/mnt_rw/extract/; # trailing / important - try_files $uri @empty; - } - - # we need to handle missing tiles as valid request returning empty string - location @empty { - default_type application/vnd.mapbox-vector-tile; - return 200 ''; - } -} diff --git a/scripts/http_host/nginx_sync/nginx_site.conf b/scripts/http_host/nginx_sync/nginx_site.conf new file mode 100644 index 0000000..69d7e38 --- /dev/null +++ b/scripts/http_host/nginx_sync/nginx_site.conf @@ -0,0 +1,19 @@ +server { + server_name ofm tiles.openfreemaps.org; + + # disabling access log by default + # access_log /data/ofm/http_host/logs_nginx/nginx-access.log access_json buffer=32k; + access_log off; + + error_log /data/ofm/http_host/logs_nginx/nginx-error.log; + + + ___LOCATION_BLOCKS___ + + + # we need to handle missing tiles as valid request returning empty string + location @empty { + default_type application/vnd.mapbox-vector-tile; + return 200 ''; + } +} diff --git a/scripts/http_host/nginx_sync/nginx_sync.py b/scripts/http_host/nginx_sync/nginx_sync.py new file mode 100755 index 0000000..38e1e6f --- /dev/null +++ b/scripts/http_host/nginx_sync/nginx_sync.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import os +import subprocess +import sys +from pathlib import Path + +import click + + +@click.command() +def cli(): + if not Path('/etc/fstab').exists(): + sys.exit('Needs to be run on Linux') + + if os.geteuid() != 0: + sys.exit('Needs sudo') + + if not Path('/mnt/ofm').exists(): + sys.exit('mounter.py needs to be run first') + + with open(Path(__file__).parent / 'nginx_site.conf') as fp: + nginx_template = fp.read() + + location_block_str = '' + help_text = '' + + for subdir in Path('/mnt/ofm').iterdir(): + if not subdir.is_dir(): + continue + + area, version = subdir.name.split('-') + + version_str = rf""" + location /{area}/{version}/ {{ + alias {subdir}; + try_files $uri @empty; + }} + """ + + location_block_str += version_str + + if not help_text: + help_text = ( + '\ntest with:\n' + f'curl -H "Host: ofm" -I http://localhost/{area}/{version}/tiles/14/8529/5975.pbf' + ) + + nginx_template = nginx_template.replace('___LOCATION_BLOCKS___', location_block_str) + + with open('/data/nginx/sites/ofm.conf', 'w') as fp: + fp.write(nginx_template) + print('nginx config written') + + subprocess.run(['nginx', '-t'], check=True) + subprocess.run(['service', 'nginx', 'reload'], check=True) + + print(help_text) + + +if __name__ == '__main__': + cli() diff --git a/ssh_lib/benchmark.py b/ssh_lib/benchmark.py index 6a793ea..5f89bb9 100644 --- a/ssh_lib/benchmark.py +++ b/ssh_lib/benchmark.py @@ -1,8 +1,11 @@ from ssh_lib.config import SCRIPTS_DIR -from ssh_lib.utils import apt_get_install, put +from ssh_lib.utils import apt_get_install, exists, put def c1000k(c): + if exists(c, 'c1000k-master'): + return + c.run('wget https://github.com/ideawu/c1000k/archive/master.zip -O tmp.zip') c.run('unzip -o tmp.zip') c.run('rm tmp.zip')