From 48736bf87b64213c35164502115bc155934d20a8 Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Thu, 21 Dec 2023 02:35:20 +0100 Subject: [PATCH] work --- init-server.py | 25 +++++++++++++++++-- scripts/tile_gen/extract_btrfs.sh | 8 +++--- .../extract_mbtiles/extract_mbtiles.py | 11 +++++--- scripts/tile_gen/planetiler_monaco.sh | 7 +++++- scripts/tile_gen/planetiler_planet.sh | 6 ++++- ssh_lib/planetiler.py | 2 -- ssh_lib/utils.py | 6 +++-- 7 files changed, 49 insertions(+), 16 deletions(-) diff --git a/init-server.py b/init-server.py index 38994cc..bf14977 100755 --- a/init-server.py +++ b/init-server.py @@ -11,7 +11,7 @@ from ssh_lib.kernel import set_cpu_governor, setup_kernel_settings from ssh_lib.nginx import certbot, nginx from ssh_lib.pkg_base import pkg_base, pkg_clean, pkg_upgrade from ssh_lib.planetiler import TILE_GEN_BIN, install_planetiler -from ssh_lib.utils import add_user, enable_sudo, put, reboot, setup_time, sudo_cmd +from ssh_lib.utils import add_user, apt_get_install, enable_sudo, put, reboot, setup_time, sudo_cmd def prepare_shared(c): @@ -30,6 +30,9 @@ def prepare_shared(c): def prepare_tile_gen(c): install_planetiler(c) + apt_get_install(c, 'btrfs-progs') + + c.sudo('chown -R ofm:ofm /data/ofm') for file in [ 'extract_btrfs.sh', @@ -38,7 +41,7 @@ def prepare_tile_gen(c): 'prepare-virtualenv.sh', 'upload_cloudflare.sh', 'extract_mbtiles/extract_mbtiles.py', - 'shrink_btrfs/shrink_btrfs.py', + 'shrink_btrfs/extract_mbtiles.py', ]: put( c, @@ -48,6 +51,24 @@ def prepare_tile_gen(c): owner='ofm', ) + put( + c, + scripts / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py', + TILE_GEN_BIN / 'extract_mbtiles', + permissions='755', + owner='ofm', + target_is_dir=True, + ) + + put( + c, + scripts / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py', + TILE_GEN_BIN / 'shrink_btrfs', + permissions='755', + owner='ofm', + target_is_dir=True, + ) + sudo_cmd(c, f'cd {TILE_GEN_BIN} && source prepare-virtualenv.sh', user='ofm') diff --git a/scripts/tile_gen/extract_btrfs.sh b/scripts/tile_gen/extract_btrfs.sh index 748a863..341136f 100644 --- a/scripts/tile_gen/extract_btrfs.sh +++ b/scripts/tile_gen/extract_btrfs.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash set -e -export TILE_GEN_BIN=/data/ofm/tile_gen/bin -export VENV_PYTHON=$TILE_GEN_BIN/venv/python +TILE_GEN_BIN=/data/ofm/tile_gen/bin +VENV_PYTHON=$TILE_GEN_BIN/venv/bin/python sudo umount mnt_rw 2> /dev/null || true sudo umount mnt_rw2 2> /dev/null || true rm -rf mnt_rw* tmp_* -rm -f "*.btrfs" "*.gz" -rm -f "*.log" "*.txt" +rm -f -- *.btrfs *.gz +rm -f -- *.log *.txt # make an empty file that's definitely bigger then the current OSM output fallocate -l 200G image.btrfs diff --git a/scripts/tile_gen/extract_mbtiles/extract_mbtiles.py b/scripts/tile_gen/extract_mbtiles/extract_mbtiles.py index 0cfeab7..ef5aa56 100755 --- a/scripts/tile_gen/extract_mbtiles/extract_mbtiles.py +++ b/scripts/tile_gen/extract_mbtiles/extract_mbtiles.py @@ -34,6 +34,7 @@ def cli(mbtiles_path: Path, dir_path: Path): write_dedupl_files(c, dir_path=dir_path) write_tile_files(c, dir_path=dir_path) + # planetiler has missing tiles by design, so disabling this # if it's a full planet run, # make sure there are exactly the right number of files generated # if 'planet' in mbtiles_path.resolve().parent.name: @@ -56,12 +57,14 @@ def write_metadata(c, *, dir_path): def write_dedupl_files(c, *, dir_path): - # dedupl files - # write out the tiles_data files into a multi-level folder + """ + dedupl files + write out the tiles_data files into a multi-level folder + """ + total = c.execute('select count(*) from tiles_data').fetchone()[0] c.execute('select tile_data_id, tile_data from tiles_data') - for i, row in enumerate(c, start=1): dedupl_id = row[0] dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id) @@ -146,7 +149,7 @@ def calculate_tiles(zoom_level): def calculate_tiles_sum(zoom_level): """ - Tiles up to zoom level (geometric series) + Sum of tiles up to zoom level (geometric series) """ return (4 ** (zoom_level + 1) - 1) // 3 diff --git a/scripts/tile_gen/planetiler_monaco.sh b/scripts/tile_gen/planetiler_monaco.sh index 9cd20a4..f4b2082 100644 --- a/scripts/tile_gen/planetiler_monaco.sh +++ b/scripts/tile_gen/planetiler_monaco.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash +set -e -DATE=$(date +"%Y%m%d_%H%M%S") TILE_GEN_BIN=/data/ofm/tile_gen/bin +DATE=$(date +"%Y%m%d_%H%M%S") RUN_FOLDER="/data/ofm/tile_gen/runs/monaco/${DATE}_pt" + mkdir -p "$RUN_FOLDER" cd "$RUN_FOLDER" || exit @@ -22,3 +24,6 @@ java -Xmx1g \ --force \ > "planetiler_out.log" 2> "planetiler_err.log" +rm -r data + +$TILE_GEN_BIN/extract_btrfs.sh \ No newline at end of file diff --git a/scripts/tile_gen/planetiler_planet.sh b/scripts/tile_gen/planetiler_planet.sh index f9be3ad..5f9522c 100644 --- a/scripts/tile_gen/planetiler_planet.sh +++ b/scripts/tile_gen/planetiler_planet.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash +set -e -DATE=$(date +"%Y%m%d_%H%M%S") TILE_GEN_BIN=/data/ofm/tile_gen/bin +DATE=$(date +"%Y%m%d_%H%M%S") RUN_FOLDER="/data/ofm/tile_gen/runs/planet/${DATE}_pt" + mkdir -p "$RUN_FOLDER" cd "$RUN_FOLDER" || exit @@ -24,3 +26,5 @@ java -Xmx30g \ --force \ > "planetiler_out.log" 2> "planetiler_err.log" +rm -r data + diff --git a/ssh_lib/planetiler.py b/ssh_lib/planetiler.py index cafaff3..a7b34b8 100644 --- a/ssh_lib/planetiler.py +++ b/ssh_lib/planetiler.py @@ -19,5 +19,3 @@ def install_planetiler(c): ) c.sudo(f'java -jar {PLANETILER_PATH} --help', hide=True) - - c.sudo('chown -R ofm:ofm /data/ofm') diff --git a/ssh_lib/utils.py b/ssh_lib/utils.py index ac801df..64bcff0 100644 --- a/ssh_lib/utils.py +++ b/ssh_lib/utils.py @@ -3,11 +3,13 @@ import secrets import string -def put(c, local_path, remote_path, permissions=None, owner='root', group=None): +def put( + c, local_path, remote_path, permissions=None, owner='root', group=None, target_is_dir=False +): tmp_path = f'/tmp/fabtmp_{random_string(8)}' c.put(local_path, tmp_path) - if is_dir(c, remote_path): + if is_dir(c, remote_path) or target_is_dir: if not remote_path.endswith('/'): remote_path += '/'