work

init-server.py

@@ -11,7 +11,7 @@ from ssh_lib.kernel import set_cpu_governor, setup_kernel_settings
 from ssh_lib.nginx import certbot, nginx
 from ssh_lib.pkg_base import pkg_base, pkg_clean, pkg_upgrade
 from ssh_lib.planetiler import TILE_GEN_BIN, install_planetiler
-from ssh_lib.utils import add_user, enable_sudo, put, reboot, setup_time, sudo_cmd
+from ssh_lib.utils import add_user, apt_get_install, enable_sudo, put, reboot, setup_time, sudo_cmd
 
 
 def prepare_shared(c):
@@ -30,6 +30,9 @@ def prepare_shared(c):
 
 def prepare_tile_gen(c):
     install_planetiler(c)
+    apt_get_install(c, 'btrfs-progs')
+
+    c.sudo('chown -R ofm:ofm /data/ofm')
 
     for file in [
         'extract_btrfs.sh',
@@ -38,7 +41,7 @@ def prepare_tile_gen(c):
         'prepare-virtualenv.sh',
         'upload_cloudflare.sh',
         'extract_mbtiles/extract_mbtiles.py',
-        'shrink_btrfs/shrink_btrfs.py',
+        'shrink_btrfs/extract_mbtiles.py',
     ]:
         put(
             c,
@@ -48,6 +51,24 @@ def prepare_tile_gen(c):
             owner='ofm',
         )
 
+    put(
+        c,
+        scripts / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py',
+        TILE_GEN_BIN / 'extract_mbtiles',
+        permissions='755',
+        owner='ofm',
+        target_is_dir=True,
+    )
+
+    put(
+        c,
+        scripts / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py',
+        TILE_GEN_BIN / 'shrink_btrfs',
+        permissions='755',
+        owner='ofm',
+        target_is_dir=True,
+    )
+
     sudo_cmd(c, f'cd {TILE_GEN_BIN} && source prepare-virtualenv.sh', user='ofm')
 
 

scripts/tile_gen/extract_btrfs.sh

@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 set -e
 
-export TILE_GEN_BIN=/data/ofm/tile_gen/bin
+TILE_GEN_BIN=/data/ofm/tile_gen/bin
-export VENV_PYTHON=$TILE_GEN_BIN/venv/python
+VENV_PYTHON=$TILE_GEN_BIN/venv/bin/python
 
 sudo umount mnt_rw 2> /dev/null || true
 sudo umount mnt_rw2 2> /dev/null || true
 rm -rf mnt_rw* tmp_*
-rm -f "*.btrfs" "*.gz"
+rm -f -- *.btrfs *.gz
-rm -f "*.log" "*.txt"
+rm -f -- *.log *.txt
 
 # make an empty file that's definitely bigger then the current OSM output
 fallocate -l 200G image.btrfs

scripts/tile_gen/extract_mbtiles/extract_mbtiles.py

@@ -34,6 +34,7 @@ def cli(mbtiles_path: Path, dir_path: Path):
     write_dedupl_files(c, dir_path=dir_path)
     write_tile_files(c, dir_path=dir_path)
 
+    # planetiler has missing tiles by design, so disabling this
     # if it's a full planet run,
     # make sure there are exactly the right number of files generated
     # if 'planet' in mbtiles_path.resolve().parent.name:
@@ -56,12 +57,14 @@ def write_metadata(c, *, dir_path):
 
 
 def write_dedupl_files(c, *, dir_path):
-    # dedupl files
+    """
-    # write out the tiles_data files into a multi-level folder
+    dedupl files
+    write out the tiles_data files into a multi-level folder
+    """
+
     total = c.execute('select count(*) from tiles_data').fetchone()[0]
 
     c.execute('select tile_data_id, tile_data from tiles_data')
-
     for i, row in enumerate(c, start=1):
         dedupl_id = row[0]
         dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
@@ -146,7 +149,7 @@ def calculate_tiles(zoom_level):
 
 def calculate_tiles_sum(zoom_level):
     """
-    Tiles up to zoom level (geometric series)
+    Sum of tiles up to zoom level (geometric series)
     """
     return (4 ** (zoom_level + 1) - 1) // 3
 

scripts/tile_gen/planetiler_monaco.sh

@@ -1,10 +1,12 @@
 #!/usr/bin/env bash
+set -e
 
-DATE=$(date +"%Y%m%d_%H%M%S")
 TILE_GEN_BIN=/data/ofm/tile_gen/bin
 
+DATE=$(date +"%Y%m%d_%H%M%S")
 RUN_FOLDER="/data/ofm/tile_gen/runs/monaco/${DATE}_pt"
 
+
 mkdir -p "$RUN_FOLDER"
 cd "$RUN_FOLDER" || exit
 
@@ -22,3 +24,6 @@ java -Xmx1g \
   --force \
   > "planetiler_out.log" 2> "planetiler_err.log"
 
+rm -r data
+
+$TILE_GEN_BIN/extract_btrfs.sh

scripts/tile_gen/planetiler_planet.sh

@@ -1,10 +1,12 @@
 #!/usr/bin/env bash
+set -e
 
-DATE=$(date +"%Y%m%d_%H%M%S")
 TILE_GEN_BIN=/data/ofm/tile_gen/bin
 
+DATE=$(date +"%Y%m%d_%H%M%S")
 RUN_FOLDER="/data/ofm/tile_gen/runs/planet/${DATE}_pt"
 
+
 mkdir -p "$RUN_FOLDER"
 cd "$RUN_FOLDER" || exit
 
@@ -24,3 +26,5 @@ java -Xmx30g \
   --force \
   > "planetiler_out.log" 2> "planetiler_err.log"
 
+rm -r data
+

ssh_lib/planetiler.py

@@ -19,5 +19,3 @@ def install_planetiler(c):
     )
 
     c.sudo(f'java -jar {PLANETILER_PATH} --help', hide=True)
-
-    c.sudo('chown -R ofm:ofm /data/ofm')

ssh_lib/utils.py

@@ -3,11 +3,13 @@ import secrets
 import string
 
 
-def put(c, local_path, remote_path, permissions=None, owner='root', group=None):
+def put(
+    c, local_path, remote_path, permissions=None, owner='root', group=None, target_is_dir=False
+):
     tmp_path = f'/tmp/fabtmp_{random_string(8)}'
     c.put(local_path, tmp_path)
 
-    if is_dir(c, remote_path):
+    if is_dir(c, remote_path) or target_is_dir:
         if not remote_path.endswith('/'):
             remote_path += '/'