refactor tile_gen in Python

ssh_lib/__init__.py

@@ -8,11 +8,16 @@ CONFIG_DIR = Path(__file__).parent.parent / 'config'
 SCRIPTS_DIR = Path(__file__).parent.parent / 'scripts'
 
 OFM_DIR = '/data/ofm'
-REMOTE_CONFIG = '/data/ofm/config'
-VENV_BIN = '/data/ofm/venv/bin'
-TILE_GEN_SRC = '/data/ofm/tile_gen/src'
-TILE_GEN_BIN = '/data/ofm/tile_gen/bin'
-HTTP_HOST_BIN = '/data/ofm/http_host/bin'
+REMOTE_CONFIG = f'{OFM_DIR}/config'
+VENV_BIN = f'{OFM_DIR}/venv/bin'
+
+TILE_GEN_DIR = f'{OFM_DIR}/tile_gen'
+TILE_GEN_BIN = f'{TILE_GEN_DIR}/bin'
+
+PLANETILER_SRC = f'{TILE_GEN_DIR}/planetiler_src'
+PLANETILER_BIN = f'{TILE_GEN_DIR}/planetiler'
+
+HTTP_HOST_BIN = f'{OFM_DIR}/http_host/bin'
 
 DOTENV_VALUES = dotenv_values(f'{CONFIG_DIR}/.env')
 

ssh_lib/planetiler.py

@@ -1,36 +1,36 @@
-from ssh_lib import TILE_GEN_BIN, TILE_GEN_SRC
+from ssh_lib import PLANETILER_BIN, PLANETILER_SRC
 from ssh_lib.utils import apt_get_install, apt_get_update, exists, sudo_cmd
 
 
-PLANETILER_COMMIT = 'cf6c55'
-PLANETILER_PATH = f'{TILE_GEN_BIN}/planetiler.jar'
+PLANETILER_COMMIT = 'cf49b86'
+PLANETILER_PATH = f'{PLANETILER_BIN}/planetiler.jar'
 
 
-def planetiler(c):
-    if exists(c, TILE_GEN_BIN):
+def install_planetiler(c):
+    if exists(c, PLANETILER_PATH):
         print('planetiler exists, skipping')
         return
 
     apt_get_update(c)
     apt_get_install(c, 'openjdk-21-jre-headless')
 
-    c.sudo(f'rm -rf {TILE_GEN_BIN} {TILE_GEN_SRC}')
-    c.sudo(f'mkdir -p {TILE_GEN_BIN} {TILE_GEN_SRC}')
+    c.sudo(f'rm -rf {PLANETILER_BIN} {PLANETILER_SRC}')
+    c.sudo(f'mkdir -p {PLANETILER_BIN} {PLANETILER_SRC}')
 
     c.sudo(
-        f'git clone --recurse-submodules https://github.com/onthegomap/planetiler.git {TILE_GEN_SRC}'
+        f'git clone --recurse-submodules https://github.com/onthegomap/planetiler.git {PLANETILER_SRC}'
     )
 
-    sudo_cmd(c, f'cd {TILE_GEN_SRC} && git checkout {PLANETILER_COMMIT}')
-    sudo_cmd(c, f'cd {TILE_GEN_SRC} && git submodule update --init --recursive')
+    sudo_cmd(c, f'cd {PLANETILER_SRC} && git checkout {PLANETILER_COMMIT}')
+    sudo_cmd(c, f'cd {PLANETILER_SRC} && git submodule update --init --recursive')
 
-    sudo_cmd(c, f'cd {TILE_GEN_SRC} && ./mvnw clean test package > {TILE_GEN_SRC}/_build.log')
+    sudo_cmd(c, f'cd {PLANETILER_SRC} && ./mvnw clean test package > {PLANETILER_SRC}/_build.log')
 
     c.sudo(
-        f'mv {TILE_GEN_SRC}/planetiler-dist/target/planetiler-dist-*-SNAPSHOT-with-deps.jar {PLANETILER_PATH}',
+        f'mv {PLANETILER_SRC}/planetiler-dist/target/planetiler-dist-*-SNAPSHOT-with-deps.jar {PLANETILER_PATH}',
         warn=True,
     )
 
     c.sudo(f'java -jar {PLANETILER_PATH} --help', hide=True)
 
-    c.sudo(f'rm -rf {TILE_GEN_SRC}')
+    c.sudo(f'rm -rf {PLANETILER_SRC}')

ssh_lib/tasks.py

@@ -15,7 +15,7 @@ from ssh_lib.benchmark import c1000k, wrk
 from ssh_lib.kernel import kernel_tweaks_ofm
 from ssh_lib.nginx import certbot, nginx
 from ssh_lib.pkg_base import pkg_base, pkg_upgrade
-from ssh_lib.planetiler import planetiler
+from ssh_lib.planetiler import install_planetiler
 from ssh_lib.rclone import rclone
 from ssh_lib.utils import add_user, enable_sudo, put, put_dir, put_str, sudo_cmd
 
@@ -48,11 +48,13 @@ def prepare_venv(c):
 
 
 def prepare_tile_gen(c):
-    planetiler(c)
+    install_planetiler(c)
+
+    c.sudo(f'rm -rf {TILE_GEN_BIN}')
 
     put_dir(c, SCRIPTS_DIR / 'tile_gen', TILE_GEN_BIN, file_permissions='755')
 
-    for dirname in ['tile_gen_lib', 'extract_mbtiles', 'shrink_btrfs']:
+    for dirname in ['tile_gen_lib', 'scripts']:
         put_dir(c, SCRIPTS_DIR / 'tile_gen' / dirname, f'{TILE_GEN_BIN}/{dirname}')
 
     if (CONFIG_DIR / 'rclone.conf').exists():
@@ -64,10 +66,10 @@ def prepare_tile_gen(c):
             user='ofm',
         )
 
-    c.sudo(f'{VENV_BIN}/pip install -e {TILE_GEN_BIN}')
+    c.sudo(f'{VENV_BIN}/pip install -e {TILE_GEN_BIN} --use-pep517')
 
     c.sudo('chown ofm:ofm /data/ofm/tile_gen')
-    c.sudo('chown ofm:ofm -R /data/ofm/tile_gen/bin')
+    c.sudo(f'chown ofm:ofm -R {TILE_GEN_BIN}')
 
 
 def upload_http_host_config(c):
@@ -121,7 +123,7 @@ def prepare_http_host(c):
     upload_http_host_files(c)
     upload_certificates(c)
 
-    c.sudo(f'{VENV_BIN}/pip install -e {HTTP_HOST_BIN}')
+    c.sudo(f'{VENV_BIN}/pip install -e {HTTP_HOST_BIN} --use-pep517')
 
 
 def run_http_host_sync(c):
@@ -243,7 +245,7 @@ def setup_loadbalancer(c):
         '/data/ofm/loadbalancer/loadbalancer_lib',
     )
 
-    c.sudo(f'{VENV_BIN}/pip install -e /data/ofm/loadbalancer')
+    c.sudo(f'{VENV_BIN}/pip install -e /data/ofm/loadbalancer --use-pep517')
 
     c.sudo('mkdir -p /data/ofm/loadbalancer/logs')
     put(c, SCRIPTS_DIR / 'loadbalancer' / 'cron.d' / 'ofm_loadbalancer', '/etc/cron.d/')