planetiler

init-server.py

@@ -2,17 +2,28 @@
 
 from fabric import Connection
 
-from openfreemaps.kernel import setup_kernel_settings
 from openfreemaps.nginx import certbot, nginx
+from openfreemaps.pkg_base import pkg_base, pkg_clean, pkg_upgrade
+from openfreemaps.planetiler import install_planetiler
+from openfreemaps.system import set_cpu_governor, setup_kernel_settings, setup_time
 
 
 def prepare_server(c):
+    pkg_upgrade(c)
+    pkg_clean(c)
+    pkg_base(c)
+
+    setup_time(c)
     setup_kernel_settings(c)
+    set_cpu_governor(c)
 
     nginx(c)
     certbot(c)
 
+    install_planetiler(c)
 
-c = Connection(host='map128')
+
+c = Connection(host='map128', port=22)
 
 prepare_server(c)
+# reboot(c)

openfreemaps/pkg_base.py

@@ -0,0 +1,31 @@
+from openfreemaps.utils import (
+    apt_get_autoremove,
+    apt_get_install,
+    apt_get_purge,
+    apt_get_update,
+    sudo_cmd,
+)
+
+
+def pkg_upgrade(c):
+    apt_get_update(c)
+    c.sudo(
+        'DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold"'
+    )
+
+
+def pkg_clean(c):
+    clean_list = [
+        # firewalls
+        'ufw',
+        'nftables',
+        'firewalld',
+    ]
+
+    apt_get_purge(c, ' '.join(clean_list))
+    apt_get_autoremove(c)
+    sudo_cmd(c, 'dpkg --list | grep "^rc" | cut -d " " -f 3 | xargs -r dpkg --purge')
+
+
+def pkg_base(c):
+    apt_get_install(c, 'nload iftop')

openfreemaps/planetiler.py

@@ -0,0 +1,23 @@
+from openfreemaps.config import templates
+from openfreemaps.utils import apt_get_install, apt_get_update, put
+
+
+PLANETILER_VERSION = '0.7.0'
+
+PLANETILER_DIR = '/data/planetiler/bin'
+PLANETILER_PATH = f'{PLANETILER_DIR}/planetiler.jar'
+
+
+def install_planetiler(c):
+    apt_get_update(c)
+    apt_get_install(c, 'openjdk-17-jdk')
+
+    c.run('mkdir -p /data/planetiler/bin')
+
+    c.run(
+        f'wget -q https://github.com/onthegomap/planetiler/releases/download/v{PLANETILER_VERSION}/planetiler.jar '
+        f'-O {PLANETILER_PATH}',
+    )
+
+    c.run(f'java -jar {PLANETILER_PATH} --help')
+    put(c, templates / 'planetiler' / 'run_planet.sh', PLANETILER_DIR, permissions='755')

openfreemaps/system.py

@@ -1,11 +1,23 @@
 from openfreemaps.config import templates
-from openfreemaps.utils import apt_get_install, apt_get_purge, put, put_str
+from openfreemaps.utils import (
+    apt_get_install,
+    apt_get_purge,
+    put,
+    put_str,
+)
+
+
+def setup_time(c):
+    apt_get_install(c, 'dbus')
+
+    c.sudo('timedatectl set-local-rtc 0')
+    c.sudo('timedatectl set-ntp 1')
+    c.sudo('timedatectl set-timezone UTC')
 
 
 def setup_kernel_settings(c):
     put(c, f'{templates}/sysctl/60-optim.conf', '/etc/sysctl.d/')
 
-    set_cpu_governor(c)
 
 
 def set_cpu_governor(c):

templates/planetiler/run_planet.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+DATE=$(date +"%Y%m%d_%H%M%S")
+
+RUN_FOLDER="/data/planetiler/runs/$DATE"
+
+mkdir -p "$RUN_FOLDER"
+cd "$RUN_FOLDER" || exit
+
+java -Xmx20g \
+  -jar /data/planetiler/bin/planetiler.jar \
+  `# Download the latest planet.osm.pbf from s3://osm-pds bucket` \
+  --area=planet --bounds=planet --download \
+  `# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \
+  --download-threads=10 --download-chunk-size-mb=1000 \
+  `# Also download name translations from wikidata` \
+  --fetch-wikidata \
+  --output=output.mbtiles \
+  `# Store temporary node locations at fixed positions in a memory-mapped file` \
+  --nodemap-type=array --storage=mmap \
+  > "output_$DATE.log"