diff --git a/init-server.py b/init-server.py index 7926c2f..af94ab0 100755 --- a/init-server.py +++ b/init-server.py @@ -2,17 +2,28 @@ from fabric import Connection -from openfreemaps.kernel import setup_kernel_settings from openfreemaps.nginx import certbot, nginx +from openfreemaps.pkg_base import pkg_base, pkg_clean, pkg_upgrade +from openfreemaps.planetiler import install_planetiler +from openfreemaps.system import set_cpu_governor, setup_kernel_settings, setup_time def prepare_server(c): + pkg_upgrade(c) + pkg_clean(c) + pkg_base(c) + + setup_time(c) setup_kernel_settings(c) + set_cpu_governor(c) nginx(c) certbot(c) + install_planetiler(c) -c = Connection(host='map128') + +c = Connection(host='map128', port=22) prepare_server(c) +# reboot(c) diff --git a/openfreemaps/pkg_base.py b/openfreemaps/pkg_base.py new file mode 100644 index 0000000..32e3644 --- /dev/null +++ b/openfreemaps/pkg_base.py @@ -0,0 +1,31 @@ +from openfreemaps.utils import ( + apt_get_autoremove, + apt_get_install, + apt_get_purge, + apt_get_update, + sudo_cmd, +) + + +def pkg_upgrade(c): + apt_get_update(c) + c.sudo( + 'DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold"' + ) + + +def pkg_clean(c): + clean_list = [ + # firewalls + 'ufw', + 'nftables', + 'firewalld', + ] + + apt_get_purge(c, ' '.join(clean_list)) + apt_get_autoremove(c) + sudo_cmd(c, 'dpkg --list | grep "^rc" | cut -d " " -f 3 | xargs -r dpkg --purge') + + +def pkg_base(c): + apt_get_install(c, 'nload iftop') diff --git a/openfreemaps/planetiler.py b/openfreemaps/planetiler.py new file mode 100644 index 0000000..cce7f38 --- /dev/null +++ b/openfreemaps/planetiler.py @@ -0,0 +1,23 @@ +from openfreemaps.config import templates +from openfreemaps.utils import apt_get_install, apt_get_update, put + + +PLANETILER_VERSION = '0.7.0' + +PLANETILER_DIR = '/data/planetiler/bin' +PLANETILER_PATH = f'{PLANETILER_DIR}/planetiler.jar' + + +def install_planetiler(c): + apt_get_update(c) + apt_get_install(c, 'openjdk-17-jdk') + + c.run('mkdir -p /data/planetiler/bin') + + c.run( + f'wget -q https://github.com/onthegomap/planetiler/releases/download/v{PLANETILER_VERSION}/planetiler.jar ' + f'-O {PLANETILER_PATH}', + ) + + c.run(f'java -jar {PLANETILER_PATH} --help') + put(c, templates / 'planetiler' / 'run_planet.sh', PLANETILER_DIR, permissions='755') diff --git a/openfreemaps/kernel.py b/openfreemaps/system.py similarity index 61% rename from openfreemaps/kernel.py rename to openfreemaps/system.py index 7f77faa..6b31703 100644 --- a/openfreemaps/kernel.py +++ b/openfreemaps/system.py @@ -1,11 +1,23 @@ from openfreemaps.config import templates -from openfreemaps.utils import apt_get_install, apt_get_purge, put, put_str +from openfreemaps.utils import ( + apt_get_install, + apt_get_purge, + put, + put_str, +) + + +def setup_time(c): + apt_get_install(c, 'dbus') + + c.sudo('timedatectl set-local-rtc 0') + c.sudo('timedatectl set-ntp 1') + c.sudo('timedatectl set-timezone UTC') def setup_kernel_settings(c): put(c, f'{templates}/sysctl/60-optim.conf', '/etc/sysctl.d/') - set_cpu_governor(c) def set_cpu_governor(c): diff --git a/templates/planetiler/run_planet.sh b/templates/planetiler/run_planet.sh new file mode 100644 index 0000000..0508efe --- /dev/null +++ b/templates/planetiler/run_planet.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +DATE=$(date +"%Y%m%d_%H%M%S") + +RUN_FOLDER="/data/planetiler/runs/$DATE" + +mkdir -p "$RUN_FOLDER" +cd "$RUN_FOLDER" || exit + +java -Xmx20g \ + -jar /data/planetiler/bin/planetiler.jar \ + `# Download the latest planet.osm.pbf from s3://osm-pds bucket` \ + --area=planet --bounds=planet --download \ + `# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \ + --download-threads=10 --download-chunk-size-mb=1000 \ + `# Also download name translations from wikidata` \ + --fetch-wikidata \ + --output=output.mbtiles \ + `# Store temporary node locations at fixed positions in a memory-mapped file` \ + --nodemap-type=array --storage=mmap \ + > "output_$DATE.log"