le dns

config/.env.sample

@@ -13,3 +13,8 @@ DOMAIN_CF=tiles.openfreemap.org
 
 # Skip the full planet download, useful for testing (true/false)
 SKIP_PLANET=false
+
+# --- Let's Encrypt DNS related variables, not needed for self-hosting
+
+DOMAIN_LE_DNS = rr.openfreemap.org
+

init-server.py

@@ -1,191 +1,25 @@
 #!/usr/bin/env python3
-import json
-import sys
 
 import click
-from dotenv import dotenv_values
 from fabric import Config, Connection
 
-from ssh_lib import CONFIG_DIR, HTTP_HOST_BIN, OFM_DIR, REMOTE_CONFIG, SCRIPTS_DIR, TILE_GEN_BIN
+from ssh_lib import dotenv_val
-from ssh_lib.benchmark import c1000k, wrk
+from ssh_lib.tasks import (
-from ssh_lib.kernel import kernel_tweaks_ofm
+    add_http_host_cron,
-from ssh_lib.nginx import certbot, lego, nginx
+    prepare_http_host,
-from ssh_lib.pkg_base import pkg_base, pkg_upgrade
+    prepare_shared,
-from ssh_lib.planetiler import planetiler
+    prepare_tile_gen,
-from ssh_lib.rclone import rclone
+    run_http_host_sync,
+    setup_le_dns_manager,
+    upload_http_host_config,
+)
 from ssh_lib.utils import (
-    add_user,
-    enable_sudo,
-    put,
-    put_dir,
-    put_str,
     sudo_cmd,
 )
 
 
-def prepare_shared(c):
-    # creates ofm user with uid=2000, disabled password and nopasswd sudo
-    add_user(c, 'ofm', uid=2000)
-    enable_sudo(c, 'ofm', nopasswd=True)
-
-    pkg_upgrade(c)
-    pkg_base(c)
-
-    kernel_tweaks_ofm(c)
-
-    c.sudo(f'mkdir -p {REMOTE_CONFIG}')
-    c.sudo('chown ofm:ofm /data/ofm/config')
-    c.sudo('chown ofm:ofm /data/ofm')
-
-    prepare_venv(c)
-
-
-def prepare_venv(c):
-    put(
-        c,
-        SCRIPTS_DIR / 'prepare-virtualenv.sh',
-        OFM_DIR,
-        permissions='755',
-        user='ofm',
-    )
-    sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh')
-
-
-def prepare_tile_gen(c):
-    planetiler(c)
-    rclone(c)
-
-    for file in [
-        'extract_btrfs.sh',
-        'planetiler_monaco.sh',
-        'planetiler_planet.sh',
-        'cloudflare_index.sh',
-        'cloudflare_upload.sh',
-    ]:
-        put(
-            c,
-            SCRIPTS_DIR / 'tile_gen' / file,
-            TILE_GEN_BIN,
-            permissions='755',
-        )
-
-    put(
-        c,
-        SCRIPTS_DIR / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py',
-        f'{TILE_GEN_BIN}/extract_mbtiles/extract_mbtiles.py',
-        create_parent_dir=True,
-    )
-
-    put(
-        c,
-        SCRIPTS_DIR / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py',
-        f'{TILE_GEN_BIN}/shrink_btrfs/shrink_btrfs.py',
-        create_parent_dir=True,
-    )
-
-    if (CONFIG_DIR / 'rclone.conf').exists():
-        put(
-            c,
-            CONFIG_DIR / 'rclone.conf',
-            f'{REMOTE_CONFIG}/rclone.conf',
-            permissions='600',
-            user='ofm',
-        )
-
-    c.sudo('chown ofm:ofm /data/ofm/tile_gen')
-    c.sudo('chown ofm:ofm -R /data/ofm/tile_gen/bin')
-
-
-def upload_http_host_config(c):
-    env_values = dotenv_values(f'{CONFIG_DIR}/.env')
-
-    domain_le = env_values.get('DOMAIN_LE', '').strip()
-    domain_cf = env_values.get('DOMAIN_CF', '').strip()
-    skip_planet = env_values.get('SKIP_PLANET', '').lower().strip() == 'true'
-    le_email = env_values.get('LE_EMAIL', '').strip()
-
-    if not (domain_le or domain_cf):
-        sys.exit('Please specify DOMAIN_LE or DOMAIN_CF in config/.env')
-
-    if domain_cf:
-        if (
-            not (CONFIG_DIR / 'certs' / 'ofm_cf.key').exists()
-            or not (CONFIG_DIR / 'certs' / 'ofm_cf.cert').exists()
-        ):
-            sys.exit(
-                'When using DOMAIN_CF, please put ofm_cf.key and ofm_cf.cert files in config/certs'
-            )
-
-    if domain_le and not le_email:
-        sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE')
-
-    host_config = {
-        'domain_le': domain_le,
-        'domain_cf': domain_cf,
-        'skip_planet': skip_planet,
-        'le_email': le_email,
-    }
-
-    host_config_str = json.dumps(host_config, indent=2, ensure_ascii=False)
-    print(host_config_str)
-    put_str(c, '/data/ofm/config/http_host.json', host_config_str)
-
-
-def prepare_http_host(c):
-    nginx(c)
-    certbot(c)
-
-    c.sudo('rm -rf /data/ofm/http_host/logs')
-    c.sudo('mkdir -p /data/ofm/http_host/logs')
-    c.sudo('chown ofm:ofm /data/ofm/http_host/logs')
-
-    c.sudo('rm -rf /data/ofm/http_host/logs_nginx')
-    c.sudo('mkdir -p /data/ofm/http_host/logs_nginx')
-    c.sudo('chown nginx:nginx /data/ofm/http_host/logs_nginx')
-
-    upload_https_host_files(c)
-    upload_certificates(c)
-
-    c.sudo('/data/ofm/venv/bin/pip install -e /data/ofm/http_host/bin')
-
-
-def add_http_host_cron(c):
-    put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
-
-
-def run_http_host_sync(c):
-    sudo_cmd(c, '/data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync')
-
-
-def upload_https_host_files(c):
-    c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
-
-    put_dir(c, SCRIPTS_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')
-    put_dir(c, SCRIPTS_DIR / 'http_host' / 'http_host_lib', f'{HTTP_HOST_BIN}/http_host_lib')
-    put_dir(
-        c,
-        SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx',
-        f'{HTTP_HOST_BIN}/http_host_lib/nginx',
-    )
-
-    c.sudo('chown -R ofm:ofm /data/ofm/http_host')
-
-
-def upload_certificates(c):
-    put_dir(c, CONFIG_DIR / 'certs', '/data/nginx/certs', file_permissions=400)
-    c.sudo('chown -R nginx:nginx /data/nginx')
-
-
-def install_benchmark(c):
-    """
-    Read docs/quick_notes/http_benchmark.md
-    """
-    c1000k(c)
-    wrk(c)
-
-
 def get_connection(hostname, user, port):
-    ssh_passwd = dotenv_values(f'{CONFIG_DIR}/.env').get('SSH_PASSWD', '').strip()
+    ssh_passwd = dotenv_val('SSH_PASSWD')
 
     if ssh_passwd:
         print('Using SSH password')
@@ -227,11 +61,11 @@ def http_host_once(hostname, user, port):
         return
 
     c = get_connection(hostname, user, port)
+
     prepare_shared(c)
-
     upload_http_host_config(c)
-    prepare_http_host(c)
 
+    prepare_http_host(c)
     run_http_host_sync(c)
 
 
@@ -242,11 +76,11 @@ def http_host_autoupdate(hostname, user, port):
         return
 
     c = get_connection(hostname, user, port)
+
     prepare_shared(c)
-
     upload_http_host_config(c)
-    prepare_http_host(c)
 
+    prepare_http_host(c)
     add_http_host_cron(c)
 
 
@@ -262,14 +96,25 @@ def tile_gen(hostname, user, port):
     prepare_tile_gen(c)
 
 
+@cli.command()
+@common_options
+def le_dns_manager(hostname, user, port):
+    if not click.confirm(f'Run script on {hostname}?'):
+        return
+
+    c = get_connection(hostname, user, port)
+
+    setup_le_dns_manager(c)
+
+
 @cli.command()
 @common_options
 def debug(hostname, user, port):
     c = get_connection(hostname, user, port)
 
     prepare_shared(c)
-
     upload_http_host_config(c)
+
     prepare_http_host(c)
 
     run_http_host_sync(c)

ssh_lib/__init__.py

@@ -1,5 +1,7 @@
 from pathlib import Path
 
+from dotenv import dotenv_values
+
 
 ASSETS_DIR = Path(__file__).parent / 'assets'
 CONFIG_DIR = Path(__file__).parent.parent / 'config'
@@ -9,3 +11,9 @@ OFM_DIR = '/data/ofm'
 REMOTE_CONFIG = '/data/ofm/config'
 TILE_GEN_BIN = '/data/ofm/tile_gen/bin'
 HTTP_HOST_BIN = '/data/ofm/http_host/bin'
+
+DOTENV_VALUES = dotenv_values(f'{CONFIG_DIR}/.env')
+
+
+def dotenv_val(key):
+    return DOTENV_VALUES.get(key, '').strip()

ssh_lib/nginx.py

@@ -69,6 +69,8 @@ def certbot(c):
 
     apt_get_purge(c, 'certbot')
     c.sudo('snap install --classic certbot', warn=True)
+    c.sudo('snap set certbot trust-plugin-with-root=ok', warn=True)
+    c.sudo('snap install certbot-dns-cloudflare', warn=True)
 
 
 def lego(c):

ssh_lib/tasks.py

@@ -0,0 +1,198 @@
+import json
+import sys
+
+from ssh_lib import (
+    CONFIG_DIR,
+    HTTP_HOST_BIN,
+    OFM_DIR,
+    REMOTE_CONFIG,
+    SCRIPTS_DIR,
+    TILE_GEN_BIN,
+    dotenv_val,
+)
+from ssh_lib.benchmark import c1000k, wrk
+from ssh_lib.kernel import kernel_tweaks_ofm
+from ssh_lib.nginx import certbot, nginx
+from ssh_lib.pkg_base import pkg_base, pkg_upgrade
+from ssh_lib.planetiler import planetiler
+from ssh_lib.rclone import rclone
+from ssh_lib.utils import add_user, enable_sudo, put, put_dir, put_str, sudo_cmd
+
+
+def prepare_shared(c):
+    # creates ofm user with uid=2000, disabled password and nopasswd sudo
+    add_user(c, 'ofm', uid=2000)
+    enable_sudo(c, 'ofm', nopasswd=True)
+
+    pkg_upgrade(c)
+    pkg_base(c)
+
+    kernel_tweaks_ofm(c)
+
+    c.sudo(f'mkdir -p {REMOTE_CONFIG}')
+    c.sudo('chown ofm:ofm /data/ofm/config')
+    c.sudo('chown ofm:ofm /data/ofm')
+
+    prepare_venv(c)
+
+
+def prepare_venv(c):
+    put(
+        c,
+        SCRIPTS_DIR / 'prepare-virtualenv.sh',
+        OFM_DIR,
+        permissions='755',
+        user='ofm',
+    )
+    sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh')
+
+
+def prepare_tile_gen(c):
+    planetiler(c)
+    rclone(c)
+
+    for file in [
+        'extract_btrfs.sh',
+        'planetiler_monaco.sh',
+        'planetiler_planet.sh',
+        'cloudflare_index.sh',
+        'cloudflare_upload.sh',
+    ]:
+        put(
+            c,
+            SCRIPTS_DIR / 'tile_gen' / file,
+            TILE_GEN_BIN,
+            permissions='755',
+        )
+
+    put(
+        c,
+        SCRIPTS_DIR / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py',
+        f'{TILE_GEN_BIN}/extract_mbtiles/extract_mbtiles.py',
+        create_parent_dir=True,
+    )
+
+    put(
+        c,
+        SCRIPTS_DIR / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py',
+        f'{TILE_GEN_BIN}/shrink_btrfs/shrink_btrfs.py',
+        create_parent_dir=True,
+    )
+
+    if (CONFIG_DIR / 'rclone.conf').exists():
+        put(
+            c,
+            CONFIG_DIR / 'rclone.conf',
+            f'{REMOTE_CONFIG}/rclone.conf',
+            permissions='600',
+            user='ofm',
+        )
+
+    c.sudo('chown ofm:ofm /data/ofm/tile_gen')
+    c.sudo('chown ofm:ofm -R /data/ofm/tile_gen/bin')
+
+
+def upload_http_host_config(c):
+    domain_le = dotenv_val('DOMAIN_LE').lower()
+    domain_cf = dotenv_val('DOMAIN_CF').lower()
+    skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
+    le_email = dotenv_val('LE_EMAIL').lower()
+
+    if not (domain_le or domain_cf):
+        sys.exit('Please specify DOMAIN_LE or DOMAIN_CF in config/.env')
+
+    if domain_cf:
+        if (
+            not (CONFIG_DIR / 'certs' / 'ofm_cf.key').exists()
+            or not (CONFIG_DIR / 'certs' / 'ofm_cf.cert').exists()
+        ):
+            sys.exit(
+                'When using DOMAIN_CF, please put ofm_cf.key and ofm_cf.cert files in config/certs'
+            )
+
+    if domain_le and not le_email:
+        sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE')
+
+    host_config = {
+        'domain_le': domain_le,
+        'domain_cf': domain_cf,
+        'skip_planet': skip_planet,
+        'le_email': le_email,
+    }
+
+    host_config_str = json.dumps(host_config, indent=2, ensure_ascii=False)
+    print(host_config_str)
+    put_str(c, '/data/ofm/config/http_host.json', host_config_str)
+
+
+def prepare_http_host(c):
+    nginx(c)
+    certbot(c)
+
+    c.sudo('rm -rf /data/ofm/http_host/logs')
+    c.sudo('mkdir -p /data/ofm/http_host/logs')
+    c.sudo('chown ofm:ofm /data/ofm/http_host/logs')
+
+    c.sudo('rm -rf /data/ofm/http_host/logs_nginx')
+    c.sudo('mkdir -p /data/ofm/http_host/logs_nginx')
+    c.sudo('chown nginx:nginx /data/ofm/http_host/logs_nginx')
+
+    upload_https_host_files(c)
+    upload_certificates(c)
+
+    c.sudo('/data/ofm/venv/bin/pip install -e /data/ofm/http_host/bin')
+
+
+def add_http_host_cron(c):
+    put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
+
+
+def run_http_host_sync(c):
+    sudo_cmd(c, '/data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync')
+
+
+def upload_https_host_files(c):
+    c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
+
+    put_dir(c, SCRIPTS_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')
+    put_dir(c, SCRIPTS_DIR / 'http_host' / 'http_host_lib', f'{HTTP_HOST_BIN}/http_host_lib')
+    put_dir(
+        c,
+        SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx',
+        f'{HTTP_HOST_BIN}/http_host_lib/nginx',
+    )
+
+    c.sudo('chown -R ofm:ofm /data/ofm/http_host')
+
+
+def upload_certificates(c):
+    put_dir(c, CONFIG_DIR / 'certs', '/data/nginx/certs', file_permissions=400)
+    c.sudo('chown -R nginx:nginx /data/nginx')
+
+
+def install_benchmark(c):
+    """
+    Read docs/quick_notes/http_benchmark.md
+    """
+    c1000k(c)
+    wrk(c)
+
+
+def setup_le_dns_manager(c):
+    le_email = dotenv_val('LE_EMAIL').lower()
+    domain_le_dns = dotenv_val('DOMAIN_LE_DNS').lower()
+
+    assert le_email
+    assert domain_le_dns
+
+    sudo_cmd(
+        c,
+        'certbot certonly '
+        '--dns-cloudflare '
+        '--dns-cloudflare-credentials ~/.secrets/certbot/cloudflare.ini '
+        '--staging '
+        f'--noninteractive -m {le_email} '
+        f'--agree-tos '
+        f'--cert-name=ofm_le_dns '
+        f'-d {domain_le_dns}',
+    )