work

init-server.py

@@ -4,12 +4,9 @@ import click
 from fabric import Config, Connection
 
 from ssh_lib import MODULES_DIR, dotenv_val
-from ssh_lib.tasks import (
-    prepare_http_host,
-    prepare_shared,
-    prepare_tile_gen,
-    run_http_host_sync,
-)
+from ssh_lib.tasks_httphost import prepare_http_host, run_http_host_sync
+from ssh_lib.tasks_tilegen import prepare_tile_gen
+from ssh_lib.tasks_shared import prepare_shared
 from ssh_lib.utils import (
     put,
 )

setup.py

@@ -9,6 +9,7 @@ requirements = [
     'ruff',
     'marko',
     'requests',
+    'jsonschema',
 ]
 
 

ssh_lib/__init__.py

@@ -1,42 +0,0 @@
-import os
-import sys
-from pathlib import Path
-
-from dotenv import dotenv_values
-
-
-ASSETS_DIR = Path(__file__).parent / 'assets'
-CONFIG_DIR = Path(__file__).parent.parent / 'config'
-MODULES_DIR = Path(__file__).parent.parent / 'modules'
-
-OFM_DIR = '/data/ofm'
-REMOTE_CONFIG = f'{OFM_DIR}/config'
-VENV_BIN = f'{OFM_DIR}/venv/bin'
-
-TILE_GEN_DIR = f'{OFM_DIR}/tile_gen'
-TILE_GEN_BIN = f'{TILE_GEN_DIR}/bin'
-
-PLANETILER_SRC = f'{TILE_GEN_DIR}/planetiler_src'
-PLANETILER_BIN = f'{TILE_GEN_DIR}/planetiler'
-
-HTTP_HOST_BIN = f'{OFM_DIR}/http_host/bin'
-
-
-# Handling multiple .env files is supported
-# or example ENV=test would use .env.test
-
-ENV = os.getenv('ENV')
-if ENV:
-    env_file_name = f'.env.{ENV}'
-else:
-    env_file_name = '.env'
-
-env_file_path = CONFIG_DIR / env_file_name
-if not env_file_path.exists():
-    sys.exit(f'config/{env_file_name} does not exist')
-
-DOTENV_VALUES = dotenv_values(CONFIG_DIR / env_file_name)
-
-
-def dotenv_val(key):
-    return DOTENV_VALUES.get(key, '').strip()

ssh_lib/config.py

@@ -0,0 +1,33 @@
+import os
+from pathlib import Path
+
+
+class Configuration:
+    # Local paths relative to this file
+    assets_dir = Path(__file__).parent / 'assets'
+    config_dir = Path(__file__).parent.parent / 'config'
+    modules_dir = Path(__file__).parent.parent / 'modules'
+
+    ENV = os.getenv('ENV')
+    if not ENV:
+        config_jsonc = config_dir / 'config.jsonc'
+    else:
+        config_jsonc = config_dir / f'config.{ENV}.jsonc'
+
+    # remote paths (always Linux /, not using pathlib)
+    ofm_dir = '/data/ofm'
+    remote_config = f'{ofm_dir}/config'
+    venv_bin = f'{ofm_dir}/venv/bin'
+
+    # remote http_host dir
+    http_host_dir = f'{ofm_dir}/http_host'
+    http_host_bin = f'{http_host_dir}/bin'
+
+    # remote tile_gen_dir
+    tile_gen_dir = f'{ofm_dir}/tile_gen'
+    tile_gen_bin = f'{tile_gen_dir}/bin'
+    planetiler_src = f'{tile_gen_dir}/planetiler_src'
+    planetiler_bin = f'{tile_gen_dir}/planetiler'
+
+
+config = Configuration()

ssh_lib/tasks.py

@@ -1,193 +1 @@
-import json
-import sys
 
-from ssh_lib import (
-    CONFIG_DIR,
-    HTTP_HOST_BIN,
-    MODULES_DIR,
-    OFM_DIR,
-    REMOTE_CONFIG,
-    TILE_GEN_BIN,
-    VENV_BIN,
-    dotenv_val,
-)
-from ssh_lib.benchmark import c1000k, wrk
-from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k
-from ssh_lib.nginx import certbot, nginx
-from ssh_lib.pkg_base import pkg_base, pkg_upgrade
-from ssh_lib.planetiler import install_planetiler
-from ssh_lib.rclone import rclone
-from ssh_lib.utils import add_user, enable_sudo, put, put_dir, put_str, sudo_cmd
-
-
-def prepare_shared(c):
-    # creates ofm user with uid=2000, disabled password and nopasswd sudo
-    add_user(c, 'ofm', uid=2000)
-    enable_sudo(c, 'ofm', nopasswd=True)
-
-    pkg_upgrade(c)
-    pkg_base(c)
-    rclone(c)
-
-    c.sudo(f'mkdir -p {REMOTE_CONFIG}')
-    c.sudo(f'chown ofm:ofm {REMOTE_CONFIG}')
-    c.sudo(f'chown ofm:ofm {OFM_DIR}')
-
-    prepare_venv(c)
-
-
-def prepare_venv(c):
-    put(
-        c,
-        MODULES_DIR / 'prepare-virtualenv.sh',
-        OFM_DIR,
-        permissions='755',
-        user='ofm',
-    )
-    sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh')
-
-
-def prepare_tile_gen(c, *, enable_cron):
-    c.sudo('rm -f /etc/cron.d/ofm_tile_gen')
-
-    install_planetiler(c)
-
-    c.sudo(f'rm -rf {TILE_GEN_BIN}')
-
-    put_dir(c, MODULES_DIR / 'tile_gen', TILE_GEN_BIN, file_permissions='755')
-
-    for dirname in ['tile_gen_lib', 'scripts']:
-        put_dir(c, MODULES_DIR / 'tile_gen' / dirname, f'{TILE_GEN_BIN}/{dirname}')
-
-    if (CONFIG_DIR / 'rclone.conf').exists():
-        put(
-            c,
-            CONFIG_DIR / 'rclone.conf',
-            f'{REMOTE_CONFIG}/rclone.conf',
-            permissions='600',
-            user='ofm',
-        )
-
-    c.sudo(f'{VENV_BIN}/pip install -e {TILE_GEN_BIN} --use-pep517')
-
-    c.sudo('rm -rf /data/ofm/tile_gen/logs')
-    c.sudo('mkdir -p /data/ofm/tile_gen/logs')
-
-    c.sudo('chown ofm:ofm /data/ofm/tile_gen/{,*}')
-    c.sudo(f'chown ofm:ofm -R {TILE_GEN_BIN}')
-
-    if enable_cron:
-        put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')
-
-
-def prepare_http_host(c):
-    kernel_somaxconn65k(c)
-    kernel_limits1m(c)
-
-    upload_config_json(c)
-
-    nginx(c)
-    certbot(c)
-
-    c.sudo('rm -rf /data/ofm/http_host/logs')
-    c.sudo('mkdir -p /data/ofm/http_host/logs')
-    c.sudo('chown ofm:ofm /data/ofm/http_host/logs')
-
-    c.sudo('rm -rf /data/ofm/http_host/logs_nginx')
-    c.sudo('mkdir -p /data/ofm/http_host/logs_nginx')
-    c.sudo('chown nginx:nginx /data/ofm/http_host/logs_nginx')
-
-    upload_http_host_files(c)
-
-    if dotenv_val('DOMAIN_ROUNDROBIN'):
-        assert (CONFIG_DIR / 'rclone.conf').exists()
-        put(
-            c,
-            CONFIG_DIR / 'rclone.conf',
-            f'{REMOTE_CONFIG}/rclone.conf',
-            permissions=400,
-        )
-        put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_roundrobin_reader', '/etc/cron.d/')
-
-    c.sudo(f'{VENV_BIN}/pip install -e {HTTP_HOST_BIN} --use-pep517')
-
-
-def run_http_host_sync(c):
-    print('Running http_host.py sync --force')
-    sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/http_host.py sync --force')
-
-
-def upload_http_host_files(c):
-    c.sudo(f'rm -rf {HTTP_HOST_BIN}')
-    c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
-
-    put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')
-
-    for dirname in ['http_host_lib', 'scripts']:
-        put_dir(c, MODULES_DIR / 'http_host' / dirname, f'{HTTP_HOST_BIN}/{dirname}')
-
-    put_dir(
-        c,
-        MODULES_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs',
-        f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs',
-    )
-
-    c.sudo('chown -R ofm:ofm /data/ofm/http_host')
-
-
-def install_benchmark(c):
-    """
-    Read docs/quick_notes/http_benchmark.md
-    """
-    c1000k(c)
-    wrk(c)
-
-
-def upload_config_json(c):
-    domain_direct = dotenv_val('DOMAIN_DIRECT').lower()
-    domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower()
-    skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
-    self_signed_certs = dotenv_val('SELF_SIGNED_CERTS').lower() == 'true'
-    letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower()
-
-    if not (domain_direct or domain_roundrobin):
-        sys.exit('Please specify DOMAIN_DIRECT or DOMAIN_ROUNDROBIN in config/.env')
-
-    if domain_direct and not letsencrypt_email and not self_signed_certs:
-        sys.exit('Please add your email to LETSENCRYPT_EMAIL when using DOMAIN_DIRECT')
-
-    http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()]
-
-    config = {
-        'domain_direct': domain_direct,
-        'domain_roundrobin': domain_roundrobin,
-        'letsencrypt_email': letsencrypt_email,
-        'skip_planet': skip_planet,
-        'self_signed_certs': self_signed_certs,
-        'http_host_list': http_host_list,
-        'telegram_token': dotenv_val('TELEGRAM_TOKEN'),
-        'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'),
-    }
-
-    config_str = json.dumps(config, indent=2, ensure_ascii=False)
-    print(config_str)
-    put_str(c, f'{REMOTE_CONFIG}/config.json', config_str)
-
-
-def setup_loadbalancer(c):
-    c.sudo('rm -f /etc/cron.d/ofm_loadbalancer')
-
-    c.sudo('rm -rf /data/ofm/loadbalancer')
-    put_dir(c, MODULES_DIR / 'loadbalancer', '/data/ofm/loadbalancer')
-    put_dir(
-        c,
-        MODULES_DIR / 'loadbalancer' / 'loadbalancer_lib',
-        '/data/ofm/loadbalancer/loadbalancer_lib',
-    )
-
-    c.sudo(f'{VENV_BIN}/pip install -e /data/ofm/loadbalancer --use-pep517')
-
-    c.sudo('mkdir -p /data/ofm/loadbalancer/logs')
-    c.sudo('chown -R ofm:ofm /data/ofm/loadbalancer')
-
-    put(c, MODULES_DIR / 'loadbalancer' / 'cron.d' / 'ofm_loadbalancer', '/etc/cron.d/')

ssh_lib/tasks_httphost.py

@@ -0,0 +1,93 @@
+import json
+import sys
+
+from ssh_lib.benchmark import c1000k, wrk
+from ssh_lib.config import config
+from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k
+from ssh_lib.nginx import certbot, nginx
+from ssh_lib.utils import put_dir, put_str, sudo_cmd
+
+
+def prepare_http_host(c):
+    kernel_somaxconn65k(c)
+    kernel_limits1m(c)
+
+    upload_config_json(c)
+
+    nginx(c)
+    certbot(c)
+
+    c.sudo(f'rm -rf {config.http_host_dir}/logs')
+    c.sudo(f'mkdir -p {config.http_host_dir}/logs')
+    c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs')
+
+    c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx')
+    c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx')
+    c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx')
+
+    upload_http_host_files(c)
+
+    c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517')
+
+
+def upload_config_json(c):
+    config.config_jsonc.is_file()
+    domain_direct = dotenv_val('DOMAIN_DIRECT').lower()
+    domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower()
+    skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
+    self_signed_certs = dotenv_val('SELF_SIGNED_CERTS').lower() == 'true'
+    letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower()
+
+    if not (domain_direct or domain_roundrobin):
+        sys.exit('Please specify DOMAIN_DIRECT or DOMAIN_ROUNDROBIN in config/.env')
+
+    if domain_direct and not letsencrypt_email and not self_signed_certs:
+        sys.exit('Please add your email to LETSENCRYPT_EMAIL when using DOMAIN_DIRECT')
+
+    http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()]
+
+    config = {
+        'domain_direct': domain_direct,
+        'domain_roundrobin': domain_roundrobin,
+        'letsencrypt_email': letsencrypt_email,
+        'skip_planet': skip_planet,
+        'self_signed_certs': self_signed_certs,
+        'http_host_list': http_host_list,
+        'telegram_token': dotenv_val('TELEGRAM_TOKEN'),
+        'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'),
+    }
+
+    config_str = json.dumps(config, indent=2, ensure_ascii=False)
+    print(config_str)
+    put_str(c, f'{REMOTE_CONFIG}/config.json', config_str)
+
+
+def run_http_host_sync(c):
+    print('Running http_host.py sync --force')
+    sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/http_host.py sync --force')
+
+
+def upload_http_host_files(c):
+    c.sudo(f'rm -rf {HTTP_HOST_BIN}')
+    c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
+
+    put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')
+
+    for dirname in ['http_host_lib', 'scripts']:
+        put_dir(c, MODULES_DIR / 'http_host' / dirname, f'{HTTP_HOST_BIN}/{dirname}')
+
+    put_dir(
+        c,
+        MODULES_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs',
+        f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs',
+    )
+
+    c.sudo('chown -R ofm:ofm /data/ofm/http_host')
+
+
+def install_benchmark(c):
+    """
+    Read docs/quick_notes/http_benchmark.md
+    """
+    c1000k(c)
+    wrk(c)

ssh_lib/tasks_shared.py

@@ -0,0 +1,31 @@
+from ssh_lib.config import config
+from ssh_lib.pkg_base import pkg_base, pkg_upgrade
+from ssh_lib.rclone import rclone
+from ssh_lib.utils import add_user, enable_sudo, put, sudo_cmd
+
+
+def prepare_shared(c):
+    # creates ofm user with uid=2000, disabled password and nopasswd sudo
+    add_user(c, 'ofm', uid=2000)
+    enable_sudo(c, 'ofm', nopasswd=True)
+
+    pkg_upgrade(c)
+    pkg_base(c)
+    rclone(c)
+
+    c.sudo(f'mkdir -p {config.remote_config}')
+    c.sudo(f'chown ofm:ofm {config.remote_config}')
+    c.sudo(f'chown ofm:ofm {config.ofm_dir}')
+
+    prepare_venv(c)
+
+
+def prepare_venv(c):
+    put(
+        c,
+        config.modules_dir / 'prepare-virtualenv.sh',
+        config.ofm_dir,
+        permissions='755',
+        user='ofm',
+    )
+    sudo_cmd(c, f'cd {config.ofm_dir} && source prepare-virtualenv.sh')

ssh_lib/tasks_tilegen.py

@@ -0,0 +1,36 @@
+from ssh_lib.config import config
+from ssh_lib.planetiler import install_planetiler
+from ssh_lib.utils import put, put_dir
+
+
+def prepare_tile_gen(c, *, enable_cron):
+    c.sudo('rm -f /etc/cron.d/ofm_tile_gen')
+
+    install_planetiler(c)
+
+    c.sudo(f'rm -rf {config.tile_gen_bin}')
+
+    put_dir(c, config.modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755')
+
+    for dirname in ['tile_gen_lib', 'scripts']:
+        put_dir(c, config.modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}')
+
+    if (config.config_dir / 'rclone.conf').exists():
+        put(
+            c,
+            config.config_dir / 'rclone.conf',
+            f'{config.remote_config}/rclone.conf',
+            permissions='600',
+            user='ofm',
+        )
+
+    c.sudo(f'{config.venv_bin}/pip install -e {config.tile_gen_bin} --use-pep517')
+
+    c.sudo('rm -rf /data/ofm/tile_gen/logs')
+    c.sudo('mkdir -p /data/ofm/tile_gen/logs')
+
+    c.sudo('chown ofm:ofm /data/ofm/tile_gen/{,*}')
+    c.sudo(f'chown ofm:ofm -R {config.tile_gen_bin}')
+
+    if enable_cron:
+        put(c, config.modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')