refactor

init-server.py

@@ -3,34 +3,35 @@
 import click
 from fabric import Config, Connection
 
-from ssh_lib import MODULES_DIR, dotenv_val
+from ssh_lib.config import config
-from ssh_lib.tasks_httphost import prepare_http_host, run_http_host_sync
+from ssh_lib.tasks_http_host import prepare_http_host, run_http_host_sync
-from ssh_lib.tasks_tilegen import prepare_tile_gen
 from ssh_lib.tasks_shared import prepare_shared
+from ssh_lib.tasks_tile_gen import prepare_tile_gen
 from ssh_lib.utils import (
     put,
 )
 
 
 def get_connection(hostname, user, port):
-    ssh_passwd = dotenv_val('SSH_PASSWD')
+    # ssh_passwd = dotenv_val('SSH_PASSWD')
 
-    if ssh_passwd:
+    # if ssh_passwd:
-        print('Using SSH password')
+    #     print('Using SSH password')
+    #
+    #     c = Connection(
+    #         host=hostname,
+    #         user=user,
+    #         port=port,
+    #         connect_kwargs={'password': ssh_passwd},
+    #         config=Config(overrides={'sudo': {'password': ssh_passwd}}),
+    #     )
+    # else:
 
-        c = Connection(
+    c = Connection(
-            host=hostname,
+        host=hostname,
-            user=user,
+        user=user,
-            port=port,
+        port=port,
-            connect_kwargs={'password': ssh_passwd},
+    )
-            config=Config(overrides={'sudo': {'password': ssh_passwd}}),
-        )
-    else:
-        c = Connection(
-            host=hostname,
-            user=user,
-            port=port,
-        )
 
     return c
 
@@ -80,7 +81,17 @@ def http_host_autoupdate(hostname, user, port, noninteractive):
 
     run_http_host_sync(c)  # disable for first install if you don't want to wait
 
-    put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
+    put(c, config.local_modules_dir / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
+
+
+@cli.command()
+@common_options
+def http_host_sync(hostname, user, port, noninteractive):
+    if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
+        return
+
+    c = get_connection(hostname, user, port)
+    run_http_host_sync(c)
 
 
 @cli.command()
@@ -108,16 +119,6 @@ def tile_gen(
     prepare_tile_gen(c, enable_cron=cron)
 
 
-@cli.command()
-@common_options
-def http_host_sync(hostname, user, port, noninteractive):
-    if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
-        return
-
-    c = get_connection(hostname, user, port)
-    run_http_host_sync(c)
-
-
 #
 # @cli.command()
 # @common_options

modules/http_host/http_host_lib/assets.py

@@ -31,7 +31,7 @@ def download_and_extract_asset_tar_gz(asset_kind):
 
     print(f'Downloading asset {asset_kind}')
 
-    asset_dir = config.assets_dir / asset_kind
+    asset_dir = config.local_assets_dir / asset_kind
     asset_dir.mkdir(exist_ok=True, parents=True)
 
     url = f'https://assets.openfreemap.com/{asset_kind}/ofm.tar.gz'
@@ -62,7 +62,7 @@ def download_sprites() -> bool:
 
     print('Downloading sprites')
 
-    sprites_dir = config.assets_dir / 'sprites'
+    sprites_dir = config.local_assets_dir / 'sprites'
     sprites_dir.mkdir(exist_ok=True, parents=True)
 
     r = requests.get('https://assets.openfreemap.com/files.txt', timeout=30)

ssh_lib/config.py

@@ -4,15 +4,15 @@ from pathlib import Path
 
 class Configuration:
     # Local paths relative to this file
-    assets_dir = Path(__file__).parent / 'assets'
+    local_assets_dir = Path(__file__).parent / 'assets'
-    config_dir = Path(__file__).parent.parent / 'config'
+    local_config_dir = Path(__file__).parent.parent / 'config'
-    modules_dir = Path(__file__).parent.parent / 'modules'
+    local_modules_dir = Path(__file__).parent.parent / 'modules'
 
     ENV = os.getenv('ENV')
     if not ENV:
-        config_jsonc = config_dir / 'config.jsonc'
+        local_config_jsonc = local_config_dir / 'config.jsonc'
     else:
-        config_jsonc = config_dir / f'config.{ENV}.jsonc'
+        local_config_jsonc = local_config_dir / f'config.{ENV}.jsonc'
 
     # remote paths (always Linux /, not using pathlib)
     ofm_dir = '/data/ofm'

ssh_lib/tasks.py

@@ -1 +0,0 @@
-

ssh_lib/tasks_http_host.py

@@ -0,0 +1,76 @@
+import json
+
+from ssh_lib.benchmark import c1000k, wrk
+from ssh_lib.config import config
+from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k
+from ssh_lib.nginx import certbot, nginx
+from ssh_lib.utils import put, put_dir, sudo_cmd
+
+
+def prepare_http_host(c):
+    kernel_somaxconn65k(c)
+    kernel_limits1m(c)
+
+    upload_config_json(c)
+
+    nginx(c)
+    certbot(c)
+
+    c.sudo(f'rm -rf {config.http_host_dir}/logs')
+    c.sudo(f'mkdir -p {config.http_host_dir}/logs')
+    c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs')
+
+    c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx')
+    c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx')
+    c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx')
+
+    upload_http_host_files(c)
+
+    c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517')
+
+
+def upload_config_json(c):
+    if not config.local_config_jsonc.is_file():
+        print(f'{config.local_config_jsonc} not found. Make sure it exists in the /config dir')
+        return
+
+    # validate using json5 + jsonschema
+    config_data = json.loads(config.local_config_jsonc.read_text())
+
+    # if ok, upload the file
+    put(
+        c,
+        config.local_config_jsonc,
+        f'{config.remote_config}/config.jsonc',
+    )
+
+
+def upload_http_host_files(c):
+    c.sudo(f'rm -rf {config.http_host_bin}')
+    c.sudo(f'mkdir -p {config.http_host_bin}')
+
+    put_dir(c, config.local_modules_dir / 'http_host', config.http_host_bin, file_permissions='755')
+
+    for dirname in ['http_host_lib', 'scripts']:
+        put_dir(c, config.local_modules_dir / 'http_host' / dirname, f'{config.http_host_bin}/{dirname}')
+
+    put_dir(
+        c,
+        config.local_modules_dir / 'http_host' / 'http_host_lib' / 'nginx_confs',
+        f'{config.http_host_bin}/http_host_lib/nginx_confs',
+    )
+
+    c.sudo('chown -R ofm:ofm /data/ofm/http_host')
+
+
+def run_http_host_sync(c):
+    print('Running http_host.py sync --force')
+    sudo_cmd(c, f'{config.venv_bin}/python -u {config.http_host_bin}/http_host.py sync --force')
+
+
+def install_benchmark(c):
+    """
+    Read docs/quick_notes/http_benchmark.md
+    """
+    c1000k(c)
+    wrk(c)

ssh_lib/tasks_httphost.py

@@ -1,93 +0,0 @@
-import json
-import sys
-
-from ssh_lib.benchmark import c1000k, wrk
-from ssh_lib.config import config
-from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k
-from ssh_lib.nginx import certbot, nginx
-from ssh_lib.utils import put_dir, put_str, sudo_cmd
-
-
-def prepare_http_host(c):
-    kernel_somaxconn65k(c)
-    kernel_limits1m(c)
-
-    upload_config_json(c)
-
-    nginx(c)
-    certbot(c)
-
-    c.sudo(f'rm -rf {config.http_host_dir}/logs')
-    c.sudo(f'mkdir -p {config.http_host_dir}/logs')
-    c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs')
-
-    c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx')
-    c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx')
-    c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx')
-
-    upload_http_host_files(c)
-
-    c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517')
-
-
-def upload_config_json(c):
-    config.config_jsonc.is_file()
-    domain_direct = dotenv_val('DOMAIN_DIRECT').lower()
-    domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower()
-    skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
-    self_signed_certs = dotenv_val('SELF_SIGNED_CERTS').lower() == 'true'
-    letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower()
-
-    if not (domain_direct or domain_roundrobin):
-        sys.exit('Please specify DOMAIN_DIRECT or DOMAIN_ROUNDROBIN in config/.env')
-
-    if domain_direct and not letsencrypt_email and not self_signed_certs:
-        sys.exit('Please add your email to LETSENCRYPT_EMAIL when using DOMAIN_DIRECT')
-
-    http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()]
-
-    config = {
-        'domain_direct': domain_direct,
-        'domain_roundrobin': domain_roundrobin,
-        'letsencrypt_email': letsencrypt_email,
-        'skip_planet': skip_planet,
-        'self_signed_certs': self_signed_certs,
-        'http_host_list': http_host_list,
-        'telegram_token': dotenv_val('TELEGRAM_TOKEN'),
-        'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'),
-    }
-
-    config_str = json.dumps(config, indent=2, ensure_ascii=False)
-    print(config_str)
-    put_str(c, f'{REMOTE_CONFIG}/config.json', config_str)
-
-
-def run_http_host_sync(c):
-    print('Running http_host.py sync --force')
-    sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/http_host.py sync --force')
-
-
-def upload_http_host_files(c):
-    c.sudo(f'rm -rf {HTTP_HOST_BIN}')
-    c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
-
-    put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')
-
-    for dirname in ['http_host_lib', 'scripts']:
-        put_dir(c, MODULES_DIR / 'http_host' / dirname, f'{HTTP_HOST_BIN}/{dirname}')
-
-    put_dir(
-        c,
-        MODULES_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs',
-        f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs',
-    )
-
-    c.sudo('chown -R ofm:ofm /data/ofm/http_host')
-
-
-def install_benchmark(c):
-    """
-    Read docs/quick_notes/http_benchmark.md
-    """
-    c1000k(c)
-    wrk(c)

ssh_lib/tasks_shared.py

@@ -23,7 +23,7 @@ def prepare_shared(c):
 def prepare_venv(c):
     put(
         c,
-        config.modules_dir / 'prepare-virtualenv.sh',
+        config.local_modules_dir / 'prepare-virtualenv.sh',
         config.ofm_dir,
         permissions='755',
         user='ofm',

ssh_lib/tasks_tile_gen.py

@@ -10,15 +10,15 @@ def prepare_tile_gen(c, *, enable_cron):
 
     c.sudo(f'rm -rf {config.tile_gen_bin}')
 
-    put_dir(c, config.modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755')
+    put_dir(c, config.local_modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755')
 
     for dirname in ['tile_gen_lib', 'scripts']:
-        put_dir(c, config.modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}')
+        put_dir(c, config.local_modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}')
 
-    if (config.config_dir / 'rclone.conf').exists():
+    if (config.local_config_dir / 'rclone.conf').exists():
         put(
             c,
-            config.config_dir / 'rclone.conf',
+            config.local_config_dir / 'rclone.conf',
             f'{config.remote_config}/rclone.conf',
             permissions='600',
             user='ofm',
@@ -33,4 +33,4 @@ def prepare_tile_gen(c, *, enable_cron):
     c.sudo(f'chown ofm:ofm -R {config.tile_gen_bin}')
 
     if enable_cron:
-        put(c, config.modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')
+        put(c, config.local_modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')