From fe30af3fb2374400db94b24c991ca57ba5edd02a Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Tue, 7 Oct 2025 16:08:53 +0200 Subject: [PATCH] work --- init-server.py | 9 +- setup.py | 1 + ssh_lib/__init__.py | 42 --------- ssh_lib/config.py | 33 +++++++ ssh_lib/tasks.py | 192 -------------------------------------- ssh_lib/tasks_httphost.py | 93 ++++++++++++++++++ ssh_lib/tasks_shared.py | 31 ++++++ ssh_lib/tasks_tilegen.py | 36 +++++++ 8 files changed, 197 insertions(+), 240 deletions(-) create mode 100644 ssh_lib/config.py create mode 100644 ssh_lib/tasks_httphost.py create mode 100644 ssh_lib/tasks_shared.py create mode 100644 ssh_lib/tasks_tilegen.py diff --git a/init-server.py b/init-server.py index 3b4ce9d..809f2cd 100755 --- a/init-server.py +++ b/init-server.py @@ -4,12 +4,9 @@ import click from fabric import Config, Connection from ssh_lib import MODULES_DIR, dotenv_val -from ssh_lib.tasks import ( - prepare_http_host, - prepare_shared, - prepare_tile_gen, - run_http_host_sync, -) +from ssh_lib.tasks_httphost import prepare_http_host, run_http_host_sync +from ssh_lib.tasks_tilegen import prepare_tile_gen +from ssh_lib.tasks_shared import prepare_shared from ssh_lib.utils import ( put, ) diff --git a/setup.py b/setup.py index 5bffac1..4ea6fb4 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ requirements = [ 'ruff', 'marko', 'requests', + 'jsonschema', ] diff --git a/ssh_lib/__init__.py b/ssh_lib/__init__.py index 668f1b2..e69de29 100644 --- a/ssh_lib/__init__.py +++ b/ssh_lib/__init__.py @@ -1,42 +0,0 @@ -import os -import sys -from pathlib import Path - -from dotenv import dotenv_values - - -ASSETS_DIR = Path(__file__).parent / 'assets' -CONFIG_DIR = Path(__file__).parent.parent / 'config' -MODULES_DIR = Path(__file__).parent.parent / 'modules' - -OFM_DIR = '/data/ofm' -REMOTE_CONFIG = f'{OFM_DIR}/config' -VENV_BIN = f'{OFM_DIR}/venv/bin' - -TILE_GEN_DIR = f'{OFM_DIR}/tile_gen' -TILE_GEN_BIN = f'{TILE_GEN_DIR}/bin' - -PLANETILER_SRC = f'{TILE_GEN_DIR}/planetiler_src' -PLANETILER_BIN = f'{TILE_GEN_DIR}/planetiler' - -HTTP_HOST_BIN = f'{OFM_DIR}/http_host/bin' - - -# Handling multiple .env files is supported -# or example ENV=test would use .env.test - -ENV = os.getenv('ENV') -if ENV: - env_file_name = f'.env.{ENV}' -else: - env_file_name = '.env' - -env_file_path = CONFIG_DIR / env_file_name -if not env_file_path.exists(): - sys.exit(f'config/{env_file_name} does not exist') - -DOTENV_VALUES = dotenv_values(CONFIG_DIR / env_file_name) - - -def dotenv_val(key): - return DOTENV_VALUES.get(key, '').strip() diff --git a/ssh_lib/config.py b/ssh_lib/config.py new file mode 100644 index 0000000..976e387 --- /dev/null +++ b/ssh_lib/config.py @@ -0,0 +1,33 @@ +import os +from pathlib import Path + + +class Configuration: + # Local paths relative to this file + assets_dir = Path(__file__).parent / 'assets' + config_dir = Path(__file__).parent.parent / 'config' + modules_dir = Path(__file__).parent.parent / 'modules' + + ENV = os.getenv('ENV') + if not ENV: + config_jsonc = config_dir / 'config.jsonc' + else: + config_jsonc = config_dir / f'config.{ENV}.jsonc' + + # remote paths (always Linux /, not using pathlib) + ofm_dir = '/data/ofm' + remote_config = f'{ofm_dir}/config' + venv_bin = f'{ofm_dir}/venv/bin' + + # remote http_host dir + http_host_dir = f'{ofm_dir}/http_host' + http_host_bin = f'{http_host_dir}/bin' + + # remote tile_gen_dir + tile_gen_dir = f'{ofm_dir}/tile_gen' + tile_gen_bin = f'{tile_gen_dir}/bin' + planetiler_src = f'{tile_gen_dir}/planetiler_src' + planetiler_bin = f'{tile_gen_dir}/planetiler' + + +config = Configuration() diff --git a/ssh_lib/tasks.py b/ssh_lib/tasks.py index 23a20bd..8b13789 100644 --- a/ssh_lib/tasks.py +++ b/ssh_lib/tasks.py @@ -1,193 +1 @@ -import json -import sys -from ssh_lib import ( - CONFIG_DIR, - HTTP_HOST_BIN, - MODULES_DIR, - OFM_DIR, - REMOTE_CONFIG, - TILE_GEN_BIN, - VENV_BIN, - dotenv_val, -) -from ssh_lib.benchmark import c1000k, wrk -from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k -from ssh_lib.nginx import certbot, nginx -from ssh_lib.pkg_base import pkg_base, pkg_upgrade -from ssh_lib.planetiler import install_planetiler -from ssh_lib.rclone import rclone -from ssh_lib.utils import add_user, enable_sudo, put, put_dir, put_str, sudo_cmd - - -def prepare_shared(c): - # creates ofm user with uid=2000, disabled password and nopasswd sudo - add_user(c, 'ofm', uid=2000) - enable_sudo(c, 'ofm', nopasswd=True) - - pkg_upgrade(c) - pkg_base(c) - rclone(c) - - c.sudo(f'mkdir -p {REMOTE_CONFIG}') - c.sudo(f'chown ofm:ofm {REMOTE_CONFIG}') - c.sudo(f'chown ofm:ofm {OFM_DIR}') - - prepare_venv(c) - - -def prepare_venv(c): - put( - c, - MODULES_DIR / 'prepare-virtualenv.sh', - OFM_DIR, - permissions='755', - user='ofm', - ) - sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh') - - -def prepare_tile_gen(c, *, enable_cron): - c.sudo('rm -f /etc/cron.d/ofm_tile_gen') - - install_planetiler(c) - - c.sudo(f'rm -rf {TILE_GEN_BIN}') - - put_dir(c, MODULES_DIR / 'tile_gen', TILE_GEN_BIN, file_permissions='755') - - for dirname in ['tile_gen_lib', 'scripts']: - put_dir(c, MODULES_DIR / 'tile_gen' / dirname, f'{TILE_GEN_BIN}/{dirname}') - - if (CONFIG_DIR / 'rclone.conf').exists(): - put( - c, - CONFIG_DIR / 'rclone.conf', - f'{REMOTE_CONFIG}/rclone.conf', - permissions='600', - user='ofm', - ) - - c.sudo(f'{VENV_BIN}/pip install -e {TILE_GEN_BIN} --use-pep517') - - c.sudo('rm -rf /data/ofm/tile_gen/logs') - c.sudo('mkdir -p /data/ofm/tile_gen/logs') - - c.sudo('chown ofm:ofm /data/ofm/tile_gen/{,*}') - c.sudo(f'chown ofm:ofm -R {TILE_GEN_BIN}') - - if enable_cron: - put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/') - - -def prepare_http_host(c): - kernel_somaxconn65k(c) - kernel_limits1m(c) - - upload_config_json(c) - - nginx(c) - certbot(c) - - c.sudo('rm -rf /data/ofm/http_host/logs') - c.sudo('mkdir -p /data/ofm/http_host/logs') - c.sudo('chown ofm:ofm /data/ofm/http_host/logs') - - c.sudo('rm -rf /data/ofm/http_host/logs_nginx') - c.sudo('mkdir -p /data/ofm/http_host/logs_nginx') - c.sudo('chown nginx:nginx /data/ofm/http_host/logs_nginx') - - upload_http_host_files(c) - - if dotenv_val('DOMAIN_ROUNDROBIN'): - assert (CONFIG_DIR / 'rclone.conf').exists() - put( - c, - CONFIG_DIR / 'rclone.conf', - f'{REMOTE_CONFIG}/rclone.conf', - permissions=400, - ) - put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_roundrobin_reader', '/etc/cron.d/') - - c.sudo(f'{VENV_BIN}/pip install -e {HTTP_HOST_BIN} --use-pep517') - - -def run_http_host_sync(c): - print('Running http_host.py sync --force') - sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/http_host.py sync --force') - - -def upload_http_host_files(c): - c.sudo(f'rm -rf {HTTP_HOST_BIN}') - c.sudo(f'mkdir -p {HTTP_HOST_BIN}') - - put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755') - - for dirname in ['http_host_lib', 'scripts']: - put_dir(c, MODULES_DIR / 'http_host' / dirname, f'{HTTP_HOST_BIN}/{dirname}') - - put_dir( - c, - MODULES_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs', - f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs', - ) - - c.sudo('chown -R ofm:ofm /data/ofm/http_host') - - -def install_benchmark(c): - """ - Read docs/quick_notes/http_benchmark.md - """ - c1000k(c) - wrk(c) - - -def upload_config_json(c): - domain_direct = dotenv_val('DOMAIN_DIRECT').lower() - domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower() - skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true' - self_signed_certs = dotenv_val('SELF_SIGNED_CERTS').lower() == 'true' - letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower() - - if not (domain_direct or domain_roundrobin): - sys.exit('Please specify DOMAIN_DIRECT or DOMAIN_ROUNDROBIN in config/.env') - - if domain_direct and not letsencrypt_email and not self_signed_certs: - sys.exit('Please add your email to LETSENCRYPT_EMAIL when using DOMAIN_DIRECT') - - http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()] - - config = { - 'domain_direct': domain_direct, - 'domain_roundrobin': domain_roundrobin, - 'letsencrypt_email': letsencrypt_email, - 'skip_planet': skip_planet, - 'self_signed_certs': self_signed_certs, - 'http_host_list': http_host_list, - 'telegram_token': dotenv_val('TELEGRAM_TOKEN'), - 'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'), - } - - config_str = json.dumps(config, indent=2, ensure_ascii=False) - print(config_str) - put_str(c, f'{REMOTE_CONFIG}/config.json', config_str) - - -def setup_loadbalancer(c): - c.sudo('rm -f /etc/cron.d/ofm_loadbalancer') - - c.sudo('rm -rf /data/ofm/loadbalancer') - put_dir(c, MODULES_DIR / 'loadbalancer', '/data/ofm/loadbalancer') - put_dir( - c, - MODULES_DIR / 'loadbalancer' / 'loadbalancer_lib', - '/data/ofm/loadbalancer/loadbalancer_lib', - ) - - c.sudo(f'{VENV_BIN}/pip install -e /data/ofm/loadbalancer --use-pep517') - - c.sudo('mkdir -p /data/ofm/loadbalancer/logs') - c.sudo('chown -R ofm:ofm /data/ofm/loadbalancer') - - put(c, MODULES_DIR / 'loadbalancer' / 'cron.d' / 'ofm_loadbalancer', '/etc/cron.d/') diff --git a/ssh_lib/tasks_httphost.py b/ssh_lib/tasks_httphost.py new file mode 100644 index 0000000..44cde6e --- /dev/null +++ b/ssh_lib/tasks_httphost.py @@ -0,0 +1,93 @@ +import json +import sys + +from ssh_lib.benchmark import c1000k, wrk +from ssh_lib.config import config +from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k +from ssh_lib.nginx import certbot, nginx +from ssh_lib.utils import put_dir, put_str, sudo_cmd + + +def prepare_http_host(c): + kernel_somaxconn65k(c) + kernel_limits1m(c) + + upload_config_json(c) + + nginx(c) + certbot(c) + + c.sudo(f'rm -rf {config.http_host_dir}/logs') + c.sudo(f'mkdir -p {config.http_host_dir}/logs') + c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs') + + c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx') + c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx') + c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx') + + upload_http_host_files(c) + + c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517') + + +def upload_config_json(c): + config.config_jsonc.is_file() + domain_direct = dotenv_val('DOMAIN_DIRECT').lower() + domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower() + skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true' + self_signed_certs = dotenv_val('SELF_SIGNED_CERTS').lower() == 'true' + letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower() + + if not (domain_direct or domain_roundrobin): + sys.exit('Please specify DOMAIN_DIRECT or DOMAIN_ROUNDROBIN in config/.env') + + if domain_direct and not letsencrypt_email and not self_signed_certs: + sys.exit('Please add your email to LETSENCRYPT_EMAIL when using DOMAIN_DIRECT') + + http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()] + + config = { + 'domain_direct': domain_direct, + 'domain_roundrobin': domain_roundrobin, + 'letsencrypt_email': letsencrypt_email, + 'skip_planet': skip_planet, + 'self_signed_certs': self_signed_certs, + 'http_host_list': http_host_list, + 'telegram_token': dotenv_val('TELEGRAM_TOKEN'), + 'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'), + } + + config_str = json.dumps(config, indent=2, ensure_ascii=False) + print(config_str) + put_str(c, f'{REMOTE_CONFIG}/config.json', config_str) + + +def run_http_host_sync(c): + print('Running http_host.py sync --force') + sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/http_host.py sync --force') + + +def upload_http_host_files(c): + c.sudo(f'rm -rf {HTTP_HOST_BIN}') + c.sudo(f'mkdir -p {HTTP_HOST_BIN}') + + put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755') + + for dirname in ['http_host_lib', 'scripts']: + put_dir(c, MODULES_DIR / 'http_host' / dirname, f'{HTTP_HOST_BIN}/{dirname}') + + put_dir( + c, + MODULES_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs', + f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs', + ) + + c.sudo('chown -R ofm:ofm /data/ofm/http_host') + + +def install_benchmark(c): + """ + Read docs/quick_notes/http_benchmark.md + """ + c1000k(c) + wrk(c) diff --git a/ssh_lib/tasks_shared.py b/ssh_lib/tasks_shared.py new file mode 100644 index 0000000..8ee2c56 --- /dev/null +++ b/ssh_lib/tasks_shared.py @@ -0,0 +1,31 @@ +from ssh_lib.config import config +from ssh_lib.pkg_base import pkg_base, pkg_upgrade +from ssh_lib.rclone import rclone +from ssh_lib.utils import add_user, enable_sudo, put, sudo_cmd + + +def prepare_shared(c): + # creates ofm user with uid=2000, disabled password and nopasswd sudo + add_user(c, 'ofm', uid=2000) + enable_sudo(c, 'ofm', nopasswd=True) + + pkg_upgrade(c) + pkg_base(c) + rclone(c) + + c.sudo(f'mkdir -p {config.remote_config}') + c.sudo(f'chown ofm:ofm {config.remote_config}') + c.sudo(f'chown ofm:ofm {config.ofm_dir}') + + prepare_venv(c) + + +def prepare_venv(c): + put( + c, + config.modules_dir / 'prepare-virtualenv.sh', + config.ofm_dir, + permissions='755', + user='ofm', + ) + sudo_cmd(c, f'cd {config.ofm_dir} && source prepare-virtualenv.sh') diff --git a/ssh_lib/tasks_tilegen.py b/ssh_lib/tasks_tilegen.py new file mode 100644 index 0000000..1281998 --- /dev/null +++ b/ssh_lib/tasks_tilegen.py @@ -0,0 +1,36 @@ +from ssh_lib.config import config +from ssh_lib.planetiler import install_planetiler +from ssh_lib.utils import put, put_dir + + +def prepare_tile_gen(c, *, enable_cron): + c.sudo('rm -f /etc/cron.d/ofm_tile_gen') + + install_planetiler(c) + + c.sudo(f'rm -rf {config.tile_gen_bin}') + + put_dir(c, config.modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755') + + for dirname in ['tile_gen_lib', 'scripts']: + put_dir(c, config.modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}') + + if (config.config_dir / 'rclone.conf').exists(): + put( + c, + config.config_dir / 'rclone.conf', + f'{config.remote_config}/rclone.conf', + permissions='600', + user='ofm', + ) + + c.sudo(f'{config.venv_bin}/pip install -e {config.tile_gen_bin} --use-pep517') + + c.sudo('rm -rf /data/ofm/tile_gen/logs') + c.sudo('mkdir -p /data/ofm/tile_gen/logs') + + c.sudo('chown ofm:ofm /data/ofm/tile_gen/{,*}') + c.sudo(f'chown ofm:ofm -R {config.tile_gen_bin}') + + if enable_cron: + put(c, config.modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')