From a36e830416abca48d169c36627a4d0518346069f Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Tue, 7 Oct 2025 18:24:21 +0200 Subject: [PATCH] work --- modules/http_host/http_host_lib/config.py | 3 +- .../http_host_lib/nginx_config_gen.py | 9 ++-- modules/http_host/setup.py | 1 - setup.py | 2 +- ssh_lib/config.py | 6 ++- .../http_host_lib => ssh_lib}/slugify.py | 0 ssh_lib/tasks_http_host.py | 47 ++++++++++++++----- 7 files changed, 46 insertions(+), 22 deletions(-) rename {modules/http_host/http_host_lib => ssh_lib}/slugify.py (100%) diff --git a/modules/http_host/http_host_lib/config.py b/modules/http_host/http_host_lib/config.py index b96b436..bc901b9 100644 --- a/modules/http_host/http_host_lib/config.py +++ b/modules/http_host/http_host_lib/config.py @@ -1,3 +1,4 @@ +import json import subprocess from pathlib import Path @@ -28,7 +29,7 @@ class Configuration: repo_root = Path(__file__).parent.parent.parent.parent ofm_config_dir = repo_root / 'config' - jsonc_config = json5.loads((ofm_config_dir / 'config.jsonc').read_text()) + json_config = json.loads((ofm_config_dir / 'config.json').read_text()) deployed_versions_dir = ofm_config_dir / 'deployed_versions' diff --git a/modules/http_host/http_host_lib/nginx_config_gen.py b/modules/http_host/http_host_lib/nginx_config_gen.py index 7f88d37..a457fb7 100644 --- a/modules/http_host/http_host_lib/nginx_config_gen.py +++ b/modules/http_host/http_host_lib/nginx_config_gen.py @@ -20,7 +20,7 @@ def write_nginx_config(): for file in config.nginx_certs_dir.glob('ofm-*'): file.unlink() - conf = config.jsonc_config + conf = config.json_config curl_help_lines = [] @@ -40,12 +40,9 @@ def write_nginx_config(): def process_domain(domain_data): - domain_slug = slugify(domain_data['domain'], separator='_') - domain_data['slug'] = domain_slug - if domain_data['cert'] == 'upload': - domain_data['cert_file'] = config.nginx_certs_dir / f'{domain_slug}.cert' - domain_data['key_file'] = config.nginx_certs_dir / f'{domain_slug}.key' + domain_data['cert_file'] = config.nginx_certs_dir / f'{domain_data["slug"]}.cert' + domain_data['key_file'] = config.nginx_certs_dir / f'{domain_data["slug"]}.key' if not domain_data['cert_file'].is_file() or not domain_data['key_file'].is_file(): sys.exit( diff --git a/modules/http_host/setup.py b/modules/http_host/setup.py index 86800de..b0aa452 100644 --- a/modules/http_host/setup.py +++ b/modules/http_host/setup.py @@ -5,7 +5,6 @@ requirements = [ 'click', 'pycurl', 'requests', - 'json5', ] diff --git a/setup.py b/setup.py index 436af7a..e36ea81 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ requirements = [ 'click', 'fabric', 'nginxfmt', - 'python-dotenv', + # 'python-dotenv', 'ruff', 'marko', 'requests', diff --git a/ssh_lib/config.py b/ssh_lib/config.py index 57c5819..3cddd92 100644 --- a/ssh_lib/config.py +++ b/ssh_lib/config.py @@ -3,7 +3,7 @@ from pathlib import Path class Configuration: - # Local paths relative to this file + # local paths relative to this file local_assets_dir = Path(__file__).parent / 'assets' local_config_dir = Path(__file__).parent.parent / 'config' local_modules_dir = Path(__file__).parent.parent / 'modules' @@ -14,7 +14,9 @@ class Configuration: else: local_config_jsonc = local_config_dir / f'config.{ENV}.jsonc' - # remote paths (always Linux /, not using pathlib) + config_schema_json = local_config_dir / 'config.schema.json' + + # remote paths (always forward / on Linux - not using pathlib) ofm_dir = '/data/ofm' remote_config = f'{ofm_dir}/config' venv_bin = f'{ofm_dir}/venv/bin' diff --git a/modules/http_host/http_host_lib/slugify.py b/ssh_lib/slugify.py similarity index 100% rename from modules/http_host/http_host_lib/slugify.py rename to ssh_lib/slugify.py diff --git a/ssh_lib/tasks_http_host.py b/ssh_lib/tasks_http_host.py index 7cd211c..6cc0ae6 100644 --- a/ssh_lib/tasks_http_host.py +++ b/ssh_lib/tasks_http_host.py @@ -1,17 +1,20 @@ import json +import json5 + from ssh_lib.benchmark import c1000k, wrk from ssh_lib.config import config from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k from ssh_lib.nginx import certbot, nginx -from ssh_lib.utils import put, put_dir, sudo_cmd +from ssh_lib.slugify import slugify +from ssh_lib.utils import put, put_dir, put_str, sudo_cmd def prepare_http_host(c): kernel_somaxconn65k(c) kernel_limits1m(c) - upload_config_json(c) + upload_config_and_certs(c) nginx(c) certbot(c) @@ -29,20 +32,40 @@ def prepare_http_host(c): c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517') -def upload_config_json(c): +def upload_config_and_certs(c): if not config.local_config_jsonc.is_file(): print(f'{config.local_config_jsonc} not found. Make sure it exists in the /config dir') return # validate using json5 + jsonschema - config_data = json.loads(config.local_config_jsonc.read_text()) + # use config.config_schema_json + config_data = json5.loads(config.local_config_jsonc.read_text()) - # if ok, upload the file - put( - c, - config.local_config_jsonc, - f'{config.remote_config}/config.jsonc', - ) + # pre-generate all the slugs + for domain_data in config_data['domains']: + domain_data['slug'] = slugify(domain_data['domain'], separator='_') + + if domain_data['cert']['type'] == 'upload': + local_cert_path = domain_data['cert']['cert_path'] + cert_basename = local_cert_path.basename + local_key_path = local_cert_path.parent / f'{cert_basename}.key' + if not local_cert_path.is_file() or local_key_path.is_file(): + print( + f'cert or key file for {domain_data["domain"]} is not found. Make sure these files exists: {local_cert_path} {local_key_path}' + ) + + remote_cert_path = f'/data/nginx/certs/ofm-{domain_data["slug"]}.cert' + remote_key_path = f'/data/nginx/certs/ofm-{domain_data["slug"]}.key' + + # TODO fix permissions + put(c, local_cert_path, remote_cert_path) + put(c, local_key_path, remote_key_path) + + + + # generate a normal JSON and upload it + config_str = json.dumps(config_data, indent=2, ensure_ascii=False) + put_str(c, f'{config.remote_config}/config.json', config_str) def upload_http_host_files(c): @@ -52,7 +75,9 @@ def upload_http_host_files(c): put_dir(c, config.local_modules_dir / 'http_host', config.http_host_bin, file_permissions='755') for dirname in ['http_host_lib', 'scripts']: - put_dir(c, config.local_modules_dir / 'http_host' / dirname, f'{config.http_host_bin}/{dirname}') + put_dir( + c, config.local_modules_dir / 'http_host' / dirname, f'{config.http_host_bin}/{dirname}' + ) put_dir( c,