From f55d9f1c8bb2fef49af410234fc4b752d9709755 Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Thu, 29 Aug 2024 01:50:52 +0200 Subject: [PATCH] config refactor, renames --- scripts/http_host/host_manager.py | 6 +++--- scripts/http_host/http_host_lib/config.py | 11 ++++++----- scripts/http_host/http_host_lib/mount.py | 2 +- scripts/http_host/http_host_lib/nginx.py | 15 ++++++++------- .../http_host_lib/{nginx => nginx_confs}/le.conf | 0 .../{nginx => nginx_confs}/ledns.conf | 0 .../{nginx => nginx_confs}/location_static.conf | 0 scripts/http_host/http_host_lib/utils.py | 12 +++++++++++- scripts/tile_gen/tile_gen_lib/config.py | 3 ++- ssh_lib/tasks.py | 5 +++-- 10 files changed, 34 insertions(+), 20 deletions(-) rename scripts/http_host/http_host_lib/{nginx => nginx_confs}/le.conf (100%) rename scripts/http_host/http_host_lib/{nginx => nginx_confs}/ledns.conf (100%) rename scripts/http_host/http_host_lib/{nginx => nginx_confs}/location_static.conf (100%) diff --git a/scripts/http_host/host_manager.py b/scripts/http_host/host_manager.py index 0c7bc19..a81cbc8 100755 --- a/scripts/http_host/host_manager.py +++ b/scripts/http_host/host_manager.py @@ -70,7 +70,7 @@ def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Pat selected_version = version if not runs_dir: - runs_dir = config.default_runs_dir + runs_dir = config.runs_dir if not runs_dir.parent.exists(): sys.exit("runs dir's parent doesn't exist") @@ -92,7 +92,7 @@ def download_assets(assets_dir: Path): print('running download_assets') if not assets_dir: - assets_dir = config.default_assets_dir + assets_dir = config.assets_dir if not assets_dir.parent.exists(): sys.exit("asset dir's parent doesn't exist") @@ -116,7 +116,7 @@ def mount(): assert_linux() assert_sudo() - if not config.default_runs_dir.exists(): + if not config.runs_dir.exists(): sys.exit(' download_tileset needs to be run first') clean_up_mounts(config.mnt_dir) diff --git a/scripts/http_host/http_host_lib/config.py b/scripts/http_host/http_host_lib/config.py index 3d7c222..00616cb 100644 --- a/scripts/http_host/http_host_lib/config.py +++ b/scripts/http_host/http_host_lib/config.py @@ -5,19 +5,20 @@ from pathlib import Path class Configuration: http_host_dir = Path('/data/ofm/http_host') - nginx_dir = Path(__file__).parent / 'nginx' + http_host_bin = http_host_dir / 'bin' + http_host_scripts_dir = http_host_bin / 'scripts' - default_runs_dir = http_host_dir / 'runs' - default_assets_dir = http_host_dir / 'assets' + runs_dir = http_host_dir / 'runs' + assets_dir = http_host_dir / 'assets' mnt_dir = Path('/mnt/ofm') ofm_config_dir = Path('/data/ofm/config') - http_host_bin = http_host_dir / 'bin' certs_dir = Path('/data/nginx/certs') + nginx_confs = Path(__file__).parent / 'nginx_confs' try: - with open('/data/ofm/config/http_host.json') as fp: + with open(ofm_config_dir / 'http_host.json') as fp: host_config = json.load(fp) except Exception: host_config = {} diff --git a/scripts/http_host/http_host_lib/mount.py b/scripts/http_host/http_host_lib/mount.py index 7bb2e24..fcec9f9 100644 --- a/scripts/http_host/http_host_lib/mount.py +++ b/scripts/http_host/http_host_lib/mount.py @@ -8,7 +8,7 @@ def create_fstab(): fstab_new = [] for area in ['planet', 'monaco']: - area_dir = (config.default_runs_dir / area).resolve() + area_dir = (config.runs_dir / area).resolve() if not area_dir.exists(): continue diff --git a/scripts/http_host/http_host_lib/nginx.py b/scripts/http_host/http_host_lib/nginx.py index 8135653..c814342 100644 --- a/scripts/http_host/http_host_lib/nginx.py +++ b/scripts/http_host/http_host_lib/nginx.py @@ -4,6 +4,7 @@ import sys from pathlib import Path from http_host_lib.config import config +from http_host_lib.utils import python_venv_executable def write_nginx_config(): @@ -29,7 +30,7 @@ def write_nginx_config(): subprocess.run(['bash', config.http_host_bin / 'ledns_reader.sh'], check=True) curl_text_mix += create_nginx_conf( - template_path=config.nginx_dir / 'ledns.conf', + template_path=config.nginx_confs / 'ledns.conf', local='ofm_ledns', domain=domain_ledns, ) @@ -44,7 +45,7 @@ def write_nginx_config(): shutil.copyfile(Path('/etc/nginx/ssl/dummy.key'), le_key) curl_text_mix += create_nginx_conf( - template_path=config.nginx_dir / 'le.conf', + template_path=config.nginx_confs / 'le.conf', local='ofm_le', domain=domain_le, ) @@ -130,7 +131,7 @@ def create_location_blocks(*, local, domain): location_str += create_latest_locations(local=local, domain=domain) - with open(config.nginx_dir / 'location_static.conf') as fp: + with open(config.nginx_confs / 'location_static.conf') as fp: location_str += '\n' + fp.read() return location_str, curl_text @@ -139,7 +140,7 @@ def create_location_blocks(*, local, domain): def create_version_location( *, area: str, version: str, subdir: Path, local: str, domain: str ) -> str: - run_dir = config.default_runs_dir / area / version + run_dir = config.runs_dir / area / version if not run_dir.is_dir(): print(f" {run_dir} doesn't exists, skipping") return '' @@ -155,8 +156,8 @@ def create_version_location( subprocess.run( [ - sys.executable, - Path(__file__).parent.parent / 'metadata_to_tilejson.py', + python_venv_executable(), + config.http_host_scripts_dir / 'metadata_to_tilejson.py', '--minify', metadata_path, tilejson_path, @@ -203,7 +204,7 @@ def create_latest_locations(*, local: str, domain: str) -> str: version = fp.read().strip() print(f' setting latest version for {area}: {version}') - run_dir = config.default_runs_dir / area / version + run_dir = config.runs_dir / area / version tilejson_path = run_dir / f'tilejson-{local}.json' assert tilejson_path.is_file() diff --git a/scripts/http_host/http_host_lib/nginx/le.conf b/scripts/http_host/http_host_lib/nginx_confs/le.conf similarity index 100% rename from scripts/http_host/http_host_lib/nginx/le.conf rename to scripts/http_host/http_host_lib/nginx_confs/le.conf diff --git a/scripts/http_host/http_host_lib/nginx/ledns.conf b/scripts/http_host/http_host_lib/nginx_confs/ledns.conf similarity index 100% rename from scripts/http_host/http_host_lib/nginx/ledns.conf rename to scripts/http_host/http_host_lib/nginx_confs/ledns.conf diff --git a/scripts/http_host/http_host_lib/nginx/location_static.conf b/scripts/http_host/http_host_lib/nginx_confs/location_static.conf similarity index 100% rename from scripts/http_host/http_host_lib/nginx/location_static.conf rename to scripts/http_host/http_host_lib/nginx_confs/location_static.conf diff --git a/scripts/http_host/http_host_lib/utils.py b/scripts/http_host/http_host_lib/utils.py index d86daf8..e4e2590 100644 --- a/scripts/http_host/http_host_lib/utils.py +++ b/scripts/http_host/http_host_lib/utils.py @@ -1,7 +1,6 @@ import os import subprocess import sys -import time from pathlib import Path import requests @@ -57,3 +56,14 @@ def download_file_aria2(url: str, local_file: Path): ], check=True, ) + + +def python_venv_executable() -> Path: + venv_path = os.environ.get('VIRTUAL_ENV') + + if venv_path: + return Path(venv_path) / 'bin' / 'python' + elif sys.prefix != sys.base_prefix: + return Path(sys.prefix) / 'bin' / 'python' + else: + return Path(sys.executable) diff --git a/scripts/tile_gen/tile_gen_lib/config.py b/scripts/tile_gen/tile_gen_lib/config.py index 09b03e4..a23c05c 100644 --- a/scripts/tile_gen/tile_gen_lib/config.py +++ b/scripts/tile_gen/tile_gen_lib/config.py @@ -12,7 +12,8 @@ class Configuration: runs_dir = tile_gen_dir / 'runs' - rclone_config = Path('/data/ofm/config/rclone.conf') + ofm_config_dir = Path('/data/ofm/config') + rclone_config = ofm_config_dir / 'rclone.conf' areas = ['planet', 'monaco'] diff --git a/ssh_lib/tasks.py b/ssh_lib/tasks.py index a495eae..69d9e0d 100644 --- a/ssh_lib/tasks.py +++ b/ssh_lib/tasks.py @@ -133,6 +133,7 @@ def prepare_http_host(c): def run_http_host_sync(c): + print('Running host_manager.py sync --force') sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/host_manager.py sync --force') @@ -146,8 +147,8 @@ def upload_http_host_files(c): put_dir( c, - SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx', - f'{HTTP_HOST_BIN}/http_host_lib/nginx', + SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs', + f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs', ) c.sudo('chown -R ofm:ofm /data/ofm/http_host')