config refactor, renames

scripts/http_host/host_manager.py

@@ -70,7 +70,7 @@ def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Pat
         selected_version = version
 
     if not runs_dir:
-        runs_dir = config.default_runs_dir
+        runs_dir = config.runs_dir
 
     if not runs_dir.parent.exists():
         sys.exit("runs dir's parent doesn't exist")
@@ -92,7 +92,7 @@ def download_assets(assets_dir: Path):
     print('running download_assets')
 
     if not assets_dir:
-        assets_dir = config.default_assets_dir
+        assets_dir = config.assets_dir
 
     if not assets_dir.parent.exists():
         sys.exit("asset dir's parent doesn't exist")
@@ -116,7 +116,7 @@ def mount():
     assert_linux()
     assert_sudo()
 
-    if not config.default_runs_dir.exists():
+    if not config.runs_dir.exists():
         sys.exit('  download_tileset needs to be run first')
 
     clean_up_mounts(config.mnt_dir)

scripts/http_host/http_host_lib/config.py

@@ -5,19 +5,20 @@ from pathlib import Path
 class Configuration:
     http_host_dir = Path('/data/ofm/http_host')
 
-    nginx_dir = Path(__file__).parent / 'nginx'
+    http_host_bin = http_host_dir / 'bin'
+    http_host_scripts_dir = http_host_bin / 'scripts'
 
-    default_runs_dir = http_host_dir / 'runs'
+    runs_dir = http_host_dir / 'runs'
-    default_assets_dir = http_host_dir / 'assets'
+    assets_dir = http_host_dir / 'assets'
 
     mnt_dir = Path('/mnt/ofm')
     ofm_config_dir = Path('/data/ofm/config')
-    http_host_bin = http_host_dir / 'bin'
 
     certs_dir = Path('/data/nginx/certs')
+    nginx_confs = Path(__file__).parent / 'nginx_confs'
 
     try:
-        with open('/data/ofm/config/http_host.json') as fp:
+        with open(ofm_config_dir / 'http_host.json') as fp:
             host_config = json.load(fp)
     except Exception:
         host_config = {}

scripts/http_host/http_host_lib/mount.py

@@ -8,7 +8,7 @@ def create_fstab():
     fstab_new = []
 
     for area in ['planet', 'monaco']:
-        area_dir = (config.default_runs_dir / area).resolve()
+        area_dir = (config.runs_dir / area).resolve()
         if not area_dir.exists():
             continue
 

scripts/http_host/http_host_lib/nginx.py

@@ -4,6 +4,7 @@ import sys
 from pathlib import Path
 
 from http_host_lib.config import config
+from http_host_lib.utils import python_venv_executable
 
 
 def write_nginx_config():
@@ -29,7 +30,7 @@ def write_nginx_config():
         subprocess.run(['bash', config.http_host_bin / 'ledns_reader.sh'], check=True)
 
         curl_text_mix += create_nginx_conf(
-            template_path=config.nginx_dir / 'ledns.conf',
+            template_path=config.nginx_confs / 'ledns.conf',
             local='ofm_ledns',
             domain=domain_ledns,
         )
@@ -44,7 +45,7 @@ def write_nginx_config():
             shutil.copyfile(Path('/etc/nginx/ssl/dummy.key'), le_key)
 
         curl_text_mix += create_nginx_conf(
-            template_path=config.nginx_dir / 'le.conf',
+            template_path=config.nginx_confs / 'le.conf',
             local='ofm_le',
             domain=domain_le,
         )
@@ -130,7 +131,7 @@ def create_location_blocks(*, local, domain):
 
     location_str += create_latest_locations(local=local, domain=domain)
 
-    with open(config.nginx_dir / 'location_static.conf') as fp:
+    with open(config.nginx_confs / 'location_static.conf') as fp:
         location_str += '\n' + fp.read()
 
     return location_str, curl_text
@@ -139,7 +140,7 @@ def create_location_blocks(*, local, domain):
 def create_version_location(
     *, area: str, version: str, subdir: Path, local: str, domain: str
 ) -> str:
-    run_dir = config.default_runs_dir / area / version
+    run_dir = config.runs_dir / area / version
     if not run_dir.is_dir():
         print(f"  {run_dir} doesn't exists, skipping")
         return ''
@@ -155,8 +156,8 @@ def create_version_location(
 
     subprocess.run(
         [
-            sys.executable,
+            python_venv_executable(),
-            Path(__file__).parent.parent / 'metadata_to_tilejson.py',
+            config.http_host_scripts_dir / 'metadata_to_tilejson.py',
             '--minify',
             metadata_path,
             tilejson_path,
@@ -203,7 +204,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
             version = fp.read().strip()
         print(f'  setting latest version for {area}: {version}')
 
-        run_dir = config.default_runs_dir / area / version
+        run_dir = config.runs_dir / area / version
         tilejson_path = run_dir / f'tilejson-{local}.json'
         assert tilejson_path.is_file()
 

scripts/http_host/http_host_lib/utils.py

@@ -1,7 +1,6 @@
 import os
 import subprocess
 import sys
-import time
 from pathlib import Path
 
 import requests
@@ -57,3 +56,14 @@ def download_file_aria2(url: str, local_file: Path):
         ],
         check=True,
     )
+
+
+def python_venv_executable() -> Path:
+    venv_path = os.environ.get('VIRTUAL_ENV')
+
+    if venv_path:
+        return Path(venv_path) / 'bin' / 'python'
+    elif sys.prefix != sys.base_prefix:
+        return Path(sys.prefix) / 'bin' / 'python'
+    else:
+        return Path(sys.executable)

scripts/tile_gen/tile_gen_lib/config.py

@@ -12,7 +12,8 @@ class Configuration:
 
     runs_dir = tile_gen_dir / 'runs'
 
-    rclone_config = Path('/data/ofm/config/rclone.conf')
+    ofm_config_dir = Path('/data/ofm/config')
+    rclone_config = ofm_config_dir / 'rclone.conf'
 
     areas = ['planet', 'monaco']
 

ssh_lib/tasks.py

@@ -133,6 +133,7 @@ def prepare_http_host(c):
 
 
 def run_http_host_sync(c):
+    print('Running host_manager.py sync --force')
     sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/host_manager.py sync --force')
 
 
@@ -146,8 +147,8 @@ def upload_http_host_files(c):
 
     put_dir(
         c,
-        SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx',
+        SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs',
-        f'{HTTP_HOST_BIN}/http_host_lib/nginx',
+        f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs',
     )
 
     c.sudo('chown -R ofm:ofm /data/ofm/http_host')