work

modules/http_host/http_host.py

@@ -7,10 +7,10 @@ from http_host_lib.assets import (
 )
 from http_host_lib.btrfs import (
     download_area_version,
-    get_versions_for_area,
 )
+from http_host_lib.get_version_shared import get_versions_for_area
 from http_host_lib.mount import auto_mount
-from http_host_lib.nginx import write_nginx_config
+from http_host_lib.nginx_config_gen import write_nginx_config
 from http_host_lib.sync import auto_clean_btrfs, full_sync
 from http_host_lib.versions import fetch_version_files
 

modules/http_host/http_host_lib/btrfs.py

@@ -3,7 +3,7 @@ import subprocess
 import sys
 
 from http_host_lib.config import config
-from http_host_lib.shared import get_versions_for_area
+from http_host_lib.get_version_shared import get_versions_for_area
 from http_host_lib.utils import download_file_aria2, get_remote_file_size
 
 

modules/http_host/http_host_lib/config.py

@@ -1,7 +1,8 @@
-import json
 import subprocess
 from pathlib import Path
 
+import json5
+
 
 class Configuration:
     areas = ['planet', 'monaco']
@@ -17,7 +18,7 @@ class Configuration:
     mnt_dir = Path('/mnt/ofm')
 
     certs_dir = Path('/data/nginx/certs')
-    nginx_confs = Path(__file__).parent / 'nginx_confs'
+    nginx_confs_templates = Path(__file__).parent / 'nginx_conf_templates'
 
     if Path('/data/ofm').exists():
         ofm_config_dir = Path('/data/ofm/config')
@@ -25,7 +26,7 @@ class Configuration:
         repo_root = Path(__file__).parent.parent.parent.parent
         ofm_config_dir = repo_root / 'config'
 
-    ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
+    jsonc_config = json5.loads((ofm_config_dir / 'config.jsonc').read_text())
 
     deployed_versions_dir = ofm_config_dir / 'deployed_versions'
 

modules/http_host/http_host_lib/get_version_shared.py

@@ -0,0 +1 @@
+../../tile_gen/tile_gen_lib/get_version_shared.py

modules/http_host/http_host_lib/nginx_conf_templates/common.conf

@@ -1,5 +1,5 @@
 server {
-    server_name __LOCAL__ __DOMAIN__;
+    server_name __DOMAIN_SLUG__ __DOMAIN__;
 
     # ssl: https://ssl-config.mozilla.org / intermediate config
 
@@ -30,8 +30,7 @@ server {
 
     add_header X-Robots-Tag "noindex, nofollow" always;
 
-
+    __DYNAMIC_BLOCKS__
-    __LOCATION_BLOCKS__
 
     location /styles/ {
         # trailing slash

modules/http_host/http_host_lib/nginx_conf_templates/letsencrypt.conf

@@ -1,5 +1,5 @@
 server {
-    server_name __LOCAL__ __DOMAIN__;
+    server_name __LOOPBACK_HOSTNAME__ __DOMAIN__;
 
     # ssl: https://ssl-config.mozilla.org / intermediate config
 
@@ -38,7 +38,7 @@ server {
         try_files $uri =404;
     }
 
-    __LOCATION_BLOCKS__
+    __DYNAMIC_BLOCKS__
 
     location /styles/ {
         # trailing slash

modules/http_host/http_host_lib/nginx_config_gen.py

@@ -1,9 +1,9 @@
-import shutil
 import subprocess
 import sys
 from pathlib import Path
 
 from http_host_lib.config import config
+from http_host_lib.slugify import slugify
 from http_host_lib.utils import python_venv_executable
 
 
@@ -13,12 +13,6 @@ def write_nginx_config():
     if not config.mnt_dir.exists():
         sys.exit('  mount needs to be run first')
 
-    curl_text_mix = ''
-
-    domain_direct = config.ofm_config['domain_direct']
-    domain_roundrobin = config.ofm_config['domain_roundrobin']
-    self_signed_certs = config.ofm_config['self_signed_certs']
-
     # remove old configs and certs
     for file in Path('/data/nginx/sites').glob('ofm_*.conf'):
         file.unlink()
@@ -26,115 +20,72 @@ def write_nginx_config():
     for file in Path('/data/nginx/certs').glob('ofm_*'):
         file.unlink()
 
-    # processing Round Robin DNS config
+    conf = config.jsonc_config
-    if domain_roundrobin:
-        if not config.rclone_config.is_file():
-            sys.exit('rclone.conf missing')
 
-        # download the roundrobin certificate from bucket using rclone
+    curl_help_lines = []
-        write_roundrobin_reader_script(domain_roundrobin)
-        subprocess.run(['bash', config.http_host_bin / 'roundrobin_reader.sh'], check=True)
 
-        curl_text_mix += create_nginx_conf(
+    for domain_data in conf['domains']:
-            template_path=config.nginx_confs / 'roundrobin.conf',
+        curl_help_lines += process_domain(domain_data)
-            local='ofm_roundrobin',
-            domain=domain_roundrobin,
-        )
-
-    # processing Let's Encrypt config
-    if domain_direct:
-        direct_cert = config.certs_dir / 'ofm_direct.cert'
-        direct_key = config.certs_dir / 'ofm_direct.key'
-
-        if not direct_cert.is_file() or not direct_key.is_file():
-            shutil.copyfile(Path('/etc/nginx/ssl/dummy.cert'), direct_cert)
-            shutil.copyfile(Path('/etc/nginx/ssl/dummy.key'), direct_key)
-
-        curl_text_mix += create_nginx_conf(
-            template_path=config.nginx_confs / 'le.conf',
-            local='ofm_direct',
-            domain=domain_direct,
-        )
-
-        subprocess.run(['nginx', '-t'], check=True)
-        subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
-
-        if not self_signed_certs:
-            subprocess.run(
-                [
-                    'certbot',
-                    'certonly',
-                    '--webroot',
-                    '--webroot-path=/data/nginx/acme-challenges',
-                    '--noninteractive',
-                    '-m',
-                    config.ofm_config['letsencrypt_email'],
-                    '--agree-tos',
-                    '--cert-name=ofm_direct',
-                    # '--staging',
-                    '--deploy-hook',
-                    'nginx -t && service nginx reload',
-                    '-d',
-                    domain_direct,
-                ],
-                check=True,
-            )
-
-            # link certs to nginx dir
-            direct_cert.unlink()
-            direct_key.unlink()
-
-            etc_cert = Path('/etc/letsencrypt/live/ofm_direct/fullchain.pem')
-            etc_key = Path('/etc/letsencrypt/live/ofm_direct/privkey.pem')
-            assert etc_cert.is_file()
-            assert etc_key.is_file()
-            direct_cert.symlink_to(etc_cert)
-            direct_key.symlink_to(etc_key)
 
     subprocess.run(['nginx', '-t'], check=True)
     subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
 
-    curl_text_lines = sorted(curl_text_mix.splitlines())
     if config.ofm_config.get('skip_planet'):
-        curl_text_lines = [l for l in curl_text_lines if '/planet' not in l]
+        curl_help_lines = [l for l in curl_help_lines if '/planet' not in l]
     else:
-        curl_text_lines = [l for l in curl_text_lines if '/monaco' not in l]
+        curl_help_lines = [l for l in curl_help_lines if '/monaco' not in l]
 
-    curl_text_mix = '\n'.join(curl_text_lines)
+    curl_help_str = '\n'.join(curl_help_lines)
-    print(f'test with:\n{curl_text_mix}')
+    print(f'test with:\n{curl_help_str}')
 
 
-def create_nginx_conf(*, template_path, local, domain):
+def process_domain(domain_data):
-    location_str, curl_text = create_location_blocks(local=local, domain=domain)
+    domain_slug = slugify(domain_data['domain'], separator='_')
+    domain_data['slug'] = domain_slug
 
-    with open(template_path) as fp:
+    if domain_data['cert'] == 'upload':
-        template = fp.read()
+        domain_data['cert_file'] = config.certs_dir / f'{domain_slug}.cert'
+        domain_data['key_file'] = config.certs_dir / f'{domain_slug}.key'
 
-    template = template.replace('__LOCATION_BLOCKS__', location_str)
+        if not domain_data['cert_file'].is_file() or not domain_data['key_file'].is_file():
-    template = template.replace('__LOCAL__', local)
+            sys.exit(
-    template = template.replace('__DOMAIN__', domain)
+                f'  cert or key file does not exist: {domain_data["cert_file"]} {domain_data["key_file"]}'
+            )
 
-    curl_text = curl_text.replace('__LOCAL__', local)
+        return create_nginx_conf(domain_data)
-    curl_text = curl_text.replace('__DOMAIN__', domain)
 
-    with open(f'/data/nginx/sites/{local}.conf', 'w') as fp:
+
+def create_nginx_conf(domain_data: dict):
+    dynamic_block_lines, curl_text = dynamic_blocks(domain_data)
+
+    template = (config.nginx_confs_templates / 'common.conf').read_text()
+
+    template = template.replace('__DYNAMIC_BLOCKS__', dynamic_block_lines)
+
+    template = template.replace('__DOMAIN_SLUG__', domain_data['slug'])
+    template = template.replace('__DOMAIN__', domain_data['domain'])
+
+    curl_text = curl_text.replace('__DOMAIN_SLUG__', domain_data['slug'])
+    curl_text = curl_text.replace('__DOMAIN__', domain_data['domain'])
+
+    with open(f'/data/nginx/sites/{domain_data["slug"]}.conf', 'w') as fp:
         fp.write(template)
-        print(f'  nginx config written: {domain} {local}')
+        print(f'  nginx config written: {domain_data["domain"]} {domain_data["slug"]}')
 
     return curl_text
 
 
-def create_location_blocks(*, local, domain):
+def dynamic_blocks(domain_data: dict):
-    location_str = ''
+    nginx_conf_lines = ''
-    curl_text = ''
+    curl_help_lines = []
 
     for subdir in config.mnt_dir.iterdir():
         if not subdir.is_dir():
             continue
         area, version = subdir.name.split('-')
 
-        location_str += create_version_location(
+        nginx_conf_lines += create_version_location(
-            area=area, version=version, mnt_dir=subdir, local=local, domain=domain
+            area=area, version=version, mnt_dir=subdir, domain_data=domain_data
         )
 
         for path in [
@@ -142,12 +93,12 @@ def create_location_blocks(*, local, domain):
             f'/{area}/{version}/14/8529/5975.pbf',
             f'/{area}/{version}/9999/9999/9999.pbf',  # empty_tile test
         ]:
-            curl_text += (
+            curl_help_lines += [
-                # f'curl -H "Host: __LOCAL__" -I http://localhost/{path}\n'
+                f'curl -H "Host: __DOMAIN_SLUG__" -I http://localhost/{path}',
-                f'curl -sI https://__DOMAIN__{path} | sort\n'
+                f'curl -sI https://__DOMAIN__{path} | sort',
-            )
+            ]
 
-    location_str += create_latest_locations(local=local, domain=domain)
+    nginx_conf_lines += create_latest_locations(domain_data=domain_data)
 
     for area in config.areas:
         for path in [
@@ -156,33 +107,30 @@ def create_location_blocks(*, local, domain):
             f'/{area}/19700101_old_version_test/14/8529/5975.pbf',
             f'/{area}/19700101_old_version_test/9999/9999/9999.pbf',  # empty_tile test
         ]:
-            curl_text += (
+            curl_help_lines += [
-                # f'curl -H "Host: __LOCAL__" -I http://localhost/{path}\n'
+                f'curl -H "Host: __DOMAIN_SLUG__" -I http://localhost/{path}',
-                f'curl -sI https://__DOMAIN__{path} | sort\n'
+                f'curl -sI https://__DOMAIN__{path} | sort',
-            )
+            ]
 
-    with open(config.nginx_confs / 'location_static.conf') as fp:
+    nginx_conf_lines += '\n' + (config.nginx_confs_templates / 'static_blocks.conf').read_text()
-        location_str += '\n' + fp.read()
 
-    return location_str, curl_text
+    return nginx_conf_lines, curl_help_lines
 
 
-def create_version_location(
+def create_version_location(*, area: str, version: str, mnt_dir: Path, domain_data: dict) -> str:
-    *, area: str, version: str, mnt_dir: Path, local: str, domain: str
-) -> str:
     run_dir = config.runs_dir / area / version
     if not run_dir.is_dir():
         print(f"  {run_dir} doesn't exist, skipping")
         return ''
 
-    tilejson_path = run_dir / f'tilejson-{local}.json'
+    tilejson_path = run_dir / f'tilejson-{domain_data["slug"]}.json'
 
     metadata_path = mnt_dir / 'metadata.json'
     if not metadata_path.is_file():
         print(f"  {metadata_path} doesn't exist, skipping")
         return ''
 
-    url_prefix = f'https://{domain}/{area}/{version}'
+    url_prefix = f'https://{domain_data["domain"]}/{area}/{version}'
 
     subprocess.run(
         [
@@ -232,7 +180,7 @@ def create_version_location(
     """
 
 
-def create_latest_locations(*, local: str, domain: str) -> str:
+def create_latest_locations(*, domain_data: dict) -> str:
     location_str = ''
 
     local_version_files = config.deployed_versions_dir.glob('*.txt')
@@ -246,7 +194,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
 
         # checking runs dir
         run_dir = config.runs_dir / area / version
-        tilejson_path = run_dir / f'tilejson-{local}.json'
+        tilejson_path = run_dir / f'tilejson-{domain_data["slug"]}.json'
         if not tilejson_path.is_file():
             print(f'    error with latest: {tilejson_path} does not exist')
             continue
@@ -285,7 +233,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
             # regex location is unreliable with alias, only root is reliable
 
             root {run_dir}; # no trailing slash
-            try_files /tilejson-{local}.json =404;
+            try_files /tilejson-{domain_data['slug']}.json =404;
 
             expires 1w;
             default_type application/json;
@@ -320,3 +268,36 @@ def create_latest_locations(*, local: str, domain: str) -> str:
         """
 
     return location_str
+
+
+# if not self_signed_certs:
+#     subprocess.run(
+#         [
+#             'certbot',
+#             'certonly',
+#             '--webroot',
+#             '--webroot-path=/data/nginx/acme-challenges',
+#             '--noninteractive',
+#             '-m',
+#             config.ofm_config['letsencrypt_email'],
+#             '--agree-tos',
+#             '--cert-name=ofm_direct',
+#             # '--staging',
+#             '--deploy-hook',
+#             'nginx -t && service nginx reload',
+#             '-d',
+#             domain_direct,
+#         ],
+#         check=True,
+#     )
+#
+#     # link certs to nginx dir
+#     direct_cert.unlink()
+#     direct_key.unlink()
+#
+#     etc_cert = Path('/etc/letsencrypt/live/ofm_direct/fullchain.pem')
+#     etc_key = Path('/etc/letsencrypt/live/ofm_direct/privkey.pem')
+#     assert etc_cert.is_file()
+#     assert etc_key.is_file()
+#     direct_cert.symlink_to(etc_cert)
+#     direct_key.symlink_to(etc_key)

modules/http_host/http_host_lib/slugify.py

@@ -0,0 +1,40 @@
+import re
+import unicodedata
+
+
+# Pre-compiled patterns for better performance
+_RE_INVALID = re.compile(r'[^a-z0-9_-]+')
+_RE_SEPARATORS = re.compile(r'[-_]+')
+
+
+def slugify(
+    value: str | bytes | int | float | None,
+    *,
+    separator: str = '-',
+) -> str:
+    if value in (None, ''):
+        return ''
+
+    if separator not in ('-', '_'):
+        raise ValueError(f"separator must be '-' or '_', got {repr(separator)}")
+
+    # 1. Normalize value to string
+    if isinstance(value, bytes):
+        value = value.decode('utf-8', 'ignore')
+    else:
+        value = str(value)
+
+    # 2. Unicode → ASCII, then lowercase
+    value = unicodedata.normalize('NFKD', value)
+    value = value.encode('ascii', 'ignore').decode('ascii').lower()
+
+    # 3. Replace invalid characters with separator
+    value = _RE_INVALID.sub(separator, value)
+
+    # 4. Collapse multiple separators
+    value = _RE_SEPARATORS.sub(separator, value)
+
+    # 5. Strip separators from edges
+    value = value.strip('-_')
+
+    return value

modules/http_host/setup.py

@@ -5,6 +5,7 @@ requirements = [
     'click',
     'pycurl',
     'requests',
+    'json5',
 ]
 
 

setup.py

@@ -10,6 +10,7 @@ requirements = [
     'marko',
     'requests',
     'jsonschema',
+    'json5',
 ]