scripts -> modules

modules/http_host/benchmark/create_path_list.py

@@ -0,0 +1,27 @@
+import json
+
+
+with open('access.jsonl') as fp:
+    json_lines = fp.readlines()
+
+paths = []
+for i, line in enumerate(json_lines):
+    log_data = json.loads(line)
+    if log_data['status'] != 200:
+        continue
+
+    if log_data['request_method'] != 'GET':
+        continue
+
+    uri = log_data['uri']
+
+    if 'tiles/' not in uri or not uri.endswith('.pbf'):
+        continue
+
+    path = log_data['uri'].split('tiles/')[1]
+    paths.append(path + '\n')
+
+    print(f'{i / len(json_lines) * 100:.1f}%')
+
+with open('path_list.txt', 'w') as fp:
+    fp.writelines(paths)

modules/http_host/benchmark/wrk_custom_list.lua

@@ -0,0 +1,39 @@
+local counter = 1
+local lines = {}
+local url_base = "/planet/20231221_134737_pt/"  -- trailing slash
+local path_list_txt = "/data/ofm/benchmark/path_list_500k.txt"
+
+for line in io.lines(path_list_txt) do
+    table.insert(lines, url_base .. line)
+end
+
+local function getNextUrl()
+    -- Get the next URL from the list
+    local url_path = lines[counter]
+    counter = counter + 1
+
+    -- If we've gone past the end of the list, wrap around to the start
+    if counter > #lines then
+        counter = 1
+    end
+
+    return url_path
+end
+
+request = function()
+    -- Return the request object with the current URL path
+    path = getNextUrl()
+    local headers = {}
+    headers["Host"] = "ofm"
+    return wrk.format('GET', path, headers, nil)
+end
+
+response = function(status)
+    if status ~= 200 then
+        print("Non-200 response")
+        print("Status: ", status)
+        -- this only works in single threaded mode (-t1)
+        print("Request path: ", path)
+    end
+end
+

modules/http_host/benchmark/wrk_usage.txt

@@ -0,0 +1,8 @@
+wrk -c10 -t4 -d10s -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost
+
+# -t1 => more correct, since the url list is loaded exactly in sequence
+# -t4 => reflecting real world usage
+
+
+
+

modules/http_host/cron.d/ofm_http_host

@@ -0,0 +1,4 @@
+# every minute sync, locking so that only one process can run at a time
+* * * * * ofm     /usr/bin/flock -n /tmp/hostmanager.lockfile -c 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync >> /data/ofm/http_host/logs/host_manager_sync.log 2>&1'
+
+

modules/http_host/cron.d/ofm_ledns_reader

@@ -0,0 +1,2 @@
+# once per day
+2 34 * * * ofm     sudo /usr/bin/bash /data/ofm/http_host/bin/ledns_reader.sh >> /data/ofm/http_host/logs/ledns_reader.log 2>&1

modules/http_host/http_host.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+import datetime
+import sys
+
+import click
+from http_host_lib.assets import (
+    download_assets,
+)
+from http_host_lib.btrfs import (
+    download_area_version,
+    get_versions_for_area,
+)
+from http_host_lib.config import config
+from http_host_lib.mount import auto_mount_unmount
+from http_host_lib.nginx import write_nginx_config
+from http_host_lib.set_tileset_versions import set_tileset_versions
+from http_host_lib.utils import assert_linux, assert_sudo
+
+
+@click.group()
+def cli():
+    """
+    Manages OpenFreeMap HTTP hosts, including:\n
+    - Downloading btrfs images\n
+    - Downloading assets\n
+    - Mounting directories\n
+    - Updating nginx config\n
+    - Getting the deployed versions of tilesets\n
+    - Running the sync cron task (called every minute)
+    """
+
+
+@cli.command()
+@click.argument('area', required=False)
+@click.option(
+    '--version', default='latest', help='Optional version string, like "20231227_043106_pt"'
+)
+def download_btrfs(area: str, version: str):
+    """
+    Downloads and uncompresses tiles.btrfs files from the btrfs bucket
+    Version can be "latest" (default) or specified, like "20231227_043106_pt"
+    Use --version=1 to list all available versions
+    """
+
+    return download_area_version(area, version)
+
+
+@cli.command(name='download-assets')
+def download_assets_():
+    """
+    Downloads and extracts assets
+    """
+
+    download_assets()
+
+
+@cli.command()
+def mount():
+    """
+    Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
+    When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
+    """
+
+    auto_mount_unmount()
+
+
+@cli.command()
+def set_latest_versions():
+    """
+    Sets the latest version of the tilesets to the version specified by
+    https://assets.openfreemap.com/versions/deployed_planet.txt
+
+    1. Checks if the given version is present on the disk and is mounted
+    2. Writes to a version file
+    """
+
+    print('running set_latest_versions')
+
+    assert_linux()
+    assert_sudo()
+
+    if not config.mnt_dir.exists():
+        sys.exit('  mount needs to be run first')
+
+    return set_tileset_versions()
+
+
+@cli.command()
+def nginx_sync():
+    """
+    Syncs the nginx config to the state of the system
+    """
+
+    print('running nginx_sync')
+
+    assert_linux()
+    assert_sudo()
+
+    if not config.mnt_dir.exists():
+        sys.exit('  mount needs to be run first')
+
+    write_nginx_config()
+
+
+@cli.command()
+@click.option('--force', is_flag=True, help='Force nginx sync run')
+@click.pass_context
+def sync(ctx, force):
+    """
+    Runs the sync task, normally called by cron every minute
+    On a new server this also takes care of everything, no need to run anything manually.
+    """
+
+    print('---')
+    print('running sync')
+    print(datetime.datetime.now(tz=datetime.timezone.utc))
+
+    assert_linux()
+    assert_sudo()
+
+    download_done = False
+    download_done += ctx.invoke(download_btrfs, area='monaco')
+
+    if not config.host_config.get('skip_planet'):
+        download_done += ctx.invoke(download_btrfs, area='planet')
+
+    if download_done:
+        ctx.invoke(mount)
+
+    ctx.invoke(download_assets)
+
+    deploy_done = ctx.invoke(set_latest_versions)
+
+    if download_done or deploy_done or force:
+        ctx.invoke(nginx_sync)
+
+
+@cli.command()
+def debug():
+    versions = get_versions_for_area('monaco')
+    print(versions)
+
+
+if __name__ == '__main__':
+    # print(config.host_config)
+    cli()

modules/http_host/http_host_lib/assets.py

@@ -0,0 +1,83 @@
+import shutil
+import subprocess
+
+import requests
+
+from http_host_lib.config import config
+from http_host_lib.utils import download_file_aria2, download_if_size_differs
+
+
+def download_assets():
+    """
+    Downloads and extracts assets
+    """
+
+    download_and_extract_asset_tar_gz('fonts')
+    download_and_extract_asset_tar_gz('styles')
+    download_and_extract_asset_tar_gz('natural_earth')
+
+    download_sprites()
+
+
+def download_and_extract_asset_tar_gz(asset_kind):
+    """
+    Download and extract asset.tgz if the file size differ or not available locally
+    """
+
+    print(f'Downloading asset {asset_kind}')
+
+    asset_dir = config.assets_dir / asset_kind
+    asset_dir.mkdir(exist_ok=True, parents=True)
+
+    url = f'https://assets.openfreemap.com/{asset_kind}/ofm.tar.gz'
+    local_file = asset_dir / 'ofm.tar.gz'
+    if not download_if_size_differs(url, local_file):
+        print(f'  skipping asset: {asset_kind}')
+        return
+
+    ofm_dir = asset_dir / 'ofm'
+    ofm_dir_bak = asset_dir / 'ofm.bak'
+    shutil.rmtree(ofm_dir_bak, ignore_errors=True)
+    if ofm_dir.exists():
+        ofm_dir.rename(ofm_dir_bak)
+
+    subprocess.run(
+        ['tar', '-xzf', local_file, '-C', asset_dir],
+        check=True,
+    )
+
+    print(f'  downloaded asset: {asset_kind}')
+
+
+def download_sprites():
+    """
+    Sprites are special assets, as we have to keep the old versions indefinitely
+    """
+
+    print('Downloading sprites')
+
+    sprites_dir = config.assets_dir / 'sprites'
+    sprites_dir.mkdir(exist_ok=True, parents=True)
+
+    r = requests.get('https://assets.openfreemap.com/files.txt', timeout=30)
+    r.raise_for_status()
+
+    sprites_remote = [l for l in r.text.splitlines() if l.startswith('sprites/')]
+
+    for sprite in sprites_remote:
+        sprite_name = sprite.split('/')[1].replace('.tar.gz', '')
+
+        if (sprites_dir / sprite_name).is_dir():
+            print(f'  skipping sprite version: {sprite_name}')
+            continue
+
+        url = f'https://assets.openfreemap.com/sprites/{sprite_name}.tar.gz'
+        local_file = sprites_dir / 'temp.tar.gz'
+        download_file_aria2(url, local_file)
+
+        subprocess.run(
+            ['tar', '-xzf', local_file, '-C', sprites_dir],
+            check=True,
+        )
+        local_file.unlink()
+        print(f'  downloaded sprite version: {sprite_name}')

modules/http_host/http_host_lib/btrfs.py

@@ -0,0 +1,84 @@
+import shutil
+import subprocess
+import sys
+
+import requests
+
+from http_host_lib.config import config
+from http_host_lib.utils import download_file_aria2, get_remote_file_size
+
+
+def download_area_version(area: str, version: str):
+    """
+    Downloads and uncompresses tiles.btrfs files from the btrfs bucket
+    """
+
+    if area not in config.areas:
+        sys.exit(f'  please specify area: {config.areas}')
+
+    versions = get_versions_for_area(area)
+
+    if version == 'latest':
+        selected_version = versions[-1]
+    else:
+        if version not in versions:
+            available_versions_str = '\n'.join(versions)
+            sys.exit(
+                f'Requested version is not available.\nAvailable versions for {area}:\n{available_versions_str}'
+            )
+        selected_version = version
+
+    return download_and_extract_btrfs(area, selected_version)
+
+
+def get_versions_for_area(area: str) -> list:
+    r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30)
+    r.raise_for_status()
+
+    versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')]
+    return sorted(versions)
+
+
+def download_and_extract_btrfs(area: str, version: str) -> bool:
+    """
+    returns True if download successful, False if skipped
+    """
+
+    print(f'downloading and extracting btrfs for: {area} {version}')
+
+    version_dir = config.runs_dir / area / version
+    btrfs_file = version_dir / 'tiles.btrfs'
+    if btrfs_file.exists():
+        print('  file exists, skipping download')
+        return False
+
+    temp_dir = config.runs_dir / '_tmp'
+    shutil.rmtree(temp_dir, ignore_errors=True)
+    temp_dir.mkdir(parents=True)
+
+    url = f'https://btrfs.openfreemap.com/areas/{area}/{version}/tiles.btrfs.gz'
+
+    # check disk space
+    disk_free = shutil.disk_usage(temp_dir).free
+    file_size = get_remote_file_size(url)
+    if not file_size:
+        raise ValueError('Cannot get remote file size')
+
+    needed_space = file_size * 3
+    if disk_free < needed_space:
+        raise ValueError(f'Not enough disk space. Needed: {needed_space}, free space: {disk_free}')
+
+    target_file = temp_dir / 'tiles.btrfs.gz'
+    download_file_aria2(url, target_file)
+
+    print('Uncompressing...')
+    subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
+    btrfs_src = temp_dir / 'tiles.btrfs'
+
+    shutil.rmtree(version_dir, ignore_errors=True)
+    version_dir.mkdir(parents=True)
+
+    btrfs_src.rename(btrfs_file)
+
+    shutil.rmtree(temp_dir)
+    return True

modules/http_host/http_host_lib/config.py

@@ -0,0 +1,29 @@
+import json
+from pathlib import Path
+
+
+class Configuration:
+    areas = ['planet', 'monaco']
+
+    http_host_dir = Path('/data/ofm/http_host')
+
+    http_host_bin = http_host_dir / 'bin'
+    http_host_scripts_dir = http_host_bin / 'scripts'
+
+    runs_dir = http_host_dir / 'runs'
+    assets_dir = http_host_dir / 'assets'
+
+    mnt_dir = Path('/mnt/ofm')
+    ofm_config_dir = Path('/data/ofm/config')
+
+    certs_dir = Path('/data/nginx/certs')
+    nginx_confs = Path(__file__).parent / 'nginx_confs'
+
+    try:
+        with open(ofm_config_dir / 'http_host.json') as fp:
+            host_config = json.load(fp)
+    except Exception:
+        host_config = {}
+
+
+config = Configuration()

modules/http_host/http_host_lib/mount.py

@@ -0,0 +1,87 @@
+import subprocess
+import sys
+from pathlib import Path
+
+from http_host_lib.config import config
+from http_host_lib.utils import assert_linux, assert_sudo
+
+
+def auto_mount_unmount():
+    """
+    Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
+    When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
+    """
+
+    print('running mount')
+
+    assert_linux()
+    assert_sudo()
+
+    if not config.runs_dir.exists():
+        sys.exit('  download-btrfs needs to be run first')
+
+    clean_up_mounts(config.mnt_dir)
+    create_fstab()
+
+    print('  running mount -a')
+    subprocess.run(['mount', '-a'], check=True)
+
+    clean_up_mounts(config.mnt_dir)
+
+
+def create_fstab():
+    fstab_new = []
+
+    for area in ['planet', 'monaco']:
+        area_dir = (config.runs_dir / area).resolve()
+        if not area_dir.exists():
+            continue
+
+        versions = sorted(area_dir.iterdir())
+        for version in versions:
+            version_str = version.name
+            btrfs_file = area_dir / version_str / 'tiles.btrfs'
+            if not btrfs_file.is_file():
+                continue
+
+            mnt_folder = config.mnt_dir / f'{area}-{version_str}'
+            mnt_folder.mkdir(exist_ok=True, parents=True)
+
+            fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n')
+            print(f'  created fstab entry for {btrfs_file} -> {mnt_folder}')
+
+    with open('/etc/fstab') as fp:
+        fstab_orig = [l for l in fp.readlines() if f'{config.mnt_dir}/' not in l]
+
+    with open('/etc/fstab', 'w') as fp:
+        fp.writelines(fstab_orig + fstab_new)
+
+
+def clean_up_mounts(mnt_dir):
+    if not mnt_dir.exists():
+        return
+
+    print('  cleaning up mounts')
+
+    # handle deleted files
+    p = subprocess.run(['mount'], capture_output=True, text=True, check=True)
+    lines = [l for l in p.stdout.splitlines() if f'{mnt_dir}/' in l and '(deleted)' in l]
+
+    for l in lines:
+        mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0])
+        print(f'  removing deleted mount {mnt_path}')
+        assert mnt_path.exists()
+        subprocess.run(['umount', mnt_path], check=True)
+        mnt_path.rmdir()
+
+    # clean all mounts not in current fstab
+    with open('/etc/fstab') as fp:
+        fstab_str = fp.read()
+
+    for subdir in mnt_dir.iterdir():
+        if f'{subdir} ' in fstab_str:
+            continue
+
+        print(f'  removing old mount {subdir}')
+        subprocess.run(['umount', subdir], check=True)
+        subdir.rmdir()

modules/http_host/http_host_lib/nginx.py

@@ -0,0 +1,235 @@
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from http_host_lib.config import config
+from http_host_lib.utils import python_venv_executable
+
+
+def write_nginx_config():
+    curl_text_mix = ''
+
+    domain_le = config.host_config['domain_le']
+    domain_ledns = config.host_config['domain_ledns']
+
+    # remove old configs and certs
+    for file in Path('/data/nginx/sites').glob('ofm_*.conf'):
+        file.unlink()
+
+    for file in Path('/data/nginx/certs').glob('ofm_*'):
+        file.unlink()
+
+    # processing Round Robin DNS config
+    if domain_ledns:
+        if not (config.ofm_config_dir / 'rclone.conf').is_file():
+            sys.exit('rclone.conf missing')
+
+        # download the ledns certificate from bucket using rclone
+        write_ledns_reader_script(domain_ledns)
+        subprocess.run(['bash', config.http_host_bin / 'ledns_reader.sh'], check=True)
+
+        curl_text_mix += create_nginx_conf(
+            template_path=config.nginx_confs / 'ledns.conf',
+            local='ofm_ledns',
+            domain=domain_ledns,
+        )
+
+    # processing Let's Encrypt config
+    if domain_le:
+        le_cert = config.certs_dir / 'ofm_le.cert'
+        le_key = config.certs_dir / 'ofm_le.key'
+
+        if not le_cert.is_file() or not le_key.is_file():
+            shutil.copyfile(Path('/etc/nginx/ssl/dummy.crt'), le_cert)
+            shutil.copyfile(Path('/etc/nginx/ssl/dummy.key'), le_key)
+
+        curl_text_mix += create_nginx_conf(
+            template_path=config.nginx_confs / 'le.conf',
+            local='ofm_le',
+            domain=domain_le,
+        )
+
+        subprocess.run(['nginx', '-t'], check=True)
+        subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
+
+        subprocess.run(
+            [
+                'certbot',
+                'certonly',
+                '--webroot',
+                '--webroot-path=/data/nginx/acme-challenges',
+                '--noninteractive',
+                '-m',
+                config.host_config['le_email'],
+                '--agree-tos',
+                '--cert-name=ofm_le',
+                # '--staging',
+                '--deploy-hook',
+                'nginx -t && service nginx reload',
+                '-d',
+                domain_le,
+            ],
+            check=True,
+        )
+
+        # link certs to nginx dir
+        le_cert.unlink()
+        le_key.unlink()
+
+        etc_cert = Path('/etc/letsencrypt/live/ofm_le/fullchain.pem')
+        etc_key = Path('/etc/letsencrypt/live/ofm_le/privkey.pem')
+        assert etc_cert.is_file()
+        assert etc_key.is_file()
+        le_cert.symlink_to(etc_cert)
+        le_key.symlink_to(etc_key)
+
+    subprocess.run(['nginx', '-t'], check=True)
+    subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
+
+    print(curl_text_mix)
+
+
+def create_nginx_conf(*, template_path, local, domain):
+    location_str, curl_text = create_location_blocks(local=local, domain=domain)
+
+    with open(template_path) as fp:
+        template = fp.read()
+
+    template = template.replace('__LOCATION_BLOCKS__', location_str)
+    template = template.replace('__LOCAL__', local)
+    template = template.replace('__DOMAIN__', domain)
+
+    curl_text = curl_text.replace('__LOCAL__', local)
+    curl_text = curl_text.replace('__DOMAIN__', domain)
+
+    with open(f'/data/nginx/sites/{local}.conf', 'w') as fp:
+        fp.write(template)
+        print(f'  nginx config written: {domain} {local}')
+
+    return curl_text
+
+
+def create_location_blocks(*, local, domain):
+    location_str = ''
+    curl_text = ''
+
+    for subdir in config.mnt_dir.iterdir():
+        if not subdir.is_dir():
+            continue
+        area, version = subdir.name.split('-')
+        location_str += create_version_location(
+            area=area, version=version, subdir=subdir, local=local, domain=domain
+        )
+
+        if not curl_text:
+            curl_text = (
+                '\ntest with:\n'
+                f'curl -H "Host: __LOCAL__" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n'
+                f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf'
+            )
+
+    location_str += create_latest_locations(local=local, domain=domain)
+
+    with open(config.nginx_confs / 'location_static.conf') as fp:
+        location_str += '\n' + fp.read()
+
+    return location_str, curl_text
+
+
+def create_version_location(
+    *, area: str, version: str, subdir: Path, local: str, domain: str
+) -> str:
+    run_dir = config.runs_dir / area / version
+    if not run_dir.is_dir():
+        print(f"  {run_dir} doesn't exists, skipping")
+        return ''
+
+    tilejson_path = run_dir / f'tilejson-{local}.json'
+
+    metadata_path = subdir / 'metadata.json'
+    if not metadata_path.is_file():
+        print(f"  {metadata_path} doesn't exists, skipping")
+        return ''
+
+    url_prefix = f'https://{domain}/{area}/{version}'
+
+    subprocess.run(
+        [
+            python_venv_executable(),
+            config.http_host_scripts_dir / 'metadata_to_tilejson.py',
+            '--minify',
+            metadata_path,
+            tilejson_path,
+            url_prefix,
+        ],
+        check=True,
+    )
+
+    return f"""
+    location = /{area}/{version} {{     # no trailing slash
+        alias {tilejson_path};          # no trailing slash
+
+        expires 1w;
+        default_type application/json;
+
+        add_header 'Access-Control-Allow-Origin' '*' always;
+        add_header Cache-Control public;
+    }}
+
+    location /{area}/{version}/ {{      # trailing slash
+        alias {subdir}/tiles/;          # trailing slash
+        try_files $uri @empty_tile;
+        add_header Content-Encoding gzip;
+
+        expires 10y;
+
+        types {{
+            application/vnd.mapbox-vector-tile pbf;
+        }}
+
+        add_header 'Access-Control-Allow-Origin' '*' always;
+        add_header Cache-Control public;
+    }}
+    """
+
+
+def create_latest_locations(*, local: str, domain: str) -> str:
+    location_str = ''
+
+    local_version_files = config.ofm_config_dir.glob('tileset_version_*.txt')
+    for file in local_version_files:
+        area = file.stem.split('_')[-1]
+        with open(file) as fp:
+            version = fp.read().strip()
+        print(f'  setting latest version for {area}: {version}')
+
+        run_dir = config.runs_dir / area / version
+        tilejson_path = run_dir / f'tilejson-{local}.json'
+        assert tilejson_path.is_file()
+
+        location_str += f"""
+        location = /{area} {{          # no trailing slash
+            alias {tilejson_path};       # no trailing slash
+
+            expires 1d;
+            default_type application/json;
+
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header Cache-Control public;
+        }}
+        """
+
+    return location_str
+
+
+def write_ledns_reader_script(domain_ledns):
+    script = f"""
+#!/usr/bin/env bash
+export RCLONE_CONFIG=/data/ofm/config/rclone.conf
+rclone copyto -v "remote:ofm-private/ledns/{domain_ledns}/ofm_ledns.cert" /data/nginx/certs/ofm_ledns.cert
+rclone copyto -v "remote:ofm-private/ledns/{domain_ledns}/ofm_ledns.key" /data/nginx/certs/ofm_ledns.key
+    """.strip()
+
+    with open(config.http_host_bin / 'ledns_reader.sh', 'w') as fp:
+        fp.write(script)

modules/http_host/http_host_lib/nginx_confs/le.conf

@@ -0,0 +1,44 @@
+server {
+    server_name __LOCAL__ __DOMAIN__;
+
+    # ssl: https://ssl-config.mozilla.org / intermediate config
+
+    listen 80;
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+
+    ssl_certificate /data/nginx/certs/ofm_le.cert;
+    ssl_certificate_key /data/nginx/certs/ofm_le.key;
+
+    ssl_session_timeout 1d;
+    ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
+    ssl_session_tickets off;
+
+    ssl_dhparam /etc/nginx/ffdhe2048.txt;
+
+    # intermediate configuration
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
+    ssl_prefer_server_ciphers off;
+
+    # access log disabled by default
+    access_log /data/ofm/http_host/logs_nginx/le-access.jsonl access_json buffer=32k;
+    #access_log off;
+
+    error_log /data/ofm/http_host/logs_nginx/le-error.log;
+
+    location ^~ /.well-known/acme-challenge/ {
+        # trailing slash
+        root /data/nginx/acme-challenges;
+        try_files $uri =404;
+    }
+
+    __LOCATION_BLOCKS__
+
+    # catch-all block to deny all other requests
+    location / {
+        deny all;
+        error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
+    }
+}

modules/http_host/http_host_lib/nginx_confs/ledns.conf

@@ -0,0 +1,38 @@
+server {
+    server_name __LOCAL__ __DOMAIN__;
+
+    # ssl: https://ssl-config.mozilla.org / intermediate config
+
+    listen 80;
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+
+    ssl_certificate /data/nginx/certs/ofm_ledns.cert;
+    ssl_certificate_key /data/nginx/certs/ofm_ledns.key;
+
+    ssl_session_timeout 1d;
+    ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
+    ssl_session_tickets off;
+
+    ssl_dhparam /etc/nginx/ffdhe2048.txt;
+
+    # intermediate configuration
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
+    ssl_prefer_server_ciphers off;
+
+    # access log disabled by default
+    access_log /data/ofm/http_host/logs_nginx/ledns-access.jsonl access_json buffer=32k;
+    #access_log off;
+
+    error_log /data/ofm/http_host/logs_nginx/ledns-error.log;
+
+    __LOCATION_BLOCKS__
+
+    # catch-all block to deny all other requests
+    location / {
+        deny all;
+        error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
+    }
+}

modules/http_host/http_host_lib/nginx_confs/location_static.conf

@@ -0,0 +1,64 @@
+location /fonts/ {
+    # trailing slash
+
+    alias /data/ofm/http_host/assets/fonts/ofm/; # trailing slash
+    try_files $uri =404;
+
+    expires 1w;
+
+    add_header 'Access-Control-Allow-Origin' '*' always;
+    add_header Cache-Control public;
+}
+
+location /styles/ {
+    # trailing slash
+
+    alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash
+    try_files $uri.json =404;
+
+    expires 1d;
+    default_type application/json;
+
+    add_header 'Access-Control-Allow-Origin' '*' always;
+    add_header Cache-Control public;
+}
+
+location /natural_earth/ {
+    # trailing slash
+
+    alias /data/ofm/http_host/assets/natural_earth/ofm/; # trailing slash
+    try_files $uri =404;
+
+    expires 10y;
+
+    add_header 'Access-Control-Allow-Origin' '*' always;
+    add_header Cache-Control public;
+}
+
+location /sprites/ {
+    # trailing slash
+
+    alias /data/ofm/http_host/assets/sprites/; # trailing slash
+    try_files $uri =404;
+
+    expires 10y;
+
+    add_header 'Access-Control-Allow-Origin' '*' always;
+    add_header Cache-Control public;
+}
+
+
+# we need to handle missing tiles as valid request returning empty string
+location @empty_tile {
+    return 200 '';
+
+    expires 10y;
+    default_type application/vnd.mapbox-vector-tile;
+
+    add_header 'Access-Control-Allow-Origin' '*' always;
+    add_header Cache-Control public;
+}
+
+location = / {
+    return 302 https://openfreemap.org;
+}

modules/http_host/http_host_lib/set_tileset_versions.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+import requests
+
+from http_host_lib.config import config
+
+
+def set_tileset_versions():
+    need_nginx_sync = False
+
+    for area in ['planet', 'monaco']:
+        r = requests.get(f'https://assets.openfreemap.com/versions/deployed_{area}.txt', timeout=30)
+        r.raise_for_status()
+        remote_version = r.text.strip()
+        print(f'  remote version for {area}: {remote_version}')
+
+        local_version_file = config.ofm_config_dir / f'tileset_version_{area}.txt'
+
+        if not local_version_file.exists():
+            local_version_start = None
+        else:
+            with open(local_version_file) as fp:
+                local_version_start = fp.read()
+
+        if not remote_version:
+            print('    remote version not specified')
+            if local_version_start is not None:
+                local_version_file.unlink()
+                need_nginx_sync = True
+            continue
+
+        mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
+        if not mnt_file.exists():
+            print('    local version does not exist')
+            if local_version_start is not None:
+                local_version_file.unlink()
+                need_nginx_sync = True
+            continue
+
+        if remote_version != local_version_start:
+            with open(local_version_file, 'w') as fp:
+                fp.write(remote_version)
+            need_nginx_sync = True
+
+    return need_nginx_sync

modules/http_host/http_host_lib/utils.py

@@ -0,0 +1,69 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import requests
+
+
+def assert_sudo():
+    if os.geteuid() != 0:
+        sys.exit('  needs sudo')
+
+
+def assert_linux():
+    if not Path('/etc/fstab').exists():
+        sys.exit('  needs to be run on Linux')
+
+
+def assert_single_process():
+    p = subprocess.run(['pgrep', '-fl', sys.argv[0]], capture_output=True, text=True)
+    lines = [l for l in p.stdout.splitlines() if 'python' in l]
+    if len(lines) >= 2:
+        sys.exit('  detected multiple processes, terminating')
+
+
+def download_if_size_differs(url: str, local_file: Path) -> bool:
+    if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url):
+        download_file_aria2(url, local_file)
+        return True
+
+    return False
+
+
+def get_remote_file_size(url: str) -> int | None:
+    r = requests.head(url, timeout=30)
+    size = r.headers.get('Content-Length')
+    return int(size) if size else None
+
+
+def download_file_aria2(url: str, local_file: Path):
+    print(f'  downloading {url} into {local_file}')
+    local_file.unlink(missing_ok=True)
+
+    subprocess.run(
+        [
+            'aria2c',
+            '--split=8',
+            '--max-connection-per-server=8',
+            '--file-allocation=none',
+            '--min-split-size=1M',
+            '-d',
+            local_file.parent,
+            '-o',
+            local_file.name,
+            url,
+        ],
+        check=True,
+    )
+
+
+def python_venv_executable() -> Path:
+    venv_path = os.environ.get('VIRTUAL_ENV')
+
+    if venv_path:
+        return Path(venv_path) / 'bin' / 'python'
+    elif sys.prefix != sys.base_prefix:
+        return Path(sys.prefix) / 'bin' / 'python'
+    else:
+        return Path(sys.executable)

modules/http_host/scripts/metadata_to_tilejson.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+import json
+from pathlib import Path
+
+import click
+
+
+@click.command()
+@click.argument(
+    'metadata_path', type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path)
+)
+@click.argument('tilejson_path', type=click.Path(path_type=Path))
+@click.argument('url_prefix')
+@click.option('--minify', is_flag=True, help='Minify the generated JSON')
+def cli(metadata_path: Path, tilejson_path: Path, url_prefix: str, minify: bool):
+    """
+    Takes a MBTiles metadata.json and generates a TileJSON 3.0.0 file
+
+    URL_PREFIX: Base URL to use as a prefix for tiles in the generated TileJSON.
+
+    Reference: https://github.com/mapbox/tilejson-spec/tree/master/3.0.0
+    """
+
+    tilejson = dict(tilejson='3.0.0')
+
+    with open(metadata_path) as fp:
+        metadata = json.load(fp)
+
+    metadata_json_key = json.loads(metadata.pop('json'))
+
+    tilejson['tiles'] = [url_prefix.rstrip('/') + '/{z}/{x}/{y}.pbf']
+
+    ''
+    tilejson['vector_layers'] = metadata_json_key.pop('vector_layers')
+    assert not metadata_json_key  # check that no more keys are left
+
+    tilejson['attribution'] = metadata.pop('attribution')
+
+    # overwriting new style OSM license, until fixed in tile_gen
+    tilejson['attribution'] = (
+        '<a href="https://openfreemap.org" target="_blank">OpenFreeMap</a> '
+        '<a href="https://www.openmaptiles.org/" target="_blank">&copy; OpenMapTiles</a> '
+        'Data from <a href="https://www.openstreetmap.org/copyright" target="_blank">OpenStreetMap</a>'
+    )
+
+    tilejson['bounds'] = [float(n) for n in metadata.pop('bounds').split(',')]
+    tilejson['center'] = [float(n) for n in metadata.pop('center').split(',')]
+    tilejson['center'][2] = 1
+
+    tilejson['description'] = metadata.pop('description')
+
+    tilejson['maxzoom'] = int(metadata.pop('maxzoom'))
+    tilejson['minzoom'] = int(metadata.pop('minzoom'))
+
+    tilejson['name'] = metadata.pop('name')
+    tilejson['version'] = metadata.pop('version')
+
+    with open(tilejson_path, 'w') as fp:
+        if minify:
+            json.dump(tilejson, fp, ensure_ascii=False, separators=(',', ':'))
+        else:
+            json.dump(tilejson, fp, ensure_ascii=False, indent=2)
+
+
+if __name__ == '__main__':
+    cli()

modules/http_host/setup.py

@@ -0,0 +1,14 @@
+from setuptools import find_packages, setup
+
+
+requirements = [
+    'click',
+    'requests',
+]
+
+
+setup(
+    python_requires='>=3.10',
+    install_requires=requirements,
+    packages=find_packages(),
+)