nginx sync

init-server.py

@@ -73,7 +73,6 @@ def prepare_tile_gen(c):
         c,
         SCRIPTS_DIR / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py',
         f'{TILE_GEN_BIN}/extract_mbtiles/extract_mbtiles.py',
-        permissions='755',
         create_parent_dir=True,
     )
 
@@ -81,7 +80,6 @@ def prepare_tile_gen(c):
         c,
         SCRIPTS_DIR / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py',
         f'{TILE_GEN_BIN}/shrink_btrfs/shrink_btrfs.py',
-        permissions='755',
         create_parent_dir=True,
     )
 
@@ -105,6 +103,7 @@ def prepare_http_host(c):
 
     prepare_venv(c)
 
+    c.sudo('mkdir -p /data/ofm/http_host/logs_nginx')
     c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
 
     for file in [
@@ -118,33 +117,25 @@ def prepare_http_host(c):
             permissions='755',
         )
 
+    for file in ['nginx_site.conf', 'nginx_sync.py']:
+        put(
+            c,
+            SCRIPTS_DIR / 'http_host' / 'nginx_sync' / file,
+            f'{HTTP_HOST_BIN}/nginx_sync/{file}',
+            create_parent_dir=True,
+        )
+
     c.sudo('chown -R ofm:ofm /data/ofm/http_host')
+    c.sudo('chown -R nginx:nginx /data/ofm/http_host/logs_nginx')
 
 
 def debug_tmp(c):
-    # for file in [
+    for file in ['nginx_site.conf', 'nginx_sync.py']:
-    #     'extract_btrfs.sh',
-    #     'planetiler_monaco.sh',
-    #     'planetiler_planet.sh',
-    #     'cloudflare_index.sh',
-    #     'cloudflare_upload.sh',
-    # ]:
-    #     put(
-    #         c,
-    #         SCRIPTS_DIR / 'tile_gen' / file,
-    #         TILE_GEN_BIN,
-    #         permissions='755',
-    #     )
-
-    for file in [
-        'downloader.py',
-        'mounter.py',
-    ]:
         put(
             c,
-            SCRIPTS_DIR / 'http_host' / file,
+            SCRIPTS_DIR / 'http_host' / 'nginx_sync' / file,
-            HTTP_HOST_BIN,
+            f'{HTTP_HOST_BIN}/nginx_sync/{file}',
-            permissions='755',
+            create_parent_dir=True,
         )
 
 

scripts/http_host/downloader.py

@@ -21,6 +21,11 @@ DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
 )
 @click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
 def cli(area: str, version: str, list_versions: bool, runs_dir: Path):
+    """
+    Downloads and extracts the latest tiles.btrfs file from the public bucket.
+    Specific version can also be specified.
+    """
+
     if area not in {'planet', 'monaco'}:
         sys.exit('Please specify are: "planet" or "monaco"')
 

scripts/http_host/mounter.py

@@ -7,34 +7,28 @@ from pathlib import Path
 import click
 
 
-DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
-
-
 @click.command()
-@click.option(
+def cli():
-    '--runs-dir',
+    """
-    help='Specify /runs directory',
+    Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
-    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
+    When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
-)
+    """
-def cli(runs_dir: Path):
-    if os.geteuid() != 0:
-        sys.exit('Needs sudo')
-
-    if not runs_dir and not Path('/data/ofm').exists():
-        sys.exit('Please specify a runs dir with --runs-dir')
 
     if not Path('/etc/fstab').exists():
         sys.exit('Needs to be run on Linux')
 
-    if not runs_dir:
+    if os.geteuid() != 0:
-        runs_dir = DEFAULT_RUNS_DIR
+        sys.exit('Needs sudo')
+
+    if not Path('/data/ofm/http_host/runs').exists():
+        sys.exit('downloader.py needs to be run first')
 
     clean_up_mounts()
 
     fstab_new = []
 
     for area in ['planet', 'monaco']:
-        area_dir = (runs_dir / area).resolve()
+        area_dir = (Path('/data/ofm/http_host/runs') / area).resolve()
         if not area_dir.exists():
             continue
 

scripts/http_host/nginx_site.conf

@@ -1,21 +0,0 @@
-server {
-    server_name ofm tiles.openfreemaps.org;
-    # test with
-    # curl -H "Host: ofm" http://localhost/planet/20231208_091355/tiles/11/637/1141.pbf
-
-    #access_log /data/ofm/logs/nginx-access.log access_json;
-    access_log off;
-    error_log /data/ofm/logs/nginx-error.log;
-
-    # trailing / important
-    location /planet/20231208_091355/ {
-        alias /data/ofm/runs/planet_20231208_091355/mnt_rw/extract/; # trailing / important
-        try_files $uri @empty;
-    }
-
-    # we need to handle missing tiles as valid request returning empty string
-    location @empty {
-        default_type application/vnd.mapbox-vector-tile;
-        return 200 '';
-    }
-}

scripts/http_host/nginx_sync/nginx_site.conf

@@ -0,0 +1,19 @@
+server {
+    server_name ofm tiles.openfreemaps.org;
+
+    # disabling access log by default
+    # access_log /data/ofm/http_host/logs_nginx/nginx-access.log access_json buffer=32k;
+    access_log off;
+
+    error_log /data/ofm/http_host/logs_nginx/nginx-error.log;
+
+
+    ___LOCATION_BLOCKS___
+
+
+    # we need to handle missing tiles as valid request returning empty string
+    location @empty {
+        default_type application/vnd.mapbox-vector-tile;
+        return 200 '';
+    }
+}

scripts/http_host/nginx_sync/nginx_sync.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+
+
+@click.command()
+def cli():
+    if not Path('/etc/fstab').exists():
+        sys.exit('Needs to be run on Linux')
+
+    if os.geteuid() != 0:
+        sys.exit('Needs sudo')
+
+    if not Path('/mnt/ofm').exists():
+        sys.exit('mounter.py needs to be run first')
+
+    with open(Path(__file__).parent / 'nginx_site.conf') as fp:
+        nginx_template = fp.read()
+
+    location_block_str = ''
+    help_text = ''
+
+    for subdir in Path('/mnt/ofm').iterdir():
+        if not subdir.is_dir():
+            continue
+
+        area, version = subdir.name.split('-')
+
+        version_str = rf"""
+            location /{area}/{version}/ {{
+                alias {subdir};
+                try_files $uri @empty;
+            }}
+            """
+
+        location_block_str += version_str
+
+        if not help_text:
+            help_text = (
+                '\ntest with:\n'
+                f'curl -H "Host: ofm" -I http://localhost/{area}/{version}/tiles/14/8529/5975.pbf'
+            )
+
+    nginx_template = nginx_template.replace('___LOCATION_BLOCKS___', location_block_str)
+
+    with open('/data/nginx/sites/ofm.conf', 'w') as fp:
+        fp.write(nginx_template)
+        print('nginx config written')
+
+    subprocess.run(['nginx', '-t'], check=True)
+    subprocess.run(['service', 'nginx', 'reload'], check=True)
+
+    print(help_text)
+
+
+if __name__ == '__main__':
+    cli()

ssh_lib/benchmark.py

@@ -1,8 +1,11 @@
 from ssh_lib.config import SCRIPTS_DIR
-from ssh_lib.utils import apt_get_install, put
+from ssh_lib.utils import apt_get_install, exists, put
 
 
 def c1000k(c):
+    if exists(c, 'c1000k-master'):
+        return
+
     c.run('wget https://github.com/ideawu/c1000k/archive/master.zip -O tmp.zip')
     c.run('unzip -o tmp.zip')
     c.run('rm tmp.zip')