work

init-server.py

@@ -84,15 +84,19 @@ def prepare_http_host(c):
 
 
 def debug_tmp(c):
-    c.sudo('rm -rf /data/ofm/logs')
-    c.sudo('mkdir -p /data/ofm/logs')
-    c.sudo('rm -f /data/nginx/logs/*')
-    put(c, f'{ASSETS_DIR}/nginx/nginx.conf', '/etc/nginx/')
-    put(c, f'{SCRIPTS_DIR}/http_host/nginx_site.conf', '/data/nginx/sites')
-    c.sudo('nginx -t')
-    c.sudo('service nginx restart')
-
-    benchmark(c)
+    for file in [
+        'extract_btrfs.sh',
+        'planetiler_monaco.sh',
+        'planetiler_planet.sh',
+        'prepare-virtualenv.sh',
+        'upload_cloudflare.sh',
+    ]:
+        put(
+            c,
+            SCRIPTS_DIR / 'tile_gen' / file,
+            TILE_GEN_BIN,
+            permissions='755',
+        )
 
 
 @click.command()

scripts/tile_gen/extract_btrfs.sh

@@ -8,7 +8,7 @@ sudo umount mnt_rw 2> /dev/null || true
 sudo umount mnt_rw2 2> /dev/null || true
 rm -rf mnt_rw* tmp_*
 rm -f -- *.btrfs *.gz
-rm -f -- *.log *.txt
+rm -rf -- *.log *.txt logs
 
 # make an empty file that's definitely bigger then the current OSM output
 fallocate -l 200G image.btrfs
@@ -44,6 +44,8 @@ $VENV_PYTHON $TILE_GEN_BIN/extract_mbtiles/extract_mbtiles.py \
   tiles.mbtiles mnt_rw/extract \
   > extract_out.log 2> extract_err.log
 
+cp mnt_rw/extract/osm_date .
+
 grep fixed extract_out.log > dedupl_fixed.log || true
 
 # Unfortunately, by deleting files from the btrfs partition, the size _grows_.
@@ -79,6 +81,7 @@ sudo btrfs filesystem usage mnt_rw2
 } > stats2.txt
 
 
+
 sudo umount mnt_rw
 sudo umount mnt_rw2
 rm -r mnt_rw*
@@ -92,4 +95,8 @@ mv image2.btrfs tiles.btrfs
 
 pigz tiles.btrfs --fast
 
+mkdir -p logs
+mv -- *.log logs
+mv -- *.txt logs
+
 echo extract_btrfs.sh DONE

scripts/tile_gen/planetiler_monaco.sh

@@ -3,8 +3,9 @@ set -e
 
 TILE_GEN_BIN=/data/ofm/tile_gen/bin
 
+AREA=monaco
 DATE=$(date +"%Y%m%d_%H%M%S")
-RUN_FOLDER="/data/ofm/tile_gen/runs/monaco/${DATE}_pt"
+RUN_FOLDER="/data/ofm/tile_gen/runs/$AREA/${DATE}_pt"
 
 
 mkdir -p "$RUN_FOLDER"
@@ -13,7 +14,7 @@ cd "$RUN_FOLDER" || exit
 java -Xmx1g \
   -jar $TILE_GEN_BIN/planetiler.jar \
   `# Download the latest osm.pbf from s3://osm-pds bucket` \
-  --area=monaco --download \
+  --area=$AREA --download \
   `# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \
   --download-threads=10 --download-chunk-size-mb=1000 \
   `# Also download name translations from wikidata` \
@@ -22,7 +23,7 @@ java -Xmx1g \
   `# Store temporary node locations at fixed positions in a memory-mapped file` \
   --nodemap-type=array --storage=mmap \
   --force \
-  > planetiler_out 2> planetiler_err
+  > planetiler.out 2> planetiler.err
 
 rm -r data
 echo planetiler.jar DONE

scripts/tile_gen/planetiler_planet.sh

@@ -3,8 +3,9 @@ set -e
 
 TILE_GEN_BIN=/data/ofm/tile_gen/bin
 
+AREA=planet
 DATE=$(date +"%Y%m%d_%H%M%S")
-RUN_FOLDER="/data/ofm/tile_gen/runs/planet/${DATE}_pt"
+RUN_FOLDER="/data/ofm/tile_gen/runs/$AREA/${DATE}_pt"
 
 
 mkdir -p "$RUN_FOLDER"
@@ -24,7 +25,7 @@ java -Xmx30g \
   `# Store temporary node locations at fixed positions in a memory-mapped file` \
   --nodemap-type=array --storage=mmap \
   --force \
-  > planetiler_out 2> planetiler_err
+  > planetiler.out 2> planetiler.err
 
 rm -r data
 echo planetiler.jar DONE