work

scripts/tile_gen/extract_btrfs.sh

@@ -1,11 +1,14 @@
 #!/usr/bin/env bash
 set -e
 
+export TILE_GEN_BIN=/data/ofm/tile_gen/bin
+export VENV_PYTHON=$TILE_GEN_BIN/venv/python
+
 sudo umount mnt_rw 2> /dev/null || true
 sudo umount mnt_rw2 2> /dev/null || true
 rm -rf mnt_rw* tmp_*
-rm -f *.btrfs *.gz
-rm -f *.log *.txt
+rm -f "*.btrfs" "*.gz"
+rm -f "*.log" "*.txt"
 
 # make an empty file that's definitely bigger then the current OSM output
 fallocate -l 200G image.btrfs
@@ -37,7 +40,8 @@ sudo mount \
 
 sudo chown ofm:ofm -R mnt_rw mnt_rw2
 
-../../tile_gen/venv/bin/python ../../tile_gen/extract_mbtiles.py output.mbtiles mnt_rw/extract \
+$VENV_PYTHON $TILE_GEN_BIN/extract_mbtiles/extract_mbtiles.py \
+  tiles.mbtiles mnt_rw/extract \
   > extract_out.log 2> extract_err.log
 
 grep fixed extract_out.log > dedupl_fixed.log || true
@@ -55,41 +59,23 @@ rsync -avH \
 {
 echo -e "df -h"
 sudo df -h mnt_rw
-
 echo -e "\n\nbtrfs filesystem df"
 sudo btrfs filesystem df mnt_rw
-
 echo -e "\n\nbtrfs filesystem show"
 sudo btrfs filesystem show mnt_rw
-
 echo -e "\n\nbtrfs filesystem usage"
 sudo btrfs filesystem usage mnt_rw
-
-# takes a lot of time, should only be used when debugging
-#echo -e "\n\nbtrfs filesystem du -s"
-#sudo btrfs filesystem du -s mnt_rw
-#echo -e "\n\ncompsize -x"
-#sudo compsize -x mnt_rw 2> /dev/null || true
 } > stats1.txt
 
 {
 echo -e "df -h"
 sudo df -h mnt_rw2
-
 echo -e "\n\nbtrfs filesystem df"
 sudo btrfs filesystem df mnt_rw2
-
 echo -e "\n\nbtrfs filesystem show"
 sudo btrfs filesystem show mnt_rw2
-
 echo -e "\n\nbtrfs filesystem usage"
 sudo btrfs filesystem usage mnt_rw2
-
-# takes a lot of time, should only be used when debugging
-#echo -e "\n\nbtrfs filesystem du -s"
-#sudo btrfs filesystem du -s mnt_rw2
-#echo -e "\n\ncompsize -x"
-#sudo compsize -x mnt_rw2 2> /dev/null || true
 } > stats2.txt
 
 
@@ -97,7 +83,7 @@ sudo umount mnt_rw
 sudo umount mnt_rw2
 rm -r mnt_rw*
 
-sudo ../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image2.btrfs \
+sudo $VENV_PYTHON $TILE_GEN_BIN/shrink_btrfs/shrink_btrfs.py image2.btrfs \
   > shrink_out.log 2> shrink_err.log
 
 

scripts/tile_gen/gen_monaco.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-DATE=$(date +"%Y%m%d_%H%M%S")
-
-RUN_FOLDER="/data/ofm/runs/monaco_$DATE"
-
-mkdir -p "$RUN_FOLDER"
-cd "$RUN_FOLDER" || exit
-
-bash /data/ofm/tile_gen/planetiler_monaco.sh "$DATE"
-

scripts/tile_gen/gen_planet.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-DATE=$(date +"%Y%m%d_%H%M%S")
-
-RUN_FOLDER="/data/ofm/runs/planet_$DATE"
-
-mkdir -p "$RUN_FOLDER"
-cd "$RUN_FOLDER" || exit
-
-bash /data/ofm/tile_gen/planetiler_planet.sh "$DATE"
-

scripts/tile_gen/planetiler_monaco.sh

@@ -1,22 +1,24 @@
 #!/usr/bin/env bash
 
-# the Xmx value below the most important parameter here
-# setting is less then 25g means there is too little memory
-# setting it to too much means there is too much memory used
+DATE=$(date +"%Y%m%d_%H%M%S")
+TILE_GEN_BIN=/data/ofm/tile_gen/bin
+
+RUN_FOLDER="/data/ofm/tile_gen/runs/monaco/${DATE}_pt"
+
+mkdir -p "$RUN_FOLDER"
+cd "$RUN_FOLDER" || exit
 
 java -Xmx1g \
-  -jar /data/ofm/tile_gen/planetiler.jar \
+  -jar $TILE_GEN_BIN/planetiler.jar \
   `# Download the latest osm.pbf from s3://osm-pds bucket` \
   --area=monaco --download \
   `# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \
   --download-threads=10 --download-chunk-size-mb=1000 \
   `# Also download name translations from wikidata` \
   --fetch-wikidata \
-  --output=output.mbtiles \
+  --output=tiles.mbtiles \
   `# Store temporary node locations at fixed positions in a memory-mapped file` \
   --nodemap-type=array --storage=mmap \
   --force \
   > "planetiler_out.log" 2> "planetiler_err.log"
 
-
-

scripts/tile_gen/planetiler_planet.sh

@@ -1,22 +1,26 @@
 #!/usr/bin/env bash
 
-# the Xmx value below the most important parameter here
-# setting is less then 25g means there is too little memory
-# setting it to too much means there is too much memory used
+DATE=$(date +"%Y%m%d_%H%M%S")
+TILE_GEN_BIN=/data/ofm/tile_gen/bin
 
+RUN_FOLDER="/data/ofm/tile_gen/runs/planet/${DATE}_pt"
+
+mkdir -p "$RUN_FOLDER"
+cd "$RUN_FOLDER" || exit
+
+# the Xmx value below the most important parameter here
+# 30 GB works well
 java -Xmx30g \
-  -jar /data/ofm/tile_gen/planetiler.jar \
+  -jar $TILE_GEN_BIN/planetiler.jar \
   `# Download the latest planet.osm.pbf from s3://osm-pds bucket` \
   --area=planet --bounds=planet --download \
   `# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \
   --download-threads=10 --download-chunk-size-mb=1000 \
   `# Also download name translations from wikidata` \
   --fetch-wikidata \
-  --output=output.mbtiles \
+  --output=tiles.mbtiles \
   `# Store temporary node locations at fixed positions in a memory-mapped file` \
   --nodemap-type=array --storage=mmap \
   --force \
   > "planetiler_out.log" 2> "planetiler_err.log"
 
-
-

scripts/tile_gen/prepare-virtualenv.sh

@@ -10,8 +10,6 @@ python3 -m venv venv
 venv/bin/pip -V
 
 venv/bin/pip install -U pip wheel setuptools
-venv/bin/pip install git+https://github.com/mapbox/mbutil.git@544c76e
-