btrfs work

docs/extract.md

@@ -8,7 +8,35 @@ Filesystem        Inodes     IUsed     IFree IUse% Mounted on
 
 
 
-### extract dedupl
+### extract dedupl ext4
 
 39,570,683 dedupl files
 
+df -h mnt
+Filesystem      Size  Used Avail Use% Mounted on
+/dev/loop0      1.4T  187G  1.2T  14% /data/ofm/runs/planet_20231208_091355/mnt
+
+df mnt
+Filesystem      1K-blocks      Used  Available Use% Mounted on
+/dev/loop0     1474386100 195624664 1278761436  14% /data/ofm/runs/planet_20231208_091355/mnt
+
+df -i mnt
+Filesystem        Inodes    IUsed     IFree IUse% Mounted on
+/dev/loop0     393216000 39614466 353601534   11% /data/ofm/runs/planet_20231208_091355/mnt
+
+--- after resize2fs ext4
+
+df -h mnt
+Filesystem      Size  Used Avail Use% Mounted on
+/dev/loop0      189G  187G  2.4G  99% /data/ofm/runs/planet_20231208_091355/mnt
+
+df mnt
+Filesystem     1K-blocks      Used Available Use% Mounted on
+/dev/loop0     198098376 195624664   2473712  99% /data/ofm/runs/planet_20231208_091355/mnt
+
+df -i mnt
+Filesystem       Inodes    IUsed    IFree IUse% Mounted on
+/dev/loop0     52854784 39614466 13240318   75% /data/ofm/runs/planet_20231208_091355/mnt
+
+
+

scripts/tile_gen/extract_btrfs.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+sudo umount mnt || true
+rm -rf mnt
+rm -f image.btrfs
+
+
+# make a sparse file
+# make sure it's bigger then the current OSM output
+fallocate -l 300G image.btrfs
+
+
+mkfs.btrfs -v image.btrfs
+
+# https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
+# compression: zstd:1 or lzo
+
+mkdir mnt
+sudo mount -v \
+  -t btrfs \
+  -o noacl,nobarrier,noatime,compress-force=lzo,max_inline=4096 \
+  image.btrfs mnt
+
+
+sudo chown ofm:ofm -R mnt
+
+../../tile_gen/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt/extract \
+  > "extract_out.log" 2> "extract_err.log"
+
+sudo umount mnt
+
+
+

scripts/tile_gen/extract_ext4.sh

@@ -26,9 +26,7 @@ rm -f image.ext4
 
 # make a sparse file
 # make sure it's bigger then the current OSM output
-# less fragmentation with fallocate
+fallocate -l 300G image.ext4
-fallocate -l 1500G image.ext4
-#truncate -s 1500G image.ext4
 
 
 mke2fs -t ext4 -v \
@@ -52,17 +50,14 @@ sudo chown ofm:ofm -R mnt
 
 sudo umount mnt
 
-resize2fs -M image.ext4
+e2fsck -vf image.ext4 && \
-e2fsck -vf image.ext4
+  resize2fs -M image.ext4 && \
+  e2fsck -vf image.ext4
+
+# default to read-only mode
+tune2fs -E mount_opts=ro image.ext4
+
 
-#fsck.ext4 -f -C 0 /dev/sda1;
-#
-#filefrag -e image.ext4
-#
-#tune2fs -E mount_opts=ro image.ext4
-#
-#cp --sparse=never sparsefile regularfile
-#