work

scripts/extract_mbtiles/extract_mbtiles.py

@@ -43,6 +43,8 @@ def cli(mbtiles_path: Path, dir_path: Path):
     if 'planet' in mbtiles_path.parent.name:
         assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14)
 
+    print('DONE')
+
 
 def write_metadata(c, *, dir_path):
     metadata = dict(c.execute('select name, value from metadata').fetchall())
@@ -72,9 +74,6 @@ def write_tile_files(c, *, dir_path):
 
     c.execute('select zoom_level, tile_column, tile_row, tile_data_id from tiles_shallow')
     for i, row in enumerate(c, start=1):
-        if i < 4678400:
-            continue
-
         z = row[0]
         x = row[1]
         y = flip_y(z, row[2])

scripts/shrink_btrfs/shrink_btrfs.py

@@ -40,25 +40,28 @@ def cli(btrfs_img: Path):
     mnt_dir = Path(tempfile.mkdtemp(dir=current_dir, prefix='tmp_shrink_'))
     subprocess.run(['mount', '-t', 'btrfs', btrfs_img, mnt_dir], check=True)
 
-    # needs to start with a balancing
+    # shink until max. 10 MB left or reached SMALLEST_SIZE or failure
-    # https://btrfs.readthedocs.io/en/latest/Balance.html
+    while True:
-    # https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html
+        # needs to start with a balancing
-    print('Starting btrfs balancing')
+        # https://btrfs.readthedocs.io/en/latest/Balance.html
-    p = subprocess.run(
+        # https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html
-        ['btrfs', 'balance', 'start', '-dusage=100', mnt_dir], capture_output=True, text=True
+        do_balancing(mnt_dir)
-    )
-    if p.returncode:
-        # subprocess.run(['umount', mnt_dir])
-        # mnt_dir.rmdir()
-        print(f'Balance error: {p.stdout} {p.stderr}')
-    print('Balancing done')
 
-    # shink until max. 10 MB left, or failure
-    free_bytes = get_usage(mnt_dir, 'Device unallocated')
-    while free_bytes > 10_000_000:
-        if not shrink(mnt_dir, int(free_bytes * 0.9)):
-            break
         free_bytes = get_usage(mnt_dir, 'Device unallocated')
+        device_size = get_usage(mnt_dir, 'Device size')
+        shrink_idea = free_bytes * 0.9
+
+        # workaround for the SMALLEST_SIZE limit
+        if device_size - free_bytes < SMALLEST_SIZE:
+            shrink_idea = (device_size - SMALLEST_SIZE) * 0.9
+
+        # stop if 10 MB left
+        if shrink_idea < 10_000_000:
+            break
+
+        # stop if process error
+        if not do_shrink(mnt_dir, shrink_idea):
+            break
 
     total_size = get_usage(mnt_dir, 'Device size')
 
@@ -80,11 +83,22 @@ def get_usage(mnt: Path, key: str):
         return free
 
 
-def shrink(mnt: Path, delta_size: int):
+def do_shrink(mnt: Path, delta_size: float):
+    delta_size = int(delta_size)
     print(f'Trying to shrink by {delta_size//1_000_000} MB')
     p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt])
     return p.returncode == 0
 
 
+def do_balancing(mnt: Path):
+    print('Starting btrfs balancing')
+    p = subprocess.run(
+        ['btrfs', 'balance', 'start', '-dusage=100', mnt], capture_output=True, text=True
+    )
+    if p.returncode:
+        print(f'Balance error: {p.stdout} {p.stderr}')
+    print('Balancing done')
+
+
 if __name__ == '__main__':
     cli()

scripts/tile_gen/extract_btrfs.sh

@@ -3,8 +3,8 @@ set -e
 
 sudo umount mnt_rw || true
 sudo umount mnt_rw2 || true
-rm -rf mnt_rw*
+rm -rf mnt_rw* tmp_*
-rm -f image*.btrfs
+rm -f *.btrfs *.gz
 rm -f *.log
 
 # make an empty file that's definitely bigger then the current OSM output
@@ -33,16 +33,42 @@ sudo mount -v \
 sudo mount -v \
   -t btrfs \
   -o noacl,nobarrier,noatime,max_inline=4096 \
-  image.btrfs mnt_rw2
+  image2.btrfs mnt_rw2
 
 sudo chown ofm:ofm -R mnt_rw mnt_rw2
 
 ../../tile_gen/venv/bin/python ../../tile_gen/extract_mbtiles.py output.mbtiles mnt_rw/extract \
   > extract_out.log 2> extract_err.log
 
-# we need to extract, delete and rsync onto a new partition
+grep fixed extract_out.log > dedupl_fixed.log || true
+
+# we need to extract, delete dedupl and rsync onto a new partition
 # otherwise the partition image stays big
-rsync -aH mnt_rw/extract/ mnt_rw2/extract/ > rsync_out.log 2> rsync_err.log
+rsync -avH mnt_rw/extract/ mnt_rw2/extract/ > rsync_out.log 2> rsync_err.log
+
+
+# collect stats
+{
+echo -e "df -h"
+df -h mnt_rw2
+
+echo -e "\n\nbtrfs filesystem df"
+btrfs filesystem df mnt_rw2
+
+echo -e "\n\nbtrfs filesystem du -s"
+btrfs filesystem du -s mnt_rw2
+
+echo -e "\n\nbtrfs filesystem show"
+btrfs filesystem show mnt_rw2
+
+echo -e "\n\nbtrfs filesystem usage"
+btrfs filesystem usage mnt_rw2
+
+echo -e "\n\ncompsize -x"
+compsize -x mnt_rw2
+} > stats.txt
+
+
 
 sudo umount mnt_rw
 sudo umount mnt_rw2
@@ -52,9 +78,12 @@ sudo ../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image2.btrfs
   > shrink_out.log 2> shrink_err.log
 
 
-#rm image.btrfs
 
-#mv image2.btrfs done.btrfs
 
-# pigz -k image.btrfs --fast
 
+rm image.btrfs
+mv image2.btrfs done.btrfs
+
+pigz done.btrfs --fast
+
+echo DONE