shrink btrfs

docs/extract.md

@@ -1,10 +1,10 @@
 ### native mapbox/mbutil
 
 Filesystem      1K-blocks       Used Available Use% Mounted on
-/dev/loop0     1,474,386,100 1,119,622,516 354,763,584  76% /data/ofm/runs/planet_20231208_091355/mnt
+/dev/loop0     1,474,386,100 1,119,622,516 354,763,584  76%
 
 Filesystem        Inodes     IUsed     IFree IUse% Mounted on
-/dev/loop0     393,216,000 269,252,174 123,963,826   69% /data/ofm/runs/planet_20231208_091355/mnt
+/dev/loop0     393,216,000 269,252,174 123,963,826   69%
 
 
 
@@ -14,29 +14,91 @@ Filesystem        Inodes     IUsed     IFree IUse% Mounted on
 
 df -h mnt
 Filesystem      Size  Used Avail Use% Mounted on
-/dev/loop0      1.4T  187G  1.2T  14% /data/ofm/runs/planet_20231208_091355/mnt
+/dev/loop0      1.4T  187G  1.2T  14%
 
 df mnt
 Filesystem      1K-blocks      Used  Available Use% Mounted on
-/dev/loop0     1474386100 195624664 1278761436  14% /data/ofm/runs/planet_20231208_091355/mnt
+/dev/loop0     1474386100 195624664 1278761436  14%
 
 df -i mnt
 Filesystem        Inodes    IUsed     IFree IUse% Mounted on
-/dev/loop0     393216000 39614466 353601534   11% /data/ofm/runs/planet_20231208_091355/mnt
+/dev/loop0     393216000 39614466 353601534   11%
 
 --- after resize2fs ext4
 
 df -h mnt
 Filesystem      Size  Used Avail Use% Mounted on
-/dev/loop0      189G  187G  2.4G  99% /data/ofm/runs/planet_20231208_091355/mnt
+/dev/loop0      189G  187G  2.4G  99%
 
 df mnt
 Filesystem     1K-blocks      Used Available Use% Mounted on
-/dev/loop0     198098376 195624664   2473712  99% /data/ofm/runs/planet_20231208_091355/mnt
+/dev/loop0     198098376 195624664   2473712  99%
 
 df -i mnt
 Filesystem       Inodes    IUsed    IFree IUse% Mounted on
-/dev/loop0     52854784 39614466 13240318   75% /data/ofm/runs/planet_20231208_091355/mnt
+/dev/loop0     52854784 39614466 13240318   75%
+
+
+### extract dedupl btrfs
+note: this test uses compress-force=lzo, but it's actually uncompressible data since the PBF files are gzipped already
+
+
+df -h mnt
+Filesystem      Size  Used Avail Use% Mounted on
+/dev/loop0      300G   97G  204G  33%
+
+df mnt
+Filesystem     1K-blocks      Used Available Use% Mounted on
+/dev/loop0     314572800 100925972 213428604  33%
+
+btrfs filesystem df mnt
+Data, single: total=48.01GiB, used=47.45GiB
+System, single: total=4.00MiB, used=16.00KiB
+Metadata, single: total=49.01GiB, used=48.32GiB
+GlobalReserve, single: total=496.86MiB, used=0.00B
+
+btrfs filesystem du -s mnt
+     Total   Exclusive  Set shared  Filename
+  47.45GiB    47.45GiB       0.00B  mnt
+
+
+sudo btrfs filesystem show mnt
+Label: none  uuid: ce7615d1-0ee5-460b-bdb0-7c4d214eecc4
+        Total devices 1 FS bytes used 95.76GiB
+        devid    1 size 300.00GiB used 97.02GiB path /dev/loop0
+
+sudo btrfs filesystem usage mnt
+Overall:
+    Device size:                 300.00GiB
+    Device allocated:             97.02GiB
+    Device unallocated:          202.98GiB
+    Device missing:                  0.00B
+    Used:                         95.76GiB
+    Free (estimated):            203.54GiB      (min: 203.54GiB)
+    Free (statfs, df):           203.54GiB
+    Data ratio:                       1.00
+    Metadata ratio:                   1.00
+    Global reserve:              501.22MiB      (used: 0.00B)
+    Multiple profiles:                  no
+
+Data,single: Size:48.01GiB, Used:47.45GiB (98.83%)
+   /dev/loop0     48.01GiB
+
+Metadata,single: Size:49.01GiB, Used:48.32GiB (98.60%)
+   /dev/loop0     49.01GiB
+
+System,single: Size:4.00MiB, Used:16.00KiB (0.39%)
+   /dev/loop0      4.00MiB
+
+Unallocated:
+   /dev/loop0    202.98GiB
 
 
 
+compsize -x mnt
+Processed 44249086 files, 3458702 regular extents (3800454 refs), 40448654 inline.
+Type       Perc     Disk Usage   Uncompressed Referenced
+TOTAL       99%       74G          74G          80G
+none       100%       74G          74G          80G
+lzo         20%      4.0K          20K          20K
+

scripts/mbtiles_extractor/extract.py

@@ -55,7 +55,7 @@ def write_dedupl_files(c, *, dir_path):
         dedupl_path.parent.mkdir(parents=True, exist_ok=True)
         with open(dedupl_path, 'wb') as fp:
             fp.write(row[1])
-            print(f'written dedupl file {i}/{total}: {dedupl_id}')
+            print(f'written dedupl file {i}/{total}')
 
 
 def write_tile_file(c, *, dir_path):

scripts/shrink_btrfs/shrink_btrfs.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import click
+
+
+@click.command()
+@click.argument(
+    'btrfs_img',
+    type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path),
+)
+def cli(btrfs_img: Path):
+    """
+    Shrinks a BTRFS image
+    // I cannot believe that BTRFS is over 15 years old,
+    // yet there is no resize2fs tool which can shrink a disk image
+    // to minimum size.
+    // It cannot even tell you how much should be the right size,
+    // it just randomly fails after which you have to umount and mount again.
+    // So we have to make a loop which tries to shrink it until it fails.
+
+    // Also, WONTFIX bugs like how instead of telling you that
+    // minimum fs size is 256 MB, it says "ERROR: unable to resize - Invalid argument"
+    // https://bugzilla.kernel.org/show_bug.cgi?id=118111
+    """
+
+    if os.geteuid() != 0:
+        sys.exit('Needs sudo')
+
+    current_dir = Path.cwd()
+
+    mnt_dir = Path(tempfile.mkdtemp(dir=current_dir, prefix='tmp_shrink_'))
+    subprocess.run(['mount', '-t', 'btrfs', btrfs_img, mnt_dir], check=True)
+
+    # needs to start with a balancing
+    # https://btrfs.readthedocs.io/en/latest/Balance.html
+    # https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html
+    print('Starting a balancing')
+    p = subprocess.run(
+        ['btrfs', 'balance', 'start', '-dusage=100', mnt_dir], capture_output=True, text=True
+    )
+    if p.returncode:
+        # subprocess.run(['umount', mnt_dir])
+        # mnt_dir.rmdir()
+        print(f'Balance error: {p.stdout} {p.stderr}')
+    print('Balancing done')
+
+    # shink until max. 10 MB left, or failure
+    free_bytes = get_usage(mnt_dir, 'Device unallocated')
+    while free_bytes > 10_000_000:
+        if not shrink(mnt_dir, int(free_bytes * 0.9)):
+            break
+        free_bytes = get_usage(mnt_dir, 'Device unallocated')
+
+    total_size = get_usage(mnt_dir, 'Device size')
+
+    subprocess.run(['umount', mnt_dir])
+    mnt_dir.rmdir()
+
+    subprocess.run(['truncate', '-s', str(total_size), btrfs_img])
+    print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size')
+
+
+def get_usage(mnt: Path, key: str):
+    p = subprocess.run(
+        ['btrfs', 'filesystem', 'usage', '-b', mnt], text=True, capture_output=True, check=True
+    )
+    for line in p.stdout.splitlines():
+        if f'{key}:' not in line:
+            continue
+        free = int(line.split(':')[1])
+        return free
+
+
+def shrink(mnt: Path, delta_size: int):
+    print(f'Trying to shrink by {delta_size//1_000_000} MB')
+    p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt])
+    return p.returncode == 0
+
+
+if __name__ == '__main__':
+    cli()

scripts/tile_gen/extract_btrfs.sh

@@ -9,17 +9,25 @@ rm -f image.btrfs
 # make sure it's bigger then the current OSM output
 fallocate -l 300G image.btrfs
 
+fallocate -l 10G image.btrfs
 
-mkfs.btrfs -v image.btrfs
+
+
+
+
+# metadata: single needed as default is now DUP
+mkfs.btrfs -v \
+  -m single \
+  image.btrfs
 
 # https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
-# compression: zstd:1 or lzo
+# compression doesn't make sense, data is already gzip compressed
 
-mkdir mnt
+mkdir -p mnt
 sudo mount -v \
   -t btrfs \
-  -o noacl,nobarrier,noatime,compress-force=lzo,max_inline=4096 \
+  -o noacl,nobarrier,noatime,max_inline=4096 \
-  image.btrfs mnt
+  image2.btrfs mnt
 
 
 sudo chown ofm:ofm -R mnt
@@ -27,7 +35,16 @@ sudo chown ofm:ofm -R mnt
 ../../tile_gen/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt/extract \
   > "extract_out.log" 2> "extract_err.log"
 
+
+# resize to min possible size
+btrfs filesystem usage -b mnt
+
+btrfs filesystem resize -10G mnt
+sudo btrfs filesystem resize -100M mnt
+
+
 sudo umount mnt
 
 
 
+