From 6d7f666255fed684e74bea01e9fbe19df3c824fa Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Sun, 10 Dec 2023 06:42:09 +0100 Subject: [PATCH] shrink btrfs --- docs/extract.md | 78 ++++++++++++++++++++++--- scripts/mbtiles_extractor/extract.py | 2 +- scripts/shrink_btrfs/shrink_btrfs.py | 86 ++++++++++++++++++++++++++++ scripts/tile_gen/extract_btrfs.sh | 27 +++++++-- 4 files changed, 179 insertions(+), 14 deletions(-) create mode 100755 scripts/shrink_btrfs/shrink_btrfs.py diff --git a/docs/extract.md b/docs/extract.md index 0a9e442..cb7c3bb 100644 --- a/docs/extract.md +++ b/docs/extract.md @@ -1,10 +1,10 @@ ### native mapbox/mbutil Filesystem 1K-blocks Used Available Use% Mounted on -/dev/loop0 1,474,386,100 1,119,622,516 354,763,584 76% /data/ofm/runs/planet_20231208_091355/mnt +/dev/loop0 1,474,386,100 1,119,622,516 354,763,584 76% Filesystem Inodes IUsed IFree IUse% Mounted on -/dev/loop0 393,216,000 269,252,174 123,963,826 69% /data/ofm/runs/planet_20231208_091355/mnt +/dev/loop0 393,216,000 269,252,174 123,963,826 69% @@ -14,29 +14,91 @@ Filesystem Inodes IUsed IFree IUse% Mounted on df -h mnt Filesystem Size Used Avail Use% Mounted on -/dev/loop0 1.4T 187G 1.2T 14% /data/ofm/runs/planet_20231208_091355/mnt +/dev/loop0 1.4T 187G 1.2T 14% df mnt Filesystem 1K-blocks Used Available Use% Mounted on -/dev/loop0 1474386100 195624664 1278761436 14% /data/ofm/runs/planet_20231208_091355/mnt +/dev/loop0 1474386100 195624664 1278761436 14% df -i mnt Filesystem Inodes IUsed IFree IUse% Mounted on -/dev/loop0 393216000 39614466 353601534 11% /data/ofm/runs/planet_20231208_091355/mnt +/dev/loop0 393216000 39614466 353601534 11% --- after resize2fs ext4 df -h mnt Filesystem Size Used Avail Use% Mounted on -/dev/loop0 189G 187G 2.4G 99% /data/ofm/runs/planet_20231208_091355/mnt +/dev/loop0 189G 187G 2.4G 99% df mnt Filesystem 1K-blocks Used Available Use% Mounted on -/dev/loop0 198098376 195624664 2473712 99% /data/ofm/runs/planet_20231208_091355/mnt +/dev/loop0 198098376 195624664 2473712 99% df -i mnt Filesystem Inodes IUsed IFree IUse% Mounted on -/dev/loop0 52854784 39614466 13240318 75% /data/ofm/runs/planet_20231208_091355/mnt +/dev/loop0 52854784 39614466 13240318 75% + + +### extract dedupl btrfs +note: this test uses compress-force=lzo, but it's actually uncompressible data since the PBF files are gzipped already + + +df -h mnt +Filesystem Size Used Avail Use% Mounted on +/dev/loop0 300G 97G 204G 33% + +df mnt +Filesystem 1K-blocks Used Available Use% Mounted on +/dev/loop0 314572800 100925972 213428604 33% + +btrfs filesystem df mnt +Data, single: total=48.01GiB, used=47.45GiB +System, single: total=4.00MiB, used=16.00KiB +Metadata, single: total=49.01GiB, used=48.32GiB +GlobalReserve, single: total=496.86MiB, used=0.00B + +btrfs filesystem du -s mnt + Total Exclusive Set shared Filename + 47.45GiB 47.45GiB 0.00B mnt + + +sudo btrfs filesystem show mnt +Label: none uuid: ce7615d1-0ee5-460b-bdb0-7c4d214eecc4 + Total devices 1 FS bytes used 95.76GiB + devid 1 size 300.00GiB used 97.02GiB path /dev/loop0 + +sudo btrfs filesystem usage mnt +Overall: + Device size: 300.00GiB + Device allocated: 97.02GiB + Device unallocated: 202.98GiB + Device missing: 0.00B + Used: 95.76GiB + Free (estimated): 203.54GiB (min: 203.54GiB) + Free (statfs, df): 203.54GiB + Data ratio: 1.00 + Metadata ratio: 1.00 + Global reserve: 501.22MiB (used: 0.00B) + Multiple profiles: no + +Data,single: Size:48.01GiB, Used:47.45GiB (98.83%) + /dev/loop0 48.01GiB + +Metadata,single: Size:49.01GiB, Used:48.32GiB (98.60%) + /dev/loop0 49.01GiB + +System,single: Size:4.00MiB, Used:16.00KiB (0.39%) + /dev/loop0 4.00MiB + +Unallocated: + /dev/loop0 202.98GiB +compsize -x mnt +Processed 44249086 files, 3458702 regular extents (3800454 refs), 40448654 inline. +Type Perc Disk Usage Uncompressed Referenced +TOTAL 99% 74G 74G 80G +none 100% 74G 74G 80G +lzo 20% 4.0K 20K 20K + diff --git a/scripts/mbtiles_extractor/extract.py b/scripts/mbtiles_extractor/extract.py index 21201e8..23a0612 100755 --- a/scripts/mbtiles_extractor/extract.py +++ b/scripts/mbtiles_extractor/extract.py @@ -55,7 +55,7 @@ def write_dedupl_files(c, *, dir_path): dedupl_path.parent.mkdir(parents=True, exist_ok=True) with open(dedupl_path, 'wb') as fp: fp.write(row[1]) - print(f'written dedupl file {i}/{total}: {dedupl_id}') + print(f'written dedupl file {i}/{total}') def write_tile_file(c, *, dir_path): diff --git a/scripts/shrink_btrfs/shrink_btrfs.py b/scripts/shrink_btrfs/shrink_btrfs.py new file mode 100755 index 0000000..32a2cc3 --- /dev/null +++ b/scripts/shrink_btrfs/shrink_btrfs.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +import click + + +@click.command() +@click.argument( + 'btrfs_img', + type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path), +) +def cli(btrfs_img: Path): + """ + Shrinks a BTRFS image + // I cannot believe that BTRFS is over 15 years old, + // yet there is no resize2fs tool which can shrink a disk image + // to minimum size. + // It cannot even tell you how much should be the right size, + // it just randomly fails after which you have to umount and mount again. + // So we have to make a loop which tries to shrink it until it fails. + + // Also, WONTFIX bugs like how instead of telling you that + // minimum fs size is 256 MB, it says "ERROR: unable to resize - Invalid argument" + // https://bugzilla.kernel.org/show_bug.cgi?id=118111 + """ + + if os.geteuid() != 0: + sys.exit('Needs sudo') + + current_dir = Path.cwd() + + mnt_dir = Path(tempfile.mkdtemp(dir=current_dir, prefix='tmp_shrink_')) + subprocess.run(['mount', '-t', 'btrfs', btrfs_img, mnt_dir], check=True) + + # needs to start with a balancing + # https://btrfs.readthedocs.io/en/latest/Balance.html + # https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html + print('Starting a balancing') + p = subprocess.run( + ['btrfs', 'balance', 'start', '-dusage=100', mnt_dir], capture_output=True, text=True + ) + if p.returncode: + # subprocess.run(['umount', mnt_dir]) + # mnt_dir.rmdir() + print(f'Balance error: {p.stdout} {p.stderr}') + print('Balancing done') + + # shink until max. 10 MB left, or failure + free_bytes = get_usage(mnt_dir, 'Device unallocated') + while free_bytes > 10_000_000: + if not shrink(mnt_dir, int(free_bytes * 0.9)): + break + free_bytes = get_usage(mnt_dir, 'Device unallocated') + + total_size = get_usage(mnt_dir, 'Device size') + + subprocess.run(['umount', mnt_dir]) + mnt_dir.rmdir() + + subprocess.run(['truncate', '-s', str(total_size), btrfs_img]) + print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size') + + +def get_usage(mnt: Path, key: str): + p = subprocess.run( + ['btrfs', 'filesystem', 'usage', '-b', mnt], text=True, capture_output=True, check=True + ) + for line in p.stdout.splitlines(): + if f'{key}:' not in line: + continue + free = int(line.split(':')[1]) + return free + + +def shrink(mnt: Path, delta_size: int): + print(f'Trying to shrink by {delta_size//1_000_000} MB') + p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt]) + return p.returncode == 0 + + +if __name__ == '__main__': + cli() diff --git a/scripts/tile_gen/extract_btrfs.sh b/scripts/tile_gen/extract_btrfs.sh index f9190a5..f4cdbb1 100644 --- a/scripts/tile_gen/extract_btrfs.sh +++ b/scripts/tile_gen/extract_btrfs.sh @@ -9,17 +9,25 @@ rm -f image.btrfs # make sure it's bigger then the current OSM output fallocate -l 300G image.btrfs +fallocate -l 10G image.btrfs -mkfs.btrfs -v image.btrfs + + + + +# metadata: single needed as default is now DUP +mkfs.btrfs -v \ + -m single \ + image.btrfs # https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options -# compression: zstd:1 or lzo +# compression doesn't make sense, data is already gzip compressed -mkdir mnt +mkdir -p mnt sudo mount -v \ -t btrfs \ - -o noacl,nobarrier,noatime,compress-force=lzo,max_inline=4096 \ - image.btrfs mnt + -o noacl,nobarrier,noatime,max_inline=4096 \ + image2.btrfs mnt sudo chown ofm:ofm -R mnt @@ -27,7 +35,16 @@ sudo chown ofm:ofm -R mnt ../../tile_gen/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt/extract \ > "extract_out.log" 2> "extract_err.log" + +# resize to min possible size +btrfs filesystem usage -b mnt + +btrfs filesystem resize -10G mnt +sudo btrfs filesystem resize -100M mnt + + sudo umount mnt +