diff --git a/init-server.py b/init-server.py index 1ab9124..a480684 100755 --- a/init-server.py +++ b/init-server.py @@ -65,6 +65,13 @@ def debug_tmp(c): permissions='755', owner='ofm', ) + put( + c, + scripts / 'shrink_btrfs' / 'shrink_btrfs.py', + TILE_GEN_BIN, + permissions='755', + owner='ofm', + ) return c.sudo('rm -rf /data/ofm/logs') diff --git a/scripts/extract_mbtiles/extract_mbtiles.py b/scripts/extract_mbtiles/extract_mbtiles.py index 8b5f13b..118ffb4 100755 --- a/scripts/extract_mbtiles/extract_mbtiles.py +++ b/scripts/extract_mbtiles/extract_mbtiles.py @@ -35,13 +35,11 @@ def cli(mbtiles_path: Path, dir_path: Path): write_dedupl_files(c, dir_path=dir_path) write_tile_files(c, dir_path=dir_path) - # remove dedupl files at the end - shutil.rmtree(dir_path / 'dedupl') - # if it's a full planet run, # make sure there are exactly the right number of files generated - if 'planet' in mbtiles_path.parent.name: + if 'planet' in mbtiles_path.resolve().parent.name: assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14) + print(f'Tile number: {calculate_tiles_sum(14)} - OK') print('DONE') diff --git a/scripts/shrink_btrfs/shrink_btrfs.py b/scripts/shrink_btrfs/shrink_btrfs.py index 1c62791..2732974 100755 --- a/scripts/shrink_btrfs/shrink_btrfs.py +++ b/scripts/shrink_btrfs/shrink_btrfs.py @@ -8,7 +8,7 @@ from pathlib import Path import click -# btrfs cannot shrink smaller than about 256 MiB +# btrfs cannot shrink smaller than 256 MiB SMALLEST_SIZE = 256 * 1024 * 1024 @@ -49,11 +49,11 @@ def cli(btrfs_img: Path): free_bytes = get_usage(mnt_dir, 'Device unallocated') device_size = get_usage(mnt_dir, 'Device size') - shrink_idea = free_bytes * 0.9 + shrink_idea = free_bytes * 0.7 # workaround for the SMALLEST_SIZE limit if device_size - free_bytes < SMALLEST_SIZE: - shrink_idea = (device_size - SMALLEST_SIZE) * 0.9 + shrink_idea = (device_size - SMALLEST_SIZE) * 0.7 # stop if 10 MB left if shrink_idea < 10_000_000: diff --git a/scripts/tile_gen/extract_btrfs.sh b/scripts/tile_gen/extract_btrfs.sh index 962ebfc..ba3490d 100644 --- a/scripts/tile_gen/extract_btrfs.sh +++ b/scripts/tile_gen/extract_btrfs.sh @@ -5,7 +5,7 @@ sudo umount mnt_rw 2> /dev/null || true sudo umount mnt_rw2 2> /dev/null || true rm -rf mnt_rw* tmp_* rm -f *.btrfs *.gz -rm -f *.log +rm -f *.log *.txt # make an empty file that's definitely bigger then the current OSM output fallocate -l 200G image.btrfs @@ -44,7 +44,10 @@ grep fixed extract_out.log > dedupl_fixed.log || true # Unfortunately, by deleting files from the btrfs partition, the size _grows_. # So we need to rsync onto a new partition. -rsync -avH mnt_rw/extract/ mnt_rw2/extract/ > rsync_out.log 2> rsync_err.log +rsync -avH \ + --exclude dedupl \ + mnt_rw/extract/ mnt_rw2/extract/ \ + > rsync_out.log 2> rsync_err.log # collect stats @@ -61,11 +64,11 @@ sudo btrfs filesystem show mnt_rw echo -e "\n\nbtrfs filesystem usage" sudo btrfs filesystem usage mnt_rw -echo -e "\n\nbtrfs filesystem du -s" -sudo btrfs filesystem du -s mnt_rw - -echo -e "\n\ncompsize -x" -sudo compsize -x mnt_rw 2> /dev/null || true +# takes a lot of time, should only be used when debugging +#echo -e "\n\nbtrfs filesystem du -s" +#sudo btrfs filesystem du -s mnt_rw +#echo -e "\n\ncompsize -x" +#sudo compsize -x mnt_rw 2> /dev/null || true } > stats1.txt { @@ -81,11 +84,11 @@ sudo btrfs filesystem show mnt_rw2 echo -e "\n\nbtrfs filesystem usage" sudo btrfs filesystem usage mnt_rw2 -echo -e "\n\nbtrfs filesystem du -s" -sudo btrfs filesystem du -s mnt_rw2 - -echo -e "\n\ncompsize -x" -sudo compsize -x mnt_rw2 2> /dev/null || true +# takes a lot of time, should only be used when debugging +#echo -e "\n\nbtrfs filesystem du -s" +#sudo btrfs filesystem du -s mnt_rw2 +#echo -e "\n\ncompsize -x" +#sudo compsize -x mnt_rw2 2> /dev/null || true } > stats2.txt