mirror of
https://github.com/hyperknot/openfreemap.git
synced 2026-05-21 14:02:15 +00:00
readme
This commit is contained in:
Zsolt Ero
5
docs/quick_notes/_not_used/ext4-ideas.txt
Normal file
5
docs/quick_notes/_not_used/ext4-ideas.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
ext4
|
||||
|
||||
Reduce the inode size: 256 -> -I 128
|
||||
|
||||
Reduce the inode ratio: The inode ratio is the number of blocks per inode. You can increase the inode ratio to reduce the number of inodes created. This can be done with the -i option when creating the filesystem. For example, -i 8192 will create one inode every 8192 blocks.
|
||||
67
docs/quick_notes/_not_used/extract_ext4.sh
Normal file
67
docs/quick_notes/_not_used/extract_ext4.sh
Normal file
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# reference:
|
||||
# https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
|
||||
# https://wiki.archlinux.org/title/ext4
|
||||
#
|
||||
# -m reserved-blocks-percentage
|
||||
# -F Force mke2fs to create a filesystem, even if the specified device is not a partition on a block special device
|
||||
#
|
||||
# -O feature
|
||||
# from /etc/mke2fs.conf
|
||||
# defaults: has_journal,extent,huge_file,flex_bg,metadata_csum,64bit,dir_nlink,extra_isize
|
||||
# disabling journalling, since it's a read-only fs, as well as other unused features
|
||||
# extent is actually needed for tail packing small files
|
||||
#
|
||||
# -E extended-options
|
||||
# lazy_itable_init - inode table is fully initialized at the time of file system creation
|
||||
# nodiscard - Do not attempt to discard blocks at mkfs time.
|
||||
#
|
||||
# inode_size = 128 (minimum)
|
||||
# inode_ratio = 16384 (default but experimenting)
|
||||
|
||||
|
||||
sudo umount mnt || true
|
||||
rm -rf mnt
|
||||
rm -f image.ext4
|
||||
|
||||
|
||||
# make a sparse file
|
||||
# make sure it's bigger then the current OSM output
|
||||
fallocate -l 200G image.ext4
|
||||
|
||||
|
||||
mke2fs -t ext4 -v \
|
||||
-m 0 \
|
||||
-F \
|
||||
-O ^has_journal,^huge_file,^metadata_csum,^64bit,^extra_isize \
|
||||
-E lazy_itable_init=0,nodiscard \
|
||||
-I 128 \
|
||||
-i 16384 \
|
||||
image.ext4
|
||||
|
||||
mkdir mnt
|
||||
sudo mount -v \
|
||||
-t ext4 \
|
||||
-o nobarrier,noatime \
|
||||
image.ext4 mnt
|
||||
|
||||
sudo chown ofm:ofm -R mnt
|
||||
|
||||
/data/ofm/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt/extract \
|
||||
> "extract_out.log" 2> "extract_err.log"
|
||||
|
||||
sudo umount mnt
|
||||
|
||||
e2fsck -vf image.ext4 && \
|
||||
resize2fs -M image.ext4 && \
|
||||
e2fsck -vf image.ext4
|
||||
|
||||
# default to read-only mode
|
||||
tune2fs -E mount_opts=ro image.ext4
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
13
docs/quick_notes/_not_used/loop_test.sh
Normal file
13
docs/quick_notes/_not_used/loop_test.sh
Normal file
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Define the source folder
|
||||
source_folder="20231228_201550_pt"
|
||||
|
||||
# Define the number of copies you want to make
|
||||
number_of_copies=40
|
||||
|
||||
# Loop and copy the folder into c1, c2, c3, c4, ...
|
||||
for i in $(seq 1 $number_of_copies); do
|
||||
cp -r "$source_folder" "c$i"
|
||||
btrfstune -m "c$i/tiles.btrfs"
|
||||
done
|
||||
5
docs/quick_notes/fs_stats/README.md
Normal file
5
docs/quick_notes/fs_stats/README.md
Normal file
@@ -0,0 +1,5 @@
|
||||
# Comparing filesystem stats after extraction
|
||||
|
||||
Run: planet/20231208_091355_pt
|
||||
|
||||
dedupl-fix.log contains the files which were created for the btrfs 64k limit workaround.
|
||||
44
docs/quick_notes/fs_stats/btrfs_after_delete.txt
Normal file
44
docs/quick_notes/fs_stats/btrfs_after_delete.txt
Normal file
@@ -0,0 +1,44 @@
|
||||
df -h
|
||||
Filesystem Size Used Avail Use% Mounted on
|
||||
/dev/loop1 200G 130G 70G 66% /data/ofm/tile_gen/runs/planet/20231221_134737_pt/mnt_rw2
|
||||
|
||||
|
||||
btrfs filesystem df
|
||||
Data, single: total=48.01GiB, used=47.54GiB
|
||||
System, single: total=4.00MiB, used=16.00KiB
|
||||
Metadata, single: total=83.01GiB, used=81.42GiB
|
||||
GlobalReserve, single: total=512.00MiB, used=0.00B
|
||||
|
||||
|
||||
btrfs filesystem show
|
||||
Label: none uuid: b9e8ae3a-aedb-4221-ab08-e1ad6095f188
|
||||
Total devices 1 FS bytes used 128.96GiB
|
||||
devid 1 size 200.00GiB used 131.02GiB path /dev/loop1
|
||||
|
||||
|
||||
|
||||
btrfs filesystem usage
|
||||
Overall:
|
||||
Device size: 200.00GiB
|
||||
Device allocated: 131.02GiB
|
||||
Device unallocated: 68.98GiB
|
||||
Device missing: 0.00B
|
||||
Used: 128.96GiB
|
||||
Free (estimated): 69.45GiB (min: 69.45GiB)
|
||||
Free (statfs, df): 69.44GiB
|
||||
Data ratio: 1.00
|
||||
Metadata ratio: 1.00
|
||||
Global reserve: 512.00MiB (used: 0.00B)
|
||||
Multiple profiles: no
|
||||
|
||||
Data,single: Size:48.01GiB, Used:47.54GiB (99.03%)
|
||||
/dev/loop1 48.01GiB
|
||||
|
||||
Metadata,single: Size:83.01GiB, Used:81.42GiB (98.09%)
|
||||
/dev/loop1 83.01GiB
|
||||
|
||||
System,single: Size:4.00MiB, Used:16.00KiB (0.39%)
|
||||
/dev/loop1 4.00MiB
|
||||
|
||||
Unallocated:
|
||||
/dev/loop1 68.98GiB
|
||||
44
docs/quick_notes/fs_stats/btrfs_before_delete.txt
Normal file
44
docs/quick_notes/fs_stats/btrfs_before_delete.txt
Normal file
@@ -0,0 +1,44 @@
|
||||
df -h
|
||||
Filesystem Size Used Avail Use% Mounted on
|
||||
/dev/loop0 200G 139G 61G 70% /data/ofm/tile_gen/runs/planet/20231221_134737_pt/mnt_rw
|
||||
|
||||
|
||||
btrfs filesystem df
|
||||
Data, single: total=48.01GiB, used=47.59GiB
|
||||
System, single: total=4.00MiB, used=16.00KiB
|
||||
Metadata, single: total=92.01GiB, used=90.52GiB
|
||||
GlobalReserve, single: total=512.00MiB, used=0.00B
|
||||
|
||||
|
||||
btrfs filesystem show
|
||||
Label: none uuid: 9c5cd306-96c8-4ee5-bfbb-b8216698f955
|
||||
Total devices 1 FS bytes used 138.11GiB
|
||||
devid 1 size 200.00GiB used 140.02GiB path /dev/loop0
|
||||
|
||||
|
||||
|
||||
btrfs filesystem usage
|
||||
Overall:
|
||||
Device size: 200.00GiB
|
||||
Device allocated: 140.02GiB
|
||||
Device unallocated: 59.98GiB
|
||||
Device missing: 0.00B
|
||||
Used: 138.11GiB
|
||||
Free (estimated): 60.40GiB (min: 60.40GiB)
|
||||
Free (statfs, df): 60.40GiB
|
||||
Data ratio: 1.00
|
||||
Metadata ratio: 1.00
|
||||
Global reserve: 512.00MiB (used: 0.00B)
|
||||
Multiple profiles: no
|
||||
|
||||
Data,single: Size:48.01GiB, Used:47.59GiB (99.12%)
|
||||
/dev/loop0 48.01GiB
|
||||
|
||||
Metadata,single: Size:92.01GiB, Used:90.52GiB (98.38%)
|
||||
/dev/loop0 92.01GiB
|
||||
|
||||
System,single: Size:4.00MiB, Used:16.00KiB (0.39%)
|
||||
/dev/loop0 4.00MiB
|
||||
|
||||
Unallocated:
|
||||
/dev/loop0 59.98GiB
|
||||
7
docs/quick_notes/fs_stats/btrfs_not_used.txt
Normal file
7
docs/quick_notes/fs_stats/btrfs_not_used.txt
Normal file
@@ -0,0 +1,7 @@
|
||||
# takes a lot of time, should only be used when debugging
|
||||
|
||||
echo -e "\n\nbtrfs filesystem du -s"
|
||||
sudo btrfs filesystem du -s mnt_rw
|
||||
|
||||
echo -e "\n\ncompsize -x"
|
||||
sudo compsize -x mnt_rw 2> /dev/null || true
|
||||
234
docs/quick_notes/fs_stats/dedupl_fixed.log
Normal file
234
docs/quick_notes/fs_stats/dedupl_fixed.log
Normal file
@@ -0,0 +1,234 @@
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-3
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-4
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-5
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-6
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-7
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-8
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-9
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-10
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-3
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-4
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-5
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-3
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-6
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-7
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-3
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-4
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-8
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-5
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-9
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-6
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-4
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-7
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-8
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-5
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-10
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-6
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-7
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-8
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-9
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-9
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-10
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-11
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-3
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-3
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-4
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-5
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-6
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-11
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-12
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-13
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-14
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-15
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-16
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-17
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-18
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-19
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-20
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-21
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-22
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-23
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-24
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-25
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-26
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-27
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-28
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-29
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-30
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-31
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-32
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-33
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-34
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-35
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-36
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-37
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-38
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/069/796.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-39
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-40
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-41
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-42
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/546.pbf-43
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/074/934.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/078/331.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/090/195.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-11
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-12
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/029/536.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-13
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/102/751.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-14
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-15
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/101/708.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-16
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/029/536.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-17
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/101/129.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-18
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-19
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-10
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-20
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-21
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-11
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-22
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/031/503.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-23
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-12
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-24
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-25
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-13
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-4
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-26
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-27
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-14
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-28
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-12
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-29
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-15
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-30
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-16
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-31
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-32
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-17
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-33
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-5
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-13
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-18
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-34
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-19
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-20
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-35
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-14
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-21
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-36
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-22
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-23
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-15
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-24
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-25
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-26
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-6
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-27
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-37
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-16
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-28
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-29
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-17
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-30
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-31
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-38
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-18
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-32
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-19
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-39
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-20
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-21
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-33
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-22
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-40
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-23
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-7
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-24
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-34
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-25
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-41
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-26
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-27
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-28
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-35
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-42
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-29
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-30
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-31
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-32
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-43
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-8
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/347.pbf-36
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-33
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-34
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-35
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-36
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-9
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-37
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-38
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-39
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/806.pbf-44
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-40
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-10
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-41
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-42
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-11
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-43
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-44
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/730.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-12
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-45
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-13
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-14
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/663.pbf-46
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-15
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/730.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/530.pbf-16
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/730.pbf-3
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/037/250.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/127/507.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/039/025.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/041/570.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/150/189.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/150/189.pbf-2
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/152/800.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/047/927.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/164/724.pbf-1
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-7
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-8
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-9
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-10
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-11
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-12
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-13
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-14
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-15
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-16
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-17
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-18
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-19
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-20
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-21
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-22
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-23
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-24
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-25
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-26
|
||||
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/518.pbf-27
|
||||
13
docs/quick_notes/fs_stats/ext4.md
Normal file
13
docs/quick_notes/fs_stats/ext4.md
Normal file
@@ -0,0 +1,13 @@
|
||||
## deduplicated on ext4
|
||||
|
||||
```df -h mnt_rw
|
||||
df -h mnt_rw
|
||||
Filesystem Size Used Avail Use% Mounted on
|
||||
/dev/loop0 1.4T 187G 1.2T 14%
|
||||
```
|
||||
|
||||
```
|
||||
df -i mnt_rw
|
||||
Filesystem Inodes IUsed IFree IUse% Mounted on
|
||||
/dev/loop0 393216000 39614466 353601534 11%
|
||||
```
|
||||
13
docs/quick_notes/fs_stats/naive_mbutil_ext4.md
Normal file
13
docs/quick_notes/fs_stats/naive_mbutil_ext4.md
Normal file
@@ -0,0 +1,13 @@
|
||||
## native mapbox/mbutil
|
||||
|
||||
// pip install git+https://github.com/mapbox/mbutil.git@544c76e
|
||||
|
||||
```
|
||||
Filesystem 1K-blocks Used Available Use% Mounted on
|
||||
/dev/loop0 1,474,386,100 1,119,622,516 354,763,584 76%
|
||||
```
|
||||
|
||||
```
|
||||
Filesystem Inodes IUsed IFree IUse% Mounted on
|
||||
/dev/loop0 393,216,000 269,252,174 123,963,826 69%
|
||||
```
|
||||
1
docs/quick_notes/fs_stats/planet_version
Normal file
1
docs/quick_notes/fs_stats/planet_version
Normal file
@@ -0,0 +1 @@
|
||||
planetiler: 2023-12-11
|
||||
37
docs/quick_notes/http_benchmark.md
Normal file
37
docs/quick_notes/http_benchmark.md
Normal file
@@ -0,0 +1,37 @@
|
||||
### wrk localhost
|
||||
|
||||
Real world usage, 500k requests replayed from server log.
|
||||
|
||||
clean cache after nginx restart.
|
||||
|
||||
```
|
||||
service nginx restart
|
||||
wrk -c10 -t4 -d60s -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost
|
||||
Running 1m test @ http://localhost
|
||||
4 threads and 10 connections
|
||||
Thread Stats Avg Stdev Max +/- Stdev
|
||||
Latency 2.02ms 7.04ms 50.43ms 93.23%
|
||||
Req/Sec 8.42k 2.01k 18.52k 69.79%
|
||||
2871265 requests in 1.00m, 230.65GB read
|
||||
Requests/sec: 47811.00
|
||||
Transfer/sec: 3.84GB
|
||||
```
|
||||
|
||||
Super much overkill, we'd only need 125 MB/s for Gigabit connection and this is 3840 MB/s.
|
||||
Also max request time is super nice + no errors.
|
||||
|
||||
### wrk over network
|
||||
|
||||
```
|
||||
wrk -c10 -t4 -d60s -s /data/ofm/benchmark/wrk_custom_list.lua http://x.x.x.x
|
||||
Running 1m test @ http://144.76.168.195
|
||||
4 threads and 10 connections
|
||||
Thread Stats Avg Stdev Max +/- Stdev
|
||||
Latency 7.57ms 6.61ms 45.34ms 84.32%
|
||||
Req/Sec 293.85 141.33 1.18k 73.07%
|
||||
71628 requests in 1.00m, 6.05GB read
|
||||
Requests/sec: 1191.88
|
||||
Transfer/sec: 103.01MB
|
||||
```
|
||||
|
||||
Realistically this is the max over Gigabit connection.
|
||||
23
docs/quick_notes/kernel-ideas.txt
Normal file
23
docs/quick_notes/kernel-ideas.txt
Normal file
@@ -0,0 +1,23 @@
|
||||
# not using
|
||||
|
||||
fs.file-max # not needed, recent Ubuntu has it on high by default
|
||||
|
||||
|
||||
# not tested
|
||||
|
||||
tcp_fin_timeout
|
||||
TCP max buffer size
|
||||
|
||||
net.core.netdev_max_backlog = 262144
|
||||
net.core.rmem_max = 16777216
|
||||
net.core.wmem_max = 16777216
|
||||
net.ipv4.tcp_max_syn_backlog = 262144
|
||||
net.ipv4.tcp_max_tw_buckets = 6000000
|
||||
net.ipv4.tcp_no_metrics_save = 1
|
||||
net.ipv4.tcp_rmem = 4096 87380 16777216
|
||||
net.ipv4.tcp_syn_retries = 2
|
||||
net.ipv4.tcp_synack_retries = 2
|
||||
net.ipv4.tcp_tw_reuse = 1
|
||||
net.ipv4.tcp_wmem = 4096 65536 16777216
|
||||
|
||||
|
||||
30
docs/quick_notes/nginx-ideas.txt
Normal file
30
docs/quick_notes/nginx-ideas.txt
Normal file
@@ -0,0 +1,30 @@
|
||||
# ideas https://calomel.org/nginx.html
|
||||
# https://www.nginx.com/blog/tuning-nginx/
|
||||
# https://github.com/denji/nginx-tuning
|
||||
|
||||
|
||||
# not using
|
||||
|
||||
sendfile_max_chunk 512k; # no need for small files
|
||||
keepalive_requests # default seems reasonable
|
||||
keepalive_timeout # default seems reasonable
|
||||
|
||||
gzip_vary on; # no need probably
|
||||
gzip_min_length 10240; # only one file
|
||||
gzip_proxied any; # no proxying
|
||||
|
||||
types_hash_max_size 2048; # default should be good for the default set
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# not tested
|
||||
|
||||
client_body_buffer_size 128k;
|
||||
client_max_body_size 128k;
|
||||
client_header_buffer_size 1k;
|
||||
large_client_header_buffers 2 1k;
|
||||
|
||||
client_header_timeout 12; # default is 60
|
||||
|
||||
Reference in New Issue
Block a user