work

scripts/benchmark/.gitignore

@@ -1 +0,0 @@
-*.txt

scripts/benchmark/command.txt

@@ -0,0 +1,4 @@
+wrk -c1000 -d10s -t1 -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost
+
+# -t1 - needs to be single treaded, otherwise the urls would be read not in sequence
+

scripts/extract_mbtiles/extract_mbtiles.py

@@ -22,20 +22,20 @@ def cli(mbtiles_path: Path, dir_path: Path):
     used for reference: https://github.com/mapbox/mbutil
     """
 
-    if dir_path.exists() and any(dir_path.iterdir()):
-        sys.exit('Dir not empty')
+    # if dir_path.exists() and any(dir_path.iterdir()):
+    #     sys.exit('Dir not empty')
 
     dir_path.mkdir(exist_ok=True)
 
     conn = sqlite3.connect(mbtiles_path)
     c = conn.cursor()
 
-    write_metadata(c, dir_path=dir_path)
-    write_dedupl_files(c, dir_path=dir_path)
-    write_tile_file(c, dir_path=dir_path)
+    # write_metadata(c, dir_path=dir_path)
+    # write_dedupl_files(c, dir_path=dir_path)
+    write_tile_files(c, dir_path=dir_path)
 
     # remove dedupl files at the end
-    shutil.rmtree(dir_path / 'dedupl')
+    # shutil.rmtree(dir_path / 'dedupl')
 
 
 def write_metadata(c, *, dir_path):
@@ -49,6 +49,7 @@ def write_dedupl_files(c, *, dir_path):
     total = c.execute('select count(*) from tiles_data').fetchone()[0]
 
     c.execute('select tile_data_id, tile_data from tiles_data')
+
     for i, row in enumerate(c, start=1):
         dedupl_id = row[0]
         dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
@@ -58,24 +59,53 @@ def write_dedupl_files(c, *, dir_path):
             print(f'written dedupl file {i}/{total}')
 
 
-def write_tile_file(c, *, dir_path):
+def write_tile_files(c, *, dir_path):
     total = c.execute('select count(*) from tiles_shallow').fetchone()[0]
 
+    bug_fix_dict = {}
+
     c.execute('select zoom_level, tile_column, tile_row, tile_data_id from tiles_shallow')
     for i, row in enumerate(c, start=1):
+        if i < 4678400:
+            continue
+
         z = row[0]
         x = row[1]
         y = flip_y(z, row[2])
         dedupl_id = row[3]
 
         dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
+        dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
 
         tile_path = dir_path / 'tiles' / str(z) / str(x) / f'{y}.pbf'
         tile_path.parent.mkdir(parents=True, exist_ok=True)
 
+        if tile_path.is_file():
+            continue
+
         # create the hard link
-        tile_path.hardlink_to(dedupl_path)
-        print(f'hard link created {i}/{total}: {tile_path}')
+        try:
+            tile_path.hardlink_to(dedupl_path_fixed)
+            print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
+        except OSError as e:
+            # fixing BTRFS's 64k max link limit
+            if e.errno == 31:
+                bug_fix_dict.setdefault(dedupl_path, 0)
+                bug_fix_dict[dedupl_path] += 1
+                fixed_path = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
+                shutil.copyfile(dedupl_path, fixed_path)
+                print(f'Created fixed dedupl file: {fixed_path}')
+            else:
+                raise
+
+    # last file: 14/16383/0.pbf
+
+
+def get_fixed_dedupl_name(bug_fix_dict, dedupl_path):
+    if dedupl_path in bug_fix_dict:
+        return dedupl_path.with_name(f'{dedupl_path.name}-{bug_fix_dict[dedupl_path]}')
+    else:
+        return dedupl_path
 
 
 def dedupl_helper_path(dedupl_id: int) -> Path:

scripts/tile_gen/extract_btrfs.sh

@@ -25,12 +25,13 @@ sudo mount -v \
 
 sudo chown ofm:ofm -R mnt_rw
 
-../../tile_gen/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt_rw/extract \
+../../tile_gen/venv/bin/python ../../tile_gen/extract_mbtiles.py output.mbtiles mnt_rw/extract \
   > "extract_out.log" 2> "extract_err.log"
 
 sudo umount mnt_rw
 
-../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image.btrfs
+sudo ../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image.btrfs \
+  > "shrink_out.log" 2> "shrink_err.log"
 
 # pigz -k image.btrfs --fast
 

scripts/tile_gen/extract_ext4.sh

@@ -11,6 +11,7 @@
 # from  /etc/mke2fs.conf
 # defaults: has_journal,extent,huge_file,flex_bg,metadata_csum,64bit,dir_nlink,extra_isize
 # disabling journalling, since it's a read-only fs, as well as other unused features
+# extent is actually needed for tail packing small files
 #
 # -E extended-options
 # lazy_itable_init - inode table is fully initialized at the time of file system creation
@@ -32,9 +33,10 @@ fallocate -l 300G image.ext4
 mke2fs -t ext4 -v \
   -m 0 \
   -F \
-  -O ^has_journal,^extent,^huge_file,^metadata_csum,^64bit,^extra_isize \
+  -O ^has_journal,^huge_file,^metadata_csum,^64bit,^extra_isize \
   -E lazy_itable_init=0,nodiscard \
   -T small \
+  -I 128 \
   image.ext4
 
 mkdir mnt