This commit is contained in:
Zsolt Ero
2023-12-18 03:06:16 +01:00
parent 1d1cae9b40
commit 818cf1e349
8 changed files with 85 additions and 22 deletions

View File

@@ -0,0 +1,234 @@
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-27
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-28
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-29
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-30
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-31
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-32
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-33
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-34
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-35
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-36
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-37
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-38
Created fixed dedupl file: mnt_rw/extract/dedupl/000/069/716.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-39
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-40
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-41
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-42
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-43
Created fixed dedupl file: mnt_rw/extract/dedupl/000/074/696.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/078/016.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/089/935.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/029/417.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/102/511.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/101/473.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/029/417.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/100/899.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/031/388.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-27
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-28
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-29
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-30
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-31
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-32
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-33
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-34
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-35
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-36
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-27
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-37
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-28
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-29
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-30
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-31
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-38
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-32
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-39
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-33
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-40
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-34
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-41
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-27
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-28
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-35
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-42
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-29
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-30
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-31
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-32
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-43
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-36
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-33
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-34
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-35
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-36
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-37
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-38
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-39
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-44
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-40
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-41
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-42
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-43
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-44
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/671.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-45
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-46
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/671.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/671.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/037/051.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/127/206.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/038/905.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/041/436.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/149/880.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/149/880.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/152/475.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/047/840.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/164/466.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-27

View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import json
import os
import shutil
import sqlite3
import sys
@@ -22,20 +23,25 @@ def cli(mbtiles_path: Path, dir_path: Path):
used for reference: https://github.com/mapbox/mbutil
"""
# if dir_path.exists() and any(dir_path.iterdir()):
# sys.exit('Dir not empty')
if dir_path.exists() and any(dir_path.iterdir()):
sys.exit('Dir not empty')
dir_path.mkdir(exist_ok=True)
conn = sqlite3.connect(mbtiles_path)
c = conn.cursor()
# write_metadata(c, dir_path=dir_path)
# write_dedupl_files(c, dir_path=dir_path)
write_metadata(c, dir_path=dir_path)
write_dedupl_files(c, dir_path=dir_path)
write_tile_files(c, dir_path=dir_path)
# remove dedupl files at the end
# shutil.rmtree(dir_path / 'dedupl')
shutil.rmtree(dir_path / 'dedupl')
# if it's a full planet run,
# make sure there are exactly the right number of files generated
if 'planet' in mbtiles_path.parent.name:
assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14)
def write_metadata(c, *, dir_path):
@@ -92,13 +98,20 @@ def write_tile_files(c, *, dir_path):
if e.errno == 31:
bug_fix_dict.setdefault(dedupl_path, 0)
bug_fix_dict[dedupl_path] += 1
fixed_path = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
shutil.copyfile(dedupl_path, fixed_path)
print(f'Created fixed dedupl file: {fixed_path}')
dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
shutil.copyfile(dedupl_path, dedupl_path_fixed)
print(f'Created fixed dedupl file: {dedupl_path_fixed}')
tile_path.hardlink_to(dedupl_path_fixed)
print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
else:
raise
# last file: 14/16383/0.pbf
def count_files(folder):
total = 0
for root, dirs, files in os.walk(folder):
total += len(files)
return
def get_fixed_dedupl_name(bug_fix_dict, dedupl_path):
@@ -124,5 +137,16 @@ def flip_y(zoom, y):
return (2**zoom - 1) - y
def calculate_tiles(zoom_level):
return (2**zoom_level) ** 2
def calculate_tiles_sum(zoom_level):
"""
Tiles up to zoom level (geometric series)
"""
return (4 ** (zoom_level + 1) - 1) // 3
if __name__ == '__main__':
cli()