sysadmin/openfreemap
1
0
Fork 0
mirror of https://github.com/hyperknot/openfreemap.git synced 2026-05-22 06:22:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

work

Browse Source
This commit is contained in:
Zsolt Ero
2023-12-17 18:53:29 +01:00
parent 71d7d7fe95
commit e2c2f1fc7e
8 changed files with 129 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
scripts/extract_mbtiles/extract_mbtiles.py
View File

@@ -22,20 +22,20 @@ def cli(mbtiles_path: Path, dir_path: Path):
used for reference: https://github.com/mapbox/mbutil
"""
if dir_path.exists() and any(dir_path.iterdir()):
sys.exit('Dir not empty')
# if dir_path.exists() and any(dir_path.iterdir()):
# sys.exit('Dir not empty')
dir_path.mkdir(exist_ok=True)
conn = sqlite3.connect(mbtiles_path)
c = conn.cursor()
write_metadata(c, dir_path=dir_path)
write_dedupl_files(c, dir_path=dir_path)
write_tile_file(c, dir_path=dir_path)
# write_metadata(c, dir_path=dir_path)
# write_dedupl_files(c, dir_path=dir_path)
write_tile_files(c, dir_path=dir_path)
# remove dedupl files at the end
shutil.rmtree(dir_path / 'dedupl')
# shutil.rmtree(dir_path / 'dedupl')
def write_metadata(c, *, dir_path):
@@ -49,6 +49,7 @@ def write_dedupl_files(c, *, dir_path):
total = c.execute('select count(*) from tiles_data').fetchone()[0]
c.execute('select tile_data_id, tile_data from tiles_data')
for i, row in enumerate(c, start=1):
dedupl_id = row[0]
dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
@@ -58,24 +59,53 @@ def write_dedupl_files(c, *, dir_path):
print(f'written dedupl file {i}/{total}')
def write_tile_file(c, *, dir_path):
def write_tile_files(c, *, dir_path):
total = c.execute('select count(*) from tiles_shallow').fetchone()[0]
bug_fix_dict = {}
c.execute('select zoom_level, tile_column, tile_row, tile_data_id from tiles_shallow')
for i, row in enumerate(c, start=1):
if i < 4678400:
continue
z = row[0]
x = row[1]
y = flip_y(z, row[2])
dedupl_id = row[3]
dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
tile_path = dir_path / 'tiles' / str(z) / str(x) / f'{y}.pbf'
tile_path.parent.mkdir(parents=True, exist_ok=True)
if tile_path.is_file():
continue
# create the hard link
tile_path.hardlink_to(dedupl_path)
print(f'hard link created {i}/{total}: {tile_path}')
try:
tile_path.hardlink_to(dedupl_path_fixed)
print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
except OSError as e:
# fixing BTRFS's 64k max link limit
if e.errno == 31:
bug_fix_dict.setdefault(dedupl_path, 0)
bug_fix_dict[dedupl_path] += 1
fixed_path = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
shutil.copyfile(dedupl_path, fixed_path)
print(f'Created fixed dedupl file: {fixed_path}')
else:
raise
# last file: 14/16383/0.pbf
def get_fixed_dedupl_name(bug_fix_dict, dedupl_path):
if dedupl_path in bug_fix_dict:
return dedupl_path.with_name(f'{dedupl_path.name}-{bug_fix_dict[dedupl_path]}')
else:
return dedupl_path
def dedupl_helper_path(dedupl_id: int) -> Path: