sysadmin/openfreemap
1
0
Fork 0
mirror of https://github.com/hyperknot/openfreemap.git synced 2026-05-21 14:02:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

work

Browse Source
This commit is contained in:
Zsolt Ero
2023-12-18 03:06:16 +01:00
parent 1d1cae9b40
commit 818cf1e349
8 changed files with 85 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
scripts/benchmark/wrk_custom_list.lua
View File

@@ -1,7 +1,7 @@
local counter = 1
local lines = {}
local base_path = "/planet/20231208_091355/tiles/"
local file_path = "/data/ofm/benchmark/path_list_small.txt"
local file_path = "/data/ofm/benchmark/path_list_100k.txt"
for line in io.lines(file_path) do
table.insert(lines, base_path .. line)
@@ -22,8 +22,18 @@ end
request = function()
-- Return the request object with the current URL path
local path = getNextUrl()
path = getNextUrl()
local headers = {}
headers["Host"] = "ofm"
return wrk.format('GET', path, headers, nil)
end
response = function(status)
if status ~= 200 then
print("Non-200 response")
print("Status: ", status)
-- this only works in single threaded mode (-t1)
print("Request path: ", path)
end
end

234
scripts/extract_mbtiles/dedupl-fix.log Normal file
View File

@@ -0,0 +1,234 @@
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-27
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-28
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-29
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-30
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-31
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-32
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-33
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-34
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-35
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-36
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-37
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-38
Created fixed dedupl file: mnt_rw/extract/dedupl/000/069/716.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-39
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-40
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-41
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-42
Created fixed dedupl file: mnt_rw/extract/dedupl/000/004/541.pbf-43
Created fixed dedupl file: mnt_rw/extract/dedupl/000/074/696.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/078/016.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/089/935.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/029/417.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/102/511.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/101/473.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/029/417.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/100/899.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/031/388.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-4
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-27
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-28
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-29
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-30
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-31
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-32
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-33
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-5
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-34
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-35
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-36
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-6
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-27
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-37
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-28
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-29
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-30
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-31
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-38
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-32
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-39
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-33
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-40
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-34
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-41
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-27
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-28
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-35
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-42
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-29
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-30
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-31
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-32
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-43
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/296.pbf-36
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-33
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-34
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-35
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-36
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-37
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-38
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-39
Created fixed dedupl file: mnt_rw/extract/dedupl/000/007/753.pbf-44
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-40
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-41
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-42
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-43
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-44
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/671.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-45
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/008/601.pbf-46
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/671.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/472.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/009/671.pbf-3
Created fixed dedupl file: mnt_rw/extract/dedupl/000/037/051.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/127/206.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/038/905.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/041/436.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/149/880.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/149/880.pbf-2
Created fixed dedupl file: mnt_rw/extract/dedupl/000/152/475.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/047/840.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/164/466.pbf-1
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-7
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-8
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-9
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-10
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-11
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-12
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-13
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-14
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-15
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-16
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-17
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-18
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-19
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-20
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-21
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-22
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-23
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-24
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-25
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-26
Created fixed dedupl file: mnt_rw/extract/dedupl/000/015/476.pbf-27

42
scripts/extract_mbtiles/extract_mbtiles.py
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import json
import os
import shutil
import sqlite3
import sys
@@ -22,20 +23,25 @@ def cli(mbtiles_path: Path, dir_path: Path):
used for reference: https://github.com/mapbox/mbutil
"""
# if dir_path.exists() and any(dir_path.iterdir()):
# sys.exit('Dir not empty')
if dir_path.exists() and any(dir_path.iterdir()):
sys.exit('Dir not empty')
dir_path.mkdir(exist_ok=True)
conn = sqlite3.connect(mbtiles_path)
c = conn.cursor()
# write_metadata(c, dir_path=dir_path)
# write_dedupl_files(c, dir_path=dir_path)
write_metadata(c, dir_path=dir_path)
write_dedupl_files(c, dir_path=dir_path)
write_tile_files(c, dir_path=dir_path)
# remove dedupl files at the end
# shutil.rmtree(dir_path / 'dedupl')
shutil.rmtree(dir_path / 'dedupl')
# if it's a full planet run,
# make sure there are exactly the right number of files generated
if 'planet' in mbtiles_path.parent.name:
assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14)
def write_metadata(c, *, dir_path):
@@ -92,13 +98,20 @@ def write_tile_files(c, *, dir_path):
if e.errno == 31:
bug_fix_dict.setdefault(dedupl_path, 0)
bug_fix_dict[dedupl_path] += 1
fixed_path = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
shutil.copyfile(dedupl_path, fixed_path)
print(f'Created fixed dedupl file: {fixed_path}')
dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
shutil.copyfile(dedupl_path, dedupl_path_fixed)
print(f'Created fixed dedupl file: {dedupl_path_fixed}')
tile_path.hardlink_to(dedupl_path_fixed)
print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
else:
raise
# last file: 14/16383/0.pbf
def count_files(folder):
total = 0
for root, dirs, files in os.walk(folder):
total += len(files)
return
def get_fixed_dedupl_name(bug_fix_dict, dedupl_path):
@@ -124,5 +137,16 @@ def flip_y(zoom, y):
return (2**zoom - 1) - y
def calculate_tiles(zoom_level):
return (2**zoom_level) ** 2
def calculate_tiles_sum(zoom_level):
"""
Tiles up to zoom level (geometric series)
"""
return (4 ** (zoom_level + 1) - 1) // 3
if __name__ == '__main__':
cli()

4
scripts/http_host/nginx_site.conf
View File

@@ -1,7 +1,7 @@
server {
server_name ofm tiles.openfreemaps.org;
# test with
# curl -H "Host: ofm" http://localhost/planet/20231208_091355/tiles/7/72/48.pbf
# curl -H "Host: ofm" http://localhost/planet/20231208_091355/tiles/11/637/1141.pbf
#access_log /data/ofm/logs/nginx-access.log access_json;
@@ -11,7 +11,7 @@ server {
location /planet/20231208_091355 {
gzip off;
alias /data/ofm/runs/planet_20231208_091355/mnt_ro/extract;
alias /data/ofm/runs/planet_20231208_091355/mnt_rw/extract;
autoindex on; # Enables listing of directory
}
}

4
scripts/shrink_btrfs/shrink_btrfs.py
View File

@@ -8,6 +8,10 @@ from pathlib import Path
import click
# btrfs cannot shrink smaller than about 268 MB
SMALLEST_SIZE = 270_000_000
@click.command()
@click.argument(
'btrfs_img',

40
scripts/tile_gen/extract_btrfs.sh
View File

@@ -1,12 +1,15 @@
#!/usr/bin/env bash
set -e
sudo umount mnt_rw || true
rm -rf mnt_rw
rm -f image.btrfs
sudo umount mnt_rw2 || true
rm -rf mnt_rw*
rm -f image*.btrfs
rm -f *.log
# make an empty file that's definitely bigger then the current OSM output
fallocate -l 200G image.btrfs
fallocate -l 200G image2.btrfs
# metadata: single needed as default is now DUP
@@ -14,25 +17,44 @@ mkfs.btrfs -v \
-m single \
image.btrfs
mkfs.btrfs -v \
-m single \
image2.btrfs
# https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
# compression doesn't make sense, data is already gzip compressed
mkdir -p mnt_rw
mkdir -p mnt_rw mnt_rw2
sudo mount -v \
-t btrfs \
-o noacl,nobarrier,noatime,max_inline=4096 \
image.btrfs mnt_rw
sudo mount -v \
-t btrfs \
-o noacl,nobarrier,noatime,max_inline=4096 \
image.btrfs mnt_rw2
sudo chown ofm:ofm -R mnt_rw
sudo chown ofm:ofm -R mnt_rw mnt_rw2
../../tile_gen/venv/bin/python ../../tile_gen/extract_mbtiles.py output.mbtiles mnt_rw/extract \
> "extract_out.log" 2> "extract_err.log"
> extract_out.log 2> extract_err.log
# we need to extract, delete and rsync onto a new partition
# otherwise the partition image stays big
rsync -aH mnt_rw/extract/ mnt_rw2/extract/ > rsync_out.log 2> rsync_err.log
sudo umount mnt_rw
sudo umount mnt_rw2
rm -r mnt_rw*
sudo ../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image.btrfs \
> "shrink_out.log" 2> "shrink_err.log"
sudo ../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image2.btrfs \
> shrink_out.log 2> shrink_err.log
#rm image.btrfs
#mv image2.btrfs done.btrfs
# pigz -k image.btrfs --fast
# rsync -avH mnt_rw/extract/ mnt_rw2/extract/ > rsync_out.log 2> rsync_err.log