mirror of
https://github.com/hyperknot/openfreemap.git
synced 2026-05-21 22:12:15 +00:00
refactor tile_gen in Python
This commit is contained in:
Zsolt Ero
18
scripts/tile_gen/tile_gen_lib/config.py
Normal file
18
scripts/tile_gen/tile_gen_lib/config.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Configuration:
|
||||
tile_gen_dir = Path('/data/ofm/tile_gen')
|
||||
|
||||
tile_gen_bin = tile_gen_dir / 'bin'
|
||||
tile_gen_scripts_dir = tile_gen_bin / 'scripts'
|
||||
|
||||
planetiler_bin = tile_gen_dir / 'planetiler'
|
||||
planetiler_path = planetiler_bin / 'planetiler.jar'
|
||||
|
||||
runs_dir = tile_gen_dir / 'runs'
|
||||
|
||||
areas = ['planet', 'monaco']
|
||||
|
||||
|
||||
config = Configuration()
|
||||
136
scripts/tile_gen/tile_gen_lib/extract.py
Normal file
136
scripts/tile_gen/tile_gen_lib/extract.py
Normal file
@@ -0,0 +1,136 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
from tile_gen_lib.utils import python_venv_executable
|
||||
|
||||
|
||||
IMAGE_SIZE = '200G'
|
||||
|
||||
|
||||
def make_btrfs(run_folder: Path):
|
||||
os.chdir(run_folder)
|
||||
|
||||
# cleanup
|
||||
for mount in ['mnt_rw', 'mnt_rw2']:
|
||||
subprocess.run(['sudo', 'umount', mount], capture_output=True)
|
||||
|
||||
for pattern in ['mnt_rw*', 'tmp_*', '*.btrfs', '*.gz', '*.log', '*.txt', 'logs', 'osm_date']:
|
||||
for item in Path().glob(pattern):
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
|
||||
# make an empty file that's definitely bigger then the current OSM output
|
||||
for image in ['image.btrfs', 'image2.btrfs']:
|
||||
subprocess.run(['fallocate', '-l', IMAGE_SIZE, image], check=True)
|
||||
subprocess.run(['mkfs.btrfs', '-m', 'single', image], check=True, capture_output=True)
|
||||
|
||||
for image, mount in [('image.btrfs', 'mnt_rw'), ('image2.btrfs', 'mnt_rw2')]:
|
||||
Path(mount).mkdir()
|
||||
|
||||
# https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
|
||||
# compression doesn't make sense, data is already gzip compressed
|
||||
subprocess.run(
|
||||
[
|
||||
'sudo',
|
||||
'mount',
|
||||
'-t',
|
||||
'btrfs',
|
||||
'-o',
|
||||
'noacl,nobarrier,noatime,max_inline=4096',
|
||||
image,
|
||||
mount,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
subprocess.run(['sudo', 'chown', 'ofm:ofm', '-R', mount], check=True)
|
||||
|
||||
# extract mbtiles
|
||||
extract_script = config.tile_gen_scripts_dir / 'extract_mbtiles.py'
|
||||
with open('extract_out.log', 'w') as out, open('extract_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
[
|
||||
python_venv_executable(),
|
||||
extract_script,
|
||||
'tiles.mbtiles',
|
||||
'mnt_rw/extract',
|
||||
],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
shutil.copy('mnt_rw/extract/osm_date', '.')
|
||||
|
||||
# process logs
|
||||
subprocess.run('grep fixed extract_out.log > dedupl_fixed.log', shell=True)
|
||||
|
||||
# unfortunately, by deleting files from the btrfs partition, the partition size grows
|
||||
# so we need to rsync onto a new partition instead of deleting
|
||||
with open('rsync_out.log', 'w') as out, open('rsync_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
[
|
||||
'rsync',
|
||||
'-avH',
|
||||
'--max-alloc=4294967296',
|
||||
'--exclude',
|
||||
'dedupl',
|
||||
'mnt_rw/extract/',
|
||||
'mnt_rw2/',
|
||||
],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
# collect stats
|
||||
for i, mount in enumerate(['mnt_rw', 'mnt_rw2'], 1):
|
||||
with open(f'stats{i}.txt', 'w') as f:
|
||||
for cmd in [
|
||||
['df', '-h', mount],
|
||||
['btrfs', 'filesystem', 'df', mount],
|
||||
['btrfs', 'filesystem', 'show', mount],
|
||||
['btrfs', 'filesystem', 'usage', mount],
|
||||
]:
|
||||
f.write(f"\n\n{' '.join(cmd)}\n")
|
||||
result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True)
|
||||
f.write(result.stdout)
|
||||
|
||||
# unmount and cleanup
|
||||
for mount in ['mnt_rw', 'mnt_rw2']:
|
||||
subprocess.run(['sudo', 'umount', mount], check=True)
|
||||
|
||||
shutil.rmtree('mnt_rw')
|
||||
shutil.rmtree('mnt_rw2')
|
||||
|
||||
# shrink btrfs
|
||||
shrink_script = config.tile_gen_scripts_dir / 'shrink_btrfs.py'
|
||||
with open('shrink_out.log', 'w') as out, open('shrink_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
['sudo', python_venv_executable(), shrink_script, 'image2.btrfs'],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
os.unlink('image.btrfs')
|
||||
shutil.move('image2.btrfs', 'tiles.btrfs')
|
||||
|
||||
# parallel gzip
|
||||
subprocess.run(['pigz', 'tiles.btrfs', '--fast'], check=True)
|
||||
|
||||
# logs
|
||||
Path('logs').mkdir()
|
||||
for pattern in ['*.log', '*.txt']:
|
||||
for file in Path().glob(pattern):
|
||||
shutil.move(file, 'logs')
|
||||
|
||||
print('extract_btrfs.py DONE')
|
||||
|
||||
return run_folder
|
||||
53
scripts/tile_gen/tile_gen_lib/planetiler.py
Normal file
53
scripts/tile_gen/tile_gen_lib/planetiler.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
|
||||
|
||||
def run_planetiler(area: str) -> Path:
|
||||
assert area in config.areas
|
||||
|
||||
date = datetime.now(tz=timezone.utc).strftime('%Y%m%d_%H%M%S')
|
||||
|
||||
# delete all previous runs for the given area
|
||||
shutil.rmtree(config.runs_dir / area, ignore_errors=True)
|
||||
|
||||
run_folder = config.runs_dir / area / f'{date}_pt'
|
||||
run_folder.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
os.chdir(run_folder)
|
||||
|
||||
java_memory_gb = 30 if area == 'planet' else 1
|
||||
|
||||
command = [
|
||||
'java',
|
||||
f'-Xmx{java_memory_gb}g',
|
||||
'-jar',
|
||||
config.planetiler_path,
|
||||
f'--area={area}',
|
||||
'--download',
|
||||
'--download-threads=10',
|
||||
'--download-chunk-size-mb=1000',
|
||||
'--fetch-wikidata',
|
||||
'--output=tiles.mbtiles',
|
||||
'--nodemap-type=array',
|
||||
'--storage=mmap',
|
||||
'--force',
|
||||
]
|
||||
|
||||
if area == 'planet':
|
||||
command += '--bounds=planet'
|
||||
|
||||
out_path = run_folder / 'planetiler.out'
|
||||
err_path = run_folder / 'planetiler.err'
|
||||
|
||||
with out_path.open('w') as out_file, err_path.open('w') as err_file:
|
||||
subprocess.run(command, stdout=out_file, stderr=err_file, check=True, cwd=run_folder)
|
||||
|
||||
shutil.rmtree(run_folder / 'data', ignore_errors=True)
|
||||
print('planetiler.jar DONE')
|
||||
|
||||
return run_folder
|
||||
93
scripts/tile_gen/tile_gen_lib/upload.py
Normal file
93
scripts/tile_gen/tile_gen_lib/upload.py
Normal file
@@ -0,0 +1,93 @@
|
||||
import subprocess
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
|
||||
|
||||
def upload_rclone(area, run):
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'sync',
|
||||
'--transfers=8',
|
||||
'--multi-thread-streams=8',
|
||||
'--fast-list',
|
||||
'-v',
|
||||
'--stats-file-name-length',
|
||||
'0',
|
||||
'--stats-one-line',
|
||||
'--log-file',
|
||||
config.runs_dir / area / run / 'logs' / 'rclone.log',
|
||||
'--exclude',
|
||||
'logs/**',
|
||||
config.runs_dir / area / run,
|
||||
f'remote:ofm-{area}/{run}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def make_indexes():
|
||||
for area in config.areas:
|
||||
print(f'creating index {area}')
|
||||
|
||||
# files
|
||||
p = subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'lsf',
|
||||
'-R',
|
||||
'--files-only',
|
||||
'--fast-list',
|
||||
'--exclude',
|
||||
'dirs.txt',
|
||||
'--exclude',
|
||||
'index.txt',
|
||||
f'remote:ofm-{area}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
index_str = p.stdout
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'rcat',
|
||||
f'remote:ofm-{area}/index.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
|
||||
check=True,
|
||||
input=index_str.encode(),
|
||||
)
|
||||
|
||||
# directories
|
||||
p = subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'lsf',
|
||||
'-R',
|
||||
'--dirs-only',
|
||||
'--dir-slash=false',
|
||||
'--fast-list',
|
||||
f'remote:ofm-{area}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
index_str = p.stdout
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'rcat',
|
||||
f'remote:ofm-{area}/dirs.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
|
||||
check=True,
|
||||
input=index_str.encode(),
|
||||
)
|
||||
14
scripts/tile_gen/tile_gen_lib/utils.py
Normal file
14
scripts/tile_gen/tile_gen_lib/utils.py
Normal file
@@ -0,0 +1,14 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def python_venv_executable() -> Path:
|
||||
venv_path = os.environ.get('VIRTUAL_ENV')
|
||||
|
||||
if venv_path:
|
||||
return Path(venv_path) / 'bin' / 'python'
|
||||
elif sys.prefix != sys.base_prefix:
|
||||
return Path(sys.prefix) / 'bin' / 'python'
|
||||
else:
|
||||
return Path(sys.executable)
|
||||
Reference in New Issue
Block a user