mirror of
https://github.com/hyperknot/openfreemap.git
synced 2026-05-22 06:22:16 +00:00
scripts -> modules
This commit is contained in:
Zsolt Ero
0
modules/tile_gen/tile_gen_lib/__init__.py
Normal file
0
modules/tile_gen/tile_gen_lib/__init__.py
Normal file
140
modules/tile_gen/tile_gen_lib/btrfs.py
Normal file
140
modules/tile_gen/tile_gen_lib/btrfs.py
Normal file
@@ -0,0 +1,140 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
from tile_gen_lib.utils import python_venv_executable
|
||||
|
||||
|
||||
IMAGE_SIZE = '200G'
|
||||
|
||||
|
||||
def make_btrfs(run_folder: Path):
|
||||
os.chdir(run_folder)
|
||||
|
||||
cleanup_folder(run_folder)
|
||||
|
||||
# make an empty file that's definitely bigger then the current OSM output
|
||||
for image in ['image.btrfs', 'image2.btrfs']:
|
||||
subprocess.run(['fallocate', '-l', IMAGE_SIZE, image], check=True)
|
||||
subprocess.run(['mkfs.btrfs', '-m', 'single', image], check=True, capture_output=True)
|
||||
|
||||
for image, mount in [('image.btrfs', 'mnt_rw'), ('image2.btrfs', 'mnt_rw2')]:
|
||||
Path(mount).mkdir()
|
||||
|
||||
# https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
|
||||
# compression doesn't make sense, data is already gzip compressed
|
||||
subprocess.run(
|
||||
[
|
||||
'sudo',
|
||||
'mount',
|
||||
'-t',
|
||||
'btrfs',
|
||||
'-o',
|
||||
'noacl,nobarrier,noatime,max_inline=4096',
|
||||
image,
|
||||
mount,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
subprocess.run(['sudo', 'chown', 'ofm:ofm', '-R', mount], check=True)
|
||||
|
||||
# extract mbtiles
|
||||
extract_script = config.tile_gen_scripts_dir / 'extract_mbtiles.py'
|
||||
with open('extract_out.log', 'w') as out, open('extract_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
[
|
||||
python_venv_executable(),
|
||||
extract_script,
|
||||
'tiles.mbtiles',
|
||||
'mnt_rw/extract',
|
||||
],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
os.unlink('tiles.mbtiles')
|
||||
|
||||
shutil.copy('mnt_rw/extract/osm_date', '.')
|
||||
|
||||
# process logs
|
||||
subprocess.run('grep fixed extract_out.log > dedupl_fixed.log', shell=True)
|
||||
|
||||
# unfortunately, by deleting files from the btrfs partition, the partition size grows
|
||||
# so we need to rsync onto a new partition instead of deleting
|
||||
with open('rsync_out.log', 'w') as out, open('rsync_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
[
|
||||
'rsync',
|
||||
'-avH',
|
||||
'--max-alloc=4294967296',
|
||||
'--exclude',
|
||||
'dedupl',
|
||||
'mnt_rw/extract/',
|
||||
'mnt_rw2/',
|
||||
],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
# collect stats
|
||||
for i, mount in enumerate(['mnt_rw', 'mnt_rw2'], 1):
|
||||
with open(f'stats{i}.txt', 'w') as f:
|
||||
for cmd in [
|
||||
['df', '-h', mount],
|
||||
['btrfs', 'filesystem', 'df', mount],
|
||||
['btrfs', 'filesystem', 'show', mount],
|
||||
['btrfs', 'filesystem', 'usage', mount],
|
||||
]:
|
||||
f.write(f"\n\n{' '.join(cmd)}\n")
|
||||
result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True)
|
||||
f.write(result.stdout)
|
||||
|
||||
# unmount and cleanup
|
||||
for mount in ['mnt_rw', 'mnt_rw2']:
|
||||
subprocess.run(['sudo', 'umount', mount], check=True)
|
||||
|
||||
shutil.rmtree('mnt_rw')
|
||||
shutil.rmtree('mnt_rw2')
|
||||
|
||||
# shrink btrfs
|
||||
shrink_script = config.tile_gen_scripts_dir / 'shrink_btrfs.py'
|
||||
with open('shrink_out.log', 'w') as out, open('shrink_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
['sudo', python_venv_executable(), shrink_script, 'image2.btrfs'],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
os.unlink('image.btrfs')
|
||||
shutil.move('image2.btrfs', 'tiles.btrfs')
|
||||
|
||||
# parallel gzip (pigz)
|
||||
subprocess.run(['pigz', 'tiles.btrfs', '--fast'], check=True)
|
||||
|
||||
# move logs
|
||||
Path('logs').mkdir()
|
||||
for pattern in ['*.log', '*.txt']:
|
||||
for file in Path().glob(pattern):
|
||||
shutil.move(file, 'logs')
|
||||
|
||||
print('extract_btrfs.py DONE')
|
||||
|
||||
|
||||
def cleanup_folder(run_folder: Path):
|
||||
print(f'cleaning up {run_folder}')
|
||||
|
||||
for mount in ['mnt_rw', 'mnt_rw2']:
|
||||
subprocess.run(['sudo', 'umount', run_folder / mount], capture_output=True)
|
||||
|
||||
for pattern in ['mnt_rw*', 'tmp_*', '*.btrfs', '*.gz', '*.log', '*.txt', 'logs', 'osm_date']:
|
||||
for item in run_folder.glob(pattern):
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
21
modules/tile_gen/tile_gen_lib/config.py
Normal file
21
modules/tile_gen/tile_gen_lib/config.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Configuration:
|
||||
areas = ['planet', 'monaco']
|
||||
|
||||
tile_gen_dir = Path('/data/ofm/tile_gen')
|
||||
|
||||
tile_gen_bin = tile_gen_dir / 'bin'
|
||||
tile_gen_scripts_dir = tile_gen_bin / 'scripts'
|
||||
|
||||
planetiler_bin = tile_gen_dir / 'planetiler'
|
||||
planetiler_path = planetiler_bin / 'planetiler.jar'
|
||||
|
||||
runs_dir = tile_gen_dir / 'runs'
|
||||
|
||||
ofm_config_dir = Path('/data/ofm/config')
|
||||
rclone_config = ofm_config_dir / 'rclone.conf'
|
||||
|
||||
|
||||
config = Configuration()
|
||||
65
modules/tile_gen/tile_gen_lib/planetiler.py
Normal file
65
modules/tile_gen/tile_gen_lib/planetiler.py
Normal file
@@ -0,0 +1,65 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
from tile_gen_lib.btrfs import cleanup_folder
|
||||
|
||||
|
||||
def run_planetiler(area: str) -> Path:
|
||||
assert area in config.areas
|
||||
|
||||
date = datetime.now(tz=timezone.utc).strftime('%Y%m%d_%H%M%S')
|
||||
|
||||
area_dir = config.runs_dir / area
|
||||
|
||||
# delete all previous runs for the given area
|
||||
for subdir in area_dir.iterdir():
|
||||
cleanup_folder(subdir)
|
||||
|
||||
print('running rmtree')
|
||||
shutil.rmtree(area_dir, ignore_errors=True)
|
||||
print('rmtree done')
|
||||
|
||||
run_folder = area_dir / f'{date}_pt'
|
||||
run_folder.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
os.chdir(run_folder)
|
||||
|
||||
# link to discussion about why exactly 30 GB
|
||||
# https://github.com/onthegomap/planetiler/discussions/690#discussioncomment-7756397
|
||||
java_memory_gb = 30 if area == 'planet' else 1
|
||||
|
||||
command = [
|
||||
'java',
|
||||
f'-Xmx{java_memory_gb}g',
|
||||
'-jar',
|
||||
config.planetiler_path,
|
||||
f'--area={area}',
|
||||
'--download',
|
||||
'--download-threads=10',
|
||||
'--download-chunk-size-mb=1000',
|
||||
'--fetch-wikidata',
|
||||
'--output=tiles.mbtiles',
|
||||
'--nodemap-type=array',
|
||||
'--storage=mmap',
|
||||
'--force',
|
||||
]
|
||||
|
||||
if area == 'planet':
|
||||
command.append('--bounds=planet')
|
||||
|
||||
print(command)
|
||||
|
||||
out_path = run_folder / 'planetiler.out'
|
||||
err_path = run_folder / 'planetiler.err'
|
||||
|
||||
with out_path.open('w') as out_file, err_path.open('w') as err_file:
|
||||
subprocess.run(command, stdout=out_file, stderr=err_file, check=True, cwd=run_folder)
|
||||
|
||||
shutil.rmtree(run_folder / 'data', ignore_errors=True)
|
||||
print('planetiler.jar DONE')
|
||||
|
||||
return run_folder
|
||||
123
modules/tile_gen/tile_gen_lib/rclone.py
Normal file
123
modules/tile_gen/tile_gen_lib/rclone.py
Normal file
@@ -0,0 +1,123 @@
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
|
||||
|
||||
def upload_area(area):
|
||||
"""
|
||||
Uploads an area, making sure there is exactly one run present
|
||||
"""
|
||||
|
||||
print(f'Uploading area: {area}')
|
||||
|
||||
assert area in config.areas
|
||||
|
||||
area_dir = config.runs_dir / area
|
||||
if not area_dir.exists():
|
||||
return
|
||||
|
||||
runs = list(area_dir.iterdir())
|
||||
if len(runs) != 1:
|
||||
print('Error: Make sure there is only one run in the given area')
|
||||
sys.exit(1)
|
||||
|
||||
run = runs[0].name
|
||||
|
||||
upload_area_run(area, run)
|
||||
make_indexes_for_bucket('ofm-btrfs')
|
||||
|
||||
|
||||
def upload_area_run(area, run):
|
||||
print(f'Uploading {area} {run} to btrfs bucket')
|
||||
|
||||
run_dir = config.runs_dir / area / run
|
||||
assert run_dir.is_dir()
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'sync',
|
||||
'--verbose=1',
|
||||
'--transfers=8',
|
||||
'--multi-thread-streams=8',
|
||||
'--fast-list',
|
||||
'--stats-file-name-length=0',
|
||||
'--stats-one-line',
|
||||
'--log-file',
|
||||
run_dir / 'logs' / 'rclone.log',
|
||||
'--exclude',
|
||||
'logs/**',
|
||||
run_dir,
|
||||
f'remote:ofm-btrfs/areas/{area}/{run}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def make_indexes_for_bucket(bucket):
|
||||
print(f'Making indexes for bucket: {bucket}')
|
||||
|
||||
# files
|
||||
p = subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'lsf',
|
||||
'--recursive',
|
||||
'--files-only',
|
||||
'--fast-list',
|
||||
'--exclude',
|
||||
'dirs.txt',
|
||||
'--exclude',
|
||||
'files.txt',
|
||||
f'remote:{bucket}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
index_str = p.stdout
|
||||
|
||||
# upload to files.txt
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'rcat',
|
||||
f'remote:{bucket}/files.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
input=index_str.encode(),
|
||||
)
|
||||
|
||||
# directories
|
||||
p = subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'lsf',
|
||||
'--recursive',
|
||||
'--dirs-only',
|
||||
'--dir-slash=false',
|
||||
'--fast-list',
|
||||
f'remote:{bucket}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
index_str = p.stdout
|
||||
|
||||
# upload to dirs.txt
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'rcat',
|
||||
f'remote:{bucket}/dirs.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
input=index_str.encode(),
|
||||
)
|
||||
14
modules/tile_gen/tile_gen_lib/utils.py
Normal file
14
modules/tile_gen/tile_gen_lib/utils.py
Normal file
@@ -0,0 +1,14 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def python_venv_executable() -> Path:
|
||||
venv_path = os.environ.get('VIRTUAL_ENV')
|
||||
|
||||
if venv_path:
|
||||
return Path(venv_path) / 'bin' / 'python'
|
||||
elif sys.prefix != sys.base_prefix:
|
||||
return Path(sys.prefix) / 'bin' / 'python'
|
||||
else:
|
||||
return Path(sys.executable)
|
||||
Reference in New Issue
Block a user