scripts -> modules

modules/tile_gen/cron.d/ofm_tile_gen

@@ -0,0 +1,8 @@
+# every hour, make a monaco run
+10 * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-tiles monaco --upload >> /data/ofm/tile_gen/logs/monaco-cron.log 2>&1
+
+
+
+# once per minute, create indexes
+* * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-indexes >> /data/ofm/tile_gen/logs/make-indexes-cron.log 2>&1
+

modules/tile_gen/scripts/README.md

@@ -0,0 +1,2 @@
+These are self contained Python scripts, they can be run outside of this project's environment.
+

modules/tile_gen/scripts/extract_mbtiles.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+import json
+import os
+import shutil
+import sqlite3
+import sys
+from pathlib import Path
+
+import click
+
+
+@click.command()
+@click.argument(
+    'mbtiles_path',
+    type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path),
+)
+@click.argument('dir_path', type=click.Path(dir_okay=True, file_okay=False, path_type=Path))
+def cli(mbtiles_path: Path, dir_path: Path):
+    """
+    Extracts a mbtiles sqlite to a folder
+    Deduplicating identical tiles as hard-links
+
+    used for reference: https://github.com/mapbox/mbutil
+    """
+
+    if dir_path.exists() and any(dir_path.iterdir()):
+        sys.exit('  dir not empty')
+
+    dir_path.mkdir(exist_ok=True)
+
+    conn = sqlite3.connect(mbtiles_path)
+    c = conn.cursor()
+
+    write_dedupl_files(c, dir_path=dir_path)
+    write_tile_files(c, dir_path=dir_path)
+
+    # planetiler has missing tiles by design, so disabling this
+    # assert_all_tiles_present(mbtiles_path, dir_path)
+
+    write_metadata(c, dir_path=dir_path)
+    conn.commit()
+
+    print('extract_mbtiles.py DONE')
+
+
+def write_metadata(c, *, dir_path):
+    metadata = dict(c.execute('select name, value from metadata').fetchall())
+    c.execute("update metadata set value='OpenFreeMap' where name='name'")
+    c.execute("update metadata set value='https://openfreemap.org' where name='description'")
+
+    if 'openfreemap' not in metadata['attribution']:
+        attr_str = (
+            '<a href="https://openfreemap.org" target="_blank">OpenFreeMap</a> '
+            + metadata['attribution']
+        )
+        c.execute("UPDATE metadata SET value = ? WHERE name = 'attribution'", (attr_str,))
+
+    if 'osm_date' not in metadata:
+        if 'planetiler:osm:osmosisreplicationtime' in metadata:
+            osm_date = metadata['planetiler:osm:osmosisreplicationtime'][:10]
+            c.execute('INSERT INTO metadata (name, value) VALUES (?, ?)', ('osm_date', osm_date))
+
+    metadata = dict(c.execute('select name, value from metadata').fetchall())
+    with open(dir_path / 'metadata.json', 'w') as fp:
+        json.dump(metadata, fp, indent=2)
+
+    with open(dir_path / 'osm_date', 'w') as fp:
+        fp.write(metadata['osm_date'])
+
+
+def write_dedupl_files(c, *, dir_path):
+    """
+    dedupl files
+    write out the tiles_data files into a multi-level folder
+    """
+
+    total = c.execute('select count(*) from tiles_data').fetchone()[0]
+
+    c.execute('select tile_data_id, tile_data from tiles_data')
+    for i, row in enumerate(c, start=1):
+        dedupl_id = row[0]
+        dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
+        dedupl_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(dedupl_path, 'wb') as fp:
+            fp.write(row[1])
+            print(f'written dedupl file {i}/{total}')
+
+
+def write_tile_files(c, *, dir_path):
+    total = c.execute('select count(*) from tiles_shallow').fetchone()[0]
+
+    bug_fix_dict = {}
+
+    c.execute('select zoom_level, tile_column, tile_row, tile_data_id from tiles_shallow')
+    for i, row in enumerate(c, start=1):
+        z = row[0]
+        x = row[1]
+        y = flip_y(z, row[2])
+        dedupl_id = row[3]
+
+        dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
+        dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
+
+        tile_path = dir_path / 'tiles' / str(z) / str(x) / f'{y}.pbf'
+        tile_path.parent.mkdir(parents=True, exist_ok=True)
+
+        if tile_path.is_file():
+            continue
+
+        # create the hard link
+        try:
+            tile_path.hardlink_to(dedupl_path_fixed)
+            print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
+        except OSError as e:
+            # fixing Btrfs's 64k max link limit
+            if e.errno == 31:
+                bug_fix_dict.setdefault(dedupl_path, 0)
+                bug_fix_dict[dedupl_path] += 1
+                dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
+                shutil.copyfile(dedupl_path, dedupl_path_fixed)
+                print(f'Created fixed dedupl file: {dedupl_path_fixed}')
+                tile_path.hardlink_to(dedupl_path_fixed)
+                print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
+            else:
+                raise
+
+
+def assert_all_tiles_present(mbtiles_path, dir_path):
+    """
+    If it's a full planet run,
+    make sure there are exactly the right number of files generated.
+    """
+
+    if 'planet' in mbtiles_path.resolve().parent.name:
+        assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14)
+        print(f'Tile number: {calculate_tiles_sum(14)} - OK')
+
+
+def count_files(folder):
+    total = 0
+    for root, dirs, files in os.walk(folder):
+        total += len(files)
+    return
+
+
+def get_fixed_dedupl_name(bug_fix_dict, dedupl_path):
+    if dedupl_path in bug_fix_dict:
+        return dedupl_path.with_name(f'{dedupl_path.name}-{bug_fix_dict[dedupl_path]}')
+    else:
+        return dedupl_path
+
+
+def dedupl_helper_path(dedupl_id: int) -> Path:
+    """
+    Naming 200 million files such that each subdir has max 1000 children
+    """
+
+    str_num = f'{dedupl_id:09}'
+    l1 = str_num[:3]
+    l2 = str_num[3:6]
+    l3 = str_num[6:]
+    return Path(l1) / l2 / f'{l3}.pbf'
+
+
+def flip_y(zoom, y):
+    return (2**zoom - 1) - y
+
+
+def calculate_tiles(zoom_level):
+    return (2**zoom_level) ** 2
+
+
+def calculate_tiles_sum(zoom_level):
+    """
+    Sum of tiles up to zoom level (geometric series)
+    """
+    return (4 ** (zoom_level + 1) - 1) // 3
+
+
+if __name__ == '__main__':
+    cli()

modules/tile_gen/scripts/shrink_btrfs.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import click
+
+
+# btrfs cannot shrink smaller than 256 MiB
+SMALLEST_SIZE = 256 * 1024 * 1024
+
+
+@click.command()
+@click.argument(
+    'btrfs_img',
+    type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path),
+)
+def cli(btrfs_img: Path):
+    """
+    Shrinks a Btrfs image
+    // I cannot believe that Btrfs is over 15 years old,
+    // yet there is no resize2fs tool which can shrink a disk image
+    // to minimum size.
+    // It cannot even tell you how much should be the right size,
+    // it just randomly fails after which you have to umount and mount again.
+    // So we have to make a loop which tries to shrink it until it fails.
+
+    // Also, WONTFIX bugs like how instead of telling you that
+    // minimum fs size is 256 MB, it says "ERROR: unable to resize - Invalid argument"
+    // https://bugzilla.kernel.org/show_bug.cgi?id=118111
+    """
+
+    if os.geteuid() != 0:
+        sys.exit('  needs sudo')
+
+    current_dir = Path.cwd()
+
+    mnt_dir = Path(tempfile.mkdtemp(dir=current_dir, prefix='tmp_shrink_'))
+    subprocess.run(['mount', '-t', 'btrfs', btrfs_img, mnt_dir], check=True)
+
+    # shink until max. 10 MB left or reached SMALLEST_SIZE or failure
+    while True:
+        # needs to start with a balancing
+        # https://btrfs.readthedocs.io/en/latest/Balance.html
+        # https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html
+        do_balancing(mnt_dir)
+
+        free_bytes = get_usage(mnt_dir, 'Device unallocated')
+        device_size = get_usage(mnt_dir, 'Device size')
+        shrink_idea = free_bytes * 0.7
+
+        # workaround for the SMALLEST_SIZE limit
+        if device_size - free_bytes < SMALLEST_SIZE:
+            shrink_idea = (device_size - SMALLEST_SIZE) * 0.7
+
+        # stop if 10 MB left
+        if shrink_idea < 10_000_000:
+            break
+
+        # stop if process error
+        if not do_shrink(mnt_dir, shrink_idea):
+            break
+
+    total_size = get_usage(mnt_dir, 'Device size')
+
+    subprocess.run(['umount', mnt_dir])
+    mnt_dir.rmdir()
+
+    subprocess.run(['truncate', '-s', str(total_size), btrfs_img])
+    print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size')
+    print('shrink_btrfs.py DONE')
+
+
+def get_usage(mnt: Path, key: str):
+    p = subprocess.run(
+        ['btrfs', 'filesystem', 'usage', '-b', mnt], text=True, capture_output=True, check=True
+    )
+    for line in p.stdout.splitlines():
+        if f'{key}:' not in line:
+            continue
+        free = int(line.split(':')[1])
+        return free
+
+
+def do_shrink(mnt: Path, delta_size: float):
+    delta_size = int(delta_size)
+    print(f'Trying to shrink by {delta_size//1_000_000} MB')
+    p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt])
+    return p.returncode == 0
+
+
+def do_balancing(mnt: Path):
+    print('Starting btrfs balancing')
+    p = subprocess.run(
+        ['btrfs', 'balance', 'start', '-dusage=100', mnt], capture_output=True, text=True
+    )
+    if p.returncode:
+        print(f'Balance error: {p.stdout} {p.stderr}')
+    print('Balancing done')
+
+
+if __name__ == '__main__':
+    cli()

modules/tile_gen/setup.py

@@ -0,0 +1,13 @@
+from setuptools import find_packages, setup
+
+
+requirements = [
+    'click',
+]
+
+
+setup(
+    python_requires='>=3.10',
+    install_requires=requirements,
+    packages=find_packages(),
+)

modules/tile_gen/tile_gen.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import click
+from tile_gen_lib.btrfs import make_btrfs
+from tile_gen_lib.planetiler import run_planetiler
+from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
+
+
+@click.group()
+def cli():
+    """
+    Generates tiles and uploads to CloudFlare
+    """
+
+
+@cli.command()
+@click.argument('area', required=True)
+@click.option('--upload', is_flag=True, help='Upload after generation is complete')
+def make_tiles(area, upload):
+    """
+    Generate tiles for a given area, optionally upload it to the btrfs bucket
+    """
+
+    run_folder = run_planetiler(area)
+    make_btrfs(run_folder)
+
+    if upload:
+        upload_area(area)
+
+
+@cli.command(name='upload-area')
+@click.argument('area', required=True)
+def upload_area_(area):
+    """
+    Upload all runs from a given area to the btrfs bucket
+    """
+
+    upload_area(area)
+
+
+@cli.command()
+def make_indexes():
+    """
+    Make indexes for all buckets
+    """
+
+    for bucket in ['ofm-btrfs', 'ofm-assets']:
+        make_indexes_for_bucket(bucket)
+
+
+if __name__ == '__main__':
+    cli()

modules/tile_gen/tile_gen_lib/btrfs.py

@@ -0,0 +1,140 @@
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from tile_gen_lib.config import config
+from tile_gen_lib.utils import python_venv_executable
+
+
+IMAGE_SIZE = '200G'
+
+
+def make_btrfs(run_folder: Path):
+    os.chdir(run_folder)
+
+    cleanup_folder(run_folder)
+
+    # make an empty file that's definitely bigger then the current OSM output
+    for image in ['image.btrfs', 'image2.btrfs']:
+        subprocess.run(['fallocate', '-l', IMAGE_SIZE, image], check=True)
+        subprocess.run(['mkfs.btrfs', '-m', 'single', image], check=True, capture_output=True)
+
+    for image, mount in [('image.btrfs', 'mnt_rw'), ('image2.btrfs', 'mnt_rw2')]:
+        Path(mount).mkdir()
+
+        # https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
+        # compression doesn't make sense, data is already gzip compressed
+        subprocess.run(
+            [
+                'sudo',
+                'mount',
+                '-t',
+                'btrfs',
+                '-o',
+                'noacl,nobarrier,noatime,max_inline=4096',
+                image,
+                mount,
+            ],
+            check=True,
+        )
+
+        subprocess.run(['sudo', 'chown', 'ofm:ofm', '-R', mount], check=True)
+
+    # extract mbtiles
+    extract_script = config.tile_gen_scripts_dir / 'extract_mbtiles.py'
+    with open('extract_out.log', 'w') as out, open('extract_err.log', 'w') as err:
+        subprocess.run(
+            [
+                python_venv_executable(),
+                extract_script,
+                'tiles.mbtiles',
+                'mnt_rw/extract',
+            ],
+            check=True,
+            stdout=out,
+            stderr=err,
+        )
+
+    os.unlink('tiles.mbtiles')
+
+    shutil.copy('mnt_rw/extract/osm_date', '.')
+
+    # process logs
+    subprocess.run('grep fixed extract_out.log > dedupl_fixed.log', shell=True)
+
+    # unfortunately, by deleting files from the btrfs partition, the partition size grows
+    # so we need to rsync onto a new partition instead of deleting
+    with open('rsync_out.log', 'w') as out, open('rsync_err.log', 'w') as err:
+        subprocess.run(
+            [
+                'rsync',
+                '-avH',
+                '--max-alloc=4294967296',
+                '--exclude',
+                'dedupl',
+                'mnt_rw/extract/',
+                'mnt_rw2/',
+            ],
+            check=True,
+            stdout=out,
+            stderr=err,
+        )
+
+    # collect stats
+    for i, mount in enumerate(['mnt_rw', 'mnt_rw2'], 1):
+        with open(f'stats{i}.txt', 'w') as f:
+            for cmd in [
+                ['df', '-h', mount],
+                ['btrfs', 'filesystem', 'df', mount],
+                ['btrfs', 'filesystem', 'show', mount],
+                ['btrfs', 'filesystem', 'usage', mount],
+            ]:
+                f.write(f"\n\n{' '.join(cmd)}\n")
+                result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True)
+                f.write(result.stdout)
+
+    # unmount and cleanup
+    for mount in ['mnt_rw', 'mnt_rw2']:
+        subprocess.run(['sudo', 'umount', mount], check=True)
+
+    shutil.rmtree('mnt_rw')
+    shutil.rmtree('mnt_rw2')
+
+    # shrink btrfs
+    shrink_script = config.tile_gen_scripts_dir / 'shrink_btrfs.py'
+    with open('shrink_out.log', 'w') as out, open('shrink_err.log', 'w') as err:
+        subprocess.run(
+            ['sudo', python_venv_executable(), shrink_script, 'image2.btrfs'],
+            check=True,
+            stdout=out,
+            stderr=err,
+        )
+
+    os.unlink('image.btrfs')
+    shutil.move('image2.btrfs', 'tiles.btrfs')
+
+    # parallel gzip (pigz)
+    subprocess.run(['pigz', 'tiles.btrfs', '--fast'], check=True)
+
+    # move logs
+    Path('logs').mkdir()
+    for pattern in ['*.log', '*.txt']:
+        for file in Path().glob(pattern):
+            shutil.move(file, 'logs')
+
+    print('extract_btrfs.py DONE')
+
+
+def cleanup_folder(run_folder: Path):
+    print(f'cleaning up {run_folder}')
+
+    for mount in ['mnt_rw', 'mnt_rw2']:
+        subprocess.run(['sudo', 'umount', run_folder / mount], capture_output=True)
+
+    for pattern in ['mnt_rw*', 'tmp_*', '*.btrfs', '*.gz', '*.log', '*.txt', 'logs', 'osm_date']:
+        for item in run_folder.glob(pattern):
+            if item.is_dir():
+                shutil.rmtree(item)
+            else:
+                item.unlink()

modules/tile_gen/tile_gen_lib/config.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+
+
+class Configuration:
+    areas = ['planet', 'monaco']
+
+    tile_gen_dir = Path('/data/ofm/tile_gen')
+
+    tile_gen_bin = tile_gen_dir / 'bin'
+    tile_gen_scripts_dir = tile_gen_bin / 'scripts'
+
+    planetiler_bin = tile_gen_dir / 'planetiler'
+    planetiler_path = planetiler_bin / 'planetiler.jar'
+
+    runs_dir = tile_gen_dir / 'runs'
+
+    ofm_config_dir = Path('/data/ofm/config')
+    rclone_config = ofm_config_dir / 'rclone.conf'
+
+
+config = Configuration()

modules/tile_gen/tile_gen_lib/planetiler.py

@@ -0,0 +1,65 @@
+import os
+import shutil
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+
+from tile_gen_lib.config import config
+from tile_gen_lib.btrfs import cleanup_folder
+
+
+def run_planetiler(area: str) -> Path:
+    assert area in config.areas
+
+    date = datetime.now(tz=timezone.utc).strftime('%Y%m%d_%H%M%S')
+
+    area_dir = config.runs_dir / area
+
+    # delete all previous runs for the given area
+    for subdir in area_dir.iterdir():
+        cleanup_folder(subdir)
+
+    print('running rmtree')
+    shutil.rmtree(area_dir, ignore_errors=True)
+    print('rmtree done')
+
+    run_folder = area_dir / f'{date}_pt'
+    run_folder.mkdir(parents=True, exist_ok=True)
+
+    os.chdir(run_folder)
+
+    # link to discussion about why exactly 30 GB
+    # https://github.com/onthegomap/planetiler/discussions/690#discussioncomment-7756397
+    java_memory_gb = 30 if area == 'planet' else 1
+
+    command = [
+        'java',
+        f'-Xmx{java_memory_gb}g',
+        '-jar',
+        config.planetiler_path,
+        f'--area={area}',
+        '--download',
+        '--download-threads=10',
+        '--download-chunk-size-mb=1000',
+        '--fetch-wikidata',
+        '--output=tiles.mbtiles',
+        '--nodemap-type=array',
+        '--storage=mmap',
+        '--force',
+    ]
+
+    if area == 'planet':
+        command.append('--bounds=planet')
+
+    print(command)
+
+    out_path = run_folder / 'planetiler.out'
+    err_path = run_folder / 'planetiler.err'
+
+    with out_path.open('w') as out_file, err_path.open('w') as err_file:
+        subprocess.run(command, stdout=out_file, stderr=err_file, check=True, cwd=run_folder)
+
+    shutil.rmtree(run_folder / 'data', ignore_errors=True)
+    print('planetiler.jar DONE')
+
+    return run_folder

modules/tile_gen/tile_gen_lib/rclone.py

@@ -0,0 +1,123 @@
+import subprocess
+import sys
+
+from tile_gen_lib.config import config
+
+
+def upload_area(area):
+    """
+    Uploads an area, making sure there is exactly one run present
+    """
+
+    print(f'Uploading area: {area}')
+
+    assert area in config.areas
+
+    area_dir = config.runs_dir / area
+    if not area_dir.exists():
+        return
+
+    runs = list(area_dir.iterdir())
+    if len(runs) != 1:
+        print('Error: Make sure there is only one run in the given area')
+        sys.exit(1)
+
+    run = runs[0].name
+
+    upload_area_run(area, run)
+    make_indexes_for_bucket('ofm-btrfs')
+
+
+def upload_area_run(area, run):
+    print(f'Uploading {area} {run} to btrfs bucket')
+
+    run_dir = config.runs_dir / area / run
+    assert run_dir.is_dir()
+
+    subprocess.run(
+        [
+            'rclone',
+            'sync',
+            '--verbose=1',
+            '--transfers=8',
+            '--multi-thread-streams=8',
+            '--fast-list',
+            '--stats-file-name-length=0',
+            '--stats-one-line',
+            '--log-file',
+            run_dir / 'logs' / 'rclone.log',
+            '--exclude',
+            'logs/**',
+            run_dir,
+            f'remote:ofm-btrfs/areas/{area}/{run}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+    )
+
+
+def make_indexes_for_bucket(bucket):
+    print(f'Making indexes for bucket: {bucket}')
+
+    # files
+    p = subprocess.run(
+        [
+            'rclone',
+            'lsf',
+            '--recursive',
+            '--files-only',
+            '--fast-list',
+            '--exclude',
+            'dirs.txt',
+            '--exclude',
+            'files.txt',
+            f'remote:{bucket}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    index_str = p.stdout
+
+    # upload to files.txt
+    subprocess.run(
+        [
+            'rclone',
+            'rcat',
+            f'remote:{bucket}/files.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=index_str.encode(),
+    )
+
+    # directories
+    p = subprocess.run(
+        [
+            'rclone',
+            'lsf',
+            '--recursive',
+            '--dirs-only',
+            '--dir-slash=false',
+            '--fast-list',
+            f'remote:{bucket}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    index_str = p.stdout
+
+    # upload to dirs.txt
+    subprocess.run(
+        [
+            'rclone',
+            'rcat',
+            f'remote:{bucket}/dirs.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=index_str.encode(),
+    )

modules/tile_gen/tile_gen_lib/utils.py

@@ -0,0 +1,14 @@
+import os
+import sys
+from pathlib import Path
+
+
+def python_venv_executable() -> Path:
+    venv_path = os.environ.get('VIRTUAL_ENV')
+
+    if venv_path:
+        return Path(venv_path) / 'bin' / 'python'
+    elif sys.prefix != sys.base_prefix:
+        return Path(sys.prefix) / 'bin' / 'python'
+    else:
+        return Path(sys.executable)