diff --git a/scripts/tile_gen/cloudflare_index.sh b/scripts/tile_gen/cloudflare_index.sh deleted file mode 100755 index 80e5f9b..0000000 --- a/scripts/tile_gen/cloudflare_index.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -e - -AREAS=('planet' 'monaco') - -export RCLONE_CONFIG=/data/ofm/config/rclone.conf - -rm -rf index -mkdir index - -for AREA in "${AREAS[@]}"; do - rclone lsf -R \ - --files-only \ - --fast-list \ - --exclude dirs.txt \ - --exclude index.txt \ - "remote:ofm-$AREA" > index/index.txt - - rclone lsf -R \ - --dirs-only \ - --dir-slash=false \ - --fast-list \ - "remote:ofm-$AREA" > index/dirs.txt - - rclone copy index "remote:ofm-$AREA" -done - -rm -rf index diff --git a/scripts/tile_gen/upload_manager.py b/scripts/tile_gen/upload_manager.py index b9de98f..a83d0ab 100755 --- a/scripts/tile_gen/upload_manager.py +++ b/scripts/tile_gen/upload_manager.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json import pathlib +import shutil import subprocess import click @@ -35,6 +36,72 @@ def upload_rclone(area, run): ) +def make_indexes(): + for area in AREAS: + print(f'creating index {area}') + + # files + p = subprocess.run( + [ + 'rclone', + 'lsf', + '-R', + '--files-only', + '--fast-list', + '--exclude', + 'dirs.txt', + '--exclude', + 'index.txt', + f'remote:ofm-{area}', + ], + env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'), + check=True, + capture_output=True, + text=True, + ) + index_str = p.stdout + + subprocess.run( + [ + 'rclone', + 'rcat', + f'remote:ofm-{area}/index.txt', + ], + env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'), + check=True, + input=index_str.encode(), + ) + + # directories + p = subprocess.run( + [ + 'rclone', + 'lsf', + '-R', + '--dirs-only', + '--dir-slash=false', + '--fast-list', + f'remote:ofm-{area}', + ], + env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'), + check=True, + capture_output=True, + text=True, + ) + index_str = p.stdout + + subprocess.run( + [ + 'rclone', + 'rcat', + f'remote:ofm-{area}/dirs.txt', + ], + env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'), + check=True, + input=index_str.encode(), + ) + + @click.group() def cli(): """ @@ -76,6 +143,17 @@ def upload_runs(): print(f'uploading {area} {run}') upload_rclone(area, run) + make_indexes() + + +@cli.command() +def index(): + """ + Run index on Cloudflare buckets + """ + + make_indexes() + if __name__ == '__main__': cli()