sysadmin/openfreemap
1
0
Fork 0
mirror of https://github.com/hyperknot/openfreemap.git synced 2026-05-22 06:22:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

upload_manager

Browse Source
This commit is contained in:
Zsolt Ero
2024-06-09 02:36:33 +02:00
parent bb73111b2b
commit d290968722
2 changed files with 78 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
scripts/tile_gen/cloudflare_index.sh
View File

@@ -1,28 +0,0 @@
#!/usr/bin/env bash
set -e
AREAS=('planet' 'monaco')
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
rm -rf index
mkdir index
for AREA in "${AREAS[@]}"; do
rclone lsf -R \
--files-only \
--fast-list \
--exclude dirs.txt \
--exclude index.txt \
"remote:ofm-$AREA" > index/index.txt
rclone lsf -R \
--dirs-only \
--dir-slash=false \
--fast-list \
"remote:ofm-$AREA" > index/dirs.txt
rclone copy index "remote:ofm-$AREA"
done
rm -rf index

78
scripts/tile_gen/upload_manager.py
View File

@@ -1,6 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import json import json
import pathlib import pathlib
import shutil
import subprocess import subprocess
import click import click
@@ -35,6 +36,72 @@ def upload_rclone(area, run):
) )
def make_indexes():
for area in AREAS:
print(f'creating index {area}')
# files
p = subprocess.run(
[
'rclone',
'lsf',
'-R',
'--files-only',
'--fast-list',
'--exclude',
'dirs.txt',
'--exclude',
'index.txt',
f'remote:ofm-{area}',
],
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
check=True,
capture_output=True,
text=True,
)
index_str = p.stdout
subprocess.run(
[
'rclone',
'rcat',
f'remote:ofm-{area}/index.txt',
],
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
check=True,
input=index_str.encode(),
)
# directories
p = subprocess.run(
[
'rclone',
'lsf',
'-R',
'--dirs-only',
'--dir-slash=false',
'--fast-list',
f'remote:ofm-{area}',
],
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
check=True,
capture_output=True,
text=True,
)
index_str = p.stdout
subprocess.run(
[
'rclone',
'rcat',
f'remote:ofm-{area}/dirs.txt',
],
env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
check=True,
input=index_str.encode(),
)
@click.group() @click.group()
def cli(): def cli():
""" """
@@ -76,6 +143,17 @@ def upload_runs():
print(f'uploading {area} {run}') print(f'uploading {area} {run}')
upload_rclone(area, run) upload_rclone(area, run)
make_indexes()
@cli.command()
def index():
"""
Run index on Cloudflare buckets
"""
make_indexes()
if __name__ == '__main__': if __name__ == '__main__':
cli() cli()