uploading and indexes

scripts/tile_gen/tile_gen.py

@@ -1,14 +1,9 @@
 #!/usr/bin/env python3
 
-import json
-import subprocess
-from pathlib import Path
-
 import click
-from tile_gen_lib.config import config
-from tile_gen_lib.extract import make_btrfs
+from tile_gen_lib.btrfs import make_btrfs
 from tile_gen_lib.planetiler import run_planetiler
-from tile_gen_lib.upload import make_indexes, upload_rclone
+from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
 
 
 @click.group()
@@ -20,60 +15,37 @@ def cli():
 
 @cli.command()
 @click.argument('area', required=True)
-def make_tiles(area):
+@click.option('--upload', is_flag=True, help='Upload after generation is complete')
+def make_tiles(area, upload):
     """
-    Generate tiles for a given area
+    Generate tiles for a given area, optionally upload it to the btrfs bucket
     """
 
     run_folder = run_planetiler(area)
     make_btrfs(run_folder)
-    # make_btrfs(Path('/data/ofm/tile_gen/runs/monaco/20240826_230406_pt'))
+
+    if upload:
+        upload_area(area)
+
+
+@cli.command(name='upload-area')
+@click.argument('area', required=True)
+def upload_area_(area):
+    """
+    Upload all runs from a given area to the btrfs bucket
+    """
+
+    upload_area(area)
 
 
 @cli.command()
-def upload_runs():
+def make_indexes():
     """
-    Upload all runs present in system
+    Make indexes for all buckets
     """
 
-    print('running upload_runs')
-
-    for area in config.areas:
-        if not (config.runs_dir / area).exists():
-            continue
-
-        p = subprocess.run(
-            [
-                'rclone',
-                'lsjson',
-                '--dirs-only',
-                '--fast-list',
-                f'remote:ofm-{area}',
-            ],
-            text=True,
-            capture_output=True,
-            check=True,
-            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-        )
-        rclone_json = json.loads(p.stdout)
-        runs_remote = {p['Path'] for p in rclone_json}
-        runs_local = {p.name for p in (config.runs_dir / area).iterdir()}
-
-        runs_to_upload = runs_local - runs_remote
-        for run in runs_to_upload:
-            print(f'uploading {area} {run}')
-            upload_rclone(area, run)
-
-    make_indexes()
-
-
-@cli.command()
-def index():
-    """
-    Run index on Cloudflare buckets
-    """
-
-    make_indexes()
+    for bucket in ['ofm-btrfs']:
+        make_indexes_for_bucket(bucket)
 
 
 if __name__ == '__main__':