uploading and indexes

scripts/tile_gen/tile_gen_lib/rclone.py

@@ -0,0 +1,123 @@
+import subprocess
+import sys
+
+from tile_gen_lib.config import config
+
+
+def upload_area(area):
+    """
+    Uploads an area, making sure there is exactly one run present
+    """
+
+    print(f'Uploading area: {area}')
+
+    assert area in config.areas
+
+    area_dir = config.runs_dir / area
+    if not area_dir.exists():
+        return
+
+    runs = list(area_dir.iterdir())
+    if len(runs) != 1:
+        print('Error: Make sure there is only one run in the given area')
+        sys.exit(1)
+
+    run = runs[0].name
+
+    upload_area_run(area, run)
+    make_indexes_for_bucket('ofm-btrfs')
+
+
+def upload_area_run(area, run):
+    print(f'Uploading {area} {run} to btrfs bucket')
+
+    run_dir = config.runs_dir / area / run
+    assert run_dir.is_dir()
+
+    subprocess.run(
+        [
+            'rclone',
+            'sync',
+            '--verbose=1',
+            '--transfers=8',
+            '--multi-thread-streams=8',
+            '--fast-list',
+            '--stats-file-name-length=0',
+            '--stats-one-line',
+            '--log-file',
+            run_dir / 'logs' / 'rclone.log',
+            '--exclude',
+            'logs/**',
+            run_dir,
+            f'remote:ofm-btrfs/areas/{area}/{run}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+    )
+
+
+def make_indexes_for_bucket(bucket):
+    print(f'Making indexes for bucket: {bucket}')
+
+    # files
+    p = subprocess.run(
+        [
+            'rclone',
+            'lsf',
+            '--recursive',
+            '--files-only',
+            '--fast-list',
+            '--exclude',
+            'dirs.txt',
+            '--exclude',
+            'files.txt',
+            f'remote:{bucket}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    index_str = p.stdout
+
+    # upload to files.txt
+    subprocess.run(
+        [
+            'rclone',
+            'rcat',
+            f'remote:{bucket}/files.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=index_str.encode(),
+    )
+
+    # directories
+    p = subprocess.run(
+        [
+            'rclone',
+            'lsf',
+            '--recursive',
+            '--dirs-only',
+            '--dir-slash=false',
+            '--fast-list',
+            f'remote:{bucket}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    index_str = p.stdout
+
+    # upload to dirs.txt
+    subprocess.run(
+        [
+            'rclone',
+            'rcat',
+            f'remote:{bucket}/dirs.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=index_str.encode(),
+    )