From aaf9d2c74ea463d49741be742d0934d0a5355df9 Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Thu, 21 Dec 2023 03:07:55 +0100 Subject: [PATCH] work --- .ruff.toml | 1 + init-server.py | 12 ++-- scripts/tile_gen/extract_btrfs.sh | 2 +- .../extract_mbtiles/extract_mbtiles.py | 2 +- scripts/tile_gen/planetiler_monaco.sh | 4 +- scripts/tile_gen/planetiler_planet.sh | 3 +- scripts/tile_gen/prepare-virtualenv.sh | 1 + scripts/tile_gen/shrink_btrfs/shrink_btrfs.py | 1 + ssh_lib/pkg_base.py | 57 ++++++++++--------- ssh_lib/utils.py | 9 ++- 10 files changed, 50 insertions(+), 42 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 3b83932..7ddd832 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -24,6 +24,7 @@ ignore = [ 'E711', 'E712', 'E741', + 'F401', # unused imports 'F841', 'PT004', 'SIM102', diff --git a/init-server.py b/init-server.py index bf14977..f7f3168 100755 --- a/init-server.py +++ b/init-server.py @@ -30,8 +30,6 @@ def prepare_shared(c): def prepare_tile_gen(c): install_planetiler(c) - apt_get_install(c, 'btrfs-progs') - c.sudo('chown -R ofm:ofm /data/ofm') for file in [ @@ -40,8 +38,6 @@ def prepare_tile_gen(c): 'planetiler_planet.sh', 'prepare-virtualenv.sh', 'upload_cloudflare.sh', - 'extract_mbtiles/extract_mbtiles.py', - 'shrink_btrfs/extract_mbtiles.py', ]: put( c, @@ -54,19 +50,19 @@ def prepare_tile_gen(c): put( c, scripts / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py', - TILE_GEN_BIN / 'extract_mbtiles', + f'{TILE_GEN_BIN}/extract_mbtiles/extract_mbtiles.py', permissions='755', owner='ofm', - target_is_dir=True, + create_parent_dir=True, ) put( c, scripts / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py', - TILE_GEN_BIN / 'shrink_btrfs', + f'{TILE_GEN_BIN}/shrink_btrfs/shrink_btrfs.py', permissions='755', owner='ofm', - target_is_dir=True, + create_parent_dir=True, ) sudo_cmd(c, f'cd {TILE_GEN_BIN} && source prepare-virtualenv.sh', user='ofm') diff --git a/scripts/tile_gen/extract_btrfs.sh b/scripts/tile_gen/extract_btrfs.sh index 341136f..9ef01dd 100644 --- a/scripts/tile_gen/extract_btrfs.sh +++ b/scripts/tile_gen/extract_btrfs.sh @@ -92,4 +92,4 @@ mv image2.btrfs tiles.btrfs pigz tiles.btrfs --fast -echo DONE +echo extract_btrfs.sh DONE diff --git a/scripts/tile_gen/extract_mbtiles/extract_mbtiles.py b/scripts/tile_gen/extract_mbtiles/extract_mbtiles.py index ef5aa56..1f5eb6a 100755 --- a/scripts/tile_gen/extract_mbtiles/extract_mbtiles.py +++ b/scripts/tile_gen/extract_mbtiles/extract_mbtiles.py @@ -42,7 +42,7 @@ def cli(mbtiles_path: Path, dir_path: Path): # print(f'Tile number: {calculate_tiles_sum(14)} - OK') write_metadata(c, dir_path=dir_path) - print('DONE') + print('extract_mbtiles.py DONE') def write_metadata(c, *, dir_path): diff --git a/scripts/tile_gen/planetiler_monaco.sh b/scripts/tile_gen/planetiler_monaco.sh index f4b2082..06d4347 100644 --- a/scripts/tile_gen/planetiler_monaco.sh +++ b/scripts/tile_gen/planetiler_monaco.sh @@ -22,8 +22,8 @@ java -Xmx1g \ `# Store temporary node locations at fixed positions in a memory-mapped file` \ --nodemap-type=array --storage=mmap \ --force \ - > "planetiler_out.log" 2> "planetiler_err.log" + > planetiler_out 2> planetiler_err rm -r data -$TILE_GEN_BIN/extract_btrfs.sh \ No newline at end of file +#$TILE_GEN_BIN/extract_btrfs.sh \ No newline at end of file diff --git a/scripts/tile_gen/planetiler_planet.sh b/scripts/tile_gen/planetiler_planet.sh index 5f9522c..773242d 100644 --- a/scripts/tile_gen/planetiler_planet.sh +++ b/scripts/tile_gen/planetiler_planet.sh @@ -24,7 +24,8 @@ java -Xmx30g \ `# Store temporary node locations at fixed positions in a memory-mapped file` \ --nodemap-type=array --storage=mmap \ --force \ - > "planetiler_out.log" 2> "planetiler_err.log" + > planetiler_out 2> planetiler_err rm -r data +#$TILE_GEN_BIN/extract_btrfs.sh \ No newline at end of file diff --git a/scripts/tile_gen/prepare-virtualenv.sh b/scripts/tile_gen/prepare-virtualenv.sh index 1c016ff..f8a51ef 100755 --- a/scripts/tile_gen/prepare-virtualenv.sh +++ b/scripts/tile_gen/prepare-virtualenv.sh @@ -11,5 +11,6 @@ venv/bin/pip -V venv/bin/pip install -U pip wheel setuptools +venv/bin/pip install click diff --git a/scripts/tile_gen/shrink_btrfs/shrink_btrfs.py b/scripts/tile_gen/shrink_btrfs/shrink_btrfs.py index 2732974..c9864f8 100755 --- a/scripts/tile_gen/shrink_btrfs/shrink_btrfs.py +++ b/scripts/tile_gen/shrink_btrfs/shrink_btrfs.py @@ -70,6 +70,7 @@ def cli(btrfs_img: Path): subprocess.run(['truncate', '-s', str(total_size), btrfs_img]) print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size') + print('shrink_btrfs.py DONE') def get_usage(mnt: Path, key: str): diff --git a/ssh_lib/pkg_base.py b/ssh_lib/pkg_base.py index c538726..0f2d465 100644 --- a/ssh_lib/pkg_base.py +++ b/ssh_lib/pkg_base.py @@ -7,11 +7,31 @@ from ssh_lib.utils import ( ) -def pkg_upgrade(c): - apt_get_update(c) - c.sudo( - 'DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold"' - ) +def pkg_base(c): + pkg_list = [ + 'lsb-release', + 'wget', + 'git', + 'build-essential', + 'unzip', + 'rsync', + 'btrfs-progs', + 'pigz', + # + 'gnupg2', + 'gnupg-agent', + 'ca-certificates', + 'ubuntu-keyring', + # + 'nload', + 'iftop', + 'vnstat', + # + 'python3', + 'python3-venv', + ] + + apt_get_install(c, ' '.join(pkg_list)) def pkg_clean(c): @@ -34,25 +54,8 @@ def pkg_clean(c): c.sudo('iptables -L', warn=True) -def pkg_base(c): - pkg_list = [ - 'lsb-release', - 'wget', - 'git', - 'build-essential', - 'unzip', - # - 'gnupg2', - 'gnupg-agent', - 'ca-certificates', - 'ubuntu-keyring', - # - 'nload', - 'iftop', - 'vnstat', - # - 'python3', - 'python3-venv', - ] - - apt_get_install(c, ' '.join(pkg_list)) +def pkg_upgrade(c): + apt_get_update(c) + c.sudo( + 'DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold"' + ) diff --git a/ssh_lib/utils.py b/ssh_lib/utils.py index 64bcff0..d8340ac 100644 --- a/ssh_lib/utils.py +++ b/ssh_lib/utils.py @@ -4,12 +4,17 @@ import string def put( - c, local_path, remote_path, permissions=None, owner='root', group=None, target_is_dir=False + c, local_path, remote_path, permissions=None, owner='root', group=None, create_parent_dir=False ): tmp_path = f'/tmp/fabtmp_{random_string(8)}' c.put(local_path, tmp_path) - if is_dir(c, remote_path) or target_is_dir: + if create_parent_dir: + dirname = os.path.dirname(remote_path) + c.sudo(f'mkdir -p {dirname}') + set_permission(c, dirname, owner=owner, group=group) + + if is_dir(c, remote_path): if not remote_path.endswith('/'): remote_path += '/'