mirror of
https://github.com/hyperknot/openfreemap.git
synced 2026-05-21 14:02:15 +00:00
host_manager
This commit is contained in:
Zsolt Ero
@@ -1,112 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import datetime
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets')
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
'--assets-dir',
|
||||
help='Specify assets directory',
|
||||
type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
|
||||
)
|
||||
def cli(assets_dir):
|
||||
"""
|
||||
Downloads and extracts assets
|
||||
"""
|
||||
|
||||
print(datetime.datetime.now(tz=datetime.timezone.utc))
|
||||
|
||||
if not assets_dir:
|
||||
assets_dir = DEFAULT_ASSETS_DIR
|
||||
|
||||
if not assets_dir.parent.exists():
|
||||
sys.exit("asset dir's parent doesn't exist")
|
||||
|
||||
download_fonts(assets_dir)
|
||||
|
||||
print('\n\n\n')
|
||||
|
||||
|
||||
def download_fonts(assets_dir):
|
||||
"""
|
||||
Download and extract font assets if their file differ.
|
||||
Making updates atomic, with extract to temp + move instead of extracting in place.
|
||||
"""
|
||||
|
||||
fonts_dir = assets_dir / 'fonts'
|
||||
fonts_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
fonts_temp = assets_dir / 'fonts_temp'
|
||||
|
||||
for font in ['ml', 'omt', 'pm']:
|
||||
url = f'https://assets.openfreemap.com/fonts/{font}.tgz'
|
||||
local_file = fonts_dir / f'{font}.tgz'
|
||||
if not download_if_size_differs(url, local_file):
|
||||
continue
|
||||
|
||||
shutil.rmtree(fonts_temp, ignore_errors=True)
|
||||
fonts_temp.mkdir()
|
||||
|
||||
subprocess.run(
|
||||
['tar', '-xzf', local_file, '-C', fonts_temp],
|
||||
check=True,
|
||||
)
|
||||
|
||||
target_dir = fonts_dir / font
|
||||
target_dir_renamed = fonts_dir / f'{font}.bak'
|
||||
temp_dir = fonts_temp / font
|
||||
|
||||
if target_dir.exists():
|
||||
target_dir.rename(target_dir_renamed)
|
||||
temp_dir.rename(target_dir)
|
||||
|
||||
shutil.rmtree(target_dir_renamed, ignore_errors=True)
|
||||
|
||||
shutil.rmtree(fonts_temp, ignore_errors=True)
|
||||
|
||||
|
||||
def download_if_size_differs(url: str, local_file: Path):
|
||||
if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url):
|
||||
download_file(url, local_file)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_remote_file_size(url: str):
|
||||
r = requests.head(url)
|
||||
size = r.headers.get('Content-Length')
|
||||
return int(size) if size else None
|
||||
|
||||
|
||||
def download_file(url, local_file):
|
||||
click.echo(f'Downloading: {url} into {local_file}')
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'aria2c',
|
||||
'--split=8',
|
||||
'--max-connection-per-server=8',
|
||||
'--file-allocation=none',
|
||||
'--min-split-size=1M',
|
||||
'-d',
|
||||
local_file.parent,
|
||||
'-o',
|
||||
local_file.name,
|
||||
url,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
@@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import datetime
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument('area', required=False)
|
||||
@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"')
|
||||
@click.option(
|
||||
'--runs-dir',
|
||||
help='Specify /runs directory',
|
||||
type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
|
||||
)
|
||||
@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
|
||||
@click.option('--run-mounter', is_flag=True, help='Run mounter.py after download is complete')
|
||||
def cli(area: str, version: str, list_versions: bool, runs_dir: Path, run_mounter: bool):
|
||||
"""
|
||||
Downloads and extracts the latest tiles.btrfs file from the public bucket.
|
||||
Specific version can also be specified.
|
||||
"""
|
||||
|
||||
print(datetime.datetime.now(tz=datetime.timezone.utc))
|
||||
|
||||
if area not in {'planet', 'monaco'}:
|
||||
sys.exit('Please specify are: "planet" or "monaco"')
|
||||
|
||||
r = requests.get(f'https://{area}.openfreemap.com/dirs.txt')
|
||||
r.raise_for_status()
|
||||
|
||||
versions = sorted(r.text.splitlines())
|
||||
|
||||
all_versions_str = '\n'.join(versions)
|
||||
if list_versions:
|
||||
print(all_versions_str)
|
||||
return
|
||||
|
||||
if version == 'latest':
|
||||
selected_version = versions[-1]
|
||||
else:
|
||||
if version not in versions:
|
||||
sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}')
|
||||
selected_version = version
|
||||
|
||||
if not runs_dir and not Path('/data/ofm').exists():
|
||||
sys.exit('Please specify a runs dir with --runs-dir')
|
||||
|
||||
changed = download(area, selected_version, runs_dir or DEFAULT_RUNS_DIR)
|
||||
|
||||
if changed and run_mounter:
|
||||
print('running mounter.py')
|
||||
subprocess.run(
|
||||
[sys.executable, Path(__file__).parent / 'mounter.py'],
|
||||
check=True,
|
||||
)
|
||||
|
||||
print('running nginx_sync.py')
|
||||
subprocess.run(
|
||||
[sys.executable, Path(__file__).parent / 'nginx_sync' / 'nginx_sync.py'],
|
||||
check=True,
|
||||
)
|
||||
|
||||
print('\n\n\n')
|
||||
|
||||
|
||||
def download(area: str, version: str, runs_dir: Path) -> bool:
|
||||
click.echo(f'Downloading: area: {area}, version: {version}')
|
||||
|
||||
version_dir = runs_dir / area / version
|
||||
btrfs_file = version_dir / 'tiles.btrfs'
|
||||
if btrfs_file.exists():
|
||||
print('File exists, skipping download')
|
||||
return False
|
||||
|
||||
temp_dir = runs_dir / '_tmp'
|
||||
if temp_dir.exists():
|
||||
sys.exit(f'{temp_dir} dir exists, please delete it first')
|
||||
|
||||
temp_dir.mkdir(parents=True)
|
||||
|
||||
url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz'
|
||||
print(url)
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'aria2c',
|
||||
'--split=8',
|
||||
'--max-connection-per-server=8',
|
||||
'--file-allocation=none',
|
||||
'--dir',
|
||||
temp_dir,
|
||||
url,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
|
||||
btrfs_src = temp_dir / 'tiles.btrfs'
|
||||
|
||||
shutil.rmtree(version_dir, ignore_errors=True)
|
||||
version_dir.mkdir(parents=True)
|
||||
|
||||
btrfs_src.rename(btrfs_file)
|
||||
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
92
scripts/http_host/host_manager.py
Executable file
92
scripts/http_host/host_manager.py
Executable file
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import requests
|
||||
from http_host_lib.download_fonts import download_fonts
|
||||
from http_host_lib.download_tileset import download_and_extract_tileset
|
||||
|
||||
|
||||
DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
|
||||
DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets')
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""
|
||||
Manages OpenFreeMap HTTP hosts, including:\n
|
||||
- Downloading tilesets\n
|
||||
- Downloading assets\n
|
||||
- Deploying the correct versions of tilesets\n
|
||||
- Mounting directories\n
|
||||
- Updating nginx config\n
|
||||
"""
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument('area', required=False)
|
||||
@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"')
|
||||
@click.option(
|
||||
'--runs-dir',
|
||||
help='Specify runs directory',
|
||||
type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
|
||||
)
|
||||
@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
|
||||
def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Path):
|
||||
"""
|
||||
Downloads and extracts the latest tiles.btrfs file from the public bucket.
|
||||
Version can also be specified.
|
||||
"""
|
||||
|
||||
if area not in {'planet', 'monaco'}:
|
||||
sys.exit('Please specify area: "planet" or "monaco"')
|
||||
|
||||
r = requests.get(f'https://{area}.openfreemap.com/dirs.txt')
|
||||
r.raise_for_status()
|
||||
|
||||
versions = sorted(r.text.splitlines())
|
||||
|
||||
all_versions_str = '\n'.join(versions)
|
||||
if list_versions:
|
||||
print(all_versions_str)
|
||||
return
|
||||
|
||||
if version == 'latest':
|
||||
selected_version = versions[-1]
|
||||
else:
|
||||
if version not in versions:
|
||||
sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}')
|
||||
selected_version = version
|
||||
|
||||
if not runs_dir:
|
||||
runs_dir = DEFAULT_RUNS_DIR
|
||||
|
||||
if not runs_dir.parent.exists():
|
||||
sys.exit("run dir's parent doesn't exist")
|
||||
|
||||
download_and_extract_tileset(area, selected_version, runs_dir)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
'--assets-dir',
|
||||
help='Specify assets directory',
|
||||
type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
|
||||
)
|
||||
def download_assets(assets_dir: Path):
|
||||
"""
|
||||
Downloads and extracts assets
|
||||
"""
|
||||
|
||||
if not assets_dir:
|
||||
assets_dir = DEFAULT_ASSETS_DIR
|
||||
|
||||
if not assets_dir.parent.exists():
|
||||
sys.exit("asset dir's parent doesn't exist")
|
||||
|
||||
download_fonts(assets_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
0
scripts/http_host/http_host_lib/__init__.py
Normal file
0
scripts/http_host/http_host_lib/__init__.py
Normal file
43
scripts/http_host/http_host_lib/download_fonts.py
Normal file
43
scripts/http_host/http_host_lib/download_fonts.py
Normal file
@@ -0,0 +1,43 @@
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from http_host_lib.utils import download_if_size_differs
|
||||
|
||||
|
||||
def download_fonts(assets_dir: Path):
|
||||
"""
|
||||
Download and extract font assets if their file size differ.
|
||||
Making updates atomic, with extraction to a temp dest + rename
|
||||
"""
|
||||
|
||||
fonts_dir = assets_dir / 'fonts'
|
||||
fonts_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
fonts_temp = assets_dir / 'fonts_temp'
|
||||
|
||||
for font in ['ml', 'omt', 'pm']:
|
||||
url = f'https://assets.openfreemap.com/fonts/{font}.tgz'
|
||||
local_file = fonts_dir / f'{font}.tgz'
|
||||
if not download_if_size_differs(url, local_file):
|
||||
continue
|
||||
|
||||
shutil.rmtree(fonts_temp, ignore_errors=True)
|
||||
fonts_temp.mkdir()
|
||||
|
||||
subprocess.run(
|
||||
['tar', '-xzf', local_file, '-C', fonts_temp],
|
||||
check=True,
|
||||
)
|
||||
|
||||
target_dir = fonts_dir / font
|
||||
target_dir_renamed = fonts_dir / f'{font}.bak'
|
||||
temp_dir = fonts_temp / font
|
||||
|
||||
if target_dir.exists():
|
||||
target_dir.rename(target_dir_renamed)
|
||||
temp_dir.rename(target_dir)
|
||||
|
||||
shutil.rmtree(target_dir_renamed, ignore_errors=True)
|
||||
|
||||
shutil.rmtree(fonts_temp, ignore_errors=True)
|
||||
42
scripts/http_host/http_host_lib/download_tileset.py
Normal file
42
scripts/http_host/http_host_lib/download_tileset.py
Normal file
@@ -0,0 +1,42 @@
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from http_host_lib.utils import download_file_aria2
|
||||
|
||||
|
||||
def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> bool:
|
||||
"""
|
||||
returns True if downloaded something
|
||||
"""
|
||||
|
||||
click.echo(f'Downloading: area: {area}, version: {version}')
|
||||
|
||||
version_dir = runs_dir / area / version
|
||||
btrfs_file = version_dir / 'tiles.btrfs'
|
||||
if btrfs_file.exists():
|
||||
print('File exists, skipping download')
|
||||
return False
|
||||
|
||||
temp_dir = runs_dir / '_tmp'
|
||||
if temp_dir.exists():
|
||||
sys.exit(f'{temp_dir} dir exists, avoiding parallel run')
|
||||
|
||||
temp_dir.mkdir(parents=True)
|
||||
|
||||
url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz'
|
||||
target_file = temp_dir / 'tiles.btrfs.gz'
|
||||
download_file_aria2(url, target_file)
|
||||
|
||||
subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
|
||||
btrfs_src = temp_dir / 'tiles.btrfs'
|
||||
|
||||
shutil.rmtree(version_dir, ignore_errors=True)
|
||||
version_dir.mkdir(parents=True)
|
||||
|
||||
btrfs_src.rename(btrfs_file)
|
||||
|
||||
shutil.rmtree(temp_dir)
|
||||
return True
|
||||
50
scripts/http_host/http_host_lib/utils.py
Normal file
50
scripts/http_host/http_host_lib/utils.py
Normal file
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def assert_sudo():
|
||||
if os.geteuid() != 0:
|
||||
sys.exit('Needs sudo')
|
||||
|
||||
|
||||
def assert_linux():
|
||||
if not Path('/etc/fstab').exists():
|
||||
sys.exit('Needs to be run on Linux')
|
||||
|
||||
|
||||
def download_if_size_differs(url: str, local_file: Path) -> bool:
|
||||
if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url):
|
||||
download_file_aria2(url, local_file)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_remote_file_size(url: str) -> int | None:
|
||||
r = requests.head(url)
|
||||
size = r.headers.get('Content-Length')
|
||||
return int(size) if size else None
|
||||
|
||||
|
||||
def download_file_aria2(url: str, local_file: Path):
|
||||
print(f'Downloading: {url} into {local_file}')
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'aria2c',
|
||||
'--split=8',
|
||||
'--max-connection-per-server=8',
|
||||
'--file-allocation=none',
|
||||
'--min-split-size=1M',
|
||||
'-d',
|
||||
local_file.parent,
|
||||
'-o',
|
||||
local_file.name,
|
||||
url,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
7
scripts/http_host/setup.py
Normal file
7
scripts/http_host/setup.py
Normal file
@@ -0,0 +1,7 @@
|
||||
from setuptools import setup
|
||||
|
||||
|
||||
setup(
|
||||
python_requires='>=3.10',
|
||||
packages=['http_host_lib'],
|
||||
)
|
||||
Reference in New Issue
Block a user