sysadmin/openfreemap
1
0
Fork 0
mirror of https://github.com/hyperknot/openfreemap.git synced 2026-05-21 14:02:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
Zsolt Ero
2025-10-07 16:22:23 +02:00
parent fe30af3fb2
commit 55dae6776f
8 changed files with 120 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
init-server.py
View File

@@ -3,34 +3,35 @@
import click
from fabric import Config, Connection
from ssh_lib import MODULES_DIR, dotenv_val
from ssh_lib.tasks_httphost import prepare_http_host, run_http_host_sync
from ssh_lib.tasks_tilegen import prepare_tile_gen
from ssh_lib.config import config
from ssh_lib.tasks_http_host import prepare_http_host, run_http_host_sync
from ssh_lib.tasks_shared import prepare_shared
from ssh_lib.tasks_tile_gen import prepare_tile_gen
from ssh_lib.utils import (
put,
)
def get_connection(hostname, user, port):
ssh_passwd = dotenv_val('SSH_PASSWD')
# ssh_passwd = dotenv_val('SSH_PASSWD')
if ssh_passwd:
print('Using SSH password')
# if ssh_passwd:
# print('Using SSH password')
#
# c = Connection(
# host=hostname,
# user=user,
# port=port,
# connect_kwargs={'password': ssh_passwd},
# config=Config(overrides={'sudo': {'password': ssh_passwd}}),
# )
# else:
c = Connection(
host=hostname,
user=user,
port=port,
connect_kwargs={'password': ssh_passwd},
config=Config(overrides={'sudo': {'password': ssh_passwd}}),
)
else:
c = Connection(
host=hostname,
user=user,
port=port,
)
c = Connection(
host=hostname,
user=user,
port=port,
)
return c
@@ -80,7 +81,17 @@ def http_host_autoupdate(hostname, user, port, noninteractive):
run_http_host_sync(c) # disable for first install if you don't want to wait
put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
put(c, config.local_modules_dir / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
@cli.command()
@common_options
def http_host_sync(hostname, user, port, noninteractive):
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
return
c = get_connection(hostname, user, port)
run_http_host_sync(c)
@cli.command()
@@ -108,16 +119,6 @@ def tile_gen(
prepare_tile_gen(c, enable_cron=cron)
@cli.command()
@common_options
def http_host_sync(hostname, user, port, noninteractive):
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
return
c = get_connection(hostname, user, port)
run_http_host_sync(c)
#
# @cli.command()
# @common_options

4
modules/http_host/http_host_lib/assets.py
View File

@@ -31,7 +31,7 @@ def download_and_extract_asset_tar_gz(asset_kind):
print(f'Downloading asset {asset_kind}')
asset_dir = config.assets_dir / asset_kind
asset_dir = config.local_assets_dir / asset_kind
asset_dir.mkdir(exist_ok=True, parents=True)
url = f'https://assets.openfreemap.com/{asset_kind}/ofm.tar.gz'
@@ -62,7 +62,7 @@ def download_sprites() -> bool:
print('Downloading sprites')
sprites_dir = config.assets_dir / 'sprites'
sprites_dir = config.local_assets_dir / 'sprites'
sprites_dir.mkdir(exist_ok=True, parents=True)
r = requests.get('https://assets.openfreemap.com/files.txt', timeout=30)

10
ssh_lib/config.py
View File

@@ -4,15 +4,15 @@ from pathlib import Path
class Configuration:
# Local paths relative to this file
assets_dir = Path(__file__).parent / 'assets'
config_dir = Path(__file__).parent.parent / 'config'
modules_dir = Path(__file__).parent.parent / 'modules'
local_assets_dir = Path(__file__).parent / 'assets'
local_config_dir = Path(__file__).parent.parent / 'config'
local_modules_dir = Path(__file__).parent.parent / 'modules'
ENV = os.getenv('ENV')
if not ENV:
config_jsonc = config_dir / 'config.jsonc'
local_config_jsonc = local_config_dir / 'config.jsonc'
else:
config_jsonc = config_dir / f'config.{ENV}.jsonc'
local_config_jsonc = local_config_dir / f'config.{ENV}.jsonc'
# remote paths (always Linux /, not using pathlib)
ofm_dir = '/data/ofm'

1
ssh_lib/tasks.py
View File

@@ -1 +0,0 @@

76
ssh_lib/tasks_http_host.py Normal file
View File

@@ -0,0 +1,76 @@
import json
from ssh_lib.benchmark import c1000k, wrk
from ssh_lib.config import config
from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k
from ssh_lib.nginx import certbot, nginx
from ssh_lib.utils import put, put_dir, sudo_cmd
def prepare_http_host(c):
kernel_somaxconn65k(c)
kernel_limits1m(c)
upload_config_json(c)
nginx(c)
certbot(c)
c.sudo(f'rm -rf {config.http_host_dir}/logs')
c.sudo(f'mkdir -p {config.http_host_dir}/logs')
c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs')
c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx')
c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx')
c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx')
upload_http_host_files(c)
c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517')
def upload_config_json(c):
if not config.local_config_jsonc.is_file():
print(f'{config.local_config_jsonc} not found. Make sure it exists in the /config dir')
return
# validate using json5 + jsonschema
config_data = json.loads(config.local_config_jsonc.read_text())
# if ok, upload the file
put(
c,
config.local_config_jsonc,
f'{config.remote_config}/config.jsonc',
)
def upload_http_host_files(c):
c.sudo(f'rm -rf {config.http_host_bin}')
c.sudo(f'mkdir -p {config.http_host_bin}')
put_dir(c, config.local_modules_dir / 'http_host', config.http_host_bin, file_permissions='755')
for dirname in ['http_host_lib', 'scripts']:
put_dir(c, config.local_modules_dir / 'http_host' / dirname, f'{config.http_host_bin}/{dirname}')
put_dir(
c,
config.local_modules_dir / 'http_host' / 'http_host_lib' / 'nginx_confs',
f'{config.http_host_bin}/http_host_lib/nginx_confs',
)
c.sudo('chown -R ofm:ofm /data/ofm/http_host')
def run_http_host_sync(c):
print('Running http_host.py sync --force')
sudo_cmd(c, f'{config.venv_bin}/python -u {config.http_host_bin}/http_host.py sync --force')
def install_benchmark(c):
"""
Read docs/quick_notes/http_benchmark.md
"""
c1000k(c)
wrk(c)

93
ssh_lib/tasks_httphost.py
View File

@@ -1,93 +0,0 @@
import json
import sys
from ssh_lib.benchmark import c1000k, wrk
from ssh_lib.config import config
from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k
from ssh_lib.nginx import certbot, nginx
from ssh_lib.utils import put_dir, put_str, sudo_cmd
def prepare_http_host(c):
kernel_somaxconn65k(c)
kernel_limits1m(c)
upload_config_json(c)
nginx(c)
certbot(c)
c.sudo(f'rm -rf {config.http_host_dir}/logs')
c.sudo(f'mkdir -p {config.http_host_dir}/logs')
c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs')
c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx')
c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx')
c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx')
upload_http_host_files(c)
c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517')
def upload_config_json(c):
config.config_jsonc.is_file()
domain_direct = dotenv_val('DOMAIN_DIRECT').lower()
domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower()
skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
self_signed_certs = dotenv_val('SELF_SIGNED_CERTS').lower() == 'true'
letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower()
if not (domain_direct or domain_roundrobin):
sys.exit('Please specify DOMAIN_DIRECT or DOMAIN_ROUNDROBIN in config/.env')
if domain_direct and not letsencrypt_email and not self_signed_certs:
sys.exit('Please add your email to LETSENCRYPT_EMAIL when using DOMAIN_DIRECT')
http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()]
config = {
'domain_direct': domain_direct,
'domain_roundrobin': domain_roundrobin,
'letsencrypt_email': letsencrypt_email,
'skip_planet': skip_planet,
'self_signed_certs': self_signed_certs,
'http_host_list': http_host_list,
'telegram_token': dotenv_val('TELEGRAM_TOKEN'),
'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'),
}
config_str = json.dumps(config, indent=2, ensure_ascii=False)
print(config_str)
put_str(c, f'{REMOTE_CONFIG}/config.json', config_str)
def run_http_host_sync(c):
print('Running http_host.py sync --force')
sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/http_host.py sync --force')
def upload_http_host_files(c):
c.sudo(f'rm -rf {HTTP_HOST_BIN}')
c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')
for dirname in ['http_host_lib', 'scripts']:
put_dir(c, MODULES_DIR / 'http_host' / dirname, f'{HTTP_HOST_BIN}/{dirname}')
put_dir(
c,
MODULES_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs',
f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs',
)
c.sudo('chown -R ofm:ofm /data/ofm/http_host')
def install_benchmark(c):
"""
Read docs/quick_notes/http_benchmark.md
"""
c1000k(c)
wrk(c)

2
ssh_lib/tasks_shared.py
View File

@@ -23,7 +23,7 @@ def prepare_shared(c):
def prepare_venv(c):
put(
c,
config.modules_dir / 'prepare-virtualenv.sh',
config.local_modules_dir / 'prepare-virtualenv.sh',
config.ofm_dir,
permissions='755',
user='ofm',

10
ssh_lib/tasks_tilegen.py → ssh_lib/tasks_tile_gen.py
View File

@@ -10,15 +10,15 @@ def prepare_tile_gen(c, *, enable_cron):
c.sudo(f'rm -rf {config.tile_gen_bin}')
put_dir(c, config.modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755')
put_dir(c, config.local_modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755')
for dirname in ['tile_gen_lib', 'scripts']:
put_dir(c, config.modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}')
put_dir(c, config.local_modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}')
if (config.config_dir / 'rclone.conf').exists():
if (config.local_config_dir / 'rclone.conf').exists():
put(
c,
config.config_dir / 'rclone.conf',
config.local_config_dir / 'rclone.conf',
f'{config.remote_config}/rclone.conf',
permissions='600',
user='ofm',
@@ -33,4 +33,4 @@ def prepare_tile_gen(c, *, enable_cron):
c.sudo(f'chown ofm:ofm -R {config.tile_gen_bin}')
if enable_cron:
put(c, config.modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')
put(c, config.local_modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')