From f0476ee03d2ec0f50483151cab20d62ffde21d1d Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Sat, 24 Feb 2024 19:53:59 +0100 Subject: [PATCH] le dns --- config/.env.sample | 5 ++ init-server.py | 209 ++++++-------------------------------------- ssh_lib/__init__.py | 8 ++ ssh_lib/nginx.py | 2 + ssh_lib/tasks.py | 198 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 240 insertions(+), 182 deletions(-) create mode 100644 ssh_lib/tasks.py diff --git a/config/.env.sample b/config/.env.sample index 1a32ecb..1a09ee4 100644 --- a/config/.env.sample +++ b/config/.env.sample @@ -13,3 +13,8 @@ DOMAIN_CF=tiles.openfreemap.org # Skip the full planet download, useful for testing (true/false) SKIP_PLANET=false + +# --- Let's Encrypt DNS related variables, not needed for self-hosting + +DOMAIN_LE_DNS = rr.openfreemap.org + diff --git a/init-server.py b/init-server.py index 3a1b19e..78b31a6 100755 --- a/init-server.py +++ b/init-server.py @@ -1,191 +1,25 @@ #!/usr/bin/env python3 -import json -import sys import click -from dotenv import dotenv_values from fabric import Config, Connection -from ssh_lib import CONFIG_DIR, HTTP_HOST_BIN, OFM_DIR, REMOTE_CONFIG, SCRIPTS_DIR, TILE_GEN_BIN -from ssh_lib.benchmark import c1000k, wrk -from ssh_lib.kernel import kernel_tweaks_ofm -from ssh_lib.nginx import certbot, lego, nginx -from ssh_lib.pkg_base import pkg_base, pkg_upgrade -from ssh_lib.planetiler import planetiler -from ssh_lib.rclone import rclone +from ssh_lib import dotenv_val +from ssh_lib.tasks import ( + add_http_host_cron, + prepare_http_host, + prepare_shared, + prepare_tile_gen, + run_http_host_sync, + setup_le_dns_manager, + upload_http_host_config, +) from ssh_lib.utils import ( - add_user, - enable_sudo, - put, - put_dir, - put_str, sudo_cmd, ) -def prepare_shared(c): - # creates ofm user with uid=2000, disabled password and nopasswd sudo - add_user(c, 'ofm', uid=2000) - enable_sudo(c, 'ofm', nopasswd=True) - - pkg_upgrade(c) - pkg_base(c) - - kernel_tweaks_ofm(c) - - c.sudo(f'mkdir -p {REMOTE_CONFIG}') - c.sudo('chown ofm:ofm /data/ofm/config') - c.sudo('chown ofm:ofm /data/ofm') - - prepare_venv(c) - - -def prepare_venv(c): - put( - c, - SCRIPTS_DIR / 'prepare-virtualenv.sh', - OFM_DIR, - permissions='755', - user='ofm', - ) - sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh') - - -def prepare_tile_gen(c): - planetiler(c) - rclone(c) - - for file in [ - 'extract_btrfs.sh', - 'planetiler_monaco.sh', - 'planetiler_planet.sh', - 'cloudflare_index.sh', - 'cloudflare_upload.sh', - ]: - put( - c, - SCRIPTS_DIR / 'tile_gen' / file, - TILE_GEN_BIN, - permissions='755', - ) - - put( - c, - SCRIPTS_DIR / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py', - f'{TILE_GEN_BIN}/extract_mbtiles/extract_mbtiles.py', - create_parent_dir=True, - ) - - put( - c, - SCRIPTS_DIR / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py', - f'{TILE_GEN_BIN}/shrink_btrfs/shrink_btrfs.py', - create_parent_dir=True, - ) - - if (CONFIG_DIR / 'rclone.conf').exists(): - put( - c, - CONFIG_DIR / 'rclone.conf', - f'{REMOTE_CONFIG}/rclone.conf', - permissions='600', - user='ofm', - ) - - c.sudo('chown ofm:ofm /data/ofm/tile_gen') - c.sudo('chown ofm:ofm -R /data/ofm/tile_gen/bin') - - -def upload_http_host_config(c): - env_values = dotenv_values(f'{CONFIG_DIR}/.env') - - domain_le = env_values.get('DOMAIN_LE', '').strip() - domain_cf = env_values.get('DOMAIN_CF', '').strip() - skip_planet = env_values.get('SKIP_PLANET', '').lower().strip() == 'true' - le_email = env_values.get('LE_EMAIL', '').strip() - - if not (domain_le or domain_cf): - sys.exit('Please specify DOMAIN_LE or DOMAIN_CF in config/.env') - - if domain_cf: - if ( - not (CONFIG_DIR / 'certs' / 'ofm_cf.key').exists() - or not (CONFIG_DIR / 'certs' / 'ofm_cf.cert').exists() - ): - sys.exit( - 'When using DOMAIN_CF, please put ofm_cf.key and ofm_cf.cert files in config/certs' - ) - - if domain_le and not le_email: - sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE') - - host_config = { - 'domain_le': domain_le, - 'domain_cf': domain_cf, - 'skip_planet': skip_planet, - 'le_email': le_email, - } - - host_config_str = json.dumps(host_config, indent=2, ensure_ascii=False) - print(host_config_str) - put_str(c, '/data/ofm/config/http_host.json', host_config_str) - - -def prepare_http_host(c): - nginx(c) - certbot(c) - - c.sudo('rm -rf /data/ofm/http_host/logs') - c.sudo('mkdir -p /data/ofm/http_host/logs') - c.sudo('chown ofm:ofm /data/ofm/http_host/logs') - - c.sudo('rm -rf /data/ofm/http_host/logs_nginx') - c.sudo('mkdir -p /data/ofm/http_host/logs_nginx') - c.sudo('chown nginx:nginx /data/ofm/http_host/logs_nginx') - - upload_https_host_files(c) - upload_certificates(c) - - c.sudo('/data/ofm/venv/bin/pip install -e /data/ofm/http_host/bin') - - -def add_http_host_cron(c): - put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') - - -def run_http_host_sync(c): - sudo_cmd(c, '/data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync') - - -def upload_https_host_files(c): - c.sudo(f'mkdir -p {HTTP_HOST_BIN}') - - put_dir(c, SCRIPTS_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755') - put_dir(c, SCRIPTS_DIR / 'http_host' / 'http_host_lib', f'{HTTP_HOST_BIN}/http_host_lib') - put_dir( - c, - SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx', - f'{HTTP_HOST_BIN}/http_host_lib/nginx', - ) - - c.sudo('chown -R ofm:ofm /data/ofm/http_host') - - -def upload_certificates(c): - put_dir(c, CONFIG_DIR / 'certs', '/data/nginx/certs', file_permissions=400) - c.sudo('chown -R nginx:nginx /data/nginx') - - -def install_benchmark(c): - """ - Read docs/quick_notes/http_benchmark.md - """ - c1000k(c) - wrk(c) - - def get_connection(hostname, user, port): - ssh_passwd = dotenv_values(f'{CONFIG_DIR}/.env').get('SSH_PASSWD', '').strip() + ssh_passwd = dotenv_val('SSH_PASSWD') if ssh_passwd: print('Using SSH password') @@ -227,11 +61,11 @@ def http_host_once(hostname, user, port): return c = get_connection(hostname, user, port) + prepare_shared(c) - upload_http_host_config(c) - prepare_http_host(c) + prepare_http_host(c) run_http_host_sync(c) @@ -242,11 +76,11 @@ def http_host_autoupdate(hostname, user, port): return c = get_connection(hostname, user, port) + prepare_shared(c) - upload_http_host_config(c) - prepare_http_host(c) + prepare_http_host(c) add_http_host_cron(c) @@ -262,14 +96,25 @@ def tile_gen(hostname, user, port): prepare_tile_gen(c) +@cli.command() +@common_options +def le_dns_manager(hostname, user, port): + if not click.confirm(f'Run script on {hostname}?'): + return + + c = get_connection(hostname, user, port) + + setup_le_dns_manager(c) + + @cli.command() @common_options def debug(hostname, user, port): c = get_connection(hostname, user, port) prepare_shared(c) - upload_http_host_config(c) + prepare_http_host(c) run_http_host_sync(c) diff --git a/ssh_lib/__init__.py b/ssh_lib/__init__.py index 2f08852..ef0bfbf 100644 --- a/ssh_lib/__init__.py +++ b/ssh_lib/__init__.py @@ -1,5 +1,7 @@ from pathlib import Path +from dotenv import dotenv_values + ASSETS_DIR = Path(__file__).parent / 'assets' CONFIG_DIR = Path(__file__).parent.parent / 'config' @@ -9,3 +11,9 @@ OFM_DIR = '/data/ofm' REMOTE_CONFIG = '/data/ofm/config' TILE_GEN_BIN = '/data/ofm/tile_gen/bin' HTTP_HOST_BIN = '/data/ofm/http_host/bin' + +DOTENV_VALUES = dotenv_values(f'{CONFIG_DIR}/.env') + + +def dotenv_val(key): + return DOTENV_VALUES.get(key, '').strip() diff --git a/ssh_lib/nginx.py b/ssh_lib/nginx.py index fb7c1fe..800def2 100644 --- a/ssh_lib/nginx.py +++ b/ssh_lib/nginx.py @@ -69,6 +69,8 @@ def certbot(c): apt_get_purge(c, 'certbot') c.sudo('snap install --classic certbot', warn=True) + c.sudo('snap set certbot trust-plugin-with-root=ok', warn=True) + c.sudo('snap install certbot-dns-cloudflare', warn=True) def lego(c): diff --git a/ssh_lib/tasks.py b/ssh_lib/tasks.py new file mode 100644 index 0000000..380a81f --- /dev/null +++ b/ssh_lib/tasks.py @@ -0,0 +1,198 @@ +import json +import sys + +from ssh_lib import ( + CONFIG_DIR, + HTTP_HOST_BIN, + OFM_DIR, + REMOTE_CONFIG, + SCRIPTS_DIR, + TILE_GEN_BIN, + dotenv_val, +) +from ssh_lib.benchmark import c1000k, wrk +from ssh_lib.kernel import kernel_tweaks_ofm +from ssh_lib.nginx import certbot, nginx +from ssh_lib.pkg_base import pkg_base, pkg_upgrade +from ssh_lib.planetiler import planetiler +from ssh_lib.rclone import rclone +from ssh_lib.utils import add_user, enable_sudo, put, put_dir, put_str, sudo_cmd + + +def prepare_shared(c): + # creates ofm user with uid=2000, disabled password and nopasswd sudo + add_user(c, 'ofm', uid=2000) + enable_sudo(c, 'ofm', nopasswd=True) + + pkg_upgrade(c) + pkg_base(c) + + kernel_tweaks_ofm(c) + + c.sudo(f'mkdir -p {REMOTE_CONFIG}') + c.sudo('chown ofm:ofm /data/ofm/config') + c.sudo('chown ofm:ofm /data/ofm') + + prepare_venv(c) + + +def prepare_venv(c): + put( + c, + SCRIPTS_DIR / 'prepare-virtualenv.sh', + OFM_DIR, + permissions='755', + user='ofm', + ) + sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh') + + +def prepare_tile_gen(c): + planetiler(c) + rclone(c) + + for file in [ + 'extract_btrfs.sh', + 'planetiler_monaco.sh', + 'planetiler_planet.sh', + 'cloudflare_index.sh', + 'cloudflare_upload.sh', + ]: + put( + c, + SCRIPTS_DIR / 'tile_gen' / file, + TILE_GEN_BIN, + permissions='755', + ) + + put( + c, + SCRIPTS_DIR / 'tile_gen' / 'extract_mbtiles' / 'extract_mbtiles.py', + f'{TILE_GEN_BIN}/extract_mbtiles/extract_mbtiles.py', + create_parent_dir=True, + ) + + put( + c, + SCRIPTS_DIR / 'tile_gen' / 'shrink_btrfs' / 'shrink_btrfs.py', + f'{TILE_GEN_BIN}/shrink_btrfs/shrink_btrfs.py', + create_parent_dir=True, + ) + + if (CONFIG_DIR / 'rclone.conf').exists(): + put( + c, + CONFIG_DIR / 'rclone.conf', + f'{REMOTE_CONFIG}/rclone.conf', + permissions='600', + user='ofm', + ) + + c.sudo('chown ofm:ofm /data/ofm/tile_gen') + c.sudo('chown ofm:ofm -R /data/ofm/tile_gen/bin') + + +def upload_http_host_config(c): + domain_le = dotenv_val('DOMAIN_LE').lower() + domain_cf = dotenv_val('DOMAIN_CF').lower() + skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true' + le_email = dotenv_val('LE_EMAIL').lower() + + if not (domain_le or domain_cf): + sys.exit('Please specify DOMAIN_LE or DOMAIN_CF in config/.env') + + if domain_cf: + if ( + not (CONFIG_DIR / 'certs' / 'ofm_cf.key').exists() + or not (CONFIG_DIR / 'certs' / 'ofm_cf.cert').exists() + ): + sys.exit( + 'When using DOMAIN_CF, please put ofm_cf.key and ofm_cf.cert files in config/certs' + ) + + if domain_le and not le_email: + sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE') + + host_config = { + 'domain_le': domain_le, + 'domain_cf': domain_cf, + 'skip_planet': skip_planet, + 'le_email': le_email, + } + + host_config_str = json.dumps(host_config, indent=2, ensure_ascii=False) + print(host_config_str) + put_str(c, '/data/ofm/config/http_host.json', host_config_str) + + +def prepare_http_host(c): + nginx(c) + certbot(c) + + c.sudo('rm -rf /data/ofm/http_host/logs') + c.sudo('mkdir -p /data/ofm/http_host/logs') + c.sudo('chown ofm:ofm /data/ofm/http_host/logs') + + c.sudo('rm -rf /data/ofm/http_host/logs_nginx') + c.sudo('mkdir -p /data/ofm/http_host/logs_nginx') + c.sudo('chown nginx:nginx /data/ofm/http_host/logs_nginx') + + upload_https_host_files(c) + upload_certificates(c) + + c.sudo('/data/ofm/venv/bin/pip install -e /data/ofm/http_host/bin') + + +def add_http_host_cron(c): + put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') + + +def run_http_host_sync(c): + sudo_cmd(c, '/data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync') + + +def upload_https_host_files(c): + c.sudo(f'mkdir -p {HTTP_HOST_BIN}') + + put_dir(c, SCRIPTS_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755') + put_dir(c, SCRIPTS_DIR / 'http_host' / 'http_host_lib', f'{HTTP_HOST_BIN}/http_host_lib') + put_dir( + c, + SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx', + f'{HTTP_HOST_BIN}/http_host_lib/nginx', + ) + + c.sudo('chown -R ofm:ofm /data/ofm/http_host') + + +def upload_certificates(c): + put_dir(c, CONFIG_DIR / 'certs', '/data/nginx/certs', file_permissions=400) + c.sudo('chown -R nginx:nginx /data/nginx') + + +def install_benchmark(c): + """ + Read docs/quick_notes/http_benchmark.md + """ + c1000k(c) + wrk(c) + + +def setup_le_dns_manager(c): + le_email = dotenv_val('LE_EMAIL').lower() + domain_le_dns = dotenv_val('DOMAIN_LE_DNS').lower() + + assert le_email + assert domain_le_dns + + sudo_cmd( + c, + 'certbot certonly ' + '--dns-cloudflare ' + '--dns-cloudflare-credentials ~/.secrets/certbot/cloudflare.ini ' + '--staging ' + f'--noninteractive -m {le_email} ' + f'--agree-tos ' + f'--cert-name=ofm_le_dns ' + f'-d {domain_le_dns}', + )