diff --git a/init-server.py b/init-server.py index 809f2cd..c8e00d5 100755 --- a/init-server.py +++ b/init-server.py @@ -3,34 +3,35 @@ import click from fabric import Config, Connection -from ssh_lib import MODULES_DIR, dotenv_val -from ssh_lib.tasks_httphost import prepare_http_host, run_http_host_sync -from ssh_lib.tasks_tilegen import prepare_tile_gen +from ssh_lib.config import config +from ssh_lib.tasks_http_host import prepare_http_host, run_http_host_sync from ssh_lib.tasks_shared import prepare_shared +from ssh_lib.tasks_tile_gen import prepare_tile_gen from ssh_lib.utils import ( put, ) def get_connection(hostname, user, port): - ssh_passwd = dotenv_val('SSH_PASSWD') + # ssh_passwd = dotenv_val('SSH_PASSWD') - if ssh_passwd: - print('Using SSH password') + # if ssh_passwd: + # print('Using SSH password') + # + # c = Connection( + # host=hostname, + # user=user, + # port=port, + # connect_kwargs={'password': ssh_passwd}, + # config=Config(overrides={'sudo': {'password': ssh_passwd}}), + # ) + # else: - c = Connection( - host=hostname, - user=user, - port=port, - connect_kwargs={'password': ssh_passwd}, - config=Config(overrides={'sudo': {'password': ssh_passwd}}), - ) - else: - c = Connection( - host=hostname, - user=user, - port=port, - ) + c = Connection( + host=hostname, + user=user, + port=port, + ) return c @@ -80,7 +81,17 @@ def http_host_autoupdate(hostname, user, port, noninteractive): run_http_host_sync(c) # disable for first install if you don't want to wait - put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') + put(c, config.local_modules_dir / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/') + + +@cli.command() +@common_options +def http_host_sync(hostname, user, port, noninteractive): + if not noninteractive and not click.confirm(f'Run script on {hostname}?'): + return + + c = get_connection(hostname, user, port) + run_http_host_sync(c) @cli.command() @@ -108,16 +119,6 @@ def tile_gen( prepare_tile_gen(c, enable_cron=cron) -@cli.command() -@common_options -def http_host_sync(hostname, user, port, noninteractive): - if not noninteractive and not click.confirm(f'Run script on {hostname}?'): - return - - c = get_connection(hostname, user, port) - run_http_host_sync(c) - - # # @cli.command() # @common_options diff --git a/modules/http_host/http_host_lib/assets.py b/modules/http_host/http_host_lib/assets.py index 01c425d..8c509b9 100644 --- a/modules/http_host/http_host_lib/assets.py +++ b/modules/http_host/http_host_lib/assets.py @@ -31,7 +31,7 @@ def download_and_extract_asset_tar_gz(asset_kind): print(f'Downloading asset {asset_kind}') - asset_dir = config.assets_dir / asset_kind + asset_dir = config.local_assets_dir / asset_kind asset_dir.mkdir(exist_ok=True, parents=True) url = f'https://assets.openfreemap.com/{asset_kind}/ofm.tar.gz' @@ -62,7 +62,7 @@ def download_sprites() -> bool: print('Downloading sprites') - sprites_dir = config.assets_dir / 'sprites' + sprites_dir = config.local_assets_dir / 'sprites' sprites_dir.mkdir(exist_ok=True, parents=True) r = requests.get('https://assets.openfreemap.com/files.txt', timeout=30) diff --git a/ssh_lib/config.py b/ssh_lib/config.py index 976e387..57c5819 100644 --- a/ssh_lib/config.py +++ b/ssh_lib/config.py @@ -4,15 +4,15 @@ from pathlib import Path class Configuration: # Local paths relative to this file - assets_dir = Path(__file__).parent / 'assets' - config_dir = Path(__file__).parent.parent / 'config' - modules_dir = Path(__file__).parent.parent / 'modules' + local_assets_dir = Path(__file__).parent / 'assets' + local_config_dir = Path(__file__).parent.parent / 'config' + local_modules_dir = Path(__file__).parent.parent / 'modules' ENV = os.getenv('ENV') if not ENV: - config_jsonc = config_dir / 'config.jsonc' + local_config_jsonc = local_config_dir / 'config.jsonc' else: - config_jsonc = config_dir / f'config.{ENV}.jsonc' + local_config_jsonc = local_config_dir / f'config.{ENV}.jsonc' # remote paths (always Linux /, not using pathlib) ofm_dir = '/data/ofm' diff --git a/ssh_lib/tasks.py b/ssh_lib/tasks.py deleted file mode 100644 index 8b13789..0000000 --- a/ssh_lib/tasks.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ssh_lib/tasks_http_host.py b/ssh_lib/tasks_http_host.py new file mode 100644 index 0000000..7cd211c --- /dev/null +++ b/ssh_lib/tasks_http_host.py @@ -0,0 +1,76 @@ +import json + +from ssh_lib.benchmark import c1000k, wrk +from ssh_lib.config import config +from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k +from ssh_lib.nginx import certbot, nginx +from ssh_lib.utils import put, put_dir, sudo_cmd + + +def prepare_http_host(c): + kernel_somaxconn65k(c) + kernel_limits1m(c) + + upload_config_json(c) + + nginx(c) + certbot(c) + + c.sudo(f'rm -rf {config.http_host_dir}/logs') + c.sudo(f'mkdir -p {config.http_host_dir}/logs') + c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs') + + c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx') + c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx') + c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx') + + upload_http_host_files(c) + + c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517') + + +def upload_config_json(c): + if not config.local_config_jsonc.is_file(): + print(f'{config.local_config_jsonc} not found. Make sure it exists in the /config dir') + return + + # validate using json5 + jsonschema + config_data = json.loads(config.local_config_jsonc.read_text()) + + # if ok, upload the file + put( + c, + config.local_config_jsonc, + f'{config.remote_config}/config.jsonc', + ) + + +def upload_http_host_files(c): + c.sudo(f'rm -rf {config.http_host_bin}') + c.sudo(f'mkdir -p {config.http_host_bin}') + + put_dir(c, config.local_modules_dir / 'http_host', config.http_host_bin, file_permissions='755') + + for dirname in ['http_host_lib', 'scripts']: + put_dir(c, config.local_modules_dir / 'http_host' / dirname, f'{config.http_host_bin}/{dirname}') + + put_dir( + c, + config.local_modules_dir / 'http_host' / 'http_host_lib' / 'nginx_confs', + f'{config.http_host_bin}/http_host_lib/nginx_confs', + ) + + c.sudo('chown -R ofm:ofm /data/ofm/http_host') + + +def run_http_host_sync(c): + print('Running http_host.py sync --force') + sudo_cmd(c, f'{config.venv_bin}/python -u {config.http_host_bin}/http_host.py sync --force') + + +def install_benchmark(c): + """ + Read docs/quick_notes/http_benchmark.md + """ + c1000k(c) + wrk(c) diff --git a/ssh_lib/tasks_httphost.py b/ssh_lib/tasks_httphost.py deleted file mode 100644 index 44cde6e..0000000 --- a/ssh_lib/tasks_httphost.py +++ /dev/null @@ -1,93 +0,0 @@ -import json -import sys - -from ssh_lib.benchmark import c1000k, wrk -from ssh_lib.config import config -from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k -from ssh_lib.nginx import certbot, nginx -from ssh_lib.utils import put_dir, put_str, sudo_cmd - - -def prepare_http_host(c): - kernel_somaxconn65k(c) - kernel_limits1m(c) - - upload_config_json(c) - - nginx(c) - certbot(c) - - c.sudo(f'rm -rf {config.http_host_dir}/logs') - c.sudo(f'mkdir -p {config.http_host_dir}/logs') - c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs') - - c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx') - c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx') - c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx') - - upload_http_host_files(c) - - c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517') - - -def upload_config_json(c): - config.config_jsonc.is_file() - domain_direct = dotenv_val('DOMAIN_DIRECT').lower() - domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower() - skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true' - self_signed_certs = dotenv_val('SELF_SIGNED_CERTS').lower() == 'true' - letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower() - - if not (domain_direct or domain_roundrobin): - sys.exit('Please specify DOMAIN_DIRECT or DOMAIN_ROUNDROBIN in config/.env') - - if domain_direct and not letsencrypt_email and not self_signed_certs: - sys.exit('Please add your email to LETSENCRYPT_EMAIL when using DOMAIN_DIRECT') - - http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()] - - config = { - 'domain_direct': domain_direct, - 'domain_roundrobin': domain_roundrobin, - 'letsencrypt_email': letsencrypt_email, - 'skip_planet': skip_planet, - 'self_signed_certs': self_signed_certs, - 'http_host_list': http_host_list, - 'telegram_token': dotenv_val('TELEGRAM_TOKEN'), - 'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'), - } - - config_str = json.dumps(config, indent=2, ensure_ascii=False) - print(config_str) - put_str(c, f'{REMOTE_CONFIG}/config.json', config_str) - - -def run_http_host_sync(c): - print('Running http_host.py sync --force') - sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/http_host.py sync --force') - - -def upload_http_host_files(c): - c.sudo(f'rm -rf {HTTP_HOST_BIN}') - c.sudo(f'mkdir -p {HTTP_HOST_BIN}') - - put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755') - - for dirname in ['http_host_lib', 'scripts']: - put_dir(c, MODULES_DIR / 'http_host' / dirname, f'{HTTP_HOST_BIN}/{dirname}') - - put_dir( - c, - MODULES_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs', - f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs', - ) - - c.sudo('chown -R ofm:ofm /data/ofm/http_host') - - -def install_benchmark(c): - """ - Read docs/quick_notes/http_benchmark.md - """ - c1000k(c) - wrk(c) diff --git a/ssh_lib/tasks_shared.py b/ssh_lib/tasks_shared.py index 8ee2c56..f4beef4 100644 --- a/ssh_lib/tasks_shared.py +++ b/ssh_lib/tasks_shared.py @@ -23,7 +23,7 @@ def prepare_shared(c): def prepare_venv(c): put( c, - config.modules_dir / 'prepare-virtualenv.sh', + config.local_modules_dir / 'prepare-virtualenv.sh', config.ofm_dir, permissions='755', user='ofm', diff --git a/ssh_lib/tasks_tilegen.py b/ssh_lib/tasks_tile_gen.py similarity index 65% rename from ssh_lib/tasks_tilegen.py rename to ssh_lib/tasks_tile_gen.py index 1281998..2834bee 100644 --- a/ssh_lib/tasks_tilegen.py +++ b/ssh_lib/tasks_tile_gen.py @@ -10,15 +10,15 @@ def prepare_tile_gen(c, *, enable_cron): c.sudo(f'rm -rf {config.tile_gen_bin}') - put_dir(c, config.modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755') + put_dir(c, config.local_modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755') for dirname in ['tile_gen_lib', 'scripts']: - put_dir(c, config.modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}') + put_dir(c, config.local_modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}') - if (config.config_dir / 'rclone.conf').exists(): + if (config.local_config_dir / 'rclone.conf').exists(): put( c, - config.config_dir / 'rclone.conf', + config.local_config_dir / 'rclone.conf', f'{config.remote_config}/rclone.conf', permissions='600', user='ofm', @@ -33,4 +33,4 @@ def prepare_tile_gen(c, *, enable_cron): c.sudo(f'chown ofm:ofm -R {config.tile_gen_bin}') if enable_cron: - put(c, config.modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/') + put(c, config.local_modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')