From 24dfa2ce37bca2cf7ff40acf19c226828c1ec5ea Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Thu, 16 Oct 2025 12:41:41 +0200 Subject: [PATCH] work --- http-host.py | 41 +++++++++++++++- setup.py | 1 + ssh_lib/pkg_base.py | 3 +- ssh_lib/pycurl.py | 99 ++++++++++++++++++++++++++++++++++++++ ssh_lib/tasks_http_host.py | 63 +++++++++++++----------- ssh_lib/utils.py | 28 +++++++++++ 6 files changed, 205 insertions(+), 30 deletions(-) create mode 100644 ssh_lib/pycurl.py diff --git a/http-host.py b/http-host.py index df9a306..55b01dc 100755 --- a/http-host.py +++ b/http-host.py @@ -1,12 +1,17 @@ #!/usr/bin/env python3 +import json +from pprint import pprint import click +from modules.http_host.http_host_lib.get_version_shared import get_deployed_version from ssh_lib.cli_helpers import common_options, get_connection from ssh_lib.config import config -from ssh_lib.tasks_http_host import prepare_http_host, run_http_host_sync +from ssh_lib.pycurl import pycurl_get +from ssh_lib.tasks_http_host import prepare_http_host, read_jsonc, run_http_host_sync from ssh_lib.tasks_shared import prepare_shared from ssh_lib.utils import ( + get_ip_from_ssh_alias, put, ) @@ -61,5 +66,39 @@ def sync(hostname, user, port, noninteractive): run_http_host_sync(c) +@cli.command() +def debug(): + config_data = read_jsonc() + + area = 'monaco' if config_data['skip_planet'] else 'planet' + version = get_deployed_version(area)['version'] + + domains = [d['domain'] for d in config_data['domains']] + + servers = [] + + for s in config_data['servers']: + hostname = s['hostname'] + ip = get_ip_from_ssh_alias(hostname) + servers.append(dict(hostname=hostname, ip=ip)) + + for domain in domains: + for server in servers: + print(domain, server) + check_host_using_tilejson( + url=f'https://{domain}/{area}/{version}', + ip=server['ip'], + version=version, + ) + + +def check_host_using_tilejson(*, url, ip, version): + tilejson_str = pycurl_get(url, ip) + tilejson = json.loads(tilejson_str) + tiles_url = tilejson['tiles'][0] + version_in_tilejson = tiles_url.split('/')[4] + assert version_in_tilejson == version + + if __name__ == '__main__': cli() diff --git a/setup.py b/setup.py index e36ea81..eed7752 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ requirements = [ 'requests', 'jsonschema', 'json5', + 'pycurl', ] diff --git a/ssh_lib/pkg_base.py b/ssh_lib/pkg_base.py index 994b115..b60fbb8 100644 --- a/ssh_lib/pkg_base.py +++ b/ssh_lib/pkg_base.py @@ -34,11 +34,11 @@ def pkg_base(c): 'python3', 'python3-venv', # + # 'ctop', # unsupported on Ubuntu 24 'acpid', 'autojump', 'bash-completion', 'btop', - # 'ctop', # unsupported on Ubuntu 24 'dbus', 'direnv', 'fd-find', @@ -54,6 +54,7 @@ def pkg_base(c): 'net-tools', 'netbase', 'nethogs', + 'nvme-cli', 'openssh-client', 'p7zip-full', 'pkg-config', diff --git a/ssh_lib/pycurl.py b/ssh_lib/pycurl.py new file mode 100644 index 0000000..7673b3f --- /dev/null +++ b/ssh_lib/pycurl.py @@ -0,0 +1,99 @@ +""" +Round-Robin DNS Bypass for Health Checking + +Check individual servers behind round-robin DNS by connecting directly to specific +IP addresses while maintaining proper HTTPS/TLS. + +Example: + pycurl_status('https://api.example.com/health', '192.168.1.101') + 200 + + pycurl_get('https://api.example.com/data', '192.168.1.102') + '{"status": "ok"}' + +How it works: + Overrides DNS resolution to connect to a specific IP while using the correct + hostname for TLS/SNI. This lets you bypass round-robin to test individual servers. +""" + +from io import BytesIO +from pathlib import Path +from urllib.parse import urlparse + +import pycurl + + +def pycurl_status(url: str, target_ip: str) -> int: + """ + Check HTTP status of a specific server behind round-robin DNS. + + Makes a HEAD request to the target IP while using the hostname for HTTPS/SNI. + + Args: + url: Full URL to request (e.g., 'https://api.example.com/health') + target_ip: IP address of specific server (e.g., '192.168.1.101') + + Returns: + HTTP status code (e.g., 200, 404, 500) + """ + parsed = urlparse(url) + hostname = parsed.hostname + port = parsed.port or (443 if parsed.scheme == 'https' else 80) + + c = pycurl.Curl() + c.setopt(c.URL, url) + + if Path('/etc/ssl/certs/ca-certificates.crt').exists(): + c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt') + + # Override DNS: map hostname:port -> target_ip + c.setopt(c.RESOLVE, [f'{hostname}:{port}:{target_ip}']) + c.setopt(c.NOBODY, True) # HEAD request + c.setopt(c.TIMEOUT, 5) + c.perform() + status_code = c.getinfo(c.RESPONSE_CODE) + c.close() + + return status_code + + +def pycurl_get(url: str, target_ip: str, binary: bool = False) -> str | bytes: + """ + Fetch content from a specific server behind round-robin DNS. + + Makes a GET request to the target IP while using the hostname for HTTPS/SNI. + + Args: + url: Full URL to request (e.g., 'https://api.example.com/data') + target_ip: IP address of specific server (e.g., '192.168.1.101') + binary: If True, return bytes; if False, decode as UTF-8 string + + Returns: + Response body as UTF-8 string (binary=False) or bytes (binary=True) + + Raises: + ValueError: If status code is not 200 + """ + parsed = urlparse(url) + hostname = parsed.hostname + port = parsed.port or (443 if parsed.scheme == 'https' else 80) + + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, url) + + if Path('/etc/ssl/certs/ca-certificates.crt').exists(): + c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt') + + c.setopt(c.RESOLVE, [f'{hostname}:{port}:{target_ip}']) + c.setopt(c.WRITEDATA, buffer) + c.setopt(c.TIMEOUT, 5) + c.perform() + status_code = c.getinfo(c.RESPONSE_CODE) + c.close() + + if status_code != 200: + raise ValueError(f'status code: {status_code}') + + body = buffer.getvalue() + return body if binary else body.decode('utf-8') diff --git a/ssh_lib/tasks_http_host.py b/ssh_lib/tasks_http_host.py index c4d1613..bf4725f 100644 --- a/ssh_lib/tasks_http_host.py +++ b/ssh_lib/tasks_http_host.py @@ -34,34 +34,7 @@ def prepare_http_host(c): def upload_config_and_certs(c): - if not config.local_config_jsonc.is_file(): - raise FileNotFoundError( - f'{config.local_config_jsonc} not found. Make sure it exists in the /config dir' - ) - - # Load and parse the JSONC/JSON5 config file - try: - config_data = json5.loads(config.local_config_jsonc.read_text()) - except Exception as e: - raise RuntimeError(f'Error parsing config file: {e}') from e - - # Load the JSON schema - try: - schema = json.loads(config.config_schema_json.read_text()) - except Exception as e: - raise RuntimeError(f'Error loading schema file: {e}') from e - - # Validate the config against the schema - try: - validate(instance=config_data, schema=schema) - print('✓ Configuration is valid') - except ValidationError as e: - error_msg = f'Configuration validation failed: {e.message}' - if e.path: - error_msg += f'\nPath: {".".join(str(p) for p in e.path)}' - raise RuntimeError(error_msg) from e - except Exception as e: - raise RuntimeError(f'Validation error: {e}') from e + config_data = read_jsonc() # clean old certs c.sudo('rm -rf /data/nginx/certs/ofm-*') @@ -100,6 +73,40 @@ def upload_config_and_certs(c): put_str(c, f'{config.remote_config}/config.json', config_str) +def read_jsonc(): + if not config.local_config_jsonc.is_file(): + raise FileNotFoundError( + f'{config.local_config_jsonc} not found. Make sure it exists in the /config dir' + ) + + # Load and parse the JSONC/JSON5 config file + try: + config_data = json5.loads(config.local_config_jsonc.read_text()) + except Exception as e: + raise RuntimeError(f'Error parsing config file: {e}') from e + + # Load the JSON schema + try: + schema = json.loads(config.config_schema_json.read_text()) + except Exception as e: + raise RuntimeError(f'Error loading schema file: {e}') from e + + # Validate the config against the schema + try: + validate(instance=config_data, schema=schema) + print('✓ Configuration is valid') + except ValidationError as e: + error_msg = f'Configuration validation failed: {e.message}' + if e.path: + error_msg += f'\nPath: {".".join(str(p) for p in e.path)}' + raise RuntimeError(error_msg) from e + + except Exception as e: + raise RuntimeError(f'Validation error: {e}') from e + + return config_data + + def upload_http_host_files(c): c.sudo(f'rm -rf {config.http_host_bin}') c.sudo(f'mkdir -p {config.http_host_bin}') diff --git a/ssh_lib/utils.py b/ssh_lib/utils.py index 01879f6..7c8c4e6 100644 --- a/ssh_lib/utils.py +++ b/ssh_lib/utils.py @@ -1,10 +1,12 @@ import os import secrets +import socket import string import sys from pathlib import Path import requests +from fabric import Connection from invoke import UnexpectedExit @@ -205,3 +207,29 @@ def get_latest_release_github(user, repo): assert data['tag_name'] == data['name'] return data['tag_name'] + + +def get_ip_from_ssh_alias(ssh_alias): + """ + Get IP address from SSH config alias. + + Args: + ssh_alias: SSH hostname/alias from ~/.ssh/config + + Returns: + str: IP address + + Raises: + socket.gaierror: If hostname cannot be resolved + """ + + # Create connection (doesn't actually connect) + conn = Connection(ssh_alias) + + # Get the resolved hostname from SSH config + hostname = conn.host + + # Resolve to IP + ip_address = socket.gethostbyname(hostname) + + return ip_address