loadbalancer

init-server.py

@@ -15,6 +15,7 @@ from ssh_lib.tasks import (
 )
 from ssh_lib.utils import (
     put,
+    put_dir,
 )
 
 
@@ -130,7 +131,13 @@ def debug(hostname, user, port):
     # upload_http_host_files(c)
     # sudo_cmd(c, f'{VENV_BIN}/python -u /data/ofm/http_host/bin/host_manager.py nginx-sync')
 
-    put(c, SCRIPTS_DIR / 'tile_gen' / 'upload_manager.py', f'{TILE_GEN_BIN}')
+    # put(c, SCRIPTS_DIR / 'tile_gen' / 'upload_manager.py', f'{TILE_GEN_BIN}')
+    put_dir(c, SCRIPTS_DIR / 'loadbalancer', '/data/ofm/loadbalancer')
+    put_dir(
+        c,
+        SCRIPTS_DIR / 'loadbalancer' / 'loadbalancer_lib',
+        '/data/ofm/loadbalancer/loadbalancer_lib',
+    )
 
 
 if __name__ == '__main__':

scripts/loadbalancer/loadbalancer.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import json
+
+import click
+import requests
+from loadbalancer_lib.curl import pycurl_get, pycurl_status
+
+
+AREAS = ['planet', 'monaco']
+
+
+@click.group()
+def cli():
+    """
+    Manages load-balancing of Round-Robin DNS records
+    """
+
+
+@cli.command()
+def run():
+    """
+    Runs load-balancing job (triggered by cron every minute)
+    """
+
+    with open('/data/ofm/config/loadbalancer.json') as fp:
+        c = json.load(fp)
+    print(c)
+
+    for area in AREAS:
+        results = run_area(c, area)
+        print(results)
+
+
+def run_area(c, area):
+    deployed_version = get_deployed_version(area)
+
+    print(f'deployed version: {area}: {deployed_version}')
+
+    results = dict()
+
+    for host_ip in c['load_balance_host_list']:
+        try:
+            check_host(c['domain_ledns'], host_ip, area, deployed_version)
+            results[host_ip] = True
+        except Exception:
+            results[host_ip] = False
+
+    return results
+
+
+def check_host(domain, host_ip, area, version):
+    # check TileJSON first
+    url = f'https://{domain}/{area}'
+    tilejson_str = pycurl_get(url, domain, host_ip)
+    tilejson = json.loads(tilejson_str)
+    tiles_url = tilejson['tiles'][0]
+    version_in_tilejson = tiles_url.split('/')[4]
+    assert version_in_tilejson == version
+
+    # check actual vector tile
+    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
+    assert pycurl_status(url, domain, host_ip) == 200
+
+
+def get_deployed_version(area):
+    url = f'https://assets.openfreemap.com/versions/deployed_{area}.txt'
+    response = requests.get(url)
+    response.raise_for_status()
+    return response.text.strip()
+
+
+if __name__ == '__main__':
+    cli()

scripts/loadbalancer/loadbalancer_lib/curl.py

@@ -0,0 +1,43 @@
+from io import BytesIO
+
+import pycurl
+
+
+def pycurl_status(url, domain, host_ip):
+    """
+    Uses pycurl to make a HTTPS HEAD request using custom resolving,
+    checks if the status code is 200
+    """
+
+    c = pycurl.Curl()
+    c.setopt(c.URL, url)
+    c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
+    c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
+    c.setopt(c.NOBODY, True)
+    c.perform()
+    status_code = c.getinfo(c.RESPONSE_CODE)
+    c.close()
+
+    return status_code
+
+
+def pycurl_get(url, domain, host_ip):
+    """
+    Uses pycurl to make a HTTPS GET request using custom resolving,
+    checks if the status code is 200, and returns the content.
+    """
+
+    buffer = BytesIO()
+    c = pycurl.Curl()
+    c.setopt(c.URL, url)
+    c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
+    c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
+    c.setopt(c.WRITEDATA, buffer)
+    c.perform()
+    status_code = c.getinfo(c.RESPONSE_CODE)
+    c.close()
+
+    if status_code != 200:
+        raise ValueError('non-200')
+
+    return buffer.getvalue().decode('utf8')

scripts/loadbalancer/setup.py

@@ -3,6 +3,8 @@ from setuptools import find_packages, setup
 
 requirements = [
     'click',
+    'requests',
+    'pycurl',
 ]
 
 

scripts/old.js

@@ -1,91 +0,0 @@
-/**
- * Welcome to Cloudflare Workers!
- *
- * This is a template for a Scheduled Worker: a Worker that can run on a
- * configurable interval:
- * https://developers.cloudflare.com/workers/platform/triggers/cron-triggers/
- *
- * - Run `npm run dev` in your terminal to start a development server
- * - Run `curl "http://localhost:8787/__scheduled?cron=*+*+*+*+*"` to see your worker in action
- * - Run `npm run deploy` to publish your worker
- *
- * Learn more at https://developers.cloudflare.com/workers/
- */
-
-const AREAS = ['planet', 'monaco']
-
-async function handleArea(area, env) {
-  const deployedVersion = await getDeployedVersion(area)
-
-  const http_hosts = env.HTTP_HOST_LIST.split(',')
-    .map(s => s.trim())
-    .filter(Boolean)
-
-  for (const host of http_hosts) {
-    await checkHost(host, area, deployedVersion)
-  }
-}
-
-async function getDeployedVersion(area) {
-  const response = await fetch(`https://assets.openfreemap.com/versions/deployed_${area}.txt`)
-
-  if (!response.ok) {
-    throw new Error(`HTTP error! Status: ${response.status}`)
-  }
-
-  const content = await response.text()
-  return content.trim()
-}
-
-async function checkHost(host, area, version) {
-  // using HTTP as the HTTPS needs custom resolvers
-  // discussion links:
-  // https://community.letsencrypt.org/t/understanding-server-name-resolving-vs-host-headers-in-https/219784
-  // https://community.cloudflare.com/t/how-to-resolve-https-in-a-js-worker/669506
-
-  const url = `http://${host}/${area}/${version}/14/8529/5975.pbf`
-  console.log(url)
-  const response = await fetch(url, {
-    method: 'HEAD',
-    headers: {
-      Host: 'direct.openfreemap.org',
-    },
-  })
-
-  console.log(response)
-}
-
-export default {
-  async fetch(req) {
-    const url = new URL(req.url)
-    url.pathname = '/__scheduled'
-    url.searchParams.append('cron', '* * * * *')
-    return new Response(
-      `To test the scheduled handler, ensure you have used the "--test-scheduled" then try running "curl ${url.href}".`,
-    )
-  },
-
-  // The scheduled handler is invoked at the interval set in our wrangler.toml's
-  // [[triggers]] configuration.
-  async scheduled(event, env, ctx) {
-    const url = 'http://direct.openfreemap.org/styles/liberty'
-
-    const response = await fetch(url, { cf: { resolveOverride: '1.1.1.1' } })
-
-    console.log(response, response.ok)
-
-    // return
-    //
-    // for (const area of AREAS) {
-    //   await handleArea(area, env)
-    // }
-    //
-    // // We'll keep it simple and make an API call to a Cloudflare API:
-    // let resp = await fetch('https://api.cloudflare.com/client/v4/ips')
-    // let wasSuccessful = resp.ok ? 'success' : 'fail'
-    //
-    // // You could store this result in KV, write to a D1 Database, or publish to a Queue.
-    // // In this template, we'll just log the result:
-    // console.log(`trigger fired at ${event.cron}: ${wasSuccessful}`)
-  },
-}

ssh_lib/tasks.py

@@ -222,12 +222,20 @@ def setup_ledns_writer(c):
 
 
 def setup_loadbalancer(c):
+    domain_ledns = dotenv_val('DOMAIN_LEDNS').lower()
     load_balance_host_list = [
         h.strip() for h in dotenv_val('LOAD_BALANCE_HOST_LIST').split(',') if h.strip()
     ]
     assert (CONFIG_DIR / 'cloudflare.ini').exists()
 
-    c.sudo(f'mkdir -p {REMOTE_CONFIG}')
+    config = {
+        'domain_ledns': domain_ledns,
+        'load_balance_host_list': load_balance_host_list,
+    }
+
+    config_str = json.dumps(config, indent=2, ensure_ascii=False)
+    print(config_str)
+    put_str(c, f'{REMOTE_CONFIG}/loadbalancer.json', config_str)
 
     put(
         c,
@@ -236,8 +244,12 @@ def setup_loadbalancer(c):
         permissions=400,
     )
 
-
     c.sudo('rm -rf /data/ofm/loadbalancer')
     put_dir(c, SCRIPTS_DIR / 'loadbalancer', '/data/ofm/loadbalancer')
+    put_dir(
+        c,
+        SCRIPTS_DIR / 'loadbalancer' / 'loadbalancer_lib',
+        '/data/ofm/loadbalancer/loadbalancer_lib',
+    )
 
     c.sudo(f'{VENV_BIN}/pip install -e /data/ofm/loadbalancer')