sysadmin/openfreemap
1
0
Fork 0
mirror of https://github.com/hyperknot/openfreemap.git synced 2026-05-21 14:02:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

removed loadbalancer and unused parts

Browse Source
This commit is contained in:
Zsolt Ero
2025-09-18 18:45:23 +02:00
parent b068aacca1
commit 604f27e7db
18 changed files with 0 additions and 1928 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config/cloudflare.ini.sample
View File

@@ -1,3 +0,0 @@
# --- Let's Encrypt DNS challenge, not needed for self-hosting
dns_cloudflare_api_token = xxx

25
init-server.py
View File

@@ -9,8 +9,6 @@ from ssh_lib.tasks import (
prepare_shared, prepare_shared,
prepare_tile_gen, prepare_tile_gen,
run_http_host_sync, run_http_host_sync,
setup_loadbalancer,
setup_roundrobin_writer,
) )
from ssh_lib.utils import ( from ssh_lib.utils import (
put, put,
@@ -113,29 +111,6 @@ def tile_gen(
prepare_tile_gen(c, enable_cron=cron) prepare_tile_gen(c, enable_cron=cron)
@cli.command()
@common_options
def roundrobin_dns_writer(hostname, user, port, noninteractive):
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
return
c = get_connection(hostname, user, port)
setup_roundrobin_writer(c)
@cli.command()
@common_options
def loadbalancer(hostname, user, port, noninteractive):
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
return
c = get_connection(hostname, user, port)
prepare_shared(c)
setup_loadbalancer(c)
@cli.command() @cli.command()
@common_options @common_options
def http_host_sync(hostname, user, port, noninteractive): def http_host_sync(hostname, user, port, noninteractive):

172
modules/debug_proxy/.gitignore vendored
View File

@@ -1,172 +0,0 @@
# Logs
logs
_.log
npm-debug.log_
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
# Runtime data
pids
_.pid
_.seed
\*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
\*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
\*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
\*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.\*
# wrangler project
.dev.vars
.wrangler/

14
modules/debug_proxy/package.json
View File

@@ -1,14 +0,0 @@
{
"name": "cf-debug-proxy",
"version": "0.0.0",
"private": true,
"scripts": {
"deploy": "wrangler deploy",
"dev": "wrangler dev",
"start": "wrangler dev"
},
"devDependencies": {
"itty-router": "^3.0.12",
"wrangler": "^3.60.3"
}
}

1142
modules/debug_proxy/pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

66
modules/debug_proxy/src/index.js
View File

@@ -1,66 +0,0 @@
async function sendTelegramMessage(message, botToken, chatId) {
const url = `https://api.telegram.org/bot${botToken}/sendMessage`
const payload = {
chat_id: chatId,
text: message,
}
try {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(payload),
})
if (!response.ok) {
console.error('Failed to send message:', await response.text())
}
} catch (error) {
console.error('Error sending Telegram message:', error)
}
}
export default {
async fetch(request, env, ctx) {
const url = new URL(request.url)
const userIP = request.headers.get('CF-Connecting-IP')
if (url.pathname === '/b') {
url.pathname = '/styles/bright'
}
// // no failure, just warning
// if (request.method !== 'GET') {
// const warningMessage = `Non-GET request ${request.method} ${url.pathname} ${userIP}`
// console.error(warningMessage)
// await sendTelegramMessage(warningMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
// }
if (!url.pathname.startsWith('/styles')) {
const errorMessage = 'Bad path'
return new Response(errorMessage, { status: 500 })
}
const proxyUrl = new URL(url.pathname, 'https://tiles.openfreemap.org')
try {
const response = await fetch(proxyUrl)
if (response.status !== 200) {
const errorMessage = `Proxy error: Bad status ${response.status} ${url.pathname} ${userIP}`
console.error(errorMessage)
await sendTelegramMessage(errorMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
return new Response('Proxy error: Bad status', { status: 500 })
}
return response
} catch (error) {
const errorMessage = `Proxy error: ${error.message} ${url.pathname} ${userIP}`
console.error(errorMessage)
await sendTelegramMessage(errorMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
return new Response('Proxy error: Fetch failed', { status: 500 })
}
},
}

107
modules/debug_proxy/wrangler.toml
View File

@@ -1,107 +0,0 @@
#:schema node_modules/wrangler/config-schema.json
name = "cf-debug-proxy"
main = "src/index.js"
compatibility_date = "2024-06-20"
# Automatically place your workloads in an optimal location to minimize latency.
# If you are running back-end logic in a Worker, running it closer to your back-end infrastructure
# rather than the end user may result in better performance.
# Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement
# [placement]
# mode = "smart"
# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
# Docs:
# - https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables
# Note: Use secrets to store sensitive data.
# - https://developers.cloudflare.com/workers/configuration/secrets/
# [vars]
# MY_VARIABLE = "production_value"
# Bind the Workers AI model catalog. Run machine learning models, powered by serverless GPUs, on Cloudflare’s global network
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#workers-ai
# [ai]
# binding = "AI"
# Bind an Analytics Engine dataset. Use Analytics Engine to write analytics within your Pages Function.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#analytics-engine-datasets
# [[analytics_engine_datasets]]
# binding = "MY_DATASET"
# Bind a headless browser instance running on Cloudflare's global network.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#browser-rendering
# [browser]
# binding = "MY_BROWSER"
# Bind a D1 database. D1 is Cloudflare’s native serverless SQL database.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#d1-databases
# [[d1_databases]]
# binding = "MY_DB"
# database_name = "my-database"
# database_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
# Bind a dispatch namespace. Use Workers for Platforms to deploy serverless functions programmatically on behalf of your customers.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#dispatch-namespace-bindings-workers-for-platforms
# [[dispatch_namespaces]]
# binding = "MY_DISPATCHER"
# namespace = "my-namespace"
# Bind a Durable Object. Durable objects are a scale-to-zero compute primitive based on the actor model.
# Durable Objects can live for as long as needed. Use these when you need a long-running "server", such as in realtime apps.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#durable-objects
# [[durable_objects.bindings]]
# name = "MY_DURABLE_OBJECT"
# class_name = "MyDurableObject"
# Durable Object migrations.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#migrations
# [[migrations]]
# tag = "v1"
# new_classes = ["MyDurableObject"]
# Bind a Hyperdrive configuration. Use to accelerate access to your existing databases from Cloudflare Workers.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#hyperdrive
# [[hyperdrive]]
# binding = "MY_HYPERDRIVE"
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
# Bind a KV Namespace. Use KV as persistent storage for small key-value pairs.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#kv-namespaces
# [[kv_namespaces]]
# binding = "MY_KV_NAMESPACE"
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
# Bind an mTLS certificate. Use to present a client certificate when communicating with another service.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#mtls-certificates
# [[mtls_certificates]]
# binding = "MY_CERTIFICATE"
# certificate_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
# Bind a Queue producer. Use this binding to schedule an arbitrary task that may be processed later by a Queue consumer.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
# [[queues.producers]]
# binding = "MY_QUEUE"
# queue = "my-queue"
# Bind a Queue consumer. Queue Consumers can retrieve tasks scheduled by Producers to act on them.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
# [[queues.consumers]]
# queue = "my-queue"
# Bind an R2 Bucket. Use R2 to store arbitrarily large blobs of data, such as files.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#r2-buckets
# [[r2_buckets]]
# binding = "MY_BUCKET"
# bucket_name = "my-bucket"
# Bind another Worker service. Use this binding to call another Worker without network overhead.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings
# [[services]]
# binding = "MY_SERVICE"
# service = "my-service"
# Bind a Vectorize index. Use to store and query vector embeddings for semantic search, classification and other vector search use-cases.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#vectorize-indexes
# [[vectorize]]
# binding = "MY_INDEX"
# index_name = "my-index"

8
modules/loadbalancer/cron.d/ofm_loadbalancer
View File

@@ -1,8 +0,0 @@
# every minute
# fix
#* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py fix >> /data/ofm/loadbalancer/logs/run.log 2>&1
# check
* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py check >> /data/ofm/loadbalancer/logs/run.log 2>&1

40
modules/loadbalancer/loadbalancer.py
View File

@@ -1,40 +0,0 @@
#!/usr/bin/env python3
from datetime import datetime, timezone
import click
from loadbalancer_lib.loadbalance import check_or_fix
now = datetime.now(timezone.utc)
@click.group()
def cli():
"""
Manages load-balancing of Round-Robin DNS records
"""
@cli.command()
def check():
"""
Runs load-balancing check
"""
print(f'---\n{now}\nStarting check')
check_or_fix(fix=False)
@cli.command()
def fix():
"""
Runs check and fixes records based on check results
"""
print(f'---\n{now}\nStarting fix')
check_or_fix(fix=True)
if __name__ == '__main__':
cli()

0
modules/loadbalancer/loadbalancer_lib/__init__.py
View File

108
modules/loadbalancer/loadbalancer_lib/cloudflare.py
View File

@@ -1,108 +0,0 @@
import requests
# docs: https://api.cloudflare.com/
def cloudflare_get(path: str, params: dict, cloudflare_api_token: str):
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
res = requests.get(
f'https://api.cloudflare.com/client/v4{path}', headers=headers, params=params
)
res.raise_for_status()
data = res.json()
assert data['success'] is True
return data
def get_zone_id(domain, cloudflare_api_token: str):
data = cloudflare_get(
'/zones', params=dict(name=domain), cloudflare_api_token=cloudflare_api_token
)
assert len(data['result']) == 1
zone_info = data['result'][0]
return zone_info['id']
def get_dns_records_round_robin(zone_id, cloudflare_api_token: str) -> dict:
data = cloudflare_get(
f'/zones/{zone_id}/dns_records',
params=dict(per_page=5000),
cloudflare_api_token=cloudflare_api_token,
)
records = data['result']
data = {}
for r in records:
if r['type'] != 'A':
continue
data.setdefault(r['name'], [])
data[r['name']].append(dict(content=r['content'], id=r['id']))
return data
def set_records_round_robin(
zone_id,
*,
name: str,
host_ip_set: set,
ttl: int = 1,
proxied: bool,
comment: str = None,
cloudflare_api_token: str,
) -> bool:
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
dns_records = get_dns_records_round_robin(zone_id, cloudflare_api_token=cloudflare_api_token)
current_records = dns_records.get(name, [])
current_ips = {r['content'] for r in current_records}
if current_ips == host_ip_set:
print(f'No need to update records: {name} currently set: {sorted(current_ips)}')
return False
# changing records
# delete all current records first
for r in current_records:
delete_record(zone_id, id_=r['id'], cloudflare_api_token=cloudflare_api_token)
# create new records
for ip in host_ip_set:
print(f'Creating record: {name} {ip}')
json_data = dict(
type='A',
name=name,
content=ip,
ttl=ttl,
proxied=proxied,
comment=comment,
)
res = requests.post(
f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records',
headers=headers,
json=json_data,
)
res.raise_for_status()
data = res.json()
assert data['success'] is True
return True
def delete_record(zone_id, *, id_: str, cloudflare_api_token: str):
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
print(f'Deleting record: {id_}')
res = requests.delete(
f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records/{id_}',
headers=headers,
json={},
)
res.raise_for_status()
data = res.json()
assert data['success'] is True

29
modules/loadbalancer/loadbalancer_lib/config.py
View File

@@ -1,29 +0,0 @@
import json
from pathlib import Path
from dotenv import dotenv_values
class Configuration:
areas = ['planet', 'monaco']
if Path('/data/ofm').exists():
ofm_config_dir = Path('/data/ofm/config')
else:
repo_root = Path(__file__).parent.parent.parent.parent
ofm_config_dir = repo_root / 'config'
ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
http_host_list = ofm_config['http_host_list']
telegram_token = ofm_config['telegram_token']
telegram_chat_id = ofm_config['telegram_chat_id']
domain_roundrobin = ofm_config['domain_roundrobin']
domain_root = '.'.join(domain_roundrobin.split('.')[-2:])
cloudflare_ini = dotenv_values(ofm_config_dir / 'cloudflare.ini')
cloudflare_api_token = cloudflare_ini['dns_cloudflare_api_token']
config = Configuration()

106
modules/loadbalancer/loadbalancer_lib/loadbalance.py
View File

@@ -1,106 +0,0 @@
from datetime import datetime, timedelta, timezone
from loadbalancer_lib.cloudflare import get_zone_id, set_records_round_robin
from loadbalancer_lib.config import config
from loadbalancer_lib.shared import check_host_latest, check_host_version, get_deployed_version
from loadbalancer_lib.telegram_ import telegram_send_message
def check_or_fix(fix=False):
if not config.http_host_list:
telegram_quick(
'OFM loadbalancer no hosts found on list, terminating',
)
return
try:
results_by_ip = {}
working_hosts = set()
for area in config.areas:
results = run_area(area)
for host_ip, host_is_ok in results.items():
results_by_ip.setdefault(host_ip, True)
results_by_ip[host_ip] &= host_is_ok
for host_ip, host_is_ok in results_by_ip.items():
if not host_is_ok:
telegram_quick(f'OFM loadbalancer ERROR with host: {host_ip}')
else:
working_hosts.add(host_ip)
except Exception as e:
telegram_quick(f'OFM loadbalancer ERROR with loadbalancer: {e}')
return
print(f'working hosts: {sorted(working_hosts)}')
if fix:
# if no hosts are detected working, probably a bug in this script
# fail-safe to include all hosts
if not working_hosts:
working_hosts = set(config.http_host_list)
telegram_quick('OFM loadbalancer FIX found no working hosts, reverting to full list!')
updated = update_records(working_hosts)
if updated:
telegram_quick(f'OFM loadbalancer FIX modified records, new records: {working_hosts}')
def run_area(area):
deployed_data = get_deployed_version(area)
version = deployed_data['version']
last_modified = deployed_data['last_modified']
if not version:
print(f' deployed version not found: {area}')
return
print(f' deployed version {area}: {version}')
# using relaxed mode for while the servers are still deploying
now = datetime.now(timezone.utc)
delta = now - last_modified
relaxed_mode = delta < timedelta(minutes=3)
if relaxed_mode:
print(' using relaxed mode')
results = {}
for host_ip in config.http_host_list:
try:
# don't check latest
if relaxed_mode:
check_host_version(config.domain_roundrobin, host_ip, area, version)
else:
check_host_latest(config.domain_roundrobin, host_ip, area, version)
results[host_ip] = True
except Exception as e:
results[host_ip] = False
print(e)
return results
def update_records(working_hosts) -> bool:
zone_id = get_zone_id(config.domain_root, cloudflare_api_token=config.cloudflare_api_token)
updated = False
updated |= set_records_round_robin(
zone_id=zone_id,
name=config.domain_roundrobin,
host_ip_set=working_hosts,
proxied=False,
ttl=300,
comment='domain_roundrobin',
cloudflare_api_token=config.cloudflare_api_token,
)
return updated
def telegram_quick(message):
telegram_send_message(message, config.telegram_token, config.telegram_chat_id)

1
modules/loadbalancer/loadbalancer_lib/shared.py
View File

@@ -1 +0,0 @@
../../tile_gen/tile_gen_lib/shared.py

16
modules/loadbalancer/loadbalancer_lib/telegram_.py
View File

@@ -1,16 +0,0 @@
import requests
def telegram_send_message(message, bot_token, chat_id):
print(message)
url = f'https://api.telegram.org/bot{bot_token}/sendMessage'
payload = {'chat_id': chat_id, 'text': message}
response = requests.post(url, data=payload)
if response.status_code == 200:
print(' Message sent successfully!')
else:
print(' Failed to send message:', response.text)

16
modules/loadbalancer/setup.py
View File

@@ -1,16 +0,0 @@
from setuptools import find_packages, setup
requirements = [
'click',
'requests',
'pycurl',
'python-dotenv',
]
setup(
python_requires='>=3.10',
install_requires=requirements,
packages=find_packages(),
)

12
modules/roundrobin/rclone_write.sh
View File

@@ -1,12 +0,0 @@
#!/usr/bin/env bash
# these are not needed as certbot generates these
#env > /data/ofm/roundrobin/env.txt
#RENEWED_DOMAINS=tiles.openfreemap.org
#RENEWED_LINEAGE=/etc/letsencrypt/live/ofm_roundrobin
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
rclone copyto -v --copy-links "$RENEWED_LINEAGE/fullchain.pem" "remote:ofm-private/roundrobin/$RENEWED_DOMAINS/ofm_roundrobin.cert"
rclone copyto -v --copy-links "$RENEWED_LINEAGE/privkey.pem" "remote:ofm-private/roundrobin/$RENEWED_DOMAINS/ofm_roundrobin.key"

63
ssh_lib/tasks.py
View File

@@ -143,62 +143,6 @@ def install_benchmark(c):
wrk(c) wrk(c)
def setup_roundrobin_writer(c):
letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower()
domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower()
assert letsencrypt_email
assert domain_roundrobin
assert (CONFIG_DIR / 'rclone.conf').exists()
assert (CONFIG_DIR / 'cloudflare.ini').exists()
rclone(c)
certbot(c)
c.sudo(f'mkdir -p {REMOTE_CONFIG}')
put(
c,
CONFIG_DIR / 'rclone.conf',
f'{REMOTE_CONFIG}/rclone.conf',
permissions=400,
)
put(
c,
CONFIG_DIR / 'cloudflare.ini',
f'{REMOTE_CONFIG}/cloudflare.ini',
permissions=400,
)
c.sudo('rm -rf /data/ofm/roundrobin')
put(
c,
MODULES_DIR / 'roundrobin' / 'rclone_write.sh',
'/data/ofm/roundrobin/rclone_write.sh',
create_parent_dir=True,
permissions=500,
)
# only use with --staging
# c.sudo('certbot delete --noninteractive --cert-name ofm_roundrobin', warn=True)
sudo_cmd(
c,
'certbot certonly '
'--dns-cloudflare '
f'--dns-cloudflare-credentials {REMOTE_CONFIG}/cloudflare.ini '
'--dns-cloudflare-propagation-seconds 20 '
f'--noninteractive '
f'-m {letsencrypt_email} '
f'--agree-tos '
f'--cert-name=ofm_roundrobin '
f'--deploy-hook /data/ofm/roundrobin/rclone_write.sh '
f'-d {domain_roundrobin}',
# f'-d {domain2_roundrobin}',
)
def upload_config_json(c): def upload_config_json(c):
domain_direct = dotenv_val('DOMAIN_DIRECT').lower() domain_direct = dotenv_val('DOMAIN_DIRECT').lower()
domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower() domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower()
@@ -233,13 +177,6 @@ def upload_config_json(c):
def setup_loadbalancer(c): def setup_loadbalancer(c):
c.sudo('rm -f /etc/cron.d/ofm_loadbalancer') c.sudo('rm -f /etc/cron.d/ofm_loadbalancer')
put(
c,
CONFIG_DIR / 'cloudflare.ini',
f'{REMOTE_CONFIG}/cloudflare.ini',
permissions=400,
)
c.sudo('rm -rf /data/ofm/loadbalancer') c.sudo('rm -rf /data/ofm/loadbalancer')
put_dir(c, MODULES_DIR / 'loadbalancer', '/data/ofm/loadbalancer') put_dir(c, MODULES_DIR / 'loadbalancer', '/data/ofm/loadbalancer')
put_dir( put_dir(