mirror of
https://github.com/hyperknot/openfreemap.git
synced 2026-05-21 14:02:15 +00:00
Compare commits
44 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b7bc98d950 | ||
|
|
b0fc592f7c | ||
|
|
aa9c32ea23 | ||
|
|
4a322a8ddd | ||
|
|
82458b9db4 | ||
|
|
a8d94319fd | ||
|
|
f8c337abe6 | ||
|
|
7757c82b59 | ||
|
|
96432037e4 | ||
|
|
ba8c766698 | ||
|
|
a28df3156f | ||
|
|
24dfa2ce37 | ||
|
|
2becae11e1 | ||
|
|
753166316c | ||
|
|
bf60c28bb5 | ||
|
|
397f56be9d | ||
|
|
dfe0a766ed | ||
|
|
2e260d30e5 | ||
|
|
407d534801 | ||
|
|
52e34fc1c9 | ||
|
|
e746b00962 | ||
|
|
8352a70111 | ||
|
|
8167f6baf9 | ||
|
|
3ace404697 | ||
|
|
c579698906 | ||
|
|
b3e8bff774 | ||
|
|
f7299f6836 | ||
|
|
c787f602d9 | ||
|
|
3a66d303c4 | ||
|
|
d9487abd97 | ||
|
|
8594d730c7 | ||
|
|
45df827cb0 | ||
|
|
154d592ace | ||
|
|
a36e830416 | ||
|
|
17d580023b | ||
|
|
377dd7f334 | ||
|
|
55dae6776f | ||
|
|
fe30af3fb2 | ||
|
|
7fa19d33d1 | ||
|
|
6eb32db16a | ||
|
|
d735f4975f | ||
|
|
9b34510c8b | ||
|
|
b24f096ad4 | ||
|
|
604f27e7db |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -21,3 +21,4 @@ venv
|
||||
/pnpm-lock.yaml
|
||||
|
||||
/deploy-*.sh
|
||||
tmp.txt
|
||||
|
||||
38
.ruff.toml
38
.ruff.toml
@@ -1,49 +1,57 @@
|
||||
target-version = "py310"
|
||||
target-version = "py313"
|
||||
unsafe-fixes = true
|
||||
line-length = 100
|
||||
extend-exclude = ["temp"]
|
||||
extend-exclude = ["alembic", "*.ipynb", "temp"]
|
||||
|
||||
|
||||
lint.select = [
|
||||
"E", # pycodestyle errors
|
||||
"W", # pycodestyle warnings
|
||||
"F", # pyflakes
|
||||
"I", # isort
|
||||
'UP', # pyupgrade
|
||||
'A', # flake8-builtins
|
||||
"C4", # flake8-comprehensions
|
||||
'DTZ', # flake8-datetimez
|
||||
"E", # pycodestyle errors
|
||||
'EXE', # flake8-executable
|
||||
"F", # Pyflakes
|
||||
'FA', # flake8-future-annotations
|
||||
"I", # isort
|
||||
'PT', # flake8-pytest-style
|
||||
'RSE', # flake8-raise
|
||||
'SIM', # flake8-simplify
|
||||
'DTZ', # flake8-datetimez, https://beta.ruff.rs/docs/rules/#flake8-datetimez-dtz
|
||||
'UP', # pyupgrade
|
||||
"W", # pycodestyle warnings
|
||||
]
|
||||
|
||||
lint.ignore = [
|
||||
'A003',
|
||||
'DTZ007',
|
||||
# 'C408', # keep dict() as-is
|
||||
'DTZ007', # naive datetime.strptime() without %z
|
||||
'E501',
|
||||
'E711',
|
||||
'E712',
|
||||
# 'E721', # type comparison
|
||||
'E721', # type() comparison
|
||||
# 'E722', # bare except
|
||||
'E741',
|
||||
'F401', # unused imports
|
||||
'EXE003', # shebang should contain "python"
|
||||
'F401', # unused imports
|
||||
'F841',
|
||||
# 'PT018', # assertion should be broken into multiple parts
|
||||
'SIM102',
|
||||
#'SIM103', # needless-bool, return the condition {condition} directly
|
||||
'SIM103', # return the condition directly
|
||||
'SIM105',
|
||||
'SIM108',
|
||||
# 'SIM110', # use any() instead of a for loop
|
||||
# 'SIM114',
|
||||
'SIM115',
|
||||
# 'DTZ007', # Naive datetime constructed using `datetime.datetime.strptime()` without %z
|
||||
# 'UP007', # use X | Y instead of Union[X, Y]
|
||||
# 'UP032', # use an f-string instead of format()
|
||||
# 'UP046', # prefer type parameters over Generic subclasses
|
||||
]
|
||||
|
||||
[format]
|
||||
quote-style = "single"
|
||||
|
||||
[lint.isort]
|
||||
known-first-party = ["ssh_lib"]
|
||||
known-first-party = ["lib", "api", "deploy", "ssh_lib"]
|
||||
lines-after-imports = 2
|
||||
|
||||
[lint.flake8-comprehensions]
|
||||
allow-dict-calls-with-keyword-arguments = true
|
||||
|
||||
|
||||
@@ -47,7 +47,6 @@ Please consider sponsoring our project on [GitHub Sponsors](https://github.com/s
|
||||
The only way this project can possibly work is to be super focused about what it is and what it isn't. OpenFreeMap has the following limitations by design:
|
||||
|
||||
1. OpenFreeMap is not providing:
|
||||
|
||||
- search or geocoding
|
||||
- route calculation, navigation or directions
|
||||
- static image generation
|
||||
@@ -190,8 +189,6 @@ See [dev setup docs](docs/dev_setup.md).
|
||||
Updated Planetiler version to latest
|
||||
Updated OpenJDK to 24 via Temurin repo
|
||||
|
||||
|
||||
|
||||
##### v0.8
|
||||
|
||||
Lot of self-hosting related fixes.
|
||||
|
||||
54
TODO.md
Normal file
54
TODO.md
Normal file
@@ -0,0 +1,54 @@
|
||||
making web maps free and fun again
|
||||
add dark themes in website
|
||||
|
||||
|
||||
|
||||
logrotate
|
||||
|
||||
/var/log/nginx/*.log /var/log/nginx/*/*.log {
|
||||
daily # rotate at least once per day
|
||||
dateext # add -YYYYMMDD to rotated files
|
||||
rotate 7 # keep up to 7 rotations (about 7 days)
|
||||
maxage 7 # hard limit: delete rotated logs older than 7 days
|
||||
missingok
|
||||
compress
|
||||
delaycompress
|
||||
notifempty
|
||||
sharedscripts
|
||||
create 0640 www-data adm # adjust user:group for your distro (e.g., nginx adm)
|
||||
postrotate
|
||||
# Tell nginx to reopen logs without losing lines
|
||||
[ -s /run/nginx.pid ] && kill -USR1 "$(cat /run/nginx.pid)" || true
|
||||
# Alternatively: nginx -s reopen
|
||||
endscript
|
||||
}
|
||||
|
||||
---
|
||||
|
||||
sudo systemctl enable --now logrotate.timer
|
||||
sudo systemctl status logrotate.timer
|
||||
|
||||
/etc/cron.daily/logrotate
|
||||
|
||||
Dry run:
|
||||
sudo logrotate -d /etc/logrotate.d/nginx
|
||||
Force a rotation immediately:
|
||||
sudo logrotate -vf /etc/logrotate.conf
|
||||
|
||||
---
|
||||
|
||||
/var/log/nginx/*.log {
|
||||
daily # Rotate every day
|
||||
rotate 7 # Keep only 7 rotated files
|
||||
missingok
|
||||
notifempty
|
||||
compress
|
||||
delaycompress
|
||||
sharedscripts
|
||||
postrotate
|
||||
[ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
|
||||
endscript
|
||||
}
|
||||
|
||||
---
|
||||
|
||||
180
biome.jsonc
180
biome.jsonc
@@ -1,28 +1,186 @@
|
||||
{
|
||||
"root": true,
|
||||
"$schema": "https://biomejs.dev/schemas/2.2.4/schema.json",
|
||||
|
||||
"vcs": {
|
||||
"enabled": false,
|
||||
"clientKind": "git",
|
||||
"useIgnoreFile": true,
|
||||
},
|
||||
"files": {
|
||||
"maxSize": 100000,
|
||||
"ignoreUnknown": true,
|
||||
"includes": [
|
||||
"**",
|
||||
//
|
||||
"!**/.pytest_cache",
|
||||
"!**/venv",
|
||||
"!**/.astro",
|
||||
"!**/.venv",
|
||||
"!**/_astro",
|
||||
"!**/_not_used",
|
||||
"!**/dist",
|
||||
"!**/dist-electron",
|
||||
"!**/node_modules",
|
||||
"!**/.pytest_cache",
|
||||
//
|
||||
],
|
||||
},
|
||||
"formatter": {
|
||||
"indentStyle": "space",
|
||||
"lineWidth": 100
|
||||
"lineWidth": 100,
|
||||
},
|
||||
"assist": { "actions": { "source": { "organizeImports": "on" } } },
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"domains": {
|
||||
"solid": "all",
|
||||
},
|
||||
"rules": {
|
||||
"recommended": true,
|
||||
"style": {
|
||||
"noNonNullAssertion": "off",
|
||||
"useConsistentArrayType": {
|
||||
"level": "error",
|
||||
"options": {
|
||||
"syntax": "generic",
|
||||
},
|
||||
},
|
||||
"useLiteralEnumMembers": "error",
|
||||
"useNodejsImportProtocol": "error",
|
||||
"useAsConstAssertion": "error",
|
||||
"useEnumInitializers": "error",
|
||||
"useSelfClosingElements": "error",
|
||||
"useSingleVarDeclarator": "error",
|
||||
"noUnusedTemplateLiteral": "error",
|
||||
"useNumberNamespace": "error",
|
||||
"noInferrableTypes": "error",
|
||||
"useExponentiationOperator": "error",
|
||||
"useTemplate": "error",
|
||||
"noParameterAssign": "error",
|
||||
"useDefaultParameterLast": "error",
|
||||
"useImportType": "error",
|
||||
"useExportType": "error",
|
||||
"useShorthandFunctionType": "error",
|
||||
"noUselessElse": "error",
|
||||
"useGroupedAccessorPairs": "error",
|
||||
"useObjectSpread": "error",
|
||||
},
|
||||
"security": {
|
||||
// "noDangerouslySetInnerHtml": "off",
|
||||
// "noBlankTarget": "off",
|
||||
},
|
||||
"a11y": {
|
||||
"useFocusableInteractive": "off", // The HTML element with the interactive role "treeitem" is not focusable.
|
||||
"useSemanticElements": "off", // SessionTreeDay / role="group"
|
||||
"noStaticElementInteractions": "off", // SessionTreeDay / To add interactivity such as a mouse or key event listener to a static element, give the element an appropriate role value.
|
||||
"useAriaPropsSupportedByRole": "off", // SessionTree / The ARIA attribute 'aria-multiselectable' is not supported by this element.
|
||||
"useKeyWithClickEvents": "off", // PasteBlock / Enforce to have the onClick mouse event with the onKeyUp, the onKeyDown, or the onKeyPress keyboard event.
|
||||
"noNoninteractiveElementToInteractiveRole": "off", // SessionTree
|
||||
// "useValidAnchor": "off",
|
||||
// "useButtonType": "off",
|
||||
// "noLabelWithoutControl": "off",
|
||||
// "noSvgWithoutTitle": "off",
|
||||
// "noNoninteractiveTabindex": "off",
|
||||
},
|
||||
"correctness": {
|
||||
"noUnusedFunctionParameters": "off",
|
||||
"noUnusedImports": "off",
|
||||
"noUnusedVariables": "off",
|
||||
"noUnusedPrivateClassMembers": "off",
|
||||
"noUnreachable": "off",
|
||||
"noSolidDestructuredProps": "error",
|
||||
"noGlobalDirnameFilename": "error",
|
||||
"useParseIntRadix": "error",
|
||||
"useSingleJsDocAsterisk": "error",
|
||||
},
|
||||
"suspicious": {
|
||||
"noExplicitAny": "off",
|
||||
"noImplicitAnyLet": "off",
|
||||
"noUselessEscapeInString": "error",
|
||||
"useStaticResponseMethods": "error",
|
||||
"useIterableCallbackReturn": "off",
|
||||
// "noAssignInExpressions": "off",
|
||||
//"noArrayIndexKey": "off",
|
||||
},
|
||||
"complexity": {
|
||||
"noForEach": "off"
|
||||
}
|
||||
"noCommaOperator": "error",
|
||||
"useNumericLiterals": "error",
|
||||
// "noArguments": "off",
|
||||
"useIndexOf": "error",
|
||||
"noImplicitCoercions": "error", // Number(), Boolean()
|
||||
// "noUselessFragments": "off"
|
||||
},
|
||||
"nursery": {
|
||||
// "noShadow": "error",
|
||||
// "useReadonlyClassProperties": "error",
|
||||
// "noImportCycles": "error",
|
||||
|
||||
// "noFloatingPromises": "error",
|
||||
// "noMisusedPromises": "error",
|
||||
|
||||
// "noUnassignedVariables": "error", // bug reported ref= not recognised
|
||||
"noUselessUndefined": "error",
|
||||
},
|
||||
"performance": { "useSolidForComponent": "error" },
|
||||
},
|
||||
"includes": ["**"]
|
||||
},
|
||||
"javascript": {
|
||||
"formatter": {
|
||||
"semicolons": "asNeeded",
|
||||
"quoteStyle": "single"
|
||||
}
|
||||
"quoteStyle": "single",
|
||||
},
|
||||
},
|
||||
"files": {
|
||||
"maxSize": 100000,
|
||||
"includes": ["**", "!**/venv", "!**/dist", "!**/.astro"]
|
||||
}
|
||||
"json": {
|
||||
"parser": {
|
||||
"allowTrailingCommas": true,
|
||||
},
|
||||
"formatter": {
|
||||
"trailingCommas": "none",
|
||||
},
|
||||
},
|
||||
"assist": {
|
||||
"enabled": true,
|
||||
"actions": {
|
||||
"source": {
|
||||
"organizeImports": "on",
|
||||
// "useSortedProperties": "on" // sort CSS props
|
||||
},
|
||||
},
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"includes": ["**/*.astro"],
|
||||
"linter": {
|
||||
"rules": {
|
||||
"correctness": {
|
||||
"noUnusedImports": "off",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"includes": ["**/*.jsonc"],
|
||||
"json": {
|
||||
"formatter": {
|
||||
"trailingCommas": "all",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"includes": ["**/*.css"],
|
||||
"linter": {
|
||||
"rules": {
|
||||
"suspicious": {
|
||||
"noDuplicateProperties": "off",
|
||||
"noEmptyBlock": "off",
|
||||
"noUnknownAtRules": "off",
|
||||
},
|
||||
"complexity": {
|
||||
"noImportantStyles": "off",
|
||||
},
|
||||
"style": { "noDescendingSpecificity": "off" },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
# Leave this empty if you use SSH keys
|
||||
SSH_PASSWD=
|
||||
|
||||
# domain/subdomain
|
||||
# Set up an A record pointing to your server's IP address and
|
||||
# write the full domain here
|
||||
DOMAIN_DIRECT=maps.example.com
|
||||
|
||||
# Your email address to be used for the Let's Encrypt certificates
|
||||
LETSENCRYPT_EMAIL=
|
||||
|
||||
# Skip the full planet download, useful for testing (true/false)
|
||||
SKIP_PLANET=false
|
||||
|
||||
# Use self-signed certs / skip the certificate management part.
|
||||
# If you are using a custom solution like VPN, Traefik,
|
||||
# or Cloudflare managed certificates, set this to true.
|
||||
# In this case, you'll have self-signed certificates after the script completes.
|
||||
SELF_SIGNED_CERTS=false
|
||||
|
||||
|
||||
|
||||
### --- Advanced setup below this line --- ###
|
||||
### --- 99.9% you don't need any of this! --- ###
|
||||
|
||||
# DOMAIN_ROUNDROBIN is a very special feature for getting certificates on one server,
|
||||
# uploading them to a bucket, and then downloading them to multiple http-host servers.
|
||||
# For a single host, you don't need it!
|
||||
DOMAIN_ROUNDROBIN=
|
||||
|
||||
# Variables used by the load balancer script - you don't need these!
|
||||
HTTP_HOST_LIST=
|
||||
TELEGRAM_TOKEN=
|
||||
TELEGRAM_CHAT_ID=
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
# --- Let's Encrypt DNS challenge, not needed for self-hosting
|
||||
|
||||
dns_cloudflare_api_token = xxx
|
||||
90
config/config.schema.json
Normal file
90
config/config.schema.json
Normal file
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"$schema": {
|
||||
"type": "string"
|
||||
},
|
||||
"domains": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": { "$ref": "#/definitions/domain" }
|
||||
},
|
||||
"servers": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": { "$ref": "#/definitions/server" }
|
||||
},
|
||||
"skip_planet": {
|
||||
"type": "boolean",
|
||||
"description": "Skip the full planet download, useful for testing"
|
||||
}
|
||||
},
|
||||
"required": ["domains", "servers"],
|
||||
"definitions": {
|
||||
"cert-upload": {
|
||||
"type": "object",
|
||||
"description": "Upload your own certificate. Ideal for Cloudflare Origin Certificates with 15 year expiry. Steps: 1) Create an Origin Certificate on Cloudflare at SSL/TLS / Origin Server. 2) Generate private key and CSR with Cloudflare: Private key type: ECC, Certificate Validity: 15 years. 3) Key format: PEM. Save origin certificate as something.cert and private key as something.key in the same directory.",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"type": { "const": "upload" },
|
||||
"cert_path": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"pattern": "^.*\\.cert$",
|
||||
"description": "Path to your certificate file (*.cert). Both absolute and relative paths are supported. The corresponding private key (.key) should be in the same directory with the same basename."
|
||||
}
|
||||
},
|
||||
"required": ["type", "cert_path"]
|
||||
},
|
||||
"cert-letsencrypt": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"type": { "const": "letsencrypt" },
|
||||
"email": { "type": "string", "format": "email" }
|
||||
},
|
||||
"required": ["type", "email"]
|
||||
},
|
||||
"cert-dummy": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"type": { "const": "dummy" }
|
||||
},
|
||||
"required": ["type"]
|
||||
},
|
||||
"cert": {
|
||||
"oneOf": [
|
||||
{ "$ref": "#/definitions/cert-upload" },
|
||||
{ "$ref": "#/definitions/cert-letsencrypt" },
|
||||
{ "$ref": "#/definitions/cert-dummy" }
|
||||
]
|
||||
},
|
||||
"domain": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"domain": { "type": "string", "format": "hostname" },
|
||||
"cert": { "$ref": "#/definitions/cert" }
|
||||
},
|
||||
"required": ["domain", "cert"]
|
||||
},
|
||||
"server": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"hostname": {
|
||||
"type": "string",
|
||||
"description": "hostname used for ssh to connect. can be an IP address"
|
||||
},
|
||||
"server_ssh_passwd": {
|
||||
"type": "string",
|
||||
"description": "Leave this empty if you use SSH keys"
|
||||
}
|
||||
},
|
||||
"required": ["hostname"]
|
||||
}
|
||||
}
|
||||
}
|
||||
16
debug.py
Normal file
16
debug.py
Normal file
@@ -0,0 +1,16 @@
|
||||
import click
|
||||
|
||||
from ssh_lib.cli_helpers import common_options, get_connection
|
||||
from ssh_lib.tasks_http_host import upload_config_and_certs
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def debug(hostname, user, port, noninteractive):
|
||||
c = get_connection(hostname, user, port)
|
||||
upload_config_and_certs(c)
|
||||
@@ -1,618 +0,0 @@
|
||||
user nginx;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
|
||||
worker_processes auto;
|
||||
worker_rlimit_nofile 300000; # needs to be < ulimit -n
|
||||
|
||||
error_log /data/nginx/logs/nginx-error.log warn;
|
||||
|
||||
events {
|
||||
worker_connections 40000;
|
||||
multi_accept off; # very important, otherwise one worker might get all the connections
|
||||
}
|
||||
|
||||
http {
|
||||
# aggressive caching for read-only sources
|
||||
open_file_cache max=1000000 inactive=60m;
|
||||
open_file_cache_valid 60m;
|
||||
open_file_cache_min_uses 1;
|
||||
open_file_cache_errors on;
|
||||
|
||||
server_tokens off;
|
||||
|
||||
include /etc/nginx/mime.types;
|
||||
types {
|
||||
application/x-protobuf pbf;
|
||||
}
|
||||
default_type application/octet-stream;
|
||||
|
||||
charset utf-8;
|
||||
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
tcp_nodelay on;
|
||||
|
||||
reset_timedout_connection on;
|
||||
send_timeout 20;
|
||||
|
||||
max_ranges 0;
|
||||
|
||||
gzip on;
|
||||
gzip_comp_level 1;
|
||||
gzip_types application/json application/x-protobuf;
|
||||
|
||||
log_format access_json '{'
|
||||
|
||||
# general
|
||||
'"time": "$time_iso8601", '
|
||||
'"status": $status, '
|
||||
#'"request_method": "$request_method", '
|
||||
#'"uri": "$uri", '
|
||||
#'"request": "$request", '
|
||||
#'"request_time": $request_time, '
|
||||
'"body_bytes_sent": $body_bytes_sent, '
|
||||
'"http_referrer": "$http_referer", '
|
||||
'"http_user_agent": "$http_user_agent", '
|
||||
#'"scheme": "$scheme", '
|
||||
#'"host": "$host", '
|
||||
#'"http_host": "$http_host", '
|
||||
|
||||
# IP address related
|
||||
# IP address logging is disabled
|
||||
#'"remote_addr": "$remote_addr", '
|
||||
#'"http_x_forwarded_for": "$http_x_forwarded_for", '
|
||||
|
||||
# CF related
|
||||
#'"http_cf_ray": "$http_cf_ray", '
|
||||
#'"http_cf_ipcountry": "$http_cf_ipcountry", '
|
||||
#'"http_cf_connecting_ip": "$http_cf_connecting_ip", '
|
||||
|
||||
'"_": "_"' # helper for no trailing comma
|
||||
'}';
|
||||
|
||||
access_log off;
|
||||
#access_log /data/nginx/logs/nginx-access.log access_json buffer=128k;
|
||||
|
||||
include /data/nginx/config/*;
|
||||
include /data/nginx/sites/*;
|
||||
}
|
||||
|
||||
# configuration file /etc/nginx/mime.types:
|
||||
types {
|
||||
|
||||
# Data interchange
|
||||
|
||||
application/atom+xml atom;
|
||||
application/json json map topojson;
|
||||
application/ld+json jsonld;
|
||||
application/rss+xml rss;
|
||||
# Normalize to standard type.
|
||||
# https://tools.ietf.org/html/rfc7946#section-12
|
||||
application/geo+json geojson;
|
||||
application/xml xml;
|
||||
# Normalize to standard type.
|
||||
# https://tools.ietf.org/html/rfc3870#section-2
|
||||
application/rdf+xml rdf;
|
||||
|
||||
|
||||
# JavaScript
|
||||
|
||||
# Servers should use text/javascript for JavaScript resources.
|
||||
# https://html.spec.whatwg.org/multipage/scripting.html#scriptingLanguages
|
||||
text/javascript js mjs;
|
||||
application/wasm wasm;
|
||||
|
||||
|
||||
# Manifest files
|
||||
|
||||
application/manifest+json webmanifest;
|
||||
application/x-web-app-manifest+json webapp;
|
||||
text/cache-manifest appcache;
|
||||
|
||||
|
||||
# Media files
|
||||
|
||||
audio/midi mid midi kar;
|
||||
audio/mp4 aac f4a f4b m4a;
|
||||
audio/mpeg mp3;
|
||||
audio/ogg oga ogg opus;
|
||||
audio/x-realaudio ra;
|
||||
audio/x-wav wav;
|
||||
image/apng apng;
|
||||
image/avif avif avifs;
|
||||
image/bmp bmp;
|
||||
image/gif gif;
|
||||
image/jpeg jpeg jpg;
|
||||
image/jxl jxl;
|
||||
image/jxr jxr hdp wdp;
|
||||
image/png png;
|
||||
image/svg+xml svg svgz;
|
||||
image/tiff tif tiff;
|
||||
image/vnd.wap.wbmp wbmp;
|
||||
image/webp webp;
|
||||
image/x-jng jng;
|
||||
video/3gpp 3gp 3gpp;
|
||||
video/mp4 f4p f4v m4v mp4;
|
||||
video/mpeg mpeg mpg;
|
||||
video/ogg ogv;
|
||||
video/quicktime mov;
|
||||
video/webm webm;
|
||||
video/x-flv flv;
|
||||
video/x-mng mng;
|
||||
video/x-ms-asf asf asx;
|
||||
video/x-msvideo avi;
|
||||
|
||||
# Serving `.ico` image files with a different media type
|
||||
# prevents Internet Explorer from displaying then as images:
|
||||
# https://github.com/h5bp/html5-boilerplate/commit/37b5fec090d00f38de64b591bcddcb205aadf8ee
|
||||
|
||||
image/x-icon cur ico;
|
||||
|
||||
|
||||
# Microsoft Office
|
||||
|
||||
application/msword doc;
|
||||
application/vnd.ms-excel xls;
|
||||
application/vnd.ms-powerpoint ppt;
|
||||
application/vnd.openxmlformats-officedocument.wordprocessingml.document docx;
|
||||
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx;
|
||||
application/vnd.openxmlformats-officedocument.presentationml.presentation pptx;
|
||||
|
||||
|
||||
# Web fonts
|
||||
|
||||
font/woff woff;
|
||||
font/woff2 woff2;
|
||||
application/vnd.ms-fontobject eot;
|
||||
font/ttf ttf;
|
||||
font/collection ttc;
|
||||
font/otf otf;
|
||||
|
||||
|
||||
# Other
|
||||
|
||||
application/java-archive ear jar war;
|
||||
application/mac-binhex40 hqx;
|
||||
application/octet-stream bin deb dll dmg exe img iso msi msm msp safariextz;
|
||||
application/pdf pdf;
|
||||
application/postscript ai eps ps;
|
||||
application/rtf rtf;
|
||||
application/vnd.google-earth.kml+xml kml;
|
||||
application/vnd.google-earth.kmz kmz;
|
||||
application/vnd.wap.wmlc wmlc;
|
||||
application/x-7z-compressed 7z;
|
||||
application/x-bb-appworld bbaw;
|
||||
application/x-bittorrent torrent;
|
||||
application/x-chrome-extension crx;
|
||||
application/x-cocoa cco;
|
||||
application/x-java-archive-diff jardiff;
|
||||
application/x-java-jnlp-file jnlp;
|
||||
application/x-makeself run;
|
||||
application/x-opera-extension oex;
|
||||
application/x-perl pl pm;
|
||||
application/x-pilot pdb prc;
|
||||
application/x-rar-compressed rar;
|
||||
application/x-redhat-package-manager rpm;
|
||||
application/x-sea sea;
|
||||
application/x-shockwave-flash swf;
|
||||
application/x-stuffit sit;
|
||||
application/x-tcl tcl tk;
|
||||
application/x-x509-ca-cert crt der pem;
|
||||
application/x-xpinstall xpi;
|
||||
application/xhtml+xml xhtml;
|
||||
application/xslt+xml xsl;
|
||||
application/zip zip;
|
||||
text/calendar ics;
|
||||
text/css css;
|
||||
text/csv csv;
|
||||
text/html htm html shtml;
|
||||
text/markdown md markdown;
|
||||
text/mathml mml;
|
||||
text/plain txt;
|
||||
text/vcard vcard vcf;
|
||||
text/vnd.rim.location.xloc xloc;
|
||||
text/vnd.sun.j2me.app-descriptor jad;
|
||||
text/vnd.wap.wml wml;
|
||||
text/vtt vtt;
|
||||
text/x-component htc;
|
||||
|
||||
}
|
||||
|
||||
# configuration file /data/nginx/sites/default_disable.conf:
|
||||
map "" $empty {
|
||||
default "";
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
listen [::]:80 default_server;
|
||||
|
||||
listen 443 ssl default_server;
|
||||
listen [::]:443 ssl default_server;
|
||||
http2 on;
|
||||
|
||||
server_name _;
|
||||
|
||||
ssl_ciphers aNULL;
|
||||
ssl_certificate /etc/nginx/ssl/dummy.crt;
|
||||
ssl_certificate_key /etc/nginx/ssl/dummy.key;
|
||||
|
||||
return 444;
|
||||
}
|
||||
|
||||
# configuration file /data/nginx/sites/ofm_roundrobin.conf:
|
||||
server {
|
||||
server_name ofm_roundrobin tiles.openfreemap.org;
|
||||
|
||||
# ssl: https://ssl-config.mozilla.org / intermediate config
|
||||
|
||||
listen 80;
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
http2 on;
|
||||
|
||||
ssl_certificate /data/nginx/certs/ofm_roundrobin.cert;
|
||||
ssl_certificate_key /data/nginx/certs/ofm_roundrobin.key;
|
||||
|
||||
ssl_session_timeout 1d;
|
||||
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
|
||||
ssl_session_tickets off;
|
||||
|
||||
ssl_dhparam /etc/nginx/ffdhe2048.txt;
|
||||
|
||||
# intermediate configuration
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
|
||||
ssl_prefer_server_ciphers off;
|
||||
|
||||
# access log doesn't contain IP address
|
||||
access_log off;
|
||||
#access_log /data/ofm/http_host/logs_nginx/roundrobin-access.jsonl access_json buffer=128k;
|
||||
|
||||
error_log /data/ofm/http_host/logs_nginx/roundrobin-error.log;
|
||||
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
|
||||
# specific JSON monaco 20250806_231001_pt
|
||||
location = /monaco/20250806_231001_pt {
|
||||
# no trailing slash
|
||||
alias /data/ofm/http_host/runs/monaco/20250806_231001_pt/tilejson-ofm_roundrobin.json; # no trailing slash
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific JSON monaco 20250806_231001_pt';
|
||||
}
|
||||
|
||||
# specific PBF monaco 20250806_231001_pt
|
||||
location ^~ /monaco/20250806_231001_pt/ {
|
||||
# trailing slash
|
||||
alias /mnt/ofm/monaco-20250806_231001_pt/tiles/; # trailing slash
|
||||
try_files $uri @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific PBF monaco 20250806_231001_pt';
|
||||
}
|
||||
|
||||
# specific JSON planet 20250806_001001_pt
|
||||
location = /planet/20250806_001001_pt {
|
||||
# no trailing slash
|
||||
alias /data/ofm/http_host/runs/planet/20250806_001001_pt/tilejson-ofm_roundrobin.json; # no trailing slash
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific JSON planet 20250806_001001_pt';
|
||||
}
|
||||
|
||||
# specific PBF planet 20250806_001001_pt
|
||||
location ^~ /planet/20250806_001001_pt/ {
|
||||
# trailing slash
|
||||
alias /mnt/ofm/planet-20250806_001001_pt/tiles/; # trailing slash
|
||||
try_files $uri @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific PBF planet 20250806_001001_pt';
|
||||
}
|
||||
|
||||
# specific JSON monaco 20250805_231001_pt
|
||||
location = /monaco/20250805_231001_pt {
|
||||
# no trailing slash
|
||||
alias /data/ofm/http_host/runs/monaco/20250805_231001_pt/tilejson-ofm_roundrobin.json; # no trailing slash
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific JSON monaco 20250805_231001_pt';
|
||||
}
|
||||
|
||||
# specific PBF monaco 20250805_231001_pt
|
||||
location ^~ /monaco/20250805_231001_pt/ {
|
||||
# trailing slash
|
||||
alias /mnt/ofm/monaco-20250805_231001_pt/tiles/; # trailing slash
|
||||
try_files $uri @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific PBF monaco 20250805_231001_pt';
|
||||
}
|
||||
|
||||
# specific JSON planet 20250730_001001_pt
|
||||
location = /planet/20250730_001001_pt {
|
||||
# no trailing slash
|
||||
alias /data/ofm/http_host/runs/planet/20250730_001001_pt/tilejson-ofm_roundrobin.json; # no trailing slash
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific JSON planet 20250730_001001_pt';
|
||||
}
|
||||
|
||||
# specific PBF planet 20250730_001001_pt
|
||||
location ^~ /planet/20250730_001001_pt/ {
|
||||
# trailing slash
|
||||
alias /mnt/ofm/planet-20250730_001001_pt/tiles/; # trailing slash
|
||||
try_files $uri @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific PBF planet 20250730_001001_pt';
|
||||
}
|
||||
|
||||
|
||||
# latest JSON monaco
|
||||
location = /monaco {
|
||||
# no trailing slash
|
||||
alias /data/ofm/http_host/runs/monaco/20250806_231001_pt/tilejson-ofm_roundrobin.json; # no trailing slash
|
||||
|
||||
expires 1d;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'latest JSON monaco';
|
||||
}
|
||||
|
||||
|
||||
# wildcard JSON monaco
|
||||
location ~ ^/monaco/([^/]+)$ {
|
||||
# regex location is unreliable with alias, only root is reliable
|
||||
|
||||
root /data/ofm/http_host/runs/monaco/20250806_231001_pt; # no trailing slash
|
||||
try_files /tilejson-ofm_roundrobin.json =404;
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'wildcard JSON monaco';
|
||||
}
|
||||
|
||||
# wildcard PBF monaco
|
||||
location ~ ^/monaco/([^/]+)/(.+)$ {
|
||||
# regex location is unreliable with alias, only root is reliable
|
||||
|
||||
root /mnt/ofm/monaco-20250806_231001_pt/tiles/; # trailing slash
|
||||
try_files /$2 @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'wildcard PBF monaco';
|
||||
}
|
||||
|
||||
|
||||
# latest JSON planet
|
||||
location = /planet {
|
||||
# no trailing slash
|
||||
alias /data/ofm/http_host/runs/planet/20250806_001001_pt/tilejson-ofm_roundrobin.json; # no trailing slash
|
||||
|
||||
expires 1d;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'latest JSON planet';
|
||||
}
|
||||
|
||||
|
||||
# wildcard JSON planet
|
||||
location ~ ^/planet/([^/]+)$ {
|
||||
# regex location is unreliable with alias, only root is reliable
|
||||
|
||||
root /data/ofm/http_host/runs/planet/20250806_001001_pt; # no trailing slash
|
||||
try_files /tilejson-ofm_roundrobin.json =404;
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'wildcard JSON planet';
|
||||
}
|
||||
|
||||
# wildcard PBF planet
|
||||
location ~ ^/planet/([^/]+)/(.+)$ {
|
||||
# regex location is unreliable with alias, only root is reliable
|
||||
|
||||
root /mnt/ofm/planet-20250806_001001_pt/tiles/; # trailing slash
|
||||
try_files /$2 @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'wildcard PBF planet';
|
||||
}
|
||||
|
||||
location /fonts/ {
|
||||
# trailing slash
|
||||
|
||||
alias /data/ofm/http_host/assets/fonts/ofm/; # trailing slash
|
||||
try_files $uri =404;
|
||||
|
||||
expires 1w;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
}
|
||||
|
||||
location /natural_earth/ {
|
||||
# trailing slash
|
||||
|
||||
alias /data/ofm/http_host/assets/natural_earth/ofm/; # trailing slash
|
||||
try_files $uri =404;
|
||||
|
||||
expires 10y;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
}
|
||||
|
||||
location /sprites/ {
|
||||
# trailing slash
|
||||
|
||||
alias /data/ofm/http_host/assets/sprites/; # trailing slash
|
||||
try_files $uri =404;
|
||||
|
||||
expires 10y;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
}
|
||||
|
||||
|
||||
# we need to handle missing tiles as valid request returning empty string
|
||||
location @empty_tile {
|
||||
return 200 '';
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'empty tile';
|
||||
}
|
||||
|
||||
location = / {
|
||||
return 302 https://openfreemap.org;
|
||||
}
|
||||
|
||||
|
||||
location /styles/ {
|
||||
# trailing slash
|
||||
|
||||
alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash
|
||||
try_files $uri.json =404;
|
||||
|
||||
expires 1d;
|
||||
default_type application/json;
|
||||
|
||||
# substitute the domain in the TileJSON
|
||||
sub_filter '__TILEJSON_DOMAIN__' 'tiles.openfreemap.org';
|
||||
sub_filter_once off;
|
||||
sub_filter_types '*';
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
}
|
||||
|
||||
# catch-all block to deny all other requests
|
||||
location / {
|
||||
deny all;
|
||||
error_log /data/ofm/http_host/logs_nginx/roundrobin-deny.log error;
|
||||
}
|
||||
}
|
||||
@@ -82,7 +82,7 @@ Run the actual deploy command and wait a few minutes
|
||||
If everything is OK, you'll have some curl lines printed. Run the first one locally and make sure it's showing HTTP/2 200. For example this is an OK response.
|
||||
|
||||
```locally to test them.
|
||||
curl -sI https://test.openfreemap.org/monaco | sort
|
||||
curl -sI https://test.openfreemap.org/monaco
|
||||
|
||||
HTTP/2 200
|
||||
access-control-allow-origin: *
|
||||
|
||||
182
http-host.py
Executable file
182
http-host.py
Executable file
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
|
||||
import click
|
||||
|
||||
from modules.http_host.http_host_lib.get_version_shared import get_deployed_version
|
||||
from ssh_lib.cli_helpers import common_options, get_connection
|
||||
from ssh_lib.config import config
|
||||
from ssh_lib.pycurl import pycurl_get
|
||||
from ssh_lib.tasks_http_host import prepare_http_host, read_jsonc, run_http_host_sync
|
||||
from ssh_lib.tasks_shared import prepare_shared
|
||||
from ssh_lib.utils import (
|
||||
get_ip_from_ssh_alias,
|
||||
put,
|
||||
)
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def init_static(hostname, user, port, noninteractive):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
|
||||
prepare_shared(c)
|
||||
prepare_http_host(c)
|
||||
|
||||
run_http_host_sync(c)
|
||||
|
||||
# Check server health after deployment
|
||||
results = check_server_health(hostname)
|
||||
print_server_health(results)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
@click.option('--sync', is_flag=True, help='Run manual sync after init')
|
||||
def init_autoupdate(hostname, user, port, noninteractive, sync):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
|
||||
c.sudo('rm -f /etc/cron.d/ofm_http_host')
|
||||
|
||||
prepare_shared(c)
|
||||
prepare_http_host(c)
|
||||
|
||||
# if --sync, run manual sync
|
||||
if sync:
|
||||
run_http_host_sync(c)
|
||||
|
||||
put(c, config.local_modules_dir / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
|
||||
|
||||
# Check server health after deployment
|
||||
results = check_server_health(hostname)
|
||||
print_server_health(results)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def sync(hostname, user, port, noninteractive):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
run_http_host_sync(c)
|
||||
|
||||
# Check server health after sync
|
||||
results = check_server_health(hostname)
|
||||
print_server_health(results)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option('--hostname', help='Check only a specific server')
|
||||
def debug(hostname):
|
||||
results = check_server_health(hostname)
|
||||
print_server_health(results)
|
||||
|
||||
|
||||
def check_server_health(hostname: str = None) -> dict:
|
||||
"""
|
||||
Check health of servers by verifying deployed version matches expected version.
|
||||
|
||||
Args:
|
||||
hostname: Optional hostname to check. If None, checks all servers in config.
|
||||
|
||||
Returns:
|
||||
dict: Results for each server with structure:
|
||||
{
|
||||
'server_hostname': {
|
||||
'ip': '1.2.3.4',
|
||||
'all_ok': True/False,
|
||||
'domains': {
|
||||
'domain.com': {'status': 'ok'/'failed', 'error': None/'error message'}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
config_data = read_jsonc()
|
||||
area = 'monaco' if config_data.get('skip_planet') else 'planet'
|
||||
version = get_deployed_version(area)['version']
|
||||
domains = [d['domain'] for d in config_data['domains']]
|
||||
|
||||
servers = [
|
||||
{'hostname': s['hostname'], 'ip': get_ip_from_ssh_alias(s['hostname'])}
|
||||
for s in config_data['servers']
|
||||
]
|
||||
|
||||
# Filter to specific server if requested
|
||||
if hostname:
|
||||
servers = [s for s in servers if s['hostname'] == hostname]
|
||||
if not servers:
|
||||
raise ValueError(f'Server {hostname} not found in config')
|
||||
|
||||
results = {}
|
||||
|
||||
for server in servers:
|
||||
server_hostname = server['hostname']
|
||||
server_ip = server['ip']
|
||||
results[server_hostname] = {'ip': server_ip, 'domains': {}, 'all_ok': True}
|
||||
|
||||
for domain in domains:
|
||||
try:
|
||||
check_host_using_tilejson(
|
||||
url=f'https://{domain}/{area}/{version}',
|
||||
ip=server_ip,
|
||||
version=version,
|
||||
)
|
||||
results[server_hostname]['domains'][domain] = {'status': 'ok', 'error': None}
|
||||
except AssertionError:
|
||||
results[server_hostname]['domains'][domain] = {
|
||||
'status': 'failed',
|
||||
'error': f'Version mismatch (expected {version})',
|
||||
}
|
||||
results[server_hostname]['all_ok'] = False
|
||||
except Exception as e:
|
||||
results[server_hostname]['domains'][domain] = {'status': 'failed', 'error': str(e)}
|
||||
results[server_hostname]['all_ok'] = False
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_server_health(results: dict) -> None:
|
||||
"""Print server health results in a human-readable format."""
|
||||
for server_hostname, server_data in results.items():
|
||||
status = (
|
||||
click.style('OK', fg='green')
|
||||
if server_data['all_ok']
|
||||
else click.style('FAILED', fg='red')
|
||||
)
|
||||
server_line = f'SERVER {server_hostname} ({server_data["ip"]})'
|
||||
print(f'{server_line:<50} {status}')
|
||||
|
||||
for domain, domain_data in server_data['domains'].items():
|
||||
domain_line = f' {domain}'
|
||||
if domain_data['status'] == 'ok':
|
||||
print(f'{domain_line:<50} {click.style("OK", fg="green")}')
|
||||
else:
|
||||
print(
|
||||
f'{domain_line:<50} {click.style("FAILED", fg="red")}\n {domain_data["error"]}'
|
||||
)
|
||||
|
||||
print()
|
||||
|
||||
|
||||
def check_host_using_tilejson(*, url: str, ip: str, version: str) -> None:
|
||||
tilejson_str = pycurl_get(url, ip)
|
||||
tilejson = json.loads(tilejson_str)
|
||||
tiles_url = tilejson['tiles'][0]
|
||||
version_in_tilejson = tiles_url.split('/')[4]
|
||||
assert version_in_tilejson == version
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
157
init-server.py
157
init-server.py
@@ -1,157 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import click
|
||||
from fabric import Config, Connection
|
||||
|
||||
from ssh_lib import MODULES_DIR, dotenv_val
|
||||
from ssh_lib.tasks import (
|
||||
prepare_http_host,
|
||||
prepare_shared,
|
||||
prepare_tile_gen,
|
||||
run_http_host_sync,
|
||||
setup_loadbalancer,
|
||||
setup_roundrobin_writer,
|
||||
)
|
||||
from ssh_lib.utils import (
|
||||
put,
|
||||
)
|
||||
|
||||
|
||||
def get_connection(hostname, user, port):
|
||||
ssh_passwd = dotenv_val('SSH_PASSWD')
|
||||
|
||||
if ssh_passwd:
|
||||
print('Using SSH password')
|
||||
|
||||
c = Connection(
|
||||
host=hostname,
|
||||
user=user,
|
||||
port=port,
|
||||
connect_kwargs={'password': ssh_passwd},
|
||||
config=Config(overrides={'sudo': {'password': ssh_passwd}}),
|
||||
)
|
||||
else:
|
||||
c = Connection(
|
||||
host=hostname,
|
||||
user=user,
|
||||
port=port,
|
||||
)
|
||||
|
||||
return c
|
||||
|
||||
|
||||
def common_options(func):
|
||||
"""Decorator to define common options."""
|
||||
func = click.argument('hostname')(func)
|
||||
func = click.option('--port', type=int, help='SSH port (if not in .ssh/config)')(func)
|
||||
func = click.option('--user', help='SSH user (if not in .ssh/config)')(func)
|
||||
func = click.option('-y', '--noninteractive', is_flag=True, help='Skip confirmation questions')(
|
||||
func
|
||||
)
|
||||
return func
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def http_host_static(hostname, user, port, noninteractive):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
|
||||
prepare_shared(c)
|
||||
prepare_http_host(c)
|
||||
|
||||
run_http_host_sync(c)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def http_host_autoupdate(hostname, user, port, noninteractive):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
|
||||
c.sudo('rm -f /etc/cron.d/ofm_http_host')
|
||||
|
||||
prepare_shared(c)
|
||||
prepare_http_host(c)
|
||||
|
||||
run_http_host_sync(c) # disable for first install if you don't want to wait
|
||||
|
||||
put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
@click.option('--cron', is_flag=True, help='Enable cron task')
|
||||
@click.option('--reinstall', is_flag=True, help='Reinstall everything in /data/ofm folder')
|
||||
def tile_gen(
|
||||
hostname,
|
||||
user,
|
||||
port,
|
||||
noninteractive,
|
||||
#
|
||||
cron,
|
||||
reinstall,
|
||||
):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
|
||||
if reinstall:
|
||||
c.sudo('rm -rf /data/ofm')
|
||||
|
||||
prepare_shared(c)
|
||||
prepare_tile_gen(c, enable_cron=cron)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def roundrobin_dns_writer(hostname, user, port, noninteractive):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
|
||||
setup_roundrobin_writer(c)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def loadbalancer(hostname, user, port, noninteractive):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
prepare_shared(c)
|
||||
|
||||
setup_loadbalancer(c)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def http_host_sync(hostname, user, port, noninteractive):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
run_http_host_sync(c)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
def debug(hostname, user, port, noninteractive):
|
||||
c = get_connection(hostname, user, port)
|
||||
run_http_host_sync(c)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
172
modules/debug_proxy/.gitignore
vendored
172
modules/debug_proxy/.gitignore
vendored
@@ -1,172 +0,0 @@
|
||||
# Logs
|
||||
|
||||
logs
|
||||
_.log
|
||||
npm-debug.log_
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
# Runtime data
|
||||
|
||||
pids
|
||||
_.pid
|
||||
_.seed
|
||||
\*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
|
||||
coverage
|
||||
\*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
|
||||
\*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
|
||||
.stylelintcache
|
||||
|
||||
# Microbundle cache
|
||||
|
||||
.rpt2_cache/
|
||||
.rts2_cache_cjs/
|
||||
.rts2_cache_es/
|
||||
.rts2_cache_umd/
|
||||
|
||||
# Optional REPL history
|
||||
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
|
||||
\*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variable files
|
||||
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
|
||||
.cache
|
||||
.parcel-cache
|
||||
|
||||
# Next.js build output
|
||||
|
||||
.next
|
||||
out
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
|
||||
.nuxt
|
||||
dist
|
||||
|
||||
# Gatsby files
|
||||
|
||||
.cache/
|
||||
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
|
||||
.temp
|
||||
.cache
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
|
||||
.dynamodb/
|
||||
|
||||
# TernJS port file
|
||||
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
|
||||
.vscode-test
|
||||
|
||||
# yarn v2
|
||||
|
||||
.yarn/cache
|
||||
.yarn/unplugged
|
||||
.yarn/build-state.yml
|
||||
.yarn/install-state.gz
|
||||
.pnp.\*
|
||||
|
||||
# wrangler project
|
||||
|
||||
.dev.vars
|
||||
.wrangler/
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"name": "cf-debug-proxy",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"deploy": "wrangler deploy",
|
||||
"dev": "wrangler dev",
|
||||
"start": "wrangler dev"
|
||||
},
|
||||
"devDependencies": {
|
||||
"itty-router": "^3.0.12",
|
||||
"wrangler": "^3.60.3"
|
||||
}
|
||||
}
|
||||
1142
modules/debug_proxy/pnpm-lock.yaml
generated
1142
modules/debug_proxy/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -1,66 +0,0 @@
|
||||
async function sendTelegramMessage(message, botToken, chatId) {
|
||||
const url = `https://api.telegram.org/bot${botToken}/sendMessage`
|
||||
const payload = {
|
||||
chat_id: chatId,
|
||||
text: message,
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
console.error('Failed to send message:', await response.text())
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error sending Telegram message:', error)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
async fetch(request, env, ctx) {
|
||||
const url = new URL(request.url)
|
||||
const userIP = request.headers.get('CF-Connecting-IP')
|
||||
|
||||
if (url.pathname === '/b') {
|
||||
url.pathname = '/styles/bright'
|
||||
}
|
||||
|
||||
// // no failure, just warning
|
||||
// if (request.method !== 'GET') {
|
||||
// const warningMessage = `Non-GET request ${request.method} ${url.pathname} ${userIP}`
|
||||
// console.error(warningMessage)
|
||||
// await sendTelegramMessage(warningMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
|
||||
// }
|
||||
|
||||
if (!url.pathname.startsWith('/styles')) {
|
||||
const errorMessage = 'Bad path'
|
||||
return new Response(errorMessage, { status: 500 })
|
||||
}
|
||||
|
||||
const proxyUrl = new URL(url.pathname, 'https://tiles.openfreemap.org')
|
||||
|
||||
try {
|
||||
const response = await fetch(proxyUrl)
|
||||
|
||||
if (response.status !== 200) {
|
||||
const errorMessage = `Proxy error: Bad status ${response.status} ${url.pathname} ${userIP}`
|
||||
console.error(errorMessage)
|
||||
await sendTelegramMessage(errorMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
|
||||
return new Response('Proxy error: Bad status', { status: 500 })
|
||||
}
|
||||
|
||||
return response
|
||||
} catch (error) {
|
||||
const errorMessage = `Proxy error: ${error.message} ${url.pathname} ${userIP}`
|
||||
console.error(errorMessage)
|
||||
await sendTelegramMessage(errorMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
|
||||
return new Response('Proxy error: Fetch failed', { status: 500 })
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
#:schema node_modules/wrangler/config-schema.json
|
||||
name = "cf-debug-proxy"
|
||||
main = "src/index.js"
|
||||
compatibility_date = "2024-06-20"
|
||||
|
||||
# Automatically place your workloads in an optimal location to minimize latency.
|
||||
# If you are running back-end logic in a Worker, running it closer to your back-end infrastructure
|
||||
# rather than the end user may result in better performance.
|
||||
# Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement
|
||||
# [placement]
|
||||
# mode = "smart"
|
||||
|
||||
# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
|
||||
# Docs:
|
||||
# - https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables
|
||||
# Note: Use secrets to store sensitive data.
|
||||
# - https://developers.cloudflare.com/workers/configuration/secrets/
|
||||
# [vars]
|
||||
# MY_VARIABLE = "production_value"
|
||||
|
||||
# Bind the Workers AI model catalog. Run machine learning models, powered by serverless GPUs, on Cloudflare’s global network
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#workers-ai
|
||||
# [ai]
|
||||
# binding = "AI"
|
||||
|
||||
# Bind an Analytics Engine dataset. Use Analytics Engine to write analytics within your Pages Function.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#analytics-engine-datasets
|
||||
# [[analytics_engine_datasets]]
|
||||
# binding = "MY_DATASET"
|
||||
|
||||
# Bind a headless browser instance running on Cloudflare's global network.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#browser-rendering
|
||||
# [browser]
|
||||
# binding = "MY_BROWSER"
|
||||
|
||||
# Bind a D1 database. D1 is Cloudflare’s native serverless SQL database.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#d1-databases
|
||||
# [[d1_databases]]
|
||||
# binding = "MY_DB"
|
||||
# database_name = "my-database"
|
||||
# database_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
|
||||
|
||||
# Bind a dispatch namespace. Use Workers for Platforms to deploy serverless functions programmatically on behalf of your customers.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#dispatch-namespace-bindings-workers-for-platforms
|
||||
# [[dispatch_namespaces]]
|
||||
# binding = "MY_DISPATCHER"
|
||||
# namespace = "my-namespace"
|
||||
|
||||
# Bind a Durable Object. Durable objects are a scale-to-zero compute primitive based on the actor model.
|
||||
# Durable Objects can live for as long as needed. Use these when you need a long-running "server", such as in realtime apps.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#durable-objects
|
||||
# [[durable_objects.bindings]]
|
||||
# name = "MY_DURABLE_OBJECT"
|
||||
# class_name = "MyDurableObject"
|
||||
|
||||
# Durable Object migrations.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#migrations
|
||||
# [[migrations]]
|
||||
# tag = "v1"
|
||||
# new_classes = ["MyDurableObject"]
|
||||
|
||||
# Bind a Hyperdrive configuration. Use to accelerate access to your existing databases from Cloudflare Workers.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#hyperdrive
|
||||
# [[hyperdrive]]
|
||||
# binding = "MY_HYPERDRIVE"
|
||||
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
|
||||
|
||||
# Bind a KV Namespace. Use KV as persistent storage for small key-value pairs.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#kv-namespaces
|
||||
# [[kv_namespaces]]
|
||||
# binding = "MY_KV_NAMESPACE"
|
||||
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
|
||||
|
||||
# Bind an mTLS certificate. Use to present a client certificate when communicating with another service.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#mtls-certificates
|
||||
# [[mtls_certificates]]
|
||||
# binding = "MY_CERTIFICATE"
|
||||
# certificate_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
|
||||
|
||||
# Bind a Queue producer. Use this binding to schedule an arbitrary task that may be processed later by a Queue consumer.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
|
||||
# [[queues.producers]]
|
||||
# binding = "MY_QUEUE"
|
||||
# queue = "my-queue"
|
||||
|
||||
# Bind a Queue consumer. Queue Consumers can retrieve tasks scheduled by Producers to act on them.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
|
||||
# [[queues.consumers]]
|
||||
# queue = "my-queue"
|
||||
|
||||
# Bind an R2 Bucket. Use R2 to store arbitrarily large blobs of data, such as files.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#r2-buckets
|
||||
# [[r2_buckets]]
|
||||
# binding = "MY_BUCKET"
|
||||
# bucket_name = "my-bucket"
|
||||
|
||||
# Bind another Worker service. Use this binding to call another Worker without network overhead.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings
|
||||
# [[services]]
|
||||
# binding = "MY_SERVICE"
|
||||
# service = "my-service"
|
||||
|
||||
# Bind a Vectorize index. Use to store and query vector embeddings for semantic search, classification and other vector search use-cases.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#vectorize-indexes
|
||||
# [[vectorize]]
|
||||
# binding = "MY_INDEX"
|
||||
# index_name = "my-index"
|
||||
@@ -1,2 +0,0 @@
|
||||
# once per day
|
||||
2 34 * * * ofm sudo /usr/bin/bash /data/ofm/http_host/bin/roundrobin_reader.sh >> /data/ofm/http_host/logs/roundrobin_reader.log 2>&1
|
||||
@@ -7,10 +7,10 @@ from http_host_lib.assets import (
|
||||
)
|
||||
from http_host_lib.btrfs import (
|
||||
download_area_version,
|
||||
get_versions_for_area,
|
||||
)
|
||||
from http_host_lib.get_version_shared import get_versions_for_area
|
||||
from http_host_lib.mount import auto_mount
|
||||
from http_host_lib.nginx import write_nginx_config
|
||||
from http_host_lib.nginx_config_gen import write_nginx_config
|
||||
from http_host_lib.sync import auto_clean_btrfs, full_sync
|
||||
from http_host_lib.versions import fetch_version_files
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import subprocess
|
||||
import sys
|
||||
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.shared import get_versions_for_area
|
||||
from http_host_lib.get_version_shared import get_versions_for_area
|
||||
from http_host_lib.utils import download_file_aria2, get_remote_file_size
|
||||
|
||||
|
||||
|
||||
@@ -16,8 +16,10 @@ class Configuration:
|
||||
|
||||
mnt_dir = Path('/mnt/ofm')
|
||||
|
||||
certs_dir = Path('/data/nginx/certs')
|
||||
nginx_confs = Path(__file__).parent / 'nginx_confs'
|
||||
nginx_templates = Path(__file__).parent / 'nginx_templates'
|
||||
|
||||
nginx_certs_dir = Path('/data/nginx/certs')
|
||||
nginx_sites_dir = Path('/data/nginx/sites')
|
||||
|
||||
if Path('/data/ofm').exists():
|
||||
ofm_config_dir = Path('/data/ofm/config')
|
||||
@@ -25,7 +27,7 @@ class Configuration:
|
||||
repo_root = Path(__file__).parent.parent.parent.parent
|
||||
ofm_config_dir = repo_root / 'config'
|
||||
|
||||
ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
|
||||
json_config = json.loads((ofm_config_dir / 'config.json').read_text())
|
||||
|
||||
deployed_versions_dir = ofm_config_dir / 'deployed_versions'
|
||||
|
||||
|
||||
1
modules/http_host/http_host_lib/get_version_shared.py
Symbolic link
1
modules/http_host/http_host_lib/get_version_shared.py
Symbolic link
@@ -0,0 +1 @@
|
||||
../../tile_gen/tile_gen_lib/get_version_shared.py
|
||||
@@ -67,11 +67,25 @@ def clean_up_mounts(mnt_dir):
|
||||
p = subprocess.run(['mount'], capture_output=True, text=True, check=True)
|
||||
lines = [l for l in p.stdout.splitlines() if f'{mnt_dir}/' in l and '(deleted)' in l]
|
||||
|
||||
# Extract unique mount paths (deduplicate)
|
||||
mount_paths = set()
|
||||
for l in lines:
|
||||
mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0])
|
||||
mount_paths.add(mnt_path)
|
||||
|
||||
# Process each unique mount path once
|
||||
for mnt_path in mount_paths:
|
||||
if not mnt_path.exists():
|
||||
print(f' skipping {mnt_path} (already removed)')
|
||||
continue
|
||||
|
||||
print(f' removing deleted mount {mnt_path}')
|
||||
assert mnt_path.exists()
|
||||
subprocess.run(['umount', mnt_path], check=True)
|
||||
|
||||
# Unmount ALL instances (handle stacked mounts)
|
||||
while subprocess.run(['mountpoint', '-q', str(mnt_path)]).returncode == 0:
|
||||
print(f' unmounting {mnt_path}')
|
||||
subprocess.run(['umount', mnt_path], check=True)
|
||||
|
||||
mnt_path.rmdir()
|
||||
|
||||
# clean all mounts not in current fstab
|
||||
@@ -83,5 +97,10 @@ def clean_up_mounts(mnt_dir):
|
||||
continue
|
||||
|
||||
print(f' removing old mount {subdir}')
|
||||
subprocess.run(['umount', subdir], check=True)
|
||||
|
||||
# Unmount ALL instances here too
|
||||
while subprocess.run(['mountpoint', '-q', str(subdir)]).returncode == 0:
|
||||
print(f' unmounting {subdir}')
|
||||
subprocess.run(['umount', subdir], check=True)
|
||||
|
||||
subdir.rmdir()
|
||||
|
||||
@@ -1,334 +0,0 @@
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.utils import python_venv_executable
|
||||
|
||||
|
||||
def write_nginx_config():
|
||||
print('Writing nginx config')
|
||||
|
||||
if not config.mnt_dir.exists():
|
||||
sys.exit(' mount needs to be run first')
|
||||
|
||||
curl_text_mix = ''
|
||||
|
||||
domain_direct = config.ofm_config['domain_direct']
|
||||
domain_roundrobin = config.ofm_config['domain_roundrobin']
|
||||
self_signed_certs = config.ofm_config['self_signed_certs']
|
||||
|
||||
# remove old configs and certs
|
||||
for file in Path('/data/nginx/sites').glob('ofm_*.conf'):
|
||||
file.unlink()
|
||||
|
||||
for file in Path('/data/nginx/certs').glob('ofm_*'):
|
||||
file.unlink()
|
||||
|
||||
# processing Round Robin DNS config
|
||||
if domain_roundrobin:
|
||||
if not config.rclone_config.is_file():
|
||||
sys.exit('rclone.conf missing')
|
||||
|
||||
# download the roundrobin certificate from bucket using rclone
|
||||
write_roundrobin_reader_script(domain_roundrobin)
|
||||
subprocess.run(['bash', config.http_host_bin / 'roundrobin_reader.sh'], check=True)
|
||||
|
||||
curl_text_mix += create_nginx_conf(
|
||||
template_path=config.nginx_confs / 'roundrobin.conf',
|
||||
local='ofm_roundrobin',
|
||||
domain=domain_roundrobin,
|
||||
)
|
||||
|
||||
# processing Let's Encrypt config
|
||||
if domain_direct:
|
||||
direct_cert = config.certs_dir / 'ofm_direct.cert'
|
||||
direct_key = config.certs_dir / 'ofm_direct.key'
|
||||
|
||||
if not direct_cert.is_file() or not direct_key.is_file():
|
||||
shutil.copyfile(Path('/etc/nginx/ssl/dummy.cert'), direct_cert)
|
||||
shutil.copyfile(Path('/etc/nginx/ssl/dummy.key'), direct_key)
|
||||
|
||||
curl_text_mix += create_nginx_conf(
|
||||
template_path=config.nginx_confs / 'le.conf',
|
||||
local='ofm_direct',
|
||||
domain=domain_direct,
|
||||
)
|
||||
|
||||
subprocess.run(['nginx', '-t'], check=True)
|
||||
subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
|
||||
|
||||
if not self_signed_certs:
|
||||
subprocess.run(
|
||||
[
|
||||
'certbot',
|
||||
'certonly',
|
||||
'--webroot',
|
||||
'--webroot-path=/data/nginx/acme-challenges',
|
||||
'--noninteractive',
|
||||
'-m',
|
||||
config.ofm_config['letsencrypt_email'],
|
||||
'--agree-tos',
|
||||
'--cert-name=ofm_direct',
|
||||
# '--staging',
|
||||
'--deploy-hook',
|
||||
'nginx -t && service nginx reload',
|
||||
'-d',
|
||||
domain_direct,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
# link certs to nginx dir
|
||||
direct_cert.unlink()
|
||||
direct_key.unlink()
|
||||
|
||||
etc_cert = Path('/etc/letsencrypt/live/ofm_direct/fullchain.pem')
|
||||
etc_key = Path('/etc/letsencrypt/live/ofm_direct/privkey.pem')
|
||||
assert etc_cert.is_file()
|
||||
assert etc_key.is_file()
|
||||
direct_cert.symlink_to(etc_cert)
|
||||
direct_key.symlink_to(etc_key)
|
||||
|
||||
subprocess.run(['nginx', '-t'], check=True)
|
||||
subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
|
||||
|
||||
curl_text_lines = sorted(curl_text_mix.splitlines())
|
||||
if config.ofm_config.get('skip_planet'):
|
||||
curl_text_lines = [l for l in curl_text_lines if '/planet' not in l]
|
||||
else:
|
||||
curl_text_lines = [l for l in curl_text_lines if '/monaco' not in l]
|
||||
|
||||
curl_text_mix = '\n'.join(curl_text_lines)
|
||||
print(f'test with:\n{curl_text_mix}')
|
||||
|
||||
|
||||
def create_nginx_conf(*, template_path, local, domain):
|
||||
location_str, curl_text = create_location_blocks(local=local, domain=domain)
|
||||
|
||||
with open(template_path) as fp:
|
||||
template = fp.read()
|
||||
|
||||
template = template.replace('__LOCATION_BLOCKS__', location_str)
|
||||
template = template.replace('__LOCAL__', local)
|
||||
template = template.replace('__DOMAIN__', domain)
|
||||
|
||||
curl_text = curl_text.replace('__LOCAL__', local)
|
||||
curl_text = curl_text.replace('__DOMAIN__', domain)
|
||||
|
||||
with open(f'/data/nginx/sites/{local}.conf', 'w') as fp:
|
||||
fp.write(template)
|
||||
print(f' nginx config written: {domain} {local}')
|
||||
|
||||
return curl_text
|
||||
|
||||
|
||||
def create_location_blocks(*, local, domain):
|
||||
location_str = ''
|
||||
curl_text = ''
|
||||
|
||||
for subdir in config.mnt_dir.iterdir():
|
||||
if not subdir.is_dir():
|
||||
continue
|
||||
area, version = subdir.name.split('-')
|
||||
|
||||
location_str += create_version_location(
|
||||
area=area, version=version, mnt_dir=subdir, local=local, domain=domain
|
||||
)
|
||||
|
||||
for path in [
|
||||
f'/{area}/{version}',
|
||||
f'/{area}/{version}/14/8529/5975.pbf',
|
||||
f'/{area}/{version}/9999/9999/9999.pbf', # empty_tile test
|
||||
]:
|
||||
curl_text += (
|
||||
# f'curl -H "Host: __LOCAL__" -I http://localhost/{path}\n'
|
||||
f'curl -sI https://__DOMAIN__{path} | sort\n'
|
||||
)
|
||||
|
||||
location_str += create_latest_locations(local=local, domain=domain)
|
||||
|
||||
for area in config.areas:
|
||||
for path in [
|
||||
f'/{area}',
|
||||
f'/{area}/19700101_old_version_test',
|
||||
f'/{area}/19700101_old_version_test/14/8529/5975.pbf',
|
||||
f'/{area}/19700101_old_version_test/9999/9999/9999.pbf', # empty_tile test
|
||||
]:
|
||||
curl_text += (
|
||||
# f'curl -H "Host: __LOCAL__" -I http://localhost/{path}\n'
|
||||
f'curl -sI https://__DOMAIN__{path} | sort\n'
|
||||
)
|
||||
|
||||
with open(config.nginx_confs / 'location_static.conf') as fp:
|
||||
location_str += '\n' + fp.read()
|
||||
|
||||
return location_str, curl_text
|
||||
|
||||
|
||||
def create_version_location(
|
||||
*, area: str, version: str, mnt_dir: Path, local: str, domain: str
|
||||
) -> str:
|
||||
run_dir = config.runs_dir / area / version
|
||||
if not run_dir.is_dir():
|
||||
print(f" {run_dir} doesn't exist, skipping")
|
||||
return ''
|
||||
|
||||
tilejson_path = run_dir / f'tilejson-{local}.json'
|
||||
|
||||
metadata_path = mnt_dir / 'metadata.json'
|
||||
if not metadata_path.is_file():
|
||||
print(f" {metadata_path} doesn't exist, skipping")
|
||||
return ''
|
||||
|
||||
url_prefix = f'https://{domain}/{area}/{version}'
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
python_venv_executable(),
|
||||
config.http_host_scripts_dir / 'metadata_to_tilejson.py',
|
||||
'--minify',
|
||||
metadata_path,
|
||||
tilejson_path,
|
||||
url_prefix,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
return f"""
|
||||
# specific JSON {area} {version}
|
||||
location = /{area}/{version} {{ # no trailing slash
|
||||
alias {tilejson_path}; # no trailing slash
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific JSON {area} {version}';
|
||||
}}
|
||||
|
||||
# specific PBF {area} {version}
|
||||
location ^~ /{area}/{version}/ {{ # trailing slash
|
||||
alias {mnt_dir}/tiles/; # trailing slash
|
||||
try_files $uri @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {{
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific PBF {area} {version}';
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
def create_latest_locations(*, local: str, domain: str) -> str:
|
||||
location_str = ''
|
||||
|
||||
local_version_files = config.deployed_versions_dir.glob('*.txt')
|
||||
|
||||
for file in local_version_files:
|
||||
area = file.stem
|
||||
with open(file) as fp:
|
||||
version = fp.read().strip()
|
||||
|
||||
print(f' linking latest version for {area}: {version}')
|
||||
|
||||
# checking runs dir
|
||||
run_dir = config.runs_dir / area / version
|
||||
tilejson_path = run_dir / f'tilejson-{local}.json'
|
||||
if not tilejson_path.is_file():
|
||||
print(f' error with latest: {tilejson_path} does not exist')
|
||||
continue
|
||||
|
||||
# checking mnt dir
|
||||
mnt_dir = Path(f'/mnt/ofm/{area}-{version}')
|
||||
mnt_file = mnt_dir / 'metadata.json'
|
||||
if not mnt_file.is_file():
|
||||
print(f' error with latest: {mnt_file} does not exist')
|
||||
continue
|
||||
|
||||
# latest
|
||||
location_str += f"""
|
||||
|
||||
# latest JSON {area}
|
||||
location = /{area} {{ # no trailing slash
|
||||
alias {tilejson_path}; # no trailing slash
|
||||
|
||||
expires 1d;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'latest JSON {area}';
|
||||
}}
|
||||
"""
|
||||
|
||||
# wildcard
|
||||
# identical to create_version_location
|
||||
location_str += f"""
|
||||
|
||||
# wildcard JSON {area}
|
||||
location ~ ^/{area}/([^/]+)$ {{
|
||||
# regex location is unreliable with alias, only root is reliable
|
||||
|
||||
root {run_dir}; # no trailing slash
|
||||
try_files /tilejson-{local}.json =404;
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'wildcard JSON {area}';
|
||||
}}
|
||||
|
||||
# wildcard PBF {area}
|
||||
location ~ ^/{area}/([^/]+)/(.+)$ {{
|
||||
# regex location is unreliable with alias, only root is reliable
|
||||
|
||||
root {mnt_dir}/tiles/; # trailing slash
|
||||
try_files /$2 @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {{
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'wildcard PBF {area}';
|
||||
}}
|
||||
"""
|
||||
|
||||
return location_str
|
||||
|
||||
|
||||
def write_roundrobin_reader_script(domain_roundrobin):
|
||||
script = f"""
|
||||
#!/usr/bin/env bash
|
||||
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
|
||||
rclone copyto -v "remote:ofm-private/roundrobin/{domain_roundrobin}/ofm_roundrobin.cert" /data/nginx/certs/ofm_roundrobin.cert
|
||||
rclone copyto -v "remote:ofm-private/roundrobin/{domain_roundrobin}/ofm_roundrobin.key" /data/nginx/certs/ofm_roundrobin.key
|
||||
""".strip()
|
||||
|
||||
with open(config.http_host_bin / 'roundrobin_reader.sh', 'w') as fp:
|
||||
fp.write(script)
|
||||
292
modules/http_host/http_host_lib/nginx_config_gen.py
Normal file
292
modules/http_host/http_host_lib/nginx_config_gen.py
Normal file
@@ -0,0 +1,292 @@
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.utils import python_venv_executable
|
||||
|
||||
|
||||
def write_nginx_config():
|
||||
print('Writing nginx config')
|
||||
|
||||
if not config.mnt_dir.exists():
|
||||
sys.exit(' mount needs to be run first')
|
||||
|
||||
# remove old configs
|
||||
for file in config.nginx_sites_dir.glob('ofm-*.conf'):
|
||||
file.unlink()
|
||||
|
||||
curl_help_text = ''
|
||||
|
||||
for domain_data in config.json_config['domains']:
|
||||
curl_help_text += process_domain(domain_data)
|
||||
|
||||
subprocess.run(['nginx', '-t'], check=True)
|
||||
subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
|
||||
|
||||
exclude_path = '/planet' if config.json_config.get('skip_planet') else '/monaco'
|
||||
curl_help_lines = [l for l in curl_help_text.splitlines() if exclude_path not in l]
|
||||
|
||||
curl_help_joined = '\n'.join(curl_help_lines)
|
||||
print(f'test with:\n{curl_help_joined}')
|
||||
|
||||
|
||||
def process_domain(domain_data) -> str:
|
||||
if domain_data['cert']['type'] == 'upload':
|
||||
if (
|
||||
not Path(domain_data['cert_file']).is_file()
|
||||
or not Path(domain_data['key_file']).is_file()
|
||||
):
|
||||
sys.exit(
|
||||
f' cert or key file does not exist: {domain_data["cert_file"]} {domain_data["key_file"]}'
|
||||
)
|
||||
|
||||
return create_nginx_conf(domain_data)
|
||||
|
||||
return ''
|
||||
|
||||
|
||||
def create_nginx_conf(domain_data: dict) -> str:
|
||||
dynamic_block_text, curl_help_text = dynamic_blocks(domain_data)
|
||||
|
||||
template = (config.nginx_templates / 'common.conf').read_text()
|
||||
|
||||
template = template.replace('__DYNAMIC_BLOCKS__', dynamic_block_text)
|
||||
|
||||
template = template.replace('__DOMAIN_SLUG__', domain_data['slug'])
|
||||
template = template.replace('__DOMAIN__', domain_data['domain'])
|
||||
|
||||
curl_help_text = curl_help_text.replace('__DOMAIN_SLUG__', domain_data['slug'])
|
||||
curl_help_text = curl_help_text.replace('__DOMAIN__', domain_data['domain'])
|
||||
|
||||
(config.nginx_sites_dir / f'ofm-{domain_data["slug"]}.conf').write_text(template)
|
||||
print(f' nginx config written: {domain_data["domain"]} {domain_data["slug"]}')
|
||||
|
||||
return curl_help_text
|
||||
|
||||
|
||||
def dynamic_blocks(domain_data: dict) -> tuple[str, str]:
|
||||
nginx_conf_text = ''
|
||||
curl_help_text = ''
|
||||
|
||||
help_area = 'monaco' if config.json_config.get('skip_planet') else 'planet'
|
||||
|
||||
for subdir in config.mnt_dir.iterdir():
|
||||
if not subdir.is_dir():
|
||||
continue
|
||||
area, version = subdir.name.split('-')
|
||||
|
||||
nginx_conf_text += create_version_location(
|
||||
area=area, version=version, mnt_dir=subdir, domain_data=domain_data
|
||||
)
|
||||
|
||||
if area == help_area:
|
||||
for path in [
|
||||
f'/{area}/{version}',
|
||||
f'/{area}/{version}/14/8529/5974.pbf',
|
||||
# f'/{area}/{version}/9999/9999/9999.pbf', # empty_tile test
|
||||
]:
|
||||
# curl_help_text += f'curl -H "Host: __DOMAIN_SLUG__" -I http://localhost{path}\n'
|
||||
curl_help_text += f'curl -sI https://__DOMAIN__{path}\n'
|
||||
|
||||
nginx_conf_text += create_latest_locations(domain_data=domain_data)
|
||||
|
||||
for path in [
|
||||
f'/{help_area}',
|
||||
f'/{help_area}/latest',
|
||||
f'/{help_area}/latest/14/8529/5974.pbf',
|
||||
# f'/{help_area}/latest/9999/9999/9999.pbf', # empty_tile test
|
||||
]:
|
||||
# curl_help_text += f'curl -H "Host: __DOMAIN_SLUG__" -I http://localhost{path}\n'
|
||||
curl_help_text += f'curl -sI https://__DOMAIN__{path}\n'
|
||||
|
||||
nginx_conf_text += '\n' + (config.nginx_templates / 'static_blocks.conf').read_text()
|
||||
return nginx_conf_text, curl_help_text
|
||||
|
||||
|
||||
def create_version_location(*, area: str, version: str, mnt_dir: Path, domain_data: dict) -> str:
|
||||
run_dir = config.runs_dir / area / version
|
||||
if not run_dir.is_dir():
|
||||
print(f" {run_dir} doesn't exist, skipping")
|
||||
return ''
|
||||
|
||||
tilejson_path = run_dir / f'tilejson-{domain_data["slug"]}.json'
|
||||
|
||||
metadata_path = mnt_dir / 'metadata.json'
|
||||
if not metadata_path.is_file():
|
||||
print(f" {metadata_path} doesn't exist, skipping")
|
||||
return ''
|
||||
|
||||
url_prefix = f'https://{domain_data["domain"]}/{area}/{version}'
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
python_venv_executable(),
|
||||
config.http_host_scripts_dir / 'metadata_to_tilejson.py',
|
||||
'--minify',
|
||||
metadata_path,
|
||||
tilejson_path,
|
||||
url_prefix,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
return f"""
|
||||
# specific JSON {area} {version}
|
||||
location = /{area}/{version} {{ # no trailing slash
|
||||
alias {tilejson_path}; # no trailing slash
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific JSON {area} {version}';
|
||||
}}
|
||||
|
||||
# specific PBF {area} {version}
|
||||
location ^~ /{area}/{version}/ {{ # trailing slash
|
||||
alias {mnt_dir}/tiles/; # trailing slash
|
||||
try_files $uri @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {{
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'specific PBF {area} {version}';
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
def create_latest_locations(*, domain_data: dict) -> str:
|
||||
location_str = ''
|
||||
|
||||
local_version_files = config.deployed_versions_dir.glob('*.txt')
|
||||
|
||||
for file in local_version_files:
|
||||
area = file.stem
|
||||
with open(file) as fp:
|
||||
version = fp.read().strip()
|
||||
|
||||
print(f' linking latest version for {area}: {version}')
|
||||
|
||||
# checking runs dir
|
||||
run_dir = config.runs_dir / area / version
|
||||
tilejson_path = run_dir / f'tilejson-{domain_data["slug"]}.json'
|
||||
if not tilejson_path.is_file():
|
||||
print(
|
||||
f' skipping latest block for {area} / {version}: {tilejson_path} does not exist'
|
||||
)
|
||||
continue
|
||||
|
||||
# checking mnt dir
|
||||
mnt_dir = Path(f'/mnt/ofm/{area}-{version}')
|
||||
mnt_file = mnt_dir / 'metadata.json'
|
||||
if not mnt_file.is_file():
|
||||
print(f' skipping latest block for {area} / {version}: {mnt_file} does not exist')
|
||||
continue
|
||||
|
||||
# latest
|
||||
location_str += f"""
|
||||
|
||||
# latest JSON {area}
|
||||
location = /{area} {{ # no trailing slash
|
||||
alias {tilejson_path}; # no trailing slash
|
||||
|
||||
expires 1d;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'latest JSON {area}';
|
||||
}}
|
||||
"""
|
||||
|
||||
# wildcard
|
||||
# identical to create_version_location
|
||||
location_str += f"""
|
||||
|
||||
# wildcard JSON {area}
|
||||
location ~ ^/{area}/([^/]+)$ {{
|
||||
# regex location is unreliable with alias, only root is reliable
|
||||
|
||||
root {run_dir}; # no trailing slash
|
||||
try_files /tilejson-{domain_data['slug']}.json =404;
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'wildcard JSON {area}';
|
||||
}}
|
||||
|
||||
# wildcard PBF {area}
|
||||
location ~ ^/{area}/([^/]+)/(.+)$ {{
|
||||
# regex location is unreliable with alias, only root is reliable
|
||||
|
||||
root {mnt_dir}/tiles/; # trailing slash
|
||||
try_files /$2 @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {{
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
add_header x-ofm-debug 'wildcard PBF {area}';
|
||||
}}
|
||||
"""
|
||||
|
||||
return location_str
|
||||
|
||||
|
||||
# if not self_signed_certs:
|
||||
# subprocess.run(
|
||||
# [
|
||||
# 'certbot',
|
||||
# 'certonly',
|
||||
# '--webroot',
|
||||
# '--webroot-path=/data/nginx/acme-challenges',
|
||||
# '--noninteractive',
|
||||
# '-m',
|
||||
# config.ofm_config['letsencrypt_email'],
|
||||
# '--agree-tos',
|
||||
# '--cert-name=ofm_direct',
|
||||
# # '--staging',
|
||||
# '--deploy-hook',
|
||||
# 'nginx -t && service nginx reload',
|
||||
# '-d',
|
||||
# domain_direct,
|
||||
# ],
|
||||
# check=True,
|
||||
# )
|
||||
#
|
||||
# # link certs to nginx dir
|
||||
# direct_cert.unlink()
|
||||
# direct_key.unlink()
|
||||
#
|
||||
# etc_cert = Path('/etc/letsencrypt/live/ofm_direct/fullchain.pem')
|
||||
# etc_key = Path('/etc/letsencrypt/live/ofm_direct/privkey.pem')
|
||||
# assert etc_cert.is_file()
|
||||
# assert etc_key.is_file()
|
||||
# direct_cert.symlink_to(etc_cert)
|
||||
# direct_key.symlink_to(etc_key)
|
||||
@@ -1,5 +1,5 @@
|
||||
server {
|
||||
server_name __LOCAL__ __DOMAIN__;
|
||||
server_name __LOOPBACK_HOSTNAME__ __DOMAIN__;
|
||||
|
||||
# ssl: https://ssl-config.mozilla.org / intermediate config
|
||||
|
||||
@@ -38,7 +38,7 @@ server {
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
__LOCATION_BLOCKS__
|
||||
__DYNAMIC_BLOCKS__
|
||||
|
||||
location /styles/ {
|
||||
# trailing slash
|
||||
@@ -1,5 +1,5 @@
|
||||
server {
|
||||
server_name __LOCAL__ __DOMAIN__;
|
||||
server_name __DOMAIN_SLUG__ __DOMAIN__;
|
||||
|
||||
# ssl: https://ssl-config.mozilla.org / intermediate config
|
||||
|
||||
@@ -8,8 +8,8 @@ server {
|
||||
listen [::]:443 ssl;
|
||||
http2 on;
|
||||
|
||||
ssl_certificate /data/nginx/certs/ofm_roundrobin.cert;
|
||||
ssl_certificate_key /data/nginx/certs/ofm_roundrobin.key;
|
||||
ssl_certificate /data/nginx/certs/ofm-__DOMAIN_SLUG__.cert;
|
||||
ssl_certificate_key /data/nginx/certs/ofm-__DOMAIN_SLUG__.key;
|
||||
|
||||
ssl_session_timeout 1d;
|
||||
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
|
||||
@@ -23,15 +23,14 @@ server {
|
||||
ssl_prefer_server_ciphers off;
|
||||
|
||||
# access log doesn't contain IP address
|
||||
access_log off;
|
||||
#access_log /data/ofm/http_host/logs_nginx/roundrobin-access.jsonl access_json buffer=128k;
|
||||
#access_log off;
|
||||
access_log /data/ofm/http_host/logs_nginx/__DOMAIN_SLUG__-access.jsonl access_json buffer=128k;
|
||||
|
||||
error_log /data/ofm/http_host/logs_nginx/roundrobin-error.log;
|
||||
error_log /data/ofm/http_host/logs_nginx/__DOMAIN_SLUG__-error.log;
|
||||
|
||||
add_header X-Robots-Tag "noindex, nofollow" always;
|
||||
|
||||
|
||||
__LOCATION_BLOCKS__
|
||||
__DYNAMIC_BLOCKS__
|
||||
|
||||
location /styles/ {
|
||||
# trailing slash
|
||||
@@ -56,6 +55,6 @@ server {
|
||||
# catch-all block to deny all other requests
|
||||
location / {
|
||||
deny all;
|
||||
error_log /data/ofm/http_host/logs_nginx/roundrobin-deny.log error;
|
||||
error_log /data/ofm/http_host/logs_nginx/__DOMAIN_SLUG__-deny.log error;
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
../../tile_gen/tile_gen_lib/shared.py
|
||||
@@ -4,7 +4,7 @@ from http_host_lib.assets import download_assets
|
||||
from http_host_lib.btrfs import download_area_version
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.mount import auto_mount, clean_up_mounts
|
||||
from http_host_lib.nginx import write_nginx_config
|
||||
from http_host_lib.nginx_config_gen import write_nginx_config
|
||||
from http_host_lib.utils import assert_linux, assert_sudo
|
||||
from http_host_lib.versions import fetch_version_files
|
||||
|
||||
@@ -18,6 +18,10 @@ def full_sync(force=False):
|
||||
assert_linux()
|
||||
assert_sudo()
|
||||
|
||||
# if it's a manual/forced run, we clean up old/deleted mounts
|
||||
if force:
|
||||
clean_up_mounts(config.mnt_dir)
|
||||
|
||||
# start
|
||||
versions_changed = fetch_version_files()
|
||||
|
||||
@@ -30,7 +34,7 @@ def full_sync(force=False):
|
||||
btrfs_downloaded += download_area_version(area='monaco', version='deployed')
|
||||
|
||||
# download latest and deployed planet
|
||||
if not config.ofm_config.get('skip_planet'):
|
||||
if not config.json_config.get('skip_planet'):
|
||||
btrfs_downloaded += download_area_version(area='planet', version='latest')
|
||||
btrfs_downloaded += download_area_version(area='planet', version='deployed')
|
||||
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import requests
|
||||
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.shared import get_deployed_version
|
||||
from http_host_lib.get_version_shared import get_deployed_version
|
||||
from http_host_lib.utils import assert_linux, assert_sudo
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# every minute
|
||||
|
||||
# fix
|
||||
#* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py fix >> /data/ofm/loadbalancer/logs/run.log 2>&1
|
||||
|
||||
|
||||
# check
|
||||
* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py check >> /data/ofm/loadbalancer/logs/run.log 2>&1
|
||||
@@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import click
|
||||
from loadbalancer_lib.loadbalance import check_or_fix
|
||||
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""
|
||||
Manages load-balancing of Round-Robin DNS records
|
||||
"""
|
||||
|
||||
|
||||
@cli.command()
|
||||
def check():
|
||||
"""
|
||||
Runs load-balancing check
|
||||
"""
|
||||
|
||||
print(f'---\n{now}\nStarting check')
|
||||
check_or_fix(fix=False)
|
||||
|
||||
|
||||
@cli.command()
|
||||
def fix():
|
||||
"""
|
||||
Runs check and fixes records based on check results
|
||||
"""
|
||||
|
||||
print(f'---\n{now}\nStarting fix')
|
||||
check_or_fix(fix=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
@@ -1,108 +0,0 @@
|
||||
import requests
|
||||
|
||||
|
||||
# docs: https://api.cloudflare.com/
|
||||
|
||||
|
||||
def cloudflare_get(path: str, params: dict, cloudflare_api_token: str):
|
||||
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
|
||||
res = requests.get(
|
||||
f'https://api.cloudflare.com/client/v4{path}', headers=headers, params=params
|
||||
)
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
assert data['success'] is True
|
||||
return data
|
||||
|
||||
|
||||
def get_zone_id(domain, cloudflare_api_token: str):
|
||||
data = cloudflare_get(
|
||||
'/zones', params=dict(name=domain), cloudflare_api_token=cloudflare_api_token
|
||||
)
|
||||
assert len(data['result']) == 1
|
||||
zone_info = data['result'][0]
|
||||
return zone_info['id']
|
||||
|
||||
|
||||
def get_dns_records_round_robin(zone_id, cloudflare_api_token: str) -> dict:
|
||||
data = cloudflare_get(
|
||||
f'/zones/{zone_id}/dns_records',
|
||||
params=dict(per_page=5000),
|
||||
cloudflare_api_token=cloudflare_api_token,
|
||||
)
|
||||
records = data['result']
|
||||
|
||||
data = {}
|
||||
|
||||
for r in records:
|
||||
if r['type'] != 'A':
|
||||
continue
|
||||
|
||||
data.setdefault(r['name'], [])
|
||||
data[r['name']].append(dict(content=r['content'], id=r['id']))
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def set_records_round_robin(
|
||||
zone_id,
|
||||
*,
|
||||
name: str,
|
||||
host_ip_set: set,
|
||||
ttl: int = 1,
|
||||
proxied: bool,
|
||||
comment: str = None,
|
||||
cloudflare_api_token: str,
|
||||
) -> bool:
|
||||
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
|
||||
|
||||
dns_records = get_dns_records_round_robin(zone_id, cloudflare_api_token=cloudflare_api_token)
|
||||
current_records = dns_records.get(name, [])
|
||||
|
||||
current_ips = {r['content'] for r in current_records}
|
||||
|
||||
if current_ips == host_ip_set:
|
||||
print(f'No need to update records: {name} currently set: {sorted(current_ips)}')
|
||||
return False
|
||||
|
||||
# changing records
|
||||
|
||||
# delete all current records first
|
||||
for r in current_records:
|
||||
delete_record(zone_id, id_=r['id'], cloudflare_api_token=cloudflare_api_token)
|
||||
|
||||
# create new records
|
||||
for ip in host_ip_set:
|
||||
print(f'Creating record: {name} {ip}')
|
||||
json_data = dict(
|
||||
type='A',
|
||||
name=name,
|
||||
content=ip,
|
||||
ttl=ttl,
|
||||
proxied=proxied,
|
||||
comment=comment,
|
||||
)
|
||||
res = requests.post(
|
||||
f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records',
|
||||
headers=headers,
|
||||
json=json_data,
|
||||
)
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
assert data['success'] is True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def delete_record(zone_id, *, id_: str, cloudflare_api_token: str):
|
||||
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
|
||||
|
||||
print(f'Deleting record: {id_}')
|
||||
res = requests.delete(
|
||||
f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records/{id_}',
|
||||
headers=headers,
|
||||
json={},
|
||||
)
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
assert data['success'] is True
|
||||
@@ -1,29 +0,0 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import dotenv_values
|
||||
|
||||
|
||||
class Configuration:
|
||||
areas = ['planet', 'monaco']
|
||||
|
||||
if Path('/data/ofm').exists():
|
||||
ofm_config_dir = Path('/data/ofm/config')
|
||||
else:
|
||||
repo_root = Path(__file__).parent.parent.parent.parent
|
||||
ofm_config_dir = repo_root / 'config'
|
||||
|
||||
ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
|
||||
|
||||
http_host_list = ofm_config['http_host_list']
|
||||
telegram_token = ofm_config['telegram_token']
|
||||
telegram_chat_id = ofm_config['telegram_chat_id']
|
||||
|
||||
domain_roundrobin = ofm_config['domain_roundrobin']
|
||||
domain_root = '.'.join(domain_roundrobin.split('.')[-2:])
|
||||
|
||||
cloudflare_ini = dotenv_values(ofm_config_dir / 'cloudflare.ini')
|
||||
cloudflare_api_token = cloudflare_ini['dns_cloudflare_api_token']
|
||||
|
||||
|
||||
config = Configuration()
|
||||
@@ -1,106 +0,0 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from loadbalancer_lib.cloudflare import get_zone_id, set_records_round_robin
|
||||
from loadbalancer_lib.config import config
|
||||
from loadbalancer_lib.shared import check_host_latest, check_host_version, get_deployed_version
|
||||
from loadbalancer_lib.telegram_ import telegram_send_message
|
||||
|
||||
|
||||
def check_or_fix(fix=False):
|
||||
if not config.http_host_list:
|
||||
telegram_quick(
|
||||
'OFM loadbalancer no hosts found on list, terminating',
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
results_by_ip = {}
|
||||
working_hosts = set()
|
||||
|
||||
for area in config.areas:
|
||||
results = run_area(area)
|
||||
for host_ip, host_is_ok in results.items():
|
||||
results_by_ip.setdefault(host_ip, True)
|
||||
results_by_ip[host_ip] &= host_is_ok
|
||||
|
||||
for host_ip, host_is_ok in results_by_ip.items():
|
||||
if not host_is_ok:
|
||||
telegram_quick(f'OFM loadbalancer ERROR with host: {host_ip}')
|
||||
else:
|
||||
working_hosts.add(host_ip)
|
||||
|
||||
except Exception as e:
|
||||
telegram_quick(f'OFM loadbalancer ERROR with loadbalancer: {e}')
|
||||
return
|
||||
|
||||
print(f'working hosts: {sorted(working_hosts)}')
|
||||
|
||||
if fix:
|
||||
# if no hosts are detected working, probably a bug in this script
|
||||
# fail-safe to include all hosts
|
||||
if not working_hosts:
|
||||
working_hosts = set(config.http_host_list)
|
||||
telegram_quick('OFM loadbalancer FIX found no working hosts, reverting to full list!')
|
||||
|
||||
updated = update_records(working_hosts)
|
||||
if updated:
|
||||
telegram_quick(f'OFM loadbalancer FIX modified records, new records: {working_hosts}')
|
||||
|
||||
|
||||
def run_area(area):
|
||||
deployed_data = get_deployed_version(area)
|
||||
version = deployed_data['version']
|
||||
last_modified = deployed_data['last_modified']
|
||||
|
||||
if not version:
|
||||
print(f' deployed version not found: {area}')
|
||||
return
|
||||
|
||||
print(f' deployed version {area}: {version}')
|
||||
|
||||
# using relaxed mode for while the servers are still deploying
|
||||
now = datetime.now(timezone.utc)
|
||||
delta = now - last_modified
|
||||
relaxed_mode = delta < timedelta(minutes=3)
|
||||
|
||||
if relaxed_mode:
|
||||
print(' using relaxed mode')
|
||||
|
||||
results = {}
|
||||
|
||||
for host_ip in config.http_host_list:
|
||||
try:
|
||||
# don't check latest
|
||||
if relaxed_mode:
|
||||
check_host_version(config.domain_roundrobin, host_ip, area, version)
|
||||
else:
|
||||
check_host_latest(config.domain_roundrobin, host_ip, area, version)
|
||||
|
||||
results[host_ip] = True
|
||||
except Exception as e:
|
||||
results[host_ip] = False
|
||||
print(e)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def update_records(working_hosts) -> bool:
|
||||
zone_id = get_zone_id(config.domain_root, cloudflare_api_token=config.cloudflare_api_token)
|
||||
|
||||
updated = False
|
||||
|
||||
updated |= set_records_round_robin(
|
||||
zone_id=zone_id,
|
||||
name=config.domain_roundrobin,
|
||||
host_ip_set=working_hosts,
|
||||
proxied=False,
|
||||
ttl=300,
|
||||
comment='domain_roundrobin',
|
||||
cloudflare_api_token=config.cloudflare_api_token,
|
||||
)
|
||||
|
||||
return updated
|
||||
|
||||
|
||||
def telegram_quick(message):
|
||||
telegram_send_message(message, config.telegram_token, config.telegram_chat_id)
|
||||
@@ -1 +0,0 @@
|
||||
../../tile_gen/tile_gen_lib/shared.py
|
||||
@@ -1,16 +0,0 @@
|
||||
import requests
|
||||
|
||||
|
||||
def telegram_send_message(message, bot_token, chat_id):
|
||||
print(message)
|
||||
|
||||
url = f'https://api.telegram.org/bot{bot_token}/sendMessage'
|
||||
|
||||
payload = {'chat_id': chat_id, 'text': message}
|
||||
|
||||
response = requests.post(url, data=payload)
|
||||
|
||||
if response.status_code == 200:
|
||||
print(' Message sent successfully!')
|
||||
else:
|
||||
print(' Failed to send message:', response.text)
|
||||
@@ -1,16 +0,0 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
||||
requirements = [
|
||||
'click',
|
||||
'requests',
|
||||
'pycurl',
|
||||
'python-dotenv',
|
||||
]
|
||||
|
||||
|
||||
setup(
|
||||
python_requires='>=3.10',
|
||||
install_requires=requirements,
|
||||
packages=find_packages(),
|
||||
)
|
||||
46
modules/mapterhorn_mirror/debug.py
Executable file
46
modules/mapterhorn_mirror/debug.py
Executable file
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.13"
|
||||
# dependencies = [
|
||||
#
|
||||
# ]
|
||||
# ///
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
rclone_config = Path('../../config/rclone.conf')
|
||||
|
||||
url = 'https://download.mapterhorn.com/planet.pmtiles'
|
||||
|
||||
parsed = urlparse(url)
|
||||
base_url = f'{parsed.scheme}://{parsed.netloc}'
|
||||
path = parsed.path.lstrip('/')
|
||||
|
||||
bucket_name = 'ofm-mapterhorn'
|
||||
remote_name = 'remote'
|
||||
destination = f'{remote_name}:{bucket_name}'
|
||||
|
||||
common_opts = [
|
||||
# '--verbose=10',
|
||||
# '--dump',
|
||||
# 'headers',
|
||||
'--progress',
|
||||
'--config',
|
||||
rclone_config,
|
||||
]
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'copy',
|
||||
'--http-url',
|
||||
base_url,
|
||||
f':http:{path}',
|
||||
destination,
|
||||
'--multi-thread-streams=8',
|
||||
'--s3-chunk-size=100M',
|
||||
*common_opts,
|
||||
]
|
||||
)
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# these are not needed as certbot generates these
|
||||
#env > /data/ofm/roundrobin/env.txt
|
||||
#RENEWED_DOMAINS=tiles.openfreemap.org
|
||||
#RENEWED_LINEAGE=/etc/letsencrypt/live/ofm_roundrobin
|
||||
|
||||
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
|
||||
|
||||
rclone copyto -v --copy-links "$RENEWED_LINEAGE/fullchain.pem" "remote:ofm-private/roundrobin/$RENEWED_DOMAINS/ofm_roundrobin.cert"
|
||||
rclone copyto -v --copy-links "$RENEWED_LINEAGE/privkey.pem" "remote:ofm-private/roundrobin/$RENEWED_DOMAINS/ofm_roundrobin.key"
|
||||
|
||||
@@ -5,8 +5,8 @@ LOG_DIR=/data/ofm/tile_gen/logs
|
||||
# every day at 23:10, make a monaco run
|
||||
10 23 * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
|
||||
|
||||
# debug monaco run, every minute
|
||||
#*/1 * * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
|
||||
# debug monaco run, normally disabled, enable to run every minute
|
||||
#* * * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
|
||||
|
||||
# every minute, set monaco to latest
|
||||
* * * * * ofm $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1
|
||||
|
||||
@@ -3,7 +3,6 @@ from setuptools import find_packages, setup
|
||||
|
||||
requirements = [
|
||||
'click',
|
||||
'pycurl',
|
||||
'requests',
|
||||
]
|
||||
|
||||
|
||||
@@ -3,9 +3,12 @@ from datetime import datetime, timezone
|
||||
|
||||
import click
|
||||
from tile_gen_lib.btrfs import make_btrfs
|
||||
from tile_gen_lib.get_version_shared import (
|
||||
get_deployed_version,
|
||||
get_versions_for_area,
|
||||
)
|
||||
from tile_gen_lib.planetiler import run_planetiler
|
||||
from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
|
||||
from tile_gen_lib.set_version import check_and_set_version
|
||||
from tile_gen_lib.rclone import make_indexes_for_bucket, set_version_on_bucket, upload_area
|
||||
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
@@ -71,7 +74,22 @@ def set_version(area, version):
|
||||
|
||||
print(f'---\n{now}\nStarting set-version {area}')
|
||||
|
||||
check_and_set_version(area, version)
|
||||
if version == 'latest':
|
||||
versions = get_versions_for_area(area)
|
||||
if not versions:
|
||||
print(f' No versions found for {area}')
|
||||
return
|
||||
|
||||
version = versions[-1]
|
||||
print(f' Latest version on bucket: {area} {version}')
|
||||
|
||||
try:
|
||||
if get_deployed_version(area)['version'] == version:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
set_version_on_bucket(area, version)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
@@ -22,8 +21,6 @@ class Configuration:
|
||||
repo_root = Path(__file__).parent.parent.parent.parent
|
||||
ofm_config_dir = repo_root / 'config'
|
||||
|
||||
ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
|
||||
|
||||
rclone_config = ofm_config_dir / 'rclone.conf'
|
||||
rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
48
modules/tile_gen/tile_gen_lib/get_version_shared.py
Normal file
48
modules/tile_gen/tile_gen_lib/get_version_shared.py
Normal file
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
This file is shared / symlinked between tile_gen_lib and http_host_lib
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def get_versions_for_area(area: str) -> list:
|
||||
"""
|
||||
Download the files.txt and check for the runs with the "done" file present
|
||||
"""
|
||||
r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30)
|
||||
r.raise_for_status()
|
||||
|
||||
versions = []
|
||||
|
||||
files = r.text.splitlines()
|
||||
for f in files:
|
||||
if not f.startswith(f'areas/{area}/'):
|
||||
continue
|
||||
if not f.endswith('/done'):
|
||||
continue
|
||||
version_str = f.split('/')[2]
|
||||
versions.append(version_str)
|
||||
|
||||
return sorted(versions)
|
||||
|
||||
|
||||
def get_deployed_version(area: str) -> dict:
|
||||
r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
|
||||
r.raise_for_status()
|
||||
version = r.text.strip()
|
||||
|
||||
last_modified_str = r.headers.get('Last-Modified')
|
||||
last_modified = parse_http_last_modified(last_modified_str)
|
||||
|
||||
return dict(
|
||||
version=version,
|
||||
last_modified=last_modified,
|
||||
)
|
||||
|
||||
|
||||
def parse_http_last_modified(date_string) -> datetime:
|
||||
parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT')
|
||||
parsed_date = parsed_date.replace(tzinfo=timezone.utc)
|
||||
return parsed_date
|
||||
@@ -132,3 +132,17 @@ def make_indexes_for_bucket(bucket):
|
||||
check=True,
|
||||
input=index_str.encode(),
|
||||
)
|
||||
|
||||
|
||||
def set_version_on_bucket(area, version):
|
||||
print(f'setting version: {area} {version}')
|
||||
subprocess.run(
|
||||
[
|
||||
config.rclone_bin,
|
||||
'rcat',
|
||||
f'remote:ofm-assets/deployed_versions/{area}.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
input=version.strip().encode(),
|
||||
)
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
import subprocess
|
||||
|
||||
from .config import config
|
||||
from .shared import check_host_version, get_deployed_version, get_versions_for_area
|
||||
|
||||
|
||||
def check_and_set_version(area, version):
|
||||
if version == 'latest':
|
||||
versions = get_versions_for_area(area)
|
||||
if not versions:
|
||||
print(f' No versions found for {area}')
|
||||
return
|
||||
|
||||
version = versions[-1]
|
||||
print(f' Latest version on bucket: {area} {version}')
|
||||
|
||||
if not check_all_hosts(area, version):
|
||||
return
|
||||
|
||||
try:
|
||||
if get_deployed_version(area)['version'] == version:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
set_version(area, version)
|
||||
|
||||
|
||||
def set_version(area, version):
|
||||
print(f'setting version: {area} {version}')
|
||||
subprocess.run(
|
||||
[
|
||||
config.rclone_bin,
|
||||
'rcat',
|
||||
f'remote:ofm-assets/deployed_versions/{area}.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
input=version.strip().encode(),
|
||||
)
|
||||
|
||||
|
||||
def check_all_hosts(area, version) -> bool:
|
||||
oc = config.ofm_config
|
||||
|
||||
domain = oc['domain_roundrobin'] or oc['domain_direct']
|
||||
print(f'Using domain: {domain}')
|
||||
|
||||
try:
|
||||
for host_ip in oc['http_host_list']:
|
||||
print(f'Checking {area} {version} on host {host_ip}')
|
||||
check_host_version(domain, host_ip, area, version)
|
||||
return True
|
||||
except Exception:
|
||||
print('Error, version not available')
|
||||
return False
|
||||
@@ -1,134 +0,0 @@
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pycurl
|
||||
import requests
|
||||
|
||||
|
||||
def get_versions_for_area(area: str) -> list:
|
||||
"""
|
||||
Download the files.txt and check for the runs with the "done" file present
|
||||
"""
|
||||
r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30)
|
||||
r.raise_for_status()
|
||||
|
||||
versions = []
|
||||
|
||||
files = r.text.splitlines()
|
||||
for f in files:
|
||||
if not f.startswith(f'areas/{area}/'):
|
||||
continue
|
||||
if not f.endswith('/done'):
|
||||
continue
|
||||
version_str = f.split('/')[2]
|
||||
versions.append(version_str)
|
||||
|
||||
return sorted(versions)
|
||||
|
||||
|
||||
def get_deployed_version(area: str) -> dict:
|
||||
r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
|
||||
r.raise_for_status()
|
||||
version = r.text.strip()
|
||||
|
||||
last_modified_str = r.headers.get('Last-Modified')
|
||||
last_modified = parse_http_last_modified(last_modified_str)
|
||||
|
||||
return dict(
|
||||
version=version,
|
||||
last_modified=last_modified,
|
||||
)
|
||||
|
||||
|
||||
def parse_http_last_modified(date_string) -> datetime:
|
||||
parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT')
|
||||
parsed_date = parsed_date.replace(tzinfo=timezone.utc)
|
||||
return parsed_date
|
||||
|
||||
|
||||
def check_host_version(domain, host_ip, area, version):
|
||||
# check versioned TileJSON
|
||||
check_tilejson(f'https://{domain}/{area}/{version}', domain, host_ip, version)
|
||||
|
||||
# check actual vector tile
|
||||
url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
|
||||
assert pycurl_status(url, domain, host_ip) == 200
|
||||
|
||||
|
||||
def check_host_latest(domain, host_ip, area, version):
|
||||
# check latest TileJSON
|
||||
check_tilejson(f'https://{domain}/{area}', domain, host_ip, version)
|
||||
|
||||
# check versioned TileJSON
|
||||
check_tilejson(f'https://{domain}/{area}/{version}', domain, host_ip, version)
|
||||
|
||||
# check actual vector tile
|
||||
url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
|
||||
assert pycurl_status(url, domain, host_ip) == 200
|
||||
|
||||
# check style
|
||||
url = f'https://{domain}/styles/bright'
|
||||
assert pycurl_status(url, domain, host_ip) == 200
|
||||
|
||||
|
||||
def check_tilejson(url, domain, host_ip, version):
|
||||
tilejson_str = pycurl_get(url, domain, host_ip)
|
||||
tilejson = json.loads(tilejson_str)
|
||||
tiles_url = tilejson['tiles'][0]
|
||||
version_in_tilejson = tiles_url.split('/')[4]
|
||||
assert version_in_tilejson == version
|
||||
|
||||
|
||||
# pycurl
|
||||
|
||||
|
||||
def pycurl_status(url, domain, host_ip):
|
||||
"""
|
||||
Uses pycurl to make a HTTPS HEAD request using custom resolving,
|
||||
checks if the status code is 200
|
||||
"""
|
||||
|
||||
c = pycurl.Curl()
|
||||
c.setopt(c.URL, url)
|
||||
|
||||
# linux needs CA certs specified manually
|
||||
if Path('/etc/ssl/certs/ca-certificates.crt').exists():
|
||||
c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
|
||||
|
||||
c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
|
||||
c.setopt(c.NOBODY, True)
|
||||
c.setopt(c.TIMEOUT, 5)
|
||||
c.perform()
|
||||
status_code = c.getinfo(c.RESPONSE_CODE)
|
||||
c.close()
|
||||
|
||||
return status_code
|
||||
|
||||
|
||||
def pycurl_get(url, domain, host_ip):
|
||||
"""
|
||||
Uses pycurl to make a HTTPS GET request using custom resolving,
|
||||
checks if the status code is 200, and returns the content.
|
||||
"""
|
||||
|
||||
buffer = BytesIO()
|
||||
c = pycurl.Curl()
|
||||
c.setopt(c.URL, url)
|
||||
|
||||
# linux needs CA certs specified manually
|
||||
if Path('/etc/ssl/certs/ca-certificates.crt').exists():
|
||||
c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
|
||||
|
||||
c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
|
||||
c.setopt(c.WRITEDATA, buffer)
|
||||
c.setopt(c.TIMEOUT, 5)
|
||||
c.perform()
|
||||
status_code = c.getinfo(c.RESPONSE_CODE)
|
||||
c.close()
|
||||
|
||||
if status_code != 200:
|
||||
raise ValueError(f'status code: {status_code}')
|
||||
|
||||
return buffer.getvalue().decode('utf8')
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"@biomejs/biome": "^1.9.2",
|
||||
"prettier": "^3.2.4",
|
||||
"@biomejs/biome": "^2.2.4",
|
||||
"prettier": "^3.6.2",
|
||||
"prettier-plugin-astro": "^0.14.0"
|
||||
},
|
||||
"packageManager": "pnpm@9.9.0+sha512.60c18acd138bff695d339be6ad13f7e936eea6745660d4cc4a776d5247c540d0edee1a563695c183a66eb917ef88f2b4feb1fc25f32a7adcadc7aaf3438e99c1"
|
||||
"packageManager": "pnpm@10.18.2"
|
||||
}
|
||||
|
||||
@@ -18,4 +18,3 @@ source .venv/bin/activate
|
||||
uv pip install -e .
|
||||
uv pip install -e modules/http_host
|
||||
uv pip install -e modules/tile_gen
|
||||
uv pip install -e modules/loadbalancer
|
||||
|
||||
6
setup.py
6
setup.py
@@ -5,10 +5,14 @@ requirements = [
|
||||
'click',
|
||||
'fabric',
|
||||
'nginxfmt',
|
||||
'python-dotenv',
|
||||
# 'python-dotenv',
|
||||
'ruff',
|
||||
'marko',
|
||||
'requests',
|
||||
'jsonschema',
|
||||
'json5',
|
||||
'pycurl',
|
||||
'certifi',
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import dotenv_values
|
||||
|
||||
|
||||
ASSETS_DIR = Path(__file__).parent / 'assets'
|
||||
CONFIG_DIR = Path(__file__).parent.parent / 'config'
|
||||
MODULES_DIR = Path(__file__).parent.parent / 'modules'
|
||||
|
||||
OFM_DIR = '/data/ofm'
|
||||
REMOTE_CONFIG = f'{OFM_DIR}/config'
|
||||
VENV_BIN = f'{OFM_DIR}/venv/bin'
|
||||
|
||||
TILE_GEN_DIR = f'{OFM_DIR}/tile_gen'
|
||||
TILE_GEN_BIN = f'{TILE_GEN_DIR}/bin'
|
||||
|
||||
PLANETILER_SRC = f'{TILE_GEN_DIR}/planetiler_src'
|
||||
PLANETILER_BIN = f'{TILE_GEN_DIR}/planetiler'
|
||||
|
||||
HTTP_HOST_BIN = f'{OFM_DIR}/http_host/bin'
|
||||
|
||||
|
||||
# Handling multiple .env files is supported
|
||||
# or example ENV=test would use .env.test
|
||||
|
||||
ENV = os.getenv('ENV')
|
||||
if ENV:
|
||||
env_file_name = f'.env.{ENV}'
|
||||
else:
|
||||
env_file_name = '.env'
|
||||
|
||||
env_file_path = CONFIG_DIR / env_file_name
|
||||
if not env_file_path.exists():
|
||||
sys.exit(f'config/{env_file_name} does not exist')
|
||||
|
||||
DOTENV_VALUES = dotenv_values(CONFIG_DIR / env_file_name)
|
||||
|
||||
|
||||
def dotenv_val(key):
|
||||
return DOTENV_VALUES.get(key, '').strip()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from ssh_lib import MODULES_DIR
|
||||
from ssh_lib.config import config
|
||||
from ssh_lib.utils import apt_get_install, exists, put
|
||||
|
||||
|
||||
@@ -20,4 +20,4 @@ def c1000k(c):
|
||||
def wrk(c):
|
||||
apt_get_install(c, 'wrk')
|
||||
c.sudo('mkdir -p /data/ofm/benchmark')
|
||||
put(c, f'{MODULES_DIR}/http_host/benchmark/wrk_custom_list.lua', '/data/ofm/benchmark')
|
||||
put(c, f'{config.modules_dir}/http_host/benchmark/wrk_custom_list.lua', '/data/ofm/benchmark')
|
||||
|
||||
38
ssh_lib/cli_helpers.py
Normal file
38
ssh_lib/cli_helpers.py
Normal file
@@ -0,0 +1,38 @@
|
||||
import os
|
||||
|
||||
import click
|
||||
from fabric import Config, Connection
|
||||
|
||||
|
||||
def get_connection(hostname, user, port):
|
||||
ssh_passwd = os.getenv('SSH_PASSWD')
|
||||
|
||||
if ssh_passwd:
|
||||
print('Using SSH password')
|
||||
|
||||
c = Connection(
|
||||
host=hostname,
|
||||
user=user,
|
||||
port=port,
|
||||
connect_kwargs={'password': ssh_passwd},
|
||||
config=Config(overrides={'sudo': {'password': ssh_passwd}}),
|
||||
)
|
||||
else:
|
||||
c = Connection(
|
||||
host=hostname,
|
||||
user=user,
|
||||
port=port,
|
||||
)
|
||||
|
||||
return c
|
||||
|
||||
|
||||
def common_options(func):
|
||||
"""Decorator to define common options."""
|
||||
func = click.argument('hostname')(func)
|
||||
func = click.option('--port', type=int, help='SSH port (if not in .ssh/config)')(func)
|
||||
func = click.option('--user', help='SSH user (if not in .ssh/config)')(func)
|
||||
func = click.option('-y', '--noninteractive', is_flag=True, help='Skip confirmation questions')(
|
||||
func
|
||||
)
|
||||
return func
|
||||
35
ssh_lib/config.py
Normal file
35
ssh_lib/config.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Configuration:
|
||||
# local paths relative to this file
|
||||
local_assets_dir = Path(__file__).parent / 'assets'
|
||||
local_config_dir = Path(__file__).parent.parent / 'config'
|
||||
local_modules_dir = Path(__file__).parent.parent / 'modules'
|
||||
|
||||
ENV = os.getenv('ENV')
|
||||
if not ENV:
|
||||
local_config_jsonc = local_config_dir / 'config.jsonc'
|
||||
else:
|
||||
local_config_jsonc = local_config_dir / f'config.{ENV}.jsonc'
|
||||
|
||||
config_schema_json = local_config_dir / 'config.schema.json'
|
||||
|
||||
# remote paths (always forward / on Linux - not using pathlib)
|
||||
ofm_dir = '/data/ofm'
|
||||
remote_config = f'{ofm_dir}/config'
|
||||
venv_bin = f'{ofm_dir}/venv/bin'
|
||||
|
||||
# remote http_host dir
|
||||
http_host_dir = f'{ofm_dir}/http_host'
|
||||
http_host_bin = f'{http_host_dir}/bin'
|
||||
|
||||
# remote tile_gen_dir
|
||||
tile_gen_dir = f'{ofm_dir}/tile_gen'
|
||||
tile_gen_bin = f'{tile_gen_dir}/bin'
|
||||
planetiler_src = f'{tile_gen_dir}/planetiler_src'
|
||||
planetiler_bin = f'{tile_gen_dir}/planetiler'
|
||||
|
||||
|
||||
config = Configuration()
|
||||
@@ -1,14 +1,16 @@
|
||||
from ssh_lib.utils import (
|
||||
from ssh_lib.apt import (
|
||||
apt_get_install,
|
||||
apt_get_purge,
|
||||
apt_get_update,
|
||||
put_str,
|
||||
sudo_cmd,
|
||||
setup_apt_repository,
|
||||
)
|
||||
from ssh_lib.utils import (
|
||||
ubuntu_codename,
|
||||
)
|
||||
|
||||
|
||||
JAVA_VER = 24
|
||||
ADOPTIUM_REPO_NAME = 'adoptium'
|
||||
|
||||
|
||||
def java(c):
|
||||
@@ -16,25 +18,17 @@ def java(c):
|
||||
# remove old Ubuntu version of OpenJDK
|
||||
apt_get_purge(c, 'openjdk* temurin*')
|
||||
|
||||
# Download and install the Eclipse Adoptium GPG key
|
||||
sudo_cmd(
|
||||
c,
|
||||
'wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public '
|
||||
'| gpg --dearmor '
|
||||
'| tee /etc/apt/trusted.gpg.d/adoptium.gpg > /dev/null',
|
||||
)
|
||||
|
||||
# Get the Ubuntu codename
|
||||
codename = ubuntu_codename(c)
|
||||
|
||||
# Configure the Eclipse Adoptium apt repository
|
||||
put_str(
|
||||
setup_apt_repository(
|
||||
c,
|
||||
'/etc/apt/sources.list.d/adoptium.list',
|
||||
f'deb https://packages.adoptium.net/artifactory/deb {codename} main',
|
||||
repo_name=ADOPTIUM_REPO_NAME,
|
||||
key_url='https://packages.adoptium.net/artifactory/api/gpg/key/public',
|
||||
repo_url='https://packages.adoptium.net/artifactory/deb',
|
||||
suite=codename,
|
||||
component='main',
|
||||
)
|
||||
|
||||
# Update package list and install Temurin JDK
|
||||
apt_get_update(c)
|
||||
apt_get_install(c, f'temurin-{JAVA_VER}-jdk')
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from ssh_lib import ASSETS_DIR
|
||||
from ssh_lib.config import config
|
||||
from ssh_lib.utils import put, put_str
|
||||
|
||||
|
||||
@@ -25,6 +25,10 @@ def kernel_vmovercommit(c):
|
||||
|
||||
def kernel_thp_fix(c):
|
||||
# transparent_hugepage
|
||||
put(c, f'{ASSETS_DIR}/kernel/thp_fix_service', '/etc/systemd/system/thp_fix.service')
|
||||
put(
|
||||
c,
|
||||
f'{config.local_assets_dir}/kernel/thp_fix_service',
|
||||
'/etc/systemd/system/thp_fix.service',
|
||||
)
|
||||
c.sudo('systemctl daemon-reload')
|
||||
c.sudo('systemctl enable thp_fix')
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from ssh_lib import ASSETS_DIR
|
||||
from ssh_lib.config import config
|
||||
from ssh_lib.utils import (
|
||||
apt_get_install,
|
||||
apt_get_purge,
|
||||
@@ -41,10 +41,10 @@ def nginx(c):
|
||||
|
||||
generate_self_signed_cert(c)
|
||||
|
||||
put(c, f'{ASSETS_DIR}/nginx/nginx.conf', '/etc/nginx/')
|
||||
put(c, f'{ASSETS_DIR}/nginx/mime.types', '/etc/nginx/')
|
||||
put(c, f'{ASSETS_DIR}/nginx/default_disable.conf', '/data/nginx/sites')
|
||||
put(c, f'{ASSETS_DIR}/nginx/cloudflare.conf', '/data/nginx/config')
|
||||
put(c, f'{config.local_assets_dir}/nginx/nginx.conf', '/etc/nginx/')
|
||||
put(c, f'{config.local_assets_dir}/nginx/mime.types', '/etc/nginx/')
|
||||
put(c, f'{config.local_assets_dir}/nginx/default_disable.conf', '/data/nginx/sites')
|
||||
put(c, f'{config.local_assets_dir}/nginx/cloudflare.conf', '/data/nginx/config')
|
||||
|
||||
sudo_cmd(c, 'curl https://ssl-config.mozilla.org/ffdhe2048.txt -o /etc/nginx/ffdhe2048.txt')
|
||||
|
||||
@@ -53,6 +53,9 @@ def nginx(c):
|
||||
|
||||
|
||||
def certbot(c):
|
||||
print('should use nginx acme')
|
||||
return
|
||||
|
||||
apt_get_install(c, 'snapd')
|
||||
|
||||
# this is silly, but needs to be run twice
|
||||
|
||||
@@ -38,9 +38,9 @@ def pkg_base(c):
|
||||
'autojump',
|
||||
'bash-completion',
|
||||
'btop',
|
||||
'ctop',
|
||||
'dbus',
|
||||
'direnv',
|
||||
'dmidecode',
|
||||
'fd-find',
|
||||
'file',
|
||||
'ioping',
|
||||
@@ -54,6 +54,7 @@ def pkg_base(c):
|
||||
'net-tools',
|
||||
'netbase',
|
||||
'nethogs',
|
||||
'nvme-cli',
|
||||
'openssh-client',
|
||||
'p7zip-full',
|
||||
'pkg-config',
|
||||
@@ -67,6 +68,7 @@ def pkg_base(c):
|
||||
# 'iperf3',
|
||||
# 'iproute2',
|
||||
# 'nasm',
|
||||
# 'ctop', # unsupported on Ubuntu 24
|
||||
]
|
||||
|
||||
apt_get_install(c, ' '.join(pkg_list))
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
from ssh_lib import PLANETILER_BIN, PLANETILER_SRC
|
||||
from ssh_lib.config import config
|
||||
from ssh_lib.java import java
|
||||
from ssh_lib.utils import exists, sudo_cmd
|
||||
|
||||
|
||||
PLANETILER_COMMIT = 'cc769c'
|
||||
PLANETILER_PATH = f'{PLANETILER_BIN}/planetiler.jar'
|
||||
PLANETILER_PATH = f'{config.planetiler_bin}/planetiler.jar'
|
||||
|
||||
|
||||
def install_planetiler(c):
|
||||
@@ -15,24 +15,24 @@ def install_planetiler(c):
|
||||
java(c)
|
||||
|
||||
c.sudo('rm -rf /root/.m2') # cleaning maven cache
|
||||
c.sudo(f'rm -rf {PLANETILER_BIN} {PLANETILER_SRC}')
|
||||
c.sudo(f'mkdir -p {PLANETILER_BIN} {PLANETILER_SRC}')
|
||||
c.sudo(f'rm -rf {config.planetiler_bin} {config.planetiler_src}')
|
||||
c.sudo(f'mkdir -p {config.planetiler_bin} {config.planetiler_src}')
|
||||
|
||||
c.sudo('git config --global advice.detachedHead false')
|
||||
c.sudo(
|
||||
f'git clone --recurse-submodules https://github.com/onthegomap/planetiler.git {PLANETILER_SRC}'
|
||||
f'git clone --recurse-submodules https://github.com/onthegomap/planetiler.git {config.planetiler_src}'
|
||||
)
|
||||
|
||||
sudo_cmd(c, f'cd {PLANETILER_SRC} && git checkout {PLANETILER_COMMIT}')
|
||||
sudo_cmd(c, f'cd {PLANETILER_SRC} && git submodule update --init --recursive')
|
||||
sudo_cmd(c, f'cd {config.planetiler_src} && git checkout {PLANETILER_COMMIT}')
|
||||
sudo_cmd(c, f'cd {config.planetiler_src} && git submodule update --init --recursive')
|
||||
|
||||
sudo_cmd(c, f'cd {PLANETILER_SRC} && ./mvnw clean test package')
|
||||
sudo_cmd(c, f'cd {config.planetiler_src} && ./mvnw clean test package')
|
||||
|
||||
c.sudo(
|
||||
f'mv {PLANETILER_SRC}/planetiler-dist/target/planetiler-dist-*-SNAPSHOT-with-deps.jar {PLANETILER_PATH}',
|
||||
f'mv {config.planetiler_src}/planetiler-dist/target/planetiler-dist-*-SNAPSHOT-with-deps.jar {PLANETILER_PATH}',
|
||||
warn=True,
|
||||
)
|
||||
|
||||
c.sudo(f'java -jar {PLANETILER_PATH} --help', hide=True)
|
||||
|
||||
c.sudo(f'rm -rf {PLANETILER_SRC}')
|
||||
c.sudo(f'rm -rf {config.planetiler_src}')
|
||||
|
||||
95
ssh_lib/pycurl.py
Normal file
95
ssh_lib/pycurl.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""
|
||||
Round-Robin DNS Bypass for Health Checking
|
||||
|
||||
Check individual servers behind round-robin DNS by connecting directly to specific
|
||||
IP addresses while maintaining proper HTTPS/TLS.
|
||||
|
||||
Example:
|
||||
pycurl_status('https://api.example.com/health', '192.168.1.101')
|
||||
200
|
||||
|
||||
pycurl_get('https://api.example.com/data', '192.168.1.102')
|
||||
'{"status": "ok"}'
|
||||
|
||||
How it works:
|
||||
Overrides DNS resolution to connect to a specific IP while using the correct
|
||||
hostname for TLS/SNI. Verifies HTTPS is working without validating certificate chain.
|
||||
"""
|
||||
|
||||
from io import BytesIO
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pycurl
|
||||
|
||||
|
||||
def pycurl_status(url: str, target_ip: str) -> int:
|
||||
"""
|
||||
Check HTTP status of a specific server behind round-robin DNS.
|
||||
|
||||
Makes a HEAD request to the target IP while using the hostname for HTTPS/SNI.
|
||||
Verifies HTTPS is configured but does not validate certificate chain.
|
||||
|
||||
Args:
|
||||
url: Full URL to request (e.g., 'https://api.example.com/health')
|
||||
target_ip: IP address of specific server (e.g., '192.168.1.101')
|
||||
|
||||
Returns:
|
||||
HTTP status code (e.g., 200, 404, 500)
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname
|
||||
port = parsed.port or (443 if parsed.scheme == 'https' else 80)
|
||||
|
||||
c = pycurl.Curl()
|
||||
c.setopt(c.URL, url)
|
||||
c.setopt(c.SSL_VERIFYPEER, 0) # Skip cert validation
|
||||
c.setopt(c.SSL_VERIFYHOST, 0) # Skip hostname validation
|
||||
c.setopt(c.RESOLVE, [f'{hostname}:{port}:{target_ip}'])
|
||||
c.setopt(c.NOBODY, True) # HEAD request
|
||||
c.setopt(c.TIMEOUT, 5)
|
||||
c.perform()
|
||||
status_code = c.getinfo(c.RESPONSE_CODE)
|
||||
c.close()
|
||||
|
||||
return status_code
|
||||
|
||||
|
||||
def pycurl_get(url: str, target_ip: str, binary: bool = False) -> str | bytes:
|
||||
"""
|
||||
Fetch content from a specific server behind round-robin DNS.
|
||||
|
||||
Makes a GET request to the target IP while using the hostname for HTTPS/SNI.
|
||||
Verifies HTTPS is configured but does not validate certificate chain.
|
||||
|
||||
Args:
|
||||
url: Full URL to request (e.g., 'https://api.example.com/data')
|
||||
target_ip: IP address of specific server (e.g., '192.168.1.101')
|
||||
binary: If True, return bytes; if False, decode as UTF-8 string
|
||||
|
||||
Returns:
|
||||
Response body as UTF-8 string (binary=False) or bytes (binary=True)
|
||||
|
||||
Raises:
|
||||
ValueError: If status code is not 200
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname
|
||||
port = parsed.port or (443 if parsed.scheme == 'https' else 80)
|
||||
|
||||
buffer = BytesIO()
|
||||
c = pycurl.Curl()
|
||||
c.setopt(c.URL, url)
|
||||
c.setopt(c.SSL_VERIFYPEER, 0) # Skip cert validation
|
||||
c.setopt(c.SSL_VERIFYHOST, 0) # Skip hostname validation
|
||||
c.setopt(c.RESOLVE, [f'{hostname}:{port}:{target_ip}'])
|
||||
c.setopt(c.WRITEDATA, buffer)
|
||||
c.setopt(c.TIMEOUT, 5)
|
||||
c.perform()
|
||||
status_code = c.getinfo(c.RESPONSE_CODE)
|
||||
c.close()
|
||||
|
||||
if status_code != 200:
|
||||
raise ValueError(f'status code: {status_code}')
|
||||
|
||||
body = buffer.getvalue()
|
||||
return body if binary else body.decode('utf-8')
|
||||
@@ -1,4 +1,5 @@
|
||||
from ssh_lib.utils import apt_get_update, exists
|
||||
from ssh_lib.apt import apt_get_update
|
||||
from ssh_lib.utils import exists
|
||||
|
||||
|
||||
def rclone(c):
|
||||
|
||||
40
ssh_lib/slugify.py
Normal file
40
ssh_lib/slugify.py
Normal file
@@ -0,0 +1,40 @@
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
|
||||
# Pre-compiled patterns for better performance
|
||||
_RE_INVALID = re.compile(r'[^a-z0-9_-]+')
|
||||
_RE_SEPARATORS = re.compile(r'[-_]+')
|
||||
|
||||
|
||||
def slugify(
|
||||
value: str | bytes | int | float | None,
|
||||
*,
|
||||
separator: str = '-',
|
||||
) -> str:
|
||||
if value in (None, ''):
|
||||
return ''
|
||||
|
||||
if separator not in ('-', '_'):
|
||||
raise ValueError(f"separator must be '-' or '_', got {repr(separator)}")
|
||||
|
||||
# 1. Normalize value to string
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode('utf-8', 'ignore')
|
||||
else:
|
||||
value = str(value)
|
||||
|
||||
# 2. Unicode → ASCII, then lowercase
|
||||
value = unicodedata.normalize('NFKD', value)
|
||||
value = value.encode('ascii', 'ignore').decode('ascii').lower()
|
||||
|
||||
# 3. Replace invalid characters with separator
|
||||
value = _RE_INVALID.sub(separator, value)
|
||||
|
||||
# 4. Collapse multiple separators
|
||||
value = _RE_SEPARATORS.sub(separator, value)
|
||||
|
||||
# 5. Strip separators from edges
|
||||
value = value.strip('-_')
|
||||
|
||||
return value
|
||||
256
ssh_lib/tasks.py
256
ssh_lib/tasks.py
@@ -1,256 +0,0 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from ssh_lib import (
|
||||
CONFIG_DIR,
|
||||
HTTP_HOST_BIN,
|
||||
MODULES_DIR,
|
||||
OFM_DIR,
|
||||
REMOTE_CONFIG,
|
||||
TILE_GEN_BIN,
|
||||
VENV_BIN,
|
||||
dotenv_val,
|
||||
)
|
||||
from ssh_lib.benchmark import c1000k, wrk
|
||||
from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k
|
||||
from ssh_lib.nginx import certbot, nginx
|
||||
from ssh_lib.pkg_base import pkg_base, pkg_upgrade
|
||||
from ssh_lib.planetiler import install_planetiler
|
||||
from ssh_lib.rclone import rclone
|
||||
from ssh_lib.utils import add_user, enable_sudo, put, put_dir, put_str, sudo_cmd
|
||||
|
||||
|
||||
def prepare_shared(c):
|
||||
# creates ofm user with uid=2000, disabled password and nopasswd sudo
|
||||
add_user(c, 'ofm', uid=2000)
|
||||
enable_sudo(c, 'ofm', nopasswd=True)
|
||||
|
||||
pkg_upgrade(c)
|
||||
pkg_base(c)
|
||||
rclone(c)
|
||||
|
||||
c.sudo(f'mkdir -p {REMOTE_CONFIG}')
|
||||
c.sudo(f'chown ofm:ofm {REMOTE_CONFIG}')
|
||||
c.sudo(f'chown ofm:ofm {OFM_DIR}')
|
||||
|
||||
upload_config_json(c)
|
||||
|
||||
prepare_venv(c)
|
||||
|
||||
|
||||
def prepare_venv(c):
|
||||
put(
|
||||
c,
|
||||
MODULES_DIR / 'prepare-virtualenv.sh',
|
||||
OFM_DIR,
|
||||
permissions='755',
|
||||
user='ofm',
|
||||
)
|
||||
sudo_cmd(c, f'cd {OFM_DIR} && source prepare-virtualenv.sh')
|
||||
|
||||
|
||||
def prepare_tile_gen(c, *, enable_cron):
|
||||
c.sudo('rm -f /etc/cron.d/ofm_tile_gen')
|
||||
|
||||
install_planetiler(c)
|
||||
|
||||
c.sudo(f'rm -rf {TILE_GEN_BIN}')
|
||||
|
||||
put_dir(c, MODULES_DIR / 'tile_gen', TILE_GEN_BIN, file_permissions='755')
|
||||
|
||||
for dirname in ['tile_gen_lib', 'scripts']:
|
||||
put_dir(c, MODULES_DIR / 'tile_gen' / dirname, f'{TILE_GEN_BIN}/{dirname}')
|
||||
|
||||
if (CONFIG_DIR / 'rclone.conf').exists():
|
||||
put(
|
||||
c,
|
||||
CONFIG_DIR / 'rclone.conf',
|
||||
f'{REMOTE_CONFIG}/rclone.conf',
|
||||
permissions='600',
|
||||
user='ofm',
|
||||
)
|
||||
|
||||
c.sudo(f'{VENV_BIN}/pip install -e {TILE_GEN_BIN} --use-pep517')
|
||||
|
||||
c.sudo('rm -rf /data/ofm/tile_gen/logs')
|
||||
c.sudo('mkdir -p /data/ofm/tile_gen/logs')
|
||||
|
||||
c.sudo('chown ofm:ofm /data/ofm/tile_gen/{,*}')
|
||||
c.sudo(f'chown ofm:ofm -R {TILE_GEN_BIN}')
|
||||
|
||||
if enable_cron:
|
||||
put(c, MODULES_DIR / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')
|
||||
|
||||
|
||||
def prepare_http_host(c):
|
||||
kernel_somaxconn65k(c)
|
||||
kernel_limits1m(c)
|
||||
|
||||
nginx(c)
|
||||
certbot(c)
|
||||
|
||||
c.sudo('rm -rf /data/ofm/http_host/logs')
|
||||
c.sudo('mkdir -p /data/ofm/http_host/logs')
|
||||
c.sudo('chown ofm:ofm /data/ofm/http_host/logs')
|
||||
|
||||
c.sudo('rm -rf /data/ofm/http_host/logs_nginx')
|
||||
c.sudo('mkdir -p /data/ofm/http_host/logs_nginx')
|
||||
c.sudo('chown nginx:nginx /data/ofm/http_host/logs_nginx')
|
||||
|
||||
upload_http_host_files(c)
|
||||
|
||||
if dotenv_val('DOMAIN_ROUNDROBIN'):
|
||||
assert (CONFIG_DIR / 'rclone.conf').exists()
|
||||
put(
|
||||
c,
|
||||
CONFIG_DIR / 'rclone.conf',
|
||||
f'{REMOTE_CONFIG}/rclone.conf',
|
||||
permissions=400,
|
||||
)
|
||||
put(c, MODULES_DIR / 'http_host' / 'cron.d' / 'ofm_roundrobin_reader', '/etc/cron.d/')
|
||||
|
||||
c.sudo(f'{VENV_BIN}/pip install -e {HTTP_HOST_BIN} --use-pep517')
|
||||
|
||||
|
||||
def run_http_host_sync(c):
|
||||
print('Running http_host.py sync --force')
|
||||
sudo_cmd(c, f'{VENV_BIN}/python -u {HTTP_HOST_BIN}/http_host.py sync --force')
|
||||
|
||||
|
||||
def upload_http_host_files(c):
|
||||
c.sudo(f'rm -rf {HTTP_HOST_BIN}')
|
||||
c.sudo(f'mkdir -p {HTTP_HOST_BIN}')
|
||||
|
||||
put_dir(c, MODULES_DIR / 'http_host', HTTP_HOST_BIN, file_permissions='755')
|
||||
|
||||
for dirname in ['http_host_lib', 'scripts']:
|
||||
put_dir(c, MODULES_DIR / 'http_host' / dirname, f'{HTTP_HOST_BIN}/{dirname}')
|
||||
|
||||
put_dir(
|
||||
c,
|
||||
MODULES_DIR / 'http_host' / 'http_host_lib' / 'nginx_confs',
|
||||
f'{HTTP_HOST_BIN}/http_host_lib/nginx_confs',
|
||||
)
|
||||
|
||||
c.sudo('chown -R ofm:ofm /data/ofm/http_host')
|
||||
|
||||
|
||||
def install_benchmark(c):
|
||||
"""
|
||||
Read docs/quick_notes/http_benchmark.md
|
||||
"""
|
||||
c1000k(c)
|
||||
wrk(c)
|
||||
|
||||
|
||||
def setup_roundrobin_writer(c):
|
||||
letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower()
|
||||
domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower()
|
||||
assert letsencrypt_email
|
||||
assert domain_roundrobin
|
||||
assert (CONFIG_DIR / 'rclone.conf').exists()
|
||||
assert (CONFIG_DIR / 'cloudflare.ini').exists()
|
||||
|
||||
rclone(c)
|
||||
certbot(c)
|
||||
|
||||
c.sudo(f'mkdir -p {REMOTE_CONFIG}')
|
||||
|
||||
put(
|
||||
c,
|
||||
CONFIG_DIR / 'rclone.conf',
|
||||
f'{REMOTE_CONFIG}/rclone.conf',
|
||||
permissions=400,
|
||||
)
|
||||
|
||||
put(
|
||||
c,
|
||||
CONFIG_DIR / 'cloudflare.ini',
|
||||
f'{REMOTE_CONFIG}/cloudflare.ini',
|
||||
permissions=400,
|
||||
)
|
||||
|
||||
c.sudo('rm -rf /data/ofm/roundrobin')
|
||||
|
||||
put(
|
||||
c,
|
||||
MODULES_DIR / 'roundrobin' / 'rclone_write.sh',
|
||||
'/data/ofm/roundrobin/rclone_write.sh',
|
||||
create_parent_dir=True,
|
||||
permissions=500,
|
||||
)
|
||||
|
||||
# only use with --staging
|
||||
# c.sudo('certbot delete --noninteractive --cert-name ofm_roundrobin', warn=True)
|
||||
|
||||
sudo_cmd(
|
||||
c,
|
||||
'certbot certonly '
|
||||
'--dns-cloudflare '
|
||||
f'--dns-cloudflare-credentials {REMOTE_CONFIG}/cloudflare.ini '
|
||||
'--dns-cloudflare-propagation-seconds 20 '
|
||||
f'--noninteractive '
|
||||
f'-m {letsencrypt_email} '
|
||||
f'--agree-tos '
|
||||
f'--cert-name=ofm_roundrobin '
|
||||
f'--deploy-hook /data/ofm/roundrobin/rclone_write.sh '
|
||||
f'-d {domain_roundrobin}',
|
||||
# f'-d {domain2_roundrobin}',
|
||||
)
|
||||
|
||||
|
||||
def upload_config_json(c):
|
||||
domain_direct = dotenv_val('DOMAIN_DIRECT').lower()
|
||||
domain_roundrobin = dotenv_val('DOMAIN_ROUNDROBIN').lower()
|
||||
skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
|
||||
self_signed_certs = dotenv_val('SELF_SIGNED_CERTS').lower() == 'true'
|
||||
letsencrypt_email = dotenv_val('LETSENCRYPT_EMAIL').lower()
|
||||
|
||||
if not (domain_direct or domain_roundrobin):
|
||||
sys.exit('Please specify DOMAIN_DIRECT or DOMAIN_ROUNDROBIN in config/.env')
|
||||
|
||||
if domain_direct and not letsencrypt_email and not self_signed_certs:
|
||||
sys.exit('Please add your email to LETSENCRYPT_EMAIL when using DOMAIN_DIRECT')
|
||||
|
||||
http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()]
|
||||
|
||||
config = {
|
||||
'domain_direct': domain_direct,
|
||||
'domain_roundrobin': domain_roundrobin,
|
||||
'letsencrypt_email': letsencrypt_email,
|
||||
'skip_planet': skip_planet,
|
||||
'self_signed_certs': self_signed_certs,
|
||||
'http_host_list': http_host_list,
|
||||
'telegram_token': dotenv_val('TELEGRAM_TOKEN'),
|
||||
'telegram_chat_id': dotenv_val('TELEGRAM_CHAT_ID'),
|
||||
}
|
||||
|
||||
config_str = json.dumps(config, indent=2, ensure_ascii=False)
|
||||
print(config_str)
|
||||
put_str(c, f'{REMOTE_CONFIG}/config.json', config_str)
|
||||
|
||||
|
||||
def setup_loadbalancer(c):
|
||||
c.sudo('rm -f /etc/cron.d/ofm_loadbalancer')
|
||||
|
||||
put(
|
||||
c,
|
||||
CONFIG_DIR / 'cloudflare.ini',
|
||||
f'{REMOTE_CONFIG}/cloudflare.ini',
|
||||
permissions=400,
|
||||
)
|
||||
|
||||
c.sudo('rm -rf /data/ofm/loadbalancer')
|
||||
put_dir(c, MODULES_DIR / 'loadbalancer', '/data/ofm/loadbalancer')
|
||||
put_dir(
|
||||
c,
|
||||
MODULES_DIR / 'loadbalancer' / 'loadbalancer_lib',
|
||||
'/data/ofm/loadbalancer/loadbalancer_lib',
|
||||
)
|
||||
|
||||
c.sudo(f'{VENV_BIN}/pip install -e /data/ofm/loadbalancer --use-pep517')
|
||||
|
||||
c.sudo('mkdir -p /data/ofm/loadbalancer/logs')
|
||||
c.sudo('chown -R ofm:ofm /data/ofm/loadbalancer')
|
||||
|
||||
put(c, MODULES_DIR / 'loadbalancer' / 'cron.d' / 'ofm_loadbalancer', '/etc/cron.d/')
|
||||
140
ssh_lib/tasks_http_host.py
Normal file
140
ssh_lib/tasks_http_host.py
Normal file
@@ -0,0 +1,140 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import json5
|
||||
from jsonschema import ValidationError, validate
|
||||
|
||||
from ssh_lib.benchmark import c1000k, wrk
|
||||
from ssh_lib.config import config
|
||||
from ssh_lib.kernel import kernel_limits1m, kernel_somaxconn65k
|
||||
from ssh_lib.nginx import nginx
|
||||
from ssh_lib.slugify import slugify
|
||||
from ssh_lib.utils import put, put_dir, put_str, sudo_cmd
|
||||
|
||||
|
||||
def prepare_http_host(c):
|
||||
kernel_somaxconn65k(c)
|
||||
kernel_limits1m(c)
|
||||
|
||||
nginx(c)
|
||||
# certbot(c)
|
||||
|
||||
c.sudo(f'rm -rf {config.http_host_dir}/logs')
|
||||
c.sudo(f'mkdir -p {config.http_host_dir}/logs')
|
||||
c.sudo(f'chown ofm:ofm {config.http_host_dir}/logs')
|
||||
|
||||
c.sudo(f'rm -rf {config.http_host_dir}/logs_nginx')
|
||||
c.sudo(f'mkdir -p {config.http_host_dir}/logs_nginx')
|
||||
c.sudo(f'chown nginx:nginx {config.http_host_dir}/logs_nginx')
|
||||
|
||||
upload_http_host_files(c)
|
||||
c.sudo(f'{config.venv_bin}/pip install -e {config.http_host_bin} --use-pep517', echo=True)
|
||||
|
||||
upload_config_and_certs(c)
|
||||
|
||||
|
||||
def upload_config_and_certs(c):
|
||||
config_data = read_jsonc()
|
||||
|
||||
# clean old certs
|
||||
c.sudo('rm -rf /data/nginx/certs/ofm-*')
|
||||
|
||||
# pre-generate all the slugs
|
||||
for domain_data in config_data['domains']:
|
||||
domain_data['slug'] = slugify(domain_data['domain'], separator='_')
|
||||
|
||||
if domain_data['cert']['type'] == 'upload':
|
||||
local_cert_path = Path(domain_data['cert']['cert_path'])
|
||||
|
||||
# handle relative paths - make them relative to config.local_config_dir
|
||||
if not local_cert_path.is_absolute():
|
||||
local_cert_path = Path(config.local_config_dir) / local_cert_path
|
||||
|
||||
cert_basename = local_cert_path.stem
|
||||
local_key_path = local_cert_path.parent / f'{cert_basename}.key'
|
||||
|
||||
if not local_cert_path.is_file() or not local_key_path.is_file():
|
||||
raise FileNotFoundError(
|
||||
f'cert or key file for {domain_data["domain"]} is not found.\n'
|
||||
f'Make sure these files exists:\n{local_cert_path}\n{local_key_path}'
|
||||
)
|
||||
|
||||
remote_cert_path = f'/data/nginx/certs/ofm-{domain_data["slug"]}.cert'
|
||||
remote_key_path = f'/data/nginx/certs/ofm-{domain_data["slug"]}.key'
|
||||
|
||||
put(c, local_cert_path, remote_cert_path)
|
||||
put(c, local_key_path, remote_key_path)
|
||||
|
||||
domain_data['cert_file'] = remote_cert_path
|
||||
domain_data['key_file'] = remote_key_path
|
||||
|
||||
# generate a normal JSON and upload it
|
||||
config_str = json.dumps(config_data, indent=2, ensure_ascii=False)
|
||||
put_str(c, f'{config.remote_config}/config.json', config_str)
|
||||
|
||||
|
||||
def read_jsonc():
|
||||
if not config.local_config_jsonc.is_file():
|
||||
raise FileNotFoundError(
|
||||
f'{config.local_config_jsonc} not found. Make sure it exists in the /config dir'
|
||||
)
|
||||
|
||||
# Load and parse the JSONC/JSON5 config file
|
||||
try:
|
||||
config_data = json5.loads(config.local_config_jsonc.read_text())
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Error parsing config file: {e}') from e
|
||||
|
||||
# Load the JSON schema
|
||||
try:
|
||||
schema = json.loads(config.config_schema_json.read_text())
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Error loading schema file: {e}') from e
|
||||
|
||||
# Validate the config against the schema
|
||||
try:
|
||||
validate(instance=config_data, schema=schema)
|
||||
print('✓ Configuration is valid')
|
||||
except ValidationError as e:
|
||||
error_msg = f'Configuration validation failed: {e.message}'
|
||||
if e.path:
|
||||
error_msg += f'\nPath: {".".join(str(p) for p in e.path)}'
|
||||
raise RuntimeError(error_msg) from e
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Validation error: {e}') from e
|
||||
|
||||
return config_data
|
||||
|
||||
|
||||
def upload_http_host_files(c):
|
||||
c.sudo(f'rm -rf {config.http_host_bin}')
|
||||
c.sudo(f'mkdir -p {config.http_host_bin}')
|
||||
|
||||
put_dir(c, config.local_modules_dir / 'http_host', config.http_host_bin, file_permissions='755')
|
||||
|
||||
for dirname in ['http_host_lib', 'scripts']:
|
||||
put_dir(
|
||||
c, config.local_modules_dir / 'http_host' / dirname, f'{config.http_host_bin}/{dirname}'
|
||||
)
|
||||
|
||||
put_dir(
|
||||
c,
|
||||
config.local_modules_dir / 'http_host' / 'http_host_lib' / 'nginx_templates',
|
||||
f'{config.http_host_bin}/http_host_lib/nginx_templates',
|
||||
)
|
||||
|
||||
c.sudo('chown -R ofm:ofm /data/ofm/http_host')
|
||||
|
||||
|
||||
def run_http_host_sync(c):
|
||||
print('Running http_host.py sync --force')
|
||||
sudo_cmd(c, f'{config.venv_bin}/python -u {config.http_host_bin}/http_host.py sync --force')
|
||||
|
||||
|
||||
def install_benchmark(c):
|
||||
"""
|
||||
Read docs/quick_notes/http_benchmark.md
|
||||
"""
|
||||
c1000k(c)
|
||||
wrk(c)
|
||||
31
ssh_lib/tasks_shared.py
Normal file
31
ssh_lib/tasks_shared.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from ssh_lib.config import config
|
||||
from ssh_lib.pkg_base import pkg_base, pkg_upgrade
|
||||
from ssh_lib.rclone import rclone
|
||||
from ssh_lib.utils import add_user, enable_sudo, put, sudo_cmd
|
||||
|
||||
|
||||
def prepare_shared(c):
|
||||
# creates ofm user with uid=2000, disabled password and nopasswd sudo
|
||||
add_user(c, 'ofm', uid=2000)
|
||||
enable_sudo(c, 'ofm', nopasswd=True)
|
||||
|
||||
pkg_upgrade(c)
|
||||
pkg_base(c)
|
||||
rclone(c)
|
||||
|
||||
c.sudo(f'mkdir -p {config.remote_config}')
|
||||
c.sudo(f'chown ofm:ofm {config.remote_config}')
|
||||
c.sudo(f'chown ofm:ofm {config.ofm_dir}')
|
||||
|
||||
prepare_venv(c)
|
||||
|
||||
|
||||
def prepare_venv(c):
|
||||
put(
|
||||
c,
|
||||
config.local_modules_dir / 'prepare-virtualenv.sh',
|
||||
config.ofm_dir,
|
||||
permissions='755',
|
||||
user='ofm',
|
||||
)
|
||||
sudo_cmd(c, f'cd {config.ofm_dir} && source prepare-virtualenv.sh')
|
||||
38
ssh_lib/tasks_tile_gen.py
Normal file
38
ssh_lib/tasks_tile_gen.py
Normal file
@@ -0,0 +1,38 @@
|
||||
from ssh_lib.config import config
|
||||
from ssh_lib.planetiler import install_planetiler
|
||||
from ssh_lib.utils import put, put_dir
|
||||
|
||||
|
||||
def prepare_tile_gen(c, *, enable_cron):
|
||||
c.sudo('rm -f /etc/cron.d/ofm_tile_gen')
|
||||
|
||||
install_planetiler(c)
|
||||
|
||||
c.sudo(f'rm -rf {config.tile_gen_bin}')
|
||||
|
||||
put_dir(c, config.local_modules_dir / 'tile_gen', config.tile_gen_bin, file_permissions='755')
|
||||
|
||||
for dirname in ['tile_gen_lib', 'scripts']:
|
||||
put_dir(
|
||||
c, config.local_modules_dir / 'tile_gen' / dirname, f'{config.tile_gen_bin}/{dirname}'
|
||||
)
|
||||
|
||||
if (config.local_config_dir / 'rclone.conf').exists():
|
||||
put(
|
||||
c,
|
||||
config.local_config_dir / 'rclone.conf',
|
||||
f'{config.remote_config}/rclone.conf',
|
||||
permissions='600',
|
||||
user='ofm',
|
||||
)
|
||||
|
||||
c.sudo(f'{config.venv_bin}/pip install -e {config.tile_gen_bin} --use-pep517')
|
||||
|
||||
c.sudo('rm -rf /data/ofm/tile_gen/logs')
|
||||
c.sudo('mkdir -p /data/ofm/tile_gen/logs')
|
||||
|
||||
c.sudo('chown ofm:ofm /data/ofm/tile_gen/{,*}')
|
||||
c.sudo(f'chown ofm:ofm -R {config.tile_gen_bin}')
|
||||
|
||||
if enable_cron:
|
||||
put(c, config.local_modules_dir / 'tile_gen' / 'cron.d' / 'ofm_tile_gen', '/etc/cron.d/')
|
||||
@@ -1,10 +1,12 @@
|
||||
import os
|
||||
import secrets
|
||||
import socket
|
||||
import string
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from fabric import Connection
|
||||
from invoke import UnexpectedExit
|
||||
|
||||
|
||||
@@ -205,3 +207,29 @@ def get_latest_release_github(user, repo):
|
||||
assert data['tag_name'] == data['name']
|
||||
|
||||
return data['tag_name']
|
||||
|
||||
|
||||
def get_ip_from_ssh_alias(ssh_alias):
|
||||
"""
|
||||
Get IP address from SSH config alias.
|
||||
|
||||
Args:
|
||||
ssh_alias: SSH hostname/alias from ~/.ssh/config
|
||||
|
||||
Returns:
|
||||
str: IP address
|
||||
|
||||
Raises:
|
||||
socket.gaierror: If hostname cannot be resolved
|
||||
"""
|
||||
|
||||
# Create connection (doesn't actually connect)
|
||||
conn = Connection(ssh_alias)
|
||||
|
||||
# Get the resolved hostname from SSH config
|
||||
hostname = conn.host
|
||||
|
||||
# Resolve to IP
|
||||
ip_address = socket.gethostbyname(hostname)
|
||||
|
||||
return ip_address
|
||||
|
||||
35
tile-gen.py
Normal file
35
tile-gen.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import click
|
||||
|
||||
from ssh_lib.cli_helpers import common_options, get_connection
|
||||
from ssh_lib.tasks_shared import prepare_shared
|
||||
from ssh_lib.tasks_tile_gen import prepare_tile_gen
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@common_options
|
||||
@click.option('--cron', is_flag=True, help='Enable cron task')
|
||||
@click.option('--reinstall', is_flag=True, help='Reinstall everything in /data/ofm folder')
|
||||
def tile_gen(
|
||||
hostname,
|
||||
user,
|
||||
port,
|
||||
noninteractive,
|
||||
#
|
||||
cron,
|
||||
reinstall,
|
||||
):
|
||||
if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
|
||||
return
|
||||
|
||||
c = get_connection(hostname, user, port)
|
||||
|
||||
if reinstall:
|
||||
c.sudo('rm -rf /data/ofm')
|
||||
|
||||
prepare_shared(c)
|
||||
prepare_tile_gen(c, enable_cron=cron)
|
||||
@@ -1,7 +1,7 @@
|
||||
// @ts-check
|
||||
import { defineConfig } from 'astro/config'
|
||||
|
||||
import sitemap from '@astrojs/sitemap'
|
||||
import { defineConfig } from 'astro/config'
|
||||
|
||||
// https://astro.build/config
|
||||
|
||||
|
||||
@@ -3,28 +3,21 @@
|
||||
"type": "module",
|
||||
"version": "0.0.1",
|
||||
"scripts": {
|
||||
"clean": "rm -rf .astro dist ___node_modules/.astro ",
|
||||
"dev": "pnpm clean; astro dev",
|
||||
"build": "pnpm clean; astro check && pnpm tsc && astro build",
|
||||
"preview": "pnpm build && astro preview",
|
||||
"preview_w": "pnpm build && wrangler dev",
|
||||
"deploy_w": "pnpm build && wrangler deploy"
|
||||
"dev": "astro dev",
|
||||
"start": "astro dev",
|
||||
"build": "astro build",
|
||||
"preview": "astro preview",
|
||||
"astro": "astro"
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/check": "^0.9.4",
|
||||
"@astrojs/sitemap": "^3.2.1",
|
||||
"astro": "^5.4.0",
|
||||
"lightningcss": "^1.29.1",
|
||||
"typescript": "^5.9.3"
|
||||
"lightningcss": "^1.29.1"
|
||||
},
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"esbuild",
|
||||
"sharp",
|
||||
"workerd"
|
||||
"sharp"
|
||||
]
|
||||
},
|
||||
"devDependencies": {
|
||||
"wrangler": "^4.43.0"
|
||||
}
|
||||
}
|
||||
|
||||
1307
website/pnpm-lock.yaml
generated
1307
website/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -1,15 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>OpenFreeMap Debug</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
|
||||
<link href="https://unpkg.com/maplibre-gl/dist/maplibre-gl.css" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="map" class="w-full h-screen"></div>
|
||||
<script src="https://unpkg.com/maplibre-gl/dist/maplibre-gl.js"></script>
|
||||
<script src="colon.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,65 +0,0 @@
|
||||
const map = new maplibregl.Map({
|
||||
container: 'map',
|
||||
style: 'https://tiles.openfreemap.org/styles/liberty',
|
||||
center: [0, 0],
|
||||
zoom: 2,
|
||||
})
|
||||
|
||||
function modifyStyle({ style, langCode }) {
|
||||
if (!langCode) {
|
||||
langCode = 'en'
|
||||
}
|
||||
|
||||
for (const layer of style.layers) {
|
||||
if (layer.source !== 'openmaptiles') continue
|
||||
if (!layer.layout) continue
|
||||
|
||||
const textField = layer.layout['text-field']
|
||||
if (!textField) continue
|
||||
|
||||
// highway numbers, etc. - skip ref-only fields
|
||||
if (JSON.stringify(textField) === JSON.stringify(['to-string', ['get', 'ref']])) continue
|
||||
|
||||
const nameUnderscore = `name_${langCode}`
|
||||
const nameColon = `name:${langCode}`
|
||||
|
||||
// Always display both values
|
||||
layer.layout['text-field'] = ['concat', ['get', nameUnderscore], '\n', ['get', nameColon]]
|
||||
|
||||
// Color red when they are different
|
||||
if (!layer.paint) layer.paint = {}
|
||||
layer.paint['text-color'] = [
|
||||
'case',
|
||||
['!=', ['get', nameUnderscore], ['get', nameColon]],
|
||||
'#ff0000', // Red when different
|
||||
'#000000', // Default color when same (adjust as needed)
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
function applyLanguage() {
|
||||
const hash = window.location.hash.substring(1) // Remove the '#'
|
||||
const langCode = hash || null
|
||||
|
||||
const style = map.getStyle()
|
||||
modifyStyle({ style, langCode })
|
||||
map.setStyle(style, { diff: false })
|
||||
}
|
||||
|
||||
map.on('load', () => {
|
||||
// Add default hash if not present
|
||||
if (!window.location.hash) {
|
||||
alert(
|
||||
'To change the map language, modify the language code in the URL #.\nLabels will be RED when different.\nname_xx on line 1, name:xx on line 2',
|
||||
)
|
||||
window.location.hash = '#en'
|
||||
// The hashchange event will trigger applyLanguage()
|
||||
} else {
|
||||
applyLanguage()
|
||||
}
|
||||
})
|
||||
|
||||
// Listen for hash changes in URL
|
||||
window.addEventListener('hashchange', () => {
|
||||
applyLanguage()
|
||||
})
|
||||
@@ -1,114 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>OpenFreeMap Debug</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
|
||||
<link href="https://unpkg.com/maplibre-gl/dist/maplibre-gl.css" rel="stylesheet" />
|
||||
</head>
|
||||
<body class="flex flex-col h-screen">
|
||||
<!-- UI Panel -->
|
||||
<div class="bg-gray-900 border-b border-gray-700 shadow-md">
|
||||
<div class="max-w-7xl mx-auto px-4 py-2">
|
||||
<div class="flex items-start gap-3">
|
||||
<div class="flex-1">
|
||||
<label for="line1" class="block text-xs font-medium text-gray-400 mb-1"> Line 1 </label>
|
||||
<input
|
||||
type="text"
|
||||
id="line1"
|
||||
class="w-full px-3 py-1.5 text-sm border border-gray-600 rounded bg-gray-800 text-gray-100 focus:outline-none focus:border-blue-500 focus:ring-1 focus:ring-blue-500 transition-all font-mono"
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
id="line1-expr"
|
||||
readonly
|
||||
class="w-full px-3 py-1 text-xs border border-gray-700 rounded bg-gray-900 text-gray-400 font-mono mt-1 cursor-default focus:outline-none focus:ring-0 focus:border-gray-700 selection:bg-gray-700 selection:text-gray-200"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="flex-1">
|
||||
<label for="line2" class="block text-xs font-medium text-gray-400 mb-1"> Line 2 </label>
|
||||
<input
|
||||
type="text"
|
||||
id="line2"
|
||||
class="w-full px-3 py-1.5 text-sm border border-gray-600 rounded bg-gray-800 text-gray-100 focus:outline-none focus:border-blue-500 focus:ring-1 focus:ring-blue-500 transition-all font-mono"
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
id="line2-expr"
|
||||
readonly
|
||||
class="w-full px-3 py-1 text-xs border border-gray-700 rounded bg-gray-900 text-gray-400 font-mono mt-1 cursor-default focus:outline-none focus:ring-0 focus:border-gray-700 selection:bg-gray-700 selection:text-gray-200"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-col gap-2">
|
||||
<div class="flex items-start gap-2">
|
||||
<div class="w-24">
|
||||
<label for="lang" class="block text-xs font-medium text-gray-400 mb-1">
|
||||
Lang
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
id="lang"
|
||||
class="w-full px-3 py-1.5 text-sm border border-gray-600 rounded bg-gray-800 text-gray-100 focus:outline-none focus:border-blue-500 focus:ring-1 focus:ring-blue-500 transition-all font-mono text-center"
|
||||
maxlength="5"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="flex gap-2">
|
||||
<button
|
||||
id="shareBtn"
|
||||
class="px-4 py-1.5 text-sm font-medium text-white bg-blue-600 rounded hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 focus:ring-offset-gray-900 transition-all mt-5"
|
||||
>
|
||||
Share
|
||||
</button>
|
||||
<button
|
||||
id="resetBtn"
|
||||
class="px-4 py-1.5 text-sm font-medium text-white bg-gray-700 rounded hover:bg-gray-600 focus:outline-none focus:ring-2 focus:ring-gray-500 focus:ring-offset-2 focus:ring-offset-gray-900 transition-all mt-5"
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<label
|
||||
class="flex items-center gap-2 text-xs text-gray-300 cursor-pointer whitespace-nowrap"
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
id="showDifferencesRed"
|
||||
class="w-4 h-4 text-blue-600 bg-gray-700 border-gray-600 rounded focus:ring-blue-500 focus:ring-2"
|
||||
/>
|
||||
<span>highlight line1 != line2</span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="map" class="flex-1"></div>
|
||||
|
||||
<!-- Modal Dialog -->
|
||||
<div id="shareModal" class="hidden fixed inset-0 z-50">
|
||||
<div id="modalOverlay" class="fixed inset-0 bg-black/50"></div>
|
||||
<div class="fixed inset-0 flex items-center justify-center p-4">
|
||||
<div class="relative bg-gray-800 rounded-lg p-8 max-w-md shadow-2xl">
|
||||
<p class="text-gray-300 text-center leading-relaxed mb-6">
|
||||
Your settings have been saved to the URL.<br />
|
||||
Copy the address bar to share this map.
|
||||
</p>
|
||||
<button
|
||||
id="closeModalBtn"
|
||||
class="w-full py-2.5 text-sm text-white bg-gray-700 rounded hover:bg-gray-600 transition-all"
|
||||
>
|
||||
Close
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="https://unpkg.com/maplibre-gl/dist/maplibre-gl.js"></script>
|
||||
<script src="mix.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,240 +0,0 @@
|
||||
// ============================================
|
||||
// 1. MAIN EXECUTION (Entry Point)
|
||||
// ============================================
|
||||
|
||||
const map = new maplibregl.Map({
|
||||
container: 'map',
|
||||
style: 'https://tiles.openfreemap.org/styles/liberty',
|
||||
center: [0, 0],
|
||||
zoom: 2,
|
||||
hash: true,
|
||||
})
|
||||
|
||||
const line1Input = document.getElementById('line1')
|
||||
const line2Input = document.getElementById('line2')
|
||||
const langInput = document.getElementById('lang')
|
||||
const line1ExprInput = document.getElementById('line1-expr')
|
||||
const line2ExprInput = document.getElementById('line2-expr')
|
||||
const showDifferencesRedCheckbox = document.getElementById('showDifferencesRed')
|
||||
|
||||
map.on('load', () => {
|
||||
const params = new URLSearchParams(window.location.search)
|
||||
|
||||
// Set defaults if no params present
|
||||
if (!params.has('line1') && !params.has('line2') && !params.has('lang')) {
|
||||
const url = new URL(window.location)
|
||||
url.searchParams.set('line1', 'colon,underscore,latin,name')
|
||||
url.searchParams.set('line2', 'nonlatin')
|
||||
url.searchParams.set('lang', 'en')
|
||||
window.history.replaceState({}, '', url)
|
||||
}
|
||||
syncInputsFromParams()
|
||||
applyConfiguration()
|
||||
initializeInputListeners()
|
||||
initializeModal()
|
||||
initializeResetButton()
|
||||
})
|
||||
|
||||
// ============================================
|
||||
// 2. UI INITIALIZATION
|
||||
// ============================================
|
||||
|
||||
function initializeInputListeners() {
|
||||
const debouncedApplyConfig = debounce(applyConfiguration, 500)
|
||||
|
||||
const handleInput = () => {
|
||||
updateParamsFromInputs()
|
||||
updateExpressionDisplays()
|
||||
debouncedApplyConfig()
|
||||
}
|
||||
|
||||
line1Input.addEventListener('input', handleInput)
|
||||
line2Input.addEventListener('input', handleInput)
|
||||
langInput.addEventListener('input', handleInput)
|
||||
showDifferencesRedCheckbox.addEventListener('change', handleInput)
|
||||
}
|
||||
|
||||
function initializeModal() {
|
||||
const modal = document.getElementById('shareModal')
|
||||
|
||||
document.getElementById('shareBtn').addEventListener('click', () => {
|
||||
modal.classList.remove('hidden')
|
||||
})
|
||||
|
||||
document.getElementById('closeModalBtn').addEventListener('click', () => {
|
||||
modal.classList.add('hidden')
|
||||
})
|
||||
|
||||
document.addEventListener('keydown', e => {
|
||||
if (e.key === 'Escape' && !modal.classList.contains('hidden')) {
|
||||
modal.classList.add('hidden')
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function initializeResetButton() {
|
||||
document.getElementById('resetBtn').addEventListener('click', () => {
|
||||
const hash = window.location.hash
|
||||
window.location.href = `${window.location.pathname}${hash}`
|
||||
})
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// 3. CONFIGURATION & SYNC
|
||||
// ============================================
|
||||
|
||||
function applyConfiguration() {
|
||||
const { line1, line2, lang, showDifferencesRed } = parseParams()
|
||||
|
||||
if (!map.getStyle()) return
|
||||
|
||||
const style = map.getStyle()
|
||||
modifyStyle({
|
||||
style,
|
||||
line1Config: line1 ?? '',
|
||||
line2Config: line2 ?? '',
|
||||
langCode: lang,
|
||||
showDifferencesRed,
|
||||
})
|
||||
map.setStyle(style, { diff: true })
|
||||
updateExpressionDisplays()
|
||||
}
|
||||
|
||||
function syncInputsFromParams() {
|
||||
const { line1, line2, lang, showDifferencesRed } = parseParams()
|
||||
line1Input.value = line1 ?? ''
|
||||
line2Input.value = line2 ?? ''
|
||||
langInput.value = lang ?? ''
|
||||
showDifferencesRedCheckbox.checked = showDifferencesRed
|
||||
}
|
||||
|
||||
function updateParamsFromInputs() {
|
||||
const params = new URLSearchParams(window.location.search)
|
||||
params.set('line1', line1Input.value)
|
||||
params.set('line2', line2Input.value)
|
||||
params.set('lang', langInput.value)
|
||||
|
||||
if (showDifferencesRedCheckbox.checked) {
|
||||
params.set('showDifferencesRed', '1')
|
||||
} else {
|
||||
params.delete('showDifferencesRed')
|
||||
}
|
||||
|
||||
const queryString = params.toString()
|
||||
const hash = window.location.hash
|
||||
const newUrl = `${window.location.pathname}?${queryString}${hash}`
|
||||
|
||||
window.history.replaceState({}, '', newUrl)
|
||||
}
|
||||
|
||||
function updateExpressionDisplays() {
|
||||
const { line1, line2, lang } = parseParams()
|
||||
const langCode = lang
|
||||
|
||||
const line1Expr = buildFieldAccessor(line1 ?? '', langCode)
|
||||
const line2Expr = buildFieldAccessor(line2 ?? '', langCode)
|
||||
|
||||
line1ExprInput.value = line1Expr ? JSON.stringify(line1Expr) : ''
|
||||
line2ExprInput.value = line2Expr ? JSON.stringify(line2Expr) : ''
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// 4. STYLE MODIFICATION
|
||||
// ============================================
|
||||
|
||||
function modifyStyle({ style, line1Config, line2Config, langCode, showDifferencesRed }) {
|
||||
for (const layer of style.layers) {
|
||||
if (layer.source !== 'openmaptiles') continue
|
||||
if (!layer.layout) continue
|
||||
|
||||
const textField = layer.layout['text-field']
|
||||
if (!textField) continue
|
||||
|
||||
if (JSON.stringify(textField) === JSON.stringify(['to-string', ['get', 'ref']])) continue
|
||||
|
||||
const id = layer.id
|
||||
const separator = id.includes('line') || id.includes('highway') ? ' ' : '\n'
|
||||
|
||||
const line1Expr = buildFieldAccessor(line1Config, langCode)
|
||||
const line2Expr = buildFieldAccessor(line2Config, langCode)
|
||||
|
||||
if (line1Expr && line2Expr) {
|
||||
layer.layout['text-field'] = ['concat', line1Expr, separator, line2Expr]
|
||||
} else if (line1Expr) {
|
||||
layer.layout['text-field'] = line1Expr
|
||||
} else if (line2Expr) {
|
||||
layer.layout['text-field'] = line2Expr
|
||||
} else {
|
||||
layer.layout['text-field'] = ['get', 'name']
|
||||
}
|
||||
|
||||
// Apply red color when differences should be shown
|
||||
if (showDifferencesRed && line1Expr && line2Expr) {
|
||||
if (!layer.paint) layer.paint = {}
|
||||
layer.paint['text-color'] = [
|
||||
'case',
|
||||
['!=', line1Expr, line2Expr],
|
||||
'#ff0000', // Red when different
|
||||
'#000000', // Black when same
|
||||
]
|
||||
} else {
|
||||
// Reset to default color if checkbox is unchecked
|
||||
if (layer.paint && layer.paint['text-color']) {
|
||||
delete layer.paint['text-color']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// 5. UTILITY FUNCTIONS
|
||||
// ============================================
|
||||
|
||||
function debounce(func, delay) {
|
||||
let timeoutId
|
||||
|
||||
return function (...args) {
|
||||
clearTimeout(timeoutId)
|
||||
timeoutId = setTimeout(() => {
|
||||
func.apply(this, args)
|
||||
}, delay)
|
||||
}
|
||||
}
|
||||
|
||||
function parseParams() {
|
||||
const params = new URLSearchParams(window.location.search)
|
||||
return {
|
||||
line1: params.get('line1'),
|
||||
line2: params.get('line2'),
|
||||
lang: params.get('lang') || 'en',
|
||||
showDifferencesRed: params.has('showDifferencesRed'),
|
||||
}
|
||||
}
|
||||
|
||||
function buildFieldAccessor(config, langCode) {
|
||||
if (!config) return null
|
||||
|
||||
const parts = []
|
||||
const fields = config
|
||||
.split(',')
|
||||
.map(f => f.trim())
|
||||
.filter(f => f)
|
||||
|
||||
for (const field of fields) {
|
||||
if (field === 'underscore') {
|
||||
parts.push(['get', `name_${langCode}`])
|
||||
} else if (field === 'colon') {
|
||||
parts.push(['get', `name:${langCode}`])
|
||||
} else if (field === 'latin') {
|
||||
parts.push(['get', 'name:latin'])
|
||||
} else if (field === 'nonlatin') {
|
||||
parts.push(['get', 'name:nonlatin'])
|
||||
} else if (field === 'name') {
|
||||
parts.push(['get', 'name'])
|
||||
} else {
|
||||
parts.push(['get', field])
|
||||
}
|
||||
}
|
||||
|
||||
return parts.length > 0 ? ['coalesce', ...parts] : null
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>OpenFreeMap Debug</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
|
||||
<link href="https://unpkg.com/maplibre-gl/dist/maplibre-gl.css" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="map" class="w-full h-screen"></div>
|
||||
<script src="https://unpkg.com/maplibre-gl/dist/maplibre-gl.js"></script>
|
||||
<script src="params.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,78 +0,0 @@
|
||||
const map = new maplibregl.Map({
|
||||
container: 'map',
|
||||
hash: 'map',
|
||||
style: 'https://tiles.openfreemap.org/styles/liberty',
|
||||
center: [0, 0],
|
||||
zoom: 2,
|
||||
})
|
||||
|
||||
function modifyStyle({ style, langCode }) {
|
||||
if (!langCode) {
|
||||
langCode = 'en'
|
||||
}
|
||||
|
||||
for (const layer of style.layers) {
|
||||
if (layer.source !== 'openmaptiles') continue
|
||||
if (!layer.layout) continue
|
||||
|
||||
const textField = layer.layout['text-field']
|
||||
if (!textField) continue
|
||||
|
||||
// highway numbers, etc. - skip ref-only fields
|
||||
if (JSON.stringify(textField) === JSON.stringify(['to-string', ['get', 'ref']])) continue
|
||||
|
||||
const nameUnderscore = `name_${langCode}`
|
||||
const nameColon = `name:${langCode}`
|
||||
|
||||
// Always display both values
|
||||
layer.layout['text-field'] = ['concat', ['get', nameUnderscore], '\n', ['get', nameColon]]
|
||||
|
||||
// Color red when they are different
|
||||
if (!layer.paint) layer.paint = {}
|
||||
layer.paint['text-color'] = [
|
||||
'case',
|
||||
['!=', ['get', nameUnderscore], ['get', nameColon]],
|
||||
'#ff0000', // Red when different
|
||||
'#000000', // Default color when same (adjust as needed)
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
function getLanguageParam() {
|
||||
const urlParams = new URLSearchParams(window.location.search)
|
||||
return urlParams.get('lang')
|
||||
}
|
||||
|
||||
function applyLanguage() {
|
||||
const langCode = getLanguageParam() || null
|
||||
|
||||
const style = map.getStyle()
|
||||
modifyStyle({ style, langCode })
|
||||
map.setStyle(style, { diff: false })
|
||||
}
|
||||
|
||||
map.on('load', () => {
|
||||
const langCode = getLanguageParam()
|
||||
|
||||
// Alert the URL param value on first load
|
||||
alert(
|
||||
`Language parameter: ${langCode || 'not set (defaulting to en)'}\n\n` +
|
||||
'To change the map language, modify the ?lang= parameter in the URL.\n' +
|
||||
'Labels will be RED when different.\n' +
|
||||
'name_xx on line 1, name:xx on line 2',
|
||||
)
|
||||
|
||||
// Add default param if not present
|
||||
if (!langCode) {
|
||||
const url = new URL(window.location)
|
||||
url.searchParams.set('lang', 'en')
|
||||
window.history.replaceState({}, '', url)
|
||||
}
|
||||
|
||||
applyLanguage()
|
||||
})
|
||||
|
||||
// Listen for URL changes (e.g., browser back/forward)
|
||||
window.addEventListener('popstate', () => {
|
||||
applyLanguage()
|
||||
})
|
||||
@@ -1,15 +1,15 @@
|
||||
<!doctype html>
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>OpenFreeMap Debug</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
|
||||
<link href="https://unpkg.com/maplibre-gl/dist/maplibre-gl.css" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
</head>
|
||||
<body>
|
||||
<div id="map" class="w-full h-screen"></div>
|
||||
<script src="https://unpkg.com/maplibre-gl/dist/maplibre-gl.js"></script>
|
||||
<script src="switch.js"></script>
|
||||
</body>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -51,8 +51,8 @@ function modifyStyle({ style, langCode }) {
|
||||
}
|
||||
|
||||
function applyLanguage() {
|
||||
const hash = window.location.hash.substring(1) // Remove the '#'
|
||||
const langCode = hash || null
|
||||
const hash = window.location.hash.substring(1); // Remove the '#'
|
||||
const langCode = hash || null;
|
||||
|
||||
const style = map.getStyle()
|
||||
modifyStyle({ style, langCode })
|
||||
@@ -62,9 +62,7 @@ function applyLanguage() {
|
||||
map.on('load', () => {
|
||||
// Add default hash if not present
|
||||
if (!window.location.hash) {
|
||||
alert(
|
||||
'To change the map language, modify the language code in the URL.\n\nExamples:\n• #en → English\n• #de → German\n• #fr → French\n• #es → Spanish\n• #int → International names',
|
||||
)
|
||||
alert('To change the map language, modify the language code in the URL.\n\nExamples:\n• #en → English\n• #de → German\n• #fr → French\n• #es → Spanish\n• #int → International names\n\nDefault language set to: en (English)')
|
||||
window.location.hash = '#es'
|
||||
// The hashchange event will trigger applyLanguage()
|
||||
} else {
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>OpenFreeMap Debug</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
|
||||
<link href="https://unpkg.com/maplibre-gl/dist/maplibre-gl.css" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="map" class="w-full h-screen"></div>
|
||||
<script src="https://unpkg.com/maplibre-gl/dist/maplibre-gl.js"></script>
|
||||
<script src="terrain.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,26 +0,0 @@
|
||||
const map = new maplibregl.Map({
|
||||
container: 'map',
|
||||
hash: 'map',
|
||||
zoom: 10.5,
|
||||
center: [9.0788, 47.1194],
|
||||
style: {
|
||||
version: 8,
|
||||
sources: {
|
||||
hillshadeSource: {
|
||||
type: 'raster-dem',
|
||||
url: 'https://tiles.mapterhorn.com/tilejson.json',
|
||||
},
|
||||
},
|
||||
layers: [
|
||||
{
|
||||
id: 'hillshade',
|
||||
type: 'hillshade',
|
||||
source: 'hillshadeSource',
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
map.on('load', () => {
|
||||
console.log('Terrain map loaded')
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
---
|
||||
import StyleUrlBug from './StyleUrlBug.astro'
|
||||
|
||||
const { showStyleURL } = Astro.props
|
||||
---
|
||||
|
||||
@@ -17,8 +18,6 @@ const { showStyleURL } = Astro.props
|
||||
<button data-style="positron" class="btn">Positron</button>
|
||||
<button data-style="bright" class="btn">Bright</button>
|
||||
<button data-style="liberty" class="btn selected">Liberty</button>
|
||||
<button data-style="dark" class="btn">Dark</button>
|
||||
<button data-style="fiord" class="btn">Fiord</button>
|
||||
<button data-style="liberty-3d" class="btn">3D</button>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -2,11 +2,10 @@
|
||||
import Donate from '../components/Donate.astro'
|
||||
import Logo from '../components/Logo.astro'
|
||||
import Map_ from '../components/Map.astro'
|
||||
import Layout from '../layouts/Layout.astro'
|
||||
|
||||
import { Content as AfterDonate } from '../content/index/after_donate.md'
|
||||
import { Content as BeforeDonate } from '../content/index/before_donate.md'
|
||||
import { Content as WhatisText } from '../content/index/whatis.md'
|
||||
import Layout from '../layouts/Layout.astro'
|
||||
---
|
||||
|
||||
<Layout title="OpenFreeMap">
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
---
|
||||
import Map_ from '../components/Map.astro'
|
||||
import Layout from '../layouts/Layout.astro'
|
||||
|
||||
import Donate from '../components/Donate.astro'
|
||||
import Logo from '../components/Logo.astro'
|
||||
import Map_ from '../components/Map.astro'
|
||||
import { Content as CustomStylesText } from '../content/how_to_use/custom_styles.md'
|
||||
import { Content as LeafletText } from '../content/how_to_use/leaflet.md'
|
||||
import { Content as MapboxText } from '../content/how_to_use/mapbox.md'
|
||||
@@ -11,6 +9,7 @@ import { Content as MaplibreText } from '../content/how_to_use/maplibre.md'
|
||||
import { Content as MobileText } from '../content/how_to_use/mobile.md'
|
||||
import { Content as OpenLayersText } from '../content/how_to_use/openlayers.md'
|
||||
import { Content as SelfHostingText } from '../content/how_to_use/self_hosting.md'
|
||||
import Layout from '../layouts/Layout.astro'
|
||||
---
|
||||
|
||||
<Layout title="OpenFreeMap Quick Start Guide">
|
||||
|
||||
@@ -24,8 +24,8 @@ h6 {
|
||||
margin-top: 3em;
|
||||
margin-bottom: 0.5em;
|
||||
line-height: 1.2;
|
||||
font-family: Seravek, "Gill Sans Nova", Ubuntu, Calibri, "DejaVu Sans", source-sans-pro,
|
||||
sans-serif;
|
||||
font-family:
|
||||
Seravek, "Gill Sans Nova", Ubuntu, Calibri, "DejaVu Sans", source-sans-pro, sans-serif;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
|
||||
@@ -25,7 +25,11 @@
|
||||
}
|
||||
|
||||
.mapbg-attrib {
|
||||
font: 12px / 20px "Helvetica Neue", Arial, Helvetica, sans-serif;
|
||||
font:
|
||||
12px / 20px "Helvetica Neue",
|
||||
Arial,
|
||||
Helvetica,
|
||||
sans-serif;
|
||||
background-color: hsla(0, 0%, 100%, 0.5);
|
||||
padding: 0 5px;
|
||||
bottom: 0;
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"$schema": "node_modules/wrangler/config-schema.json",
|
||||
"name": "openfreemap-website",
|
||||
"compatibility_date": "2025-10-17",
|
||||
"assets": {
|
||||
"directory": "./dist"
|
||||
},
|
||||
"account_id": "99fde2e5efdeb199c6910cdeaa276a97",
|
||||
"workers_dev": false
|
||||
}
|
||||
Reference in New Issue
Block a user