sysadmin/openfreemap
1
0
Fork 0
mirror of https://github.com/hyperknot/openfreemap.git synced 2026-05-21 14:02:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

work

Browse Source
This commit is contained in:
Zsolt Ero
2025-09-18 19:35:56 +02:00
parent 604f27e7db
commit b24f096ad4
13 changed files with 93 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -190,8 +190,6 @@ See [dev setup docs](docs/dev_setup.md).
Updated Planetiler version to latest
Updated OpenJDK to 24 via Temurin repo
##### v0.8
Lot of self-hosting related fixes.

11
init-server.py
View File

@@ -121,11 +121,12 @@ def http_host_sync(hostname, user, port, noninteractive):
run_http_host_sync(c)
@cli.command()
@common_options
def debug(hostname, user, port, noninteractive):
c = get_connection(hostname, user, port)
run_http_host_sync(c)
#
# @cli.command()
# @common_options
# def debug(hostname, user, port, noninteractive):
# c = get_connection(hostname, user, port)
# run_http_host_sync(c)
if __name__ == '__main__':

12
modules/http_host/http_host_lib/nginx.py
View File

@@ -320,15 +320,3 @@ def create_latest_locations(*, local: str, domain: str) -> str:
"""
return location_str
def write_roundrobin_reader_script(domain_roundrobin):
script = f"""
#!/usr/bin/env bash
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
rclone copyto -v "remote:ofm-private/roundrobin/{domain_roundrobin}/ofm_roundrobin.cert" /data/nginx/certs/ofm_roundrobin.cert
rclone copyto -v "remote:ofm-private/roundrobin/{domain_roundrobin}/ofm_roundrobin.key" /data/nginx/certs/ofm_roundrobin.key
""".strip()
with open(config.http_host_bin / 'roundrobin_reader.sh', 'w') as fp:
fp.write(script)

1
modules/http_host/http_host_lib/shared.py
View File

@@ -1 +0,0 @@
../../tile_gen/tile_gen_lib/shared.py

4
modules/tile_gen/cron.d/ofm_tile_gen
View File

@@ -5,8 +5,8 @@ LOG_DIR=/data/ofm/tile_gen/logs
# every day at 23:10, make a monaco run
10 23 * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
# debug monaco run, every minute
#*/1 * * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
# debug monaco run, normally disabled, enable to run every minute
#* * * * * ofm $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
# every minute, set monaco to latest
* * * * * ofm $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1

1
modules/tile_gen/setup.py
View File

@@ -3,7 +3,6 @@ from setuptools import find_packages, setup
requirements = [
'click',
'pycurl',
'requests',
]

24
modules/tile_gen/tile_gen.py
View File

@@ -3,9 +3,12 @@ from datetime import datetime, timezone
import click
from tile_gen_lib.btrfs import make_btrfs
from tile_gen_lib.get_version_shared import (
get_deployed_version,
get_versions_for_area,
)
from tile_gen_lib.planetiler import run_planetiler
from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
from tile_gen_lib.set_version import check_and_set_version
from tile_gen_lib.rclone import make_indexes_for_bucket, set_version_on_bucket, upload_area
now = datetime.now(timezone.utc)
@@ -71,7 +74,22 @@ def set_version(area, version):
print(f'---\n{now}\nStarting set-version {area}')
check_and_set_version(area, version)
if version == 'latest':
versions = get_versions_for_area(area)
if not versions:
print(f' No versions found for {area}')
return
version = versions[-1]
print(f' Latest version on bucket: {area} {version}')
try:
if get_deployed_version(area)['version'] == version:
return
except Exception:
pass
set_version_on_bucket(area, version)
if __name__ == '__main__':

3
modules/tile_gen/tile_gen_lib/config.py
View File

@@ -1,4 +1,3 @@
import json
import subprocess
from pathlib import Path
@@ -22,8 +21,6 @@ class Configuration:
repo_root = Path(__file__).parent.parent.parent.parent
ofm_config_dir = repo_root / 'config'
ofm_config = json.loads((ofm_config_dir / 'config.json').read_text())
rclone_config = ofm_config_dir / 'rclone.conf'
rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip()

48
modules/tile_gen/tile_gen_lib/get_version_shared.py Normal file
View File

@@ -0,0 +1,48 @@
"""
This file is shared / symlinked between tile_gen_lib and http_host_lib
"""
from datetime import datetime, timezone
import requests
def get_versions_for_area(area: str) -> list:
"""
Download the files.txt and check for the runs with the "done" file present
"""
r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30)
r.raise_for_status()
versions = []
files = r.text.splitlines()
for f in files:
if not f.startswith(f'areas/{area}/'):
continue
if not f.endswith('/done'):
continue
version_str = f.split('/')[2]
versions.append(version_str)
return sorted(versions)
def get_deployed_version(area: str) -> dict:
r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
r.raise_for_status()
version = r.text.strip()
last_modified_str = r.headers.get('Last-Modified')
last_modified = parse_http_last_modified(last_modified_str)
return dict(
version=version,
last_modified=last_modified,
)
def parse_http_last_modified(date_string) -> datetime:
parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT')
parsed_date = parsed_date.replace(tzinfo=timezone.utc)
return parsed_date

14
modules/tile_gen/tile_gen_lib/rclone.py
View File

@@ -132,3 +132,17 @@ def make_indexes_for_bucket(bucket):
check=True,
input=index_str.encode(),
)
def set_version_on_bucket(area, version):
print(f'setting version: {area} {version}')
subprocess.run(
[
config.rclone_bin,
'rcat',
f'remote:ofm-assets/deployed_versions/{area}.txt',
],
env=dict(RCLONE_CONFIG=config.rclone_config),
check=True,
input=version.strip().encode(),
)

56
modules/tile_gen/tile_gen_lib/set_version.py
View File

@@ -1,56 +0,0 @@
import subprocess
from .config import config
from .shared import check_host_version, get_deployed_version, get_versions_for_area
def check_and_set_version(area, version):
if version == 'latest':
versions = get_versions_for_area(area)
if not versions:
print(f' No versions found for {area}')
return
version = versions[-1]
print(f' Latest version on bucket: {area} {version}')
if not check_all_hosts(area, version):
return
try:
if get_deployed_version(area)['version'] == version:
return
except Exception:
pass
set_version(area, version)
def set_version(area, version):
print(f'setting version: {area} {version}')
subprocess.run(
[
config.rclone_bin,
'rcat',
f'remote:ofm-assets/deployed_versions/{area}.txt',
],
env=dict(RCLONE_CONFIG=config.rclone_config),
check=True,
input=version.strip().encode(),
)
def check_all_hosts(area, version) -> bool:
oc = config.ofm_config
domain = oc['domain_roundrobin'] or oc['domain_direct']
print(f'Using domain: {domain}')
try:
for host_ip in oc['http_host_list']:
print(f'Checking {area} {version} on host {host_ip}')
check_host_version(domain, host_ip, area, version)
return True
except Exception:
print('Error, version not available')
return False

134
modules/tile_gen/tile_gen_lib/shared.py
View File

@@ -1,134 +0,0 @@
import json
from datetime import datetime, timezone
from io import BytesIO
from pathlib import Path
import pycurl
import requests
def get_versions_for_area(area: str) -> list:
"""
Download the files.txt and check for the runs with the "done" file present
"""
r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30)
r.raise_for_status()
versions = []
files = r.text.splitlines()
for f in files:
if not f.startswith(f'areas/{area}/'):
continue
if not f.endswith('/done'):
continue
version_str = f.split('/')[2]
versions.append(version_str)
return sorted(versions)
def get_deployed_version(area: str) -> dict:
r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
r.raise_for_status()
version = r.text.strip()
last_modified_str = r.headers.get('Last-Modified')
last_modified = parse_http_last_modified(last_modified_str)
return dict(
version=version,
last_modified=last_modified,
)
def parse_http_last_modified(date_string) -> datetime:
parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT')
parsed_date = parsed_date.replace(tzinfo=timezone.utc)
return parsed_date
def check_host_version(domain, host_ip, area, version):
# check versioned TileJSON
check_tilejson(f'https://{domain}/{area}/{version}', domain, host_ip, version)
# check actual vector tile
url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
assert pycurl_status(url, domain, host_ip) == 200
def check_host_latest(domain, host_ip, area, version):
# check latest TileJSON
check_tilejson(f'https://{domain}/{area}', domain, host_ip, version)
# check versioned TileJSON
check_tilejson(f'https://{domain}/{area}/{version}', domain, host_ip, version)
# check actual vector tile
url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
assert pycurl_status(url, domain, host_ip) == 200
# check style
url = f'https://{domain}/styles/bright'
assert pycurl_status(url, domain, host_ip) == 200
def check_tilejson(url, domain, host_ip, version):
tilejson_str = pycurl_get(url, domain, host_ip)
tilejson = json.loads(tilejson_str)
tiles_url = tilejson['tiles'][0]
version_in_tilejson = tiles_url.split('/')[4]
assert version_in_tilejson == version
# pycurl
def pycurl_status(url, domain, host_ip):
"""
Uses pycurl to make a HTTPS HEAD request using custom resolving,
checks if the status code is 200
"""
c = pycurl.Curl()
c.setopt(c.URL, url)
# linux needs CA certs specified manually
if Path('/etc/ssl/certs/ca-certificates.crt').exists():
c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
c.setopt(c.NOBODY, True)
c.setopt(c.TIMEOUT, 5)
c.perform()
status_code = c.getinfo(c.RESPONSE_CODE)
c.close()
return status_code
def pycurl_get(url, domain, host_ip):
"""
Uses pycurl to make a HTTPS GET request using custom resolving,
checks if the status code is 200, and returns the content.
"""
buffer = BytesIO()
c = pycurl.Curl()
c.setopt(c.URL, url)
# linux needs CA certs specified manually
if Path('/etc/ssl/certs/ca-certificates.crt').exists():
c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.TIMEOUT, 5)
c.perform()
status_code = c.getinfo(c.RESPONSE_CODE)
c.close()
if status_code != 200:
raise ValueError(f'status code: {status_code}')
return buffer.getvalue().decode('utf8')

4
ssh_lib/tasks.py
View File

@@ -33,8 +33,6 @@ def prepare_shared(c):
c.sudo(f'chown ofm:ofm {REMOTE_CONFIG}')
c.sudo(f'chown ofm:ofm {OFM_DIR}')
upload_config_json(c)
prepare_venv(c)
@@ -86,6 +84,8 @@ def prepare_http_host(c):
kernel_somaxconn65k(c)
kernel_limits1m(c)
upload_config_json(c)
nginx(c)
certbot(c)