readme

README.md

@@ -126,13 +126,6 @@ There are three public buckets:
 - https://planet.openfreemap.com - full planet runs. index: [dirs](https://planet.openfreemap.com/dirs.txt), [files](https://planet.openfreemap.com/index.txt)
 - https://monaco.openfreemap.com - identical runs to the full planet, but only for Monaco area. Very tiny, ideal for development. index: [dirs](https://monaco.openfreemap.com/dirs.txt), [files](https://monaco.openfreemap.com/index.txt)
 
-### Domains and Cloudflare
-
-- `tiles.openfreemap.org` - Cloudflare proxied
-- `direct.openfreemap.org` - direct connection, Round-Robin DNS
-
-The project has been designed in such a way that we can migrate away from Cloudflare if needed. This is the reason why there are a .com and a .org domain: the .com will always stay on Cloudflare to host the R2 buckets, while the .org domain is independent.
-
 ### What about PMTiles?
 
 I would have loved to use PMTiles; they are a brilliant idea!
@@ -165,6 +158,12 @@ See [dev setup docs](docs/dev_setup.md).
 
 ## Changelog
 
+##### v0.3
+
+Lot of performance related problems with Cloudflare when using Round-Robin DNS. Works much better without any Cloudflare proxying, the browsers actually do a great job of client-side failover and selecting the best host.
+
+Load-balancing script running in check mode again.
+
 ##### v0.2
 
 Load-balancing script is running in write mode, updating records when needed.

config/.env.sample

@@ -7,9 +7,7 @@ DOMAIN_LE=
 # Let's Encrypt account email
 LE_EMAIL=
 
-# CloudFlare subdomain, using origin certificates
-# Please put ofm_cf.key and ofm_cf.cert files in config/certs
-DOMAIN_CF=tiles.openfreemap.org
+
 
 # Skip the full planet download, useful for testing (true/false)
 SKIP_PLANET=false
@@ -17,7 +15,7 @@ SKIP_PLANET=false
 
 # --- Let's Encrypt DNS related variables, not needed for self-hosting
 
-DOMAIN_LEDNS=direct.openfreemap.org
+DOMAIN_LEDNS=
 
 # --- host list
 
@@ -28,3 +26,4 @@ HTTP_HOST_LIST=
 
 TELEGRAM_TOKEN=
 TELEGRAM_CHAT_ID=
+

docs/self_hosting.md

@@ -46,9 +46,7 @@ It's recommended to use [direnv](https://direnv.net/), to have automatic venv ac
 
 1. Copy `.env.sample` to `.env` and set the values.
 
-   DOMAIN_LE - Use this to specify a domain to be used with Let's Encrypt. Recommended.
-
-   DOMAIN_CF - Use this if you want to use long term CloudFlare Origin certificates. You have to upload the certs into `config/certs`
+   DOMAIN_LE - Use this to specify a domain to be used with Let's Encrypt.
 
 1. If you want to run tile generation and upload via rclone, you can copy the `rclone.conf.sample` file as well. For simple self-hosting there is no need for this.
 

init-server.py

@@ -3,7 +3,7 @@
 import click
 from fabric import Config, Connection
 
-from ssh_lib import SCRIPTS_DIR, TILE_GEN_BIN, dotenv_val
+from ssh_lib import SCRIPTS_DIR, dotenv_val
 from ssh_lib.tasks import (
     prepare_http_host,
     prepare_shared,
@@ -101,7 +101,7 @@ def tile_gen(hostname, user, port):
 
 @cli.command()
 @common_options
-def ledns_writer(hostname, user, port):
+def ledns(hostname, user, port):
     if not click.confirm(f'Run script on {hostname}?'):
         return
 

prepare-virtualenv.sh

@@ -13,6 +13,8 @@ pip install -U pip wheel setuptools
 pip install -e .
 pip install -e scripts/http_host
 pip install -e scripts/tile_gen
+pip install -e scripts/loadbalancer
+pip install -e scripts/setversion
 
 
 

scripts/http_host/host_manager.py

@@ -198,5 +198,5 @@ def sync(ctx):
 
 
 if __name__ == '__main__':
-    print(HOST_CONFIG)
+    # print(HOST_CONFIG)
     cli()

scripts/http_host/http_host_lib/nginx.py

@@ -17,22 +17,17 @@ from http_host_lib import (
 def write_nginx_config():
     curl_text_mix = ''
 
-    domain_cf = HOST_CONFIG['domain_cf']
     domain_le = HOST_CONFIG['domain_le']
     domain_ledns = HOST_CONFIG['domain_ledns']
 
-    # processing Cloudflare config
-    if domain_cf:
-        if not (CERTS_DIR / 'ofm_cf.cert').is_file() or not (CERTS_DIR / 'ofm_cf.key').is_file():
-            sys.exit('ofm_cf.cert or ofm_cf.key missing')
+    # remove old configs and certs
+    for file in Path('/data/nginx/sites').glob('ofm_*.conf'):
+        file.unlink()
 
-        curl_text_mix += create_nginx_conf(
-            template_path=NGINX_DIR / 'cf.conf',
-            local='ofm_cf',
-            domain=domain_cf,
-        )
+    for file in Path('/data/nginx/certs').glob('ofm_*'):
+        file.unlink()
 
-    # processing Cloudflare config
+    # processing Round Robin DNS config
     if domain_ledns:
         if not (OFM_CONFIG_DIR / 'rclone.conf').is_file():
             sys.exit('rclone.conf missing')

scripts/loadbalancer/cron.d/ofm_loadbalancer

@@ -1,4 +1,8 @@
 # every minute
-* * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py fix >> /data/ofm/loadbalancer/logs/check.log 2>&1
+
+# fix
+#* * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py fix >> /data/ofm/loadbalancer/logs/run.log 2>&1
 
 
+# check
+* * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py check >> /data/ofm/loadbalancer/logs/run.log 2>&1

scripts/loadbalancer/loadbalancer.py

@@ -5,6 +5,7 @@ import json
 import click
 import requests
 from dotenv import dotenv_values
+from loadbalancer_lib import OFM_CONFIG_DIR
 from loadbalancer_lib.cloudflare import get_zone_id, set_records_round_robin
 from loadbalancer_lib.curl import pycurl_get, pycurl_status
 from loadbalancer_lib.telegram_ import telegram_send_message
@@ -41,10 +42,18 @@ def fix():
 
 
 def check_or_fix(fix=False):
-    with open('/data/ofm/config/loadbalancer.json') as fp:
+    with open(OFM_CONFIG_DIR / 'loadbalancer.json') as fp:
         c = json.load(fp)
         # print(c)
 
+    if not c['http_host_list']:
+        telegram_send_message(
+            'OFM loadbalancer no hosts found on list, terminating',
+            c['telegram_token'],
+            c['telegram_chat_id'],
+        )
+        return
+
     try:
         results_by_ip = {}
         working_hosts = set()
@@ -56,15 +65,13 @@ def check_or_fix(fix=False):
 
         for host_ip, host_is_ok in results_by_ip.items():
             if not host_is_ok:
-                message = f'OFM ERROR with host: {host_ip}'
-                print(message)
+                message = f'OFM loadbalancer ERROR with host: {host_ip}'
                 telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
             else:
                 working_hosts.add(host_ip)
 
     except Exception as e:
-        message = f'OFM ERROR with loadbalancer: {e}'
-        print(message)
+        message = f'OFM loadbalancer ERROR with loadbalancer: {e}'
         telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
         return
 
@@ -77,13 +84,11 @@ def check_or_fix(fix=False):
             working_hosts = set(c['http_host_list'])
 
             message = 'OFM loadbalancer FIX found no working hosts, reverting to full list!'
-            print(message)
             telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
 
         updated = update_records(c, working_hosts)
         if updated:
             message = f'OFM loadbalancer FIX modified records, new records: {working_hosts}'
-            print(message)
             telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
 
 
@@ -98,8 +103,9 @@ def run_area(c, area):
         try:
             check_host(c['domain_ledns'], host_ip, area, target_version)
             results[host_ip] = True
-        except Exception:
+        except Exception as e:
             results[host_ip] = False
+            print(e)
 
     return results
 
@@ -126,7 +132,7 @@ def get_target_version(area):
 
 
 def update_records(c, working_hosts) -> bool:
-    config = dotenv_values('/data/ofm/config/cloudflare.ini')
+    config = dotenv_values(OFM_CONFIG_DIR / 'cloudflare.ini')
     cloudflare_api_token = config['dns_cloudflare_api_token']
 
     domain = '.'.join(c['domain_ledns'].split('.')[-2:])
@@ -144,15 +150,6 @@ def update_records(c, working_hosts) -> bool:
         cloudflare_api_token=cloudflare_api_token,
     )
 
-    updated |= set_records_round_robin(
-        zone_id=zone_id,
-        name=c['domain_cf'],
-        host_ip_set=working_hosts,
-        proxied=True,
-        comment='domain_cf',
-        cloudflare_api_token=cloudflare_api_token,
-    )
-
     return updated
 
 

scripts/loadbalancer/loadbalancer_lib/__init__.py

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+
+if Path('/data/ofm/config').exists():
+    OFM_CONFIG_DIR = Path('/data/ofm/config')
+else:
+    OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
+
+assert OFM_CONFIG_DIR.exists()

scripts/loadbalancer/loadbalancer_lib/curl.py

@@ -1,4 +1,5 @@
 from io import BytesIO
+from pathlib import Path
 
 import pycurl
 
@@ -11,7 +12,11 @@ def pycurl_status(url, domain, host_ip):
 
     c = pycurl.Curl()
     c.setopt(c.URL, url)
-    c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
+
+    # linux needs CA certs specified manually
+    if Path('/etc/ssl/certs/ca-certificates.crt').exists():
+        c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
+
     c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
     c.setopt(c.NOBODY, True)
     c.setopt(c.TIMEOUT, 5)
@@ -31,7 +36,11 @@ def pycurl_get(url, domain, host_ip):
     buffer = BytesIO()
     c = pycurl.Curl()
     c.setopt(c.URL, url)
-    c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
+
+    # linux needs CA certs specified manually
+    if Path('/etc/ssl/certs/ca-certificates.crt').exists():
+        c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
+
     c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
     c.setopt(c.WRITEDATA, buffer)
     c.setopt(c.TIMEOUT, 5)
@@ -40,6 +49,6 @@ def pycurl_get(url, domain, host_ip):
     c.close()
 
     if status_code != 200:
-        raise ValueError('non-200')
+        raise ValueError(f'status code: {status_code}')
 
     return buffer.getvalue().decode('utf8')

scripts/loadbalancer/loadbalancer_lib/telegram_.py

@@ -2,6 +2,8 @@ import requests
 
 
 def telegram_send_message(message, bot_token, chat_id):
+    print(message)
+
     url = f'https://api.telegram.org/bot{bot_token}/sendMessage'
 
     payload = {'chat_id': chat_id, 'text': message}
@@ -9,6 +11,6 @@ def telegram_send_message(message, bot_token, chat_id):
     response = requests.post(url, data=payload)
 
     if response.status_code == 200:
-        print('Message sent successfully!')
+        print('  Message sent successfully!')
     else:
-        print('Failed to send message:', response.text)
+        print('  Failed to send message:', response.text)

scripts/setversion/setup.py

@@ -0,0 +1,17 @@
+from setuptools import find_packages, setup
+
+
+requirements = [
+    'click',
+    'requests',
+    'pycurl',
+    'python-dotenv',
+    'questionary',
+]
+
+
+setup(
+    python_requires='>=3.10',
+    install_requires=requirements,
+    packages=find_packages(),
+)

scripts/setversion/setversion.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import subprocess
+
+import click
+import questionary
+from setversion_lib import RCLONE_BIN, RCLONE_CONF
+
+
+@click.group()
+def cli():
+    """
+    Sets deployed reference versions
+    """
+
+
+@cli.command()
+@click.argument('area', required=True)
+def interactive(area):
+    versions = get_available_versions(area)[::-1]
+
+    choices = [questionary.Choice(title=r, value=i) for i, r in enumerate(versions)]
+    answer = questionary.select(f'Select version for: {area}', choices=choices).ask()
+
+    selected = versions[answer]
+
+    set_version(area, selected)
+
+
+def get_available_versions(area):
+    p = subprocess.run(
+        [
+            RCLONE_BIN,
+            'cat',
+            f'remote:ofm-{area}/dirs.txt',
+        ],
+        env=dict(RCLONE_CONFIG=RCLONE_CONF),
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    versions = [l.strip() for l in p.stdout.strip().splitlines()]
+    versions.sort()
+
+    return versions
+
+
+def set_version(area, version):
+    subprocess.run(
+        [
+            RCLONE_BIN,
+            'rcat',
+            f'remote:ofm-assets/versions/deployed_{area}.txt',
+        ],
+        env=dict(RCLONE_CONFIG=RCLONE_CONF),
+        check=True,
+        input=version.encode(),
+    )
+
+
+if __name__ == '__main__':
+    cli()

scripts/setversion/setversion_lib/__init__.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+
+
+if Path('/data/ofm/config').exists():
+    OFM_CONFIG_DIR = Path('/data/ofm/config')
+else:
+    OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
+
+assert OFM_CONFIG_DIR.exists()
+
+RCLONE_CONF = OFM_CONFIG_DIR / 'rclone.conf'
+
+if Path('/opt/homebrew/bin/rclone').exists():
+    RCLONE_BIN = '/opt/homebrew/bin/rclone'
+else:
+    RCLONE_BIN = 'rclone'

scripts/tile_gen/upload_manager.py

@@ -155,43 +155,5 @@ def index():
     make_indexes()
 
 
-@cli.command()
-def set_latest_versions():
-    """
-    Sets the latest version as the deployed one
-    """
-
-    for area in AREAS:
-        print(f'setting latest version for {area}')
-
-        p = subprocess.run(
-            [
-                'rclone',
-                'cat',
-                f'remote:ofm-{area}/dirs.txt',
-            ],
-            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-            check=True,
-            capture_output=True,
-            text=True,
-        )
-        versions = [l.strip() for l in p.stdout.strip().splitlines()]
-        versions.sort(reverse=True)
-
-        latest_version = versions[0]
-        print(latest_version)
-
-        subprocess.run(
-            [
-                'rclone',
-                'rcat',
-                f'remote:ofm-assets/versions/deployed_{area}.txt',
-            ],
-            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-            check=True,
-            input=latest_version.encode(),
-        )
-
-
 if __name__ == '__main__':
     cli()

ssh_lib/assets/nginx/cloudflare.conf

@@ -1,29 +0,0 @@
-# https://www.cloudflare.com/ips/
-
-set_real_ip_from 103.21.244.0/22;
-set_real_ip_from 103.22.200.0/22;
-set_real_ip_from 103.31.4.0/22;
-set_real_ip_from 104.16.0.0/13;
-set_real_ip_from 104.24.0.0/14;
-set_real_ip_from 108.162.192.0/18;
-set_real_ip_from 131.0.72.0/22;
-set_real_ip_from 141.101.64.0/18;
-set_real_ip_from 162.158.0.0/15;
-set_real_ip_from 172.64.0.0/13;
-set_real_ip_from 173.245.48.0/20;
-set_real_ip_from 188.114.96.0/20;
-set_real_ip_from 190.93.240.0/20;
-set_real_ip_from 197.234.240.0/22;
-set_real_ip_from 198.41.128.0/17;
-
-set_real_ip_from 2400:cb00::/32;
-set_real_ip_from 2405:8100::/32;
-set_real_ip_from 2405:b500::/32;
-set_real_ip_from 2606:4700::/32;
-set_real_ip_from 2803:f800::/32;
-set_real_ip_from 2a06:98c0::/29;
-set_real_ip_from 2c0f:f248::/32;
-
-# use any of the following two
-real_ip_header CF-Connecting-IP;
-#real_ip_header X-Forwarded-For;

ssh_lib/nginx.py

@@ -50,7 +50,7 @@ def nginx(c):
     put(c, f'{ASSETS_DIR}/nginx/nginx.conf', '/etc/nginx/')
     put(c, f'{ASSETS_DIR}/nginx/mime.types', '/etc/nginx/')
     put(c, f'{ASSETS_DIR}/nginx/default_disable.conf', '/data/nginx/sites')
-    put(c, f'{ASSETS_DIR}/nginx/cloudflare.conf', '/data/nginx/config')
+    # put(c, f'{ASSETS_DIR}/nginx/cloudflare.conf', '/data/nginx/config')
 
     sudo_cmd(c, 'curl https://ssl-config.mozilla.org/ffdhe2048.txt -o /etc/nginx/ffdhe2048.txt')
 

ssh_lib/tasks.py

@@ -72,29 +72,18 @@ def prepare_tile_gen(c):
 
 def upload_http_host_config(c):
     domain_le = dotenv_val('DOMAIN_LE').lower()
-    domain_cf = dotenv_val('DOMAIN_CF').lower()
     domain_ledns = dotenv_val('DOMAIN_LEDNS').lower()
     skip_planet = dotenv_val('SKIP_PLANET').lower() == 'true'
     le_email = dotenv_val('LE_EMAIL').lower()
 
-    if not (domain_le or domain_cf):
-        sys.exit('Please specify DOMAIN_LE or DOMAIN_CF in config/.env')
-
-    if domain_cf:
-        if (
-            not (CONFIG_DIR / 'certs' / 'ofm_cf.key').exists()
-            or not (CONFIG_DIR / 'certs' / 'ofm_cf.cert').exists()
-        ):
-            sys.exit(
-                'When using DOMAIN_CF, please put ofm_cf.key and ofm_cf.cert files in config/certs'
-            )
+    if not (domain_le or domain_ledns):
+        sys.exit('Please specify DOMAIN_LE or DOMAIN_LEDNS in config/.env')
 
     if domain_le and not le_email:
         sys.exit('Please add your email to LE_EMAIL when using DOMAIN_LE')
 
     host_config = {
         'domain_le': domain_le,
-        'domain_cf': domain_cf,
         'domain_ledns': domain_ledns,
         'skip_planet': skip_planet,
         'le_email': le_email,
@@ -222,13 +211,11 @@ def setup_ledns_writer(c):
 
 
 def setup_loadbalancer(c):
-    domain_cf = dotenv_val('DOMAIN_CF').lower()
     domain_ledns = dotenv_val('DOMAIN_LEDNS').lower()
     http_host_list = [h.strip() for h in dotenv_val('HTTP_HOST_LIST').split(',') if h.strip()]
     assert (CONFIG_DIR / 'cloudflare.ini').exists()
 
     config = {
-        'domain_cf': domain_cf,
         'domain_ledns': domain_ledns,
         'http_host_list': http_host_list,
         'telegram_token': dotenv_val('TELEGRAM_TOKEN'),

website/src/components/Logo.astro

@@ -4,7 +4,7 @@ const { title } = Astro.props
 
 <img src="/logo.jpg" alt="logo" height="200" class="logo" />
 
-<h1>{title}</h1>
+<h1 set:html={title} />
 
 <div class="icons">
   <a href="https://github.com/hyperknot/openfreemap" target="_blank"

website/src/content/index/rest.md

@@ -22,7 +22,7 @@ I waited for years for someone to offer this service but realized that no one wa
 
 I'll share more about the reasons in a future [blog post](https://blog.hyperknot.com/). Feel free to subscribe.
 
-## How can this work? How can a one-person project offer unlimited map hosting for free?
+## How can this work [technically and financially]?
 
 There is no technical reason why map hosting costs as much as it does today. Vector tiles are just static files. It's true that serving hundreds of millions of files is not easy, but at the end of the day, they are just files.
 
@@ -40,12 +40,6 @@ Special thanks go to [Michael Barry](https://github.com/msbarry) for developing
 
 The [styles](https://github.com/hyperknot/openfreemap-styles) are forked and heavily modified.
 
-## Domains
-
-`tiles.openfreemap.org` - Cloudflare proxied
-
-`direct.openfreemap.org` - direct connection, Round-Robin DNS
-
 ## Attribution
 
 Attribution is required. If you are using MapLibre, they are automatically added, you have nothing to do.

website/src/pages/index.astro

@@ -21,7 +21,7 @@ import { Content as RestText } from '../content/index/rest.md'
   <div class="container">
     <p>
       Have a look at the default styles and read more about how to integrate it to your website or
-      app here:
+      app:
     </p>
 
     <a class="quick-start-button" href="/quick_start">Quick Start Guide</a>

website/src/pages/quick_start.astro

@@ -13,7 +13,7 @@ import Donate from '../components/Donate.astro'
 ---
 
 <Layout title="OpenFreeMap Quick Start Guide">
-  <Logo title="OpenFreeMap Quick Start Guide" />
+  <Logo title="OpenFreeMap<br>Quick Start Guide" />
 
   <div class="container" style="margin-top:30px; margin-bottom: 30px;">
     <p>

website/src/styles/global.css

@@ -104,21 +104,23 @@ hr {
 }
 
 .quick-start-button {
-  /*display: block;*/
+  display: block;
   text-decoration: none;
   color: white;
   font-weight: bold;
   letter-spacing: 0.05rem;
-  font-size: 15px;
+  font-size: 18px;
   border-radius: 20px;
-  padding: 6px 16px;
-  margin-top: 15px;
-  background: linear-gradient(32deg, #03a9f4, transparent) #f441a5;
+  padding: 15px 0;
+  margin: 2em auto 5em;
+  width: 230px;
+  text-align: center;
+  background: linear-gradient(32deg, #0070a2, transparent) #59c15a;
   transition: background-color 1s;
 
   &:hover,
   &:focus {
-    background-color: #fdb900;
+    background-color: #dea31d;
   }
 }