From b41ced91dea49fe2ec415f79d487b7f80718d0ee Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Sat, 24 Feb 2024 01:59:13 +0100 Subject: [PATCH] nginx rework --- README.md | 21 +--- config/.env.sample | 1 + docs/dev_setup.md | 3 - docs/quick_notes/http_benchmark.md | 13 +-- docs/self_hosting.md | 34 ++---- init-server.py | 8 +- scripts/http_host/host_manager.py | 13 +-- scripts/http_host/http_host_lib/__init__.py | 9 +- scripts/http_host/http_host_lib/nginx.py | 40 ++++--- scripts/http_host/http_host_lib/nginx/cf.conf | 27 +++++ .../http_host_lib/nginx/location_static.conf | 71 +++++++++++++ .../http_host_lib/templates/nginx_cf.conf | 100 ------------------ 12 files changed, 158 insertions(+), 182 deletions(-) create mode 100644 scripts/http_host/http_host_lib/nginx/cf.conf create mode 100644 scripts/http_host/http_host_lib/nginx/location_static.conf delete mode 100644 scripts/http_host/http_host_lib/templates/nginx_cf.conf diff --git a/README.md b/README.md index d689166..0bff0ed 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ It is truly **open-source**: everything, including the full production setup, is ## Goals of this project -The goal of this project is to provide free, production-quality vector-tile hosting using existing tools. +The goal of this project is to provide free, production-quality vector-tile hosting using existing tools. Currently these tools are: [OpenStreetMap](https://www.openstreetmap.org/copyright), [OpenMapTiles](https://github.com/openmaptiles/openmaptiles), [Planetiler](https://github.com/onthegomap/planetiler) , [MapLibre](https://maplibre.org/) and [Natural Earth](https://www.naturalearthdata.com/) and soon [tilemaker](https://github.com/systemed/tilemaker). OFM does not want to be an alternative to any of these projects. If the community decides, we can replace any of these tools. @@ -29,6 +29,7 @@ Contributions are more than welcome! The only way this project can possibly work is to be super focused about what it is and what it isn't. OFM has the following limitations by design: 1. OFM is not providing: + - search or geocoding - route calculation, navigation or directions - static image generation @@ -41,7 +42,7 @@ The only way this project can possibly work is to be super focused about what it This repo is also Docker free. If someone wants to make a Docker-based version of this, I'm more than happy to link it here. -3. OFM does not promise worry-free automatic updates for self-hosters. Only use the autoupdate version of http-host if you keep a close eye on this repo. +3. OFM does not promise worry-free automatic updates for self-hosters. Only use the autoupdate version of http-host if you keep a close eye on this repo. ## Code structure @@ -53,7 +54,7 @@ This sets up everything on a clean Ubuntu server. You run it locally and it sets #### HTTP host - scripts/http_host -Inside `http_host`, all work is done by `host_manager.py`. +Inside `http_host`, all work is done by `host_manager.py`. It does the following: @@ -68,7 +69,7 @@ You can run `./host_manager.py --help` to see which options are available. Some #### tile generation - scripts/tile_gen -*note: Tile generation is 100% optional, as we are providing the processed full planet files for public download.* +_note: Tile generation is 100% optional, as we are providing the processed full planet files for public download._ The `tile_gen` scripts downloads a full planet OSM extract and runs it through Planetiler (or soon tilemaker). @@ -80,14 +81,10 @@ Finally, it's uploaded to a public Cloudflare R2 bucket using rclone. A very important part, probably needs the most work in the long term future. - - ## Self hosting See [self hosting docs](docs/self_hosting.md). - - ## BTRFS images Production-quality hosting of 300 million tiny files is hard. The average file size is just 450 byte. Dozens of tile servers have been written to tackle this problem, but they all have their limitations. @@ -98,8 +95,6 @@ This replaces a running service with a pure, file-system-level implementation. S I run some [benchmarks](docs/quick_notes/http_benchmark.md) on a Hetzner server, the aim was to saturate a gigabit connection. At the end, it was able to serve 30 Gbit on localhost, on a cold nginx cache. - - ## FAQ ### Full planet downloads @@ -141,8 +136,6 @@ Unfortunately, making range requests in 80 GB files just doesn't work in product If PMTiles implements splitting to <10 MB files, it can be a valid alternative to running servers. - - ## Contributing Contributors welcome! @@ -168,14 +161,10 @@ Tasks outside the scope of this project: See [dev setup docs](docs/dev_setup.md). - - ## Changelog v0.1 - everything works. 1 server for tile gen, 1 server for HTTP host. <- we are here! - - ## Attribution Attribution is required. If you are using MapLibre, they are automatically added, you have nothing to do. diff --git a/config/.env.sample b/config/.env.sample index ba14d7d..020695e 100644 --- a/config/.env.sample +++ b/config/.env.sample @@ -5,6 +5,7 @@ SSH_PASSWD= DOMAIN_DIRECT=direct.openfreemap.org # Domain via CloudFlare, using origin certificates +# Please put cf.key and cf.cert files in config/certs DOMAIN_CF=tiles.openfreemap.org # Skip the full planet download, useful for testing (true/false) diff --git a/docs/dev_setup.md b/docs/dev_setup.md index 3fc23e7..09220a2 100644 --- a/docs/dev_setup.md +++ b/docs/dev_setup.md @@ -1,7 +1,5 @@ # dev setup - - ### macOS On macOS, I recommend [OrbStack](https://orbstack.dev/). @@ -30,4 +28,3 @@ Then I run commands like the following: ./init-server.py http-host-once orb_my ./init-server.py debug orb_my ``` - diff --git a/docs/quick_notes/http_benchmark.md b/docs/quick_notes/http_benchmark.md index b8d976c..da09b8b 100644 --- a/docs/quick_notes/http_benchmark.md +++ b/docs/quick_notes/http_benchmark.md @@ -1,14 +1,12 @@ # wrk benchmarks -Real world usage, 500k requests replayed from server log. - - +Real world usage, 500k requests replayed from server log. ### Hetnzer dedicated server with NVME ssd #### localhost -clean cache after nginx restart. +clean cache after nginx restart. ``` service nginx restart @@ -44,8 +42,6 @@ Realistically this is the max over Gigabit connection. --- - - ### BuyVM KVM machine with 1 TB BuyVM Block Storage Slab Advertisement: 40Gbit+ InfiniBand RDMA Storage Fabric giving near local storage performance. @@ -73,8 +69,3 @@ Transfer/sec: 23.07MB ``` Abandoned the idea of using BuyVM, even though their unlimited bandwidth is quite unique in this price range in USA. - - - - - diff --git a/docs/self_hosting.md b/docs/self_hosting.md index 5a6bfc0..3e28efd 100644 --- a/docs/self_hosting.md +++ b/docs/self_hosting.md @@ -1,8 +1,6 @@ # Self-hosting Howto -*note: For most users, **you don't need to run anything**! The tiles are hosted free of charge, without registration. Read the "How can I use it?" section on https://openfreemap.org* - - +_note: For most users, **you don't need to run anything**! The tiles are hosted free of charge, without registration. Read the "How can I use it?" section on https://openfreemap.org_ When self-hosting, there are two tasks you can set up on a server (see details in the repo README). @@ -12,27 +10,23 @@ When self-hosting, there are two tasks you can set up on a server (see details i note: Tile generation is 100% optional, as we are providing the processed full planet files for public download. It also requires a beefy machine, see below. - - ### System requirements ##### Disk space -​ **http-host**: 300 GB for hosting a single run +​ **http-host**: 300 GB for hosting a single run -​ **tile-gen**: 500 GB for +​ **tile-gen**: 500 GB for ##### RAM -​ **http-host**: 4 GB +​ **http-host**: 4 GB -​ **tile-gen**: 64 GB+ RAM. +​ **tile-gen**: 64 GB+ RAM. ##### OS -​ **Ubuntu 22+** - - +​ **Ubuntu 22+** ### Limitations @@ -42,8 +36,6 @@ There are two limitations in the current beta version: - The domain is hard-coded to `tiles.openfreemap.org` - you have to edit this. - - --- ### Warning @@ -52,12 +44,9 @@ This project is made to run on clean servers or virtual machines dedicated for t --- - - - ## Instructions -Create virtualenv using: `source prepare-virtualenv.sh` +Create virtualenv using: `source prepare-virtualenv.sh` It's recommended to use [direnv](https://direnv.net/), to have automatic venv activation. @@ -71,11 +60,10 @@ It's recommended to use [direnv](https://direnv.net/), to have automatic venv ac Currently the domain is hard coded to `tiles.openfreemap.org`. Please search & replace this. -The script is made with long expiry CloudFlare origin certificates in mind, which are placed in the `config/certs` folder. For self-hosting you may want to use Let's Encrypt or similar automated tool. +The script is made with long expiry CloudFlare origin certificates in mind, which are placed in the `config/certs` folder. For self-hosting you may want to use Let's Encrypt or similar automated tool. If you know how to make Let's Encrypt work with Round Robin DNS, please comment in the Discussions. - #### 3. Deploy a http-host You run the deploy script locally. It'll connect to an SSH server, like this @@ -86,14 +74,10 @@ After this, go for a walk and by the time you come back it should be up and runn #### 4. Deploy tile-gen server (optional) -If you have a really beefy machine (see above) and you want to generate tiles yourself, you can run `./init-server.py tile-gen HOSTNAME`. +If you have a really beefy machine (see above) and you want to generate tiles yourself, you can run `./init-server.py tile-gen HOSTNAME`. Trigger a run manually, by running `planetiler_{area}.sh`. Recommended to use tmux or similar, as it can take days. - - - - ### HTTPS certs The current HTTPS system is made to use long term Cloudflare origin certificates. The same certificates are uploaded to all the servers. This is only possible because CF certs are valid for 15 years. diff --git a/init-server.py b/init-server.py index 54fa430..aac3541 100755 --- a/init-server.py +++ b/init-server.py @@ -150,8 +150,8 @@ def upload_https_host_files(c): put_dir(c, SCRIPTS_DIR / 'http_host' / 'http_host_lib', f'{HTTP_HOST_BIN}/http_host_lib') put_dir( c, - SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'templates', - f'{HTTP_HOST_BIN}/http_host_lib/templates', + SCRIPTS_DIR / 'http_host' / 'http_host_lib' / 'nginx', + f'{HTTP_HOST_BIN}/http_host_lib/nginx', ) c.sudo('chown -R ofm:ofm /data/ofm/http_host') @@ -256,7 +256,9 @@ def debug(hostname, user, port): upload_http_host_config(c) upload_https_host_files(c) - run_http_host_sync(c) + # run_http_host_sync(c) + + sudo_cmd(c, '/data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py nginx-sync') if __name__ == '__main__': diff --git a/scripts/http_host/host_manager.py b/scripts/http_host/host_manager.py index 041ba13..3a6f356 100755 --- a/scripts/http_host/host_manager.py +++ b/scripts/http_host/host_manager.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 import datetime -import json import subprocess import sys from pathlib import Path import click import requests -from http_host_lib import DEFAULT_ASSETS_DIR, DEFAULT_RUNS_DIR, MNT_DIR +from http_host_lib import DEFAULT_ASSETS_DIR, DEFAULT_RUNS_DIR, HOST_CONFIG, MNT_DIR from http_host_lib.download_assets import ( download_and_extract_asset_tar_gz, download_sprites, @@ -184,7 +183,7 @@ def sync(ctx): download_done = False download_done += ctx.invoke(download_tileset, area='monaco') - if not host_config.get('skip_planet'): + if not HOST_CONFIG.get('skip_planet'): download_done += ctx.invoke(download_tileset, area='planet') if download_done: @@ -199,11 +198,5 @@ def sync(ctx): if __name__ == '__main__': - try: - with open('/data/ofm/config/http_host.json') as fp: - host_config = json.load(fp) - except Exception: - host_config = {} - - print(host_config) + print(HOST_CONFIG) cli() diff --git a/scripts/http_host/http_host_lib/__init__.py b/scripts/http_host/http_host_lib/__init__.py index 63f4722..7d41ac3 100644 --- a/scripts/http_host/http_host_lib/__init__.py +++ b/scripts/http_host/http_host_lib/__init__.py @@ -1,10 +1,17 @@ +import json from pathlib import Path -TEMPLATES_DIR = Path(__file__).parent / 'templates' +NGINX_DIR = Path(__file__).parent / 'nginx' DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs') DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets') MNT_DIR = Path('/mnt/ofm') OFM_CONFIG_DIR = Path('/data/ofm/config') + +try: + with open('/data/ofm/config/http_host.json') as fp: + HOST_CONFIG = json.load(fp) +except Exception: + HOST_CONFIG = {} diff --git a/scripts/http_host/http_host_lib/nginx.py b/scripts/http_host/http_host_lib/nginx.py index ed10636..db644f6 100644 --- a/scripts/http_host/http_host_lib/nginx.py +++ b/scripts/http_host/http_host_lib/nginx.py @@ -2,13 +2,33 @@ import subprocess import sys from pathlib import Path -from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR, OFM_CONFIG_DIR, TEMPLATES_DIR +from http_host_lib import DEFAULT_RUNS_DIR, HOST_CONFIG, MNT_DIR, NGINX_DIR, OFM_CONFIG_DIR def write_nginx_config(): - with open(TEMPLATES_DIR / 'nginx_cf.conf') as fp: - nginx_template = fp.read() + location_str, curl_text = create_location_blocks() + curl_text_mix = '' + if HOST_CONFIG['domain_cf']: + with open(NGINX_DIR / 'cf.conf') as fp: + cf_template = fp.read() + + cf_template = cf_template.replace('__LOCATION_BLOCKS__', location_str) + cf_template = cf_template.replace('__DOMAIN__', HOST_CONFIG['domain_cf']) + + curl_text_mix += curl_text.replace('__DOMAIN__', HOST_CONFIG['domain_cf']) + + with open('/data/nginx/sites/cf.conf', 'w') as fp: + fp.write(cf_template) + print(' nginx config written') + + subprocess.run(['nginx', '-t'], check=True) + subprocess.run(['systemctl', 'reload', 'nginx'], check=True) + + print(curl_text_mix) + + +def create_location_blocks(): location_str = '' curl_text = '' @@ -22,21 +42,15 @@ def write_nginx_config(): curl_text = ( '\ntest with:\n' f'curl -H "Host: ofm" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n' - f'curl -I https://tiles.openfreemap.org/{area}/{version}/14/8529/5975.pbf' + f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf' ) location_str += create_latest_locations() - nginx_template = nginx_template.replace('___LOCATION_BLOCKS___', location_str) + with open(NGINX_DIR / 'location_static.conf') as fp: + location_str += '\n' + fp.read() - with open('/data/nginx/sites/ofm-tiles-org.conf', 'w') as fp: - fp.write(nginx_template) - print(' nginx config written') - - subprocess.run(['nginx', '-t'], check=True) - subprocess.run(['systemctl', 'reload', 'nginx'], check=True) - - print(curl_text) + return location_str, curl_text def create_version_location(area: str, version: str, subdir: Path) -> str: diff --git a/scripts/http_host/http_host_lib/nginx/cf.conf b/scripts/http_host/http_host_lib/nginx/cf.conf new file mode 100644 index 0000000..378811b --- /dev/null +++ b/scripts/http_host/http_host_lib/nginx/cf.conf @@ -0,0 +1,27 @@ +server { + server_name ofm __DOMAIN__; + + # ssl: https://ssl-config.mozilla.org / modern config + # to be used with the Cloudflare proxied endpoint + + listen 80; + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + + ssl_certificate /data/nginx/certs/cf.cert; + ssl_certificate_key /data/nginx/certs/cf.key; + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + ssl_protocols TLSv1.3; + ssl_prefer_server_ciphers off; + + # access log disabled by default + #access_log /data/ofm/http_host/logs_nginx/cf-access.log access_json buffer=32k; + access_log off; + + error_log /data/ofm/http_host/logs_nginx/cf-error.log; + + __LOCATION_BLOCKS__ +} diff --git a/scripts/http_host/http_host_lib/nginx/location_static.conf b/scripts/http_host/http_host_lib/nginx/location_static.conf new file mode 100644 index 0000000..f60792e --- /dev/null +++ b/scripts/http_host/http_host_lib/nginx/location_static.conf @@ -0,0 +1,71 @@ +location /fonts/ { + # trailing slash + + alias /data/ofm/http_host/assets/fonts/ofm/; # trailing slash + try_files $uri =404; + + expires 1w; + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; +} + +location /styles/ { + # trailing slash + + alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash + try_files $uri.json =404; + + expires 1d; + default_type application/json; + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; +} + +location /natural_earth/ { + # trailing slash + + alias /data/ofm/http_host/assets/natural_earth/ofm/; # trailing slash + try_files $uri =404; + + expires 10y; + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; +} + +location /sprites/ { + # trailing slash + + alias /data/ofm/http_host/assets/sprites/; # trailing slash + try_files $uri =404; + + expires 10y; + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; +} + + +# we need to handle missing tiles as valid request returning empty string +location @empty_tile { + return 200 ''; + + expires 10y; + default_type application/vnd.mapbox-vector-tile; + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header Cache-Control public; +} + +location = / { + return 302 https://openfreemap.org; +} + +# catch-all block to deny all other requests +location / { + deny all; + + error_log /data/ofm/http_host/logs_nginx/tiles-org-error.log error; +} diff --git a/scripts/http_host/http_host_lib/templates/nginx_cf.conf b/scripts/http_host/http_host_lib/templates/nginx_cf.conf deleted file mode 100644 index 81c7982..0000000 --- a/scripts/http_host/http_host_lib/templates/nginx_cf.conf +++ /dev/null @@ -1,100 +0,0 @@ -server { - server_name ofm tiles.openfreemap.org; - - # ssl: https://ssl-config.mozilla.org / modern config - # to be used with the Cloudflare proxied endpoint - - listen 80; - listen 443 ssl; - listen [::]:443 ssl; - http2 on; - - ssl_certificate /data/nginx/certs/cf.cert; - ssl_certificate_key /data/nginx/certs/cf.key; - ssl_session_timeout 1d; - ssl_session_cache shared:MozSSL:10m; # about 40000 sessions - ssl_session_tickets off; - ssl_protocols TLSv1.3; - ssl_prefer_server_ciphers off; - - # access log disabled by default - #access_log /data/ofm/http_host/logs_nginx/tiles-org-access.log access_json buffer=32k; - access_log off; - - error_log /data/ofm/http_host/logs_nginx/tiles-org-error.log; - - ___LOCATION_BLOCKS___ - - location /fonts/ { - # trailing slash - - alias /data/ofm/http_host/assets/fonts/ofm/; # trailing slash - try_files $uri =404; - - expires 1w; - - add_header 'Access-Control-Allow-Origin' '*' always; - add_header Cache-Control public; - } - - location /styles/ { - # trailing slash - - alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash - try_files $uri.json =404; - - expires 1d; - default_type application/json; - - add_header 'Access-Control-Allow-Origin' '*' always; - add_header Cache-Control public; - } - - location /natural_earth/ { - # trailing slash - - alias /data/ofm/http_host/assets/natural_earth/ofm/; # trailing slash - try_files $uri =404; - - expires 10y; - - add_header 'Access-Control-Allow-Origin' '*' always; - add_header Cache-Control public; - } - - location /sprites/ { - # trailing slash - - alias /data/ofm/http_host/assets/sprites/; # trailing slash - try_files $uri =404; - - expires 10y; - - add_header 'Access-Control-Allow-Origin' '*' always; - add_header Cache-Control public; - } - - - - # we need to handle missing tiles as valid request returning empty string - location @empty_tile { - return 200 ''; - - expires 10y; - default_type application/vnd.mapbox-vector-tile; - - add_header 'Access-Control-Allow-Origin' '*' always; - add_header Cache-Control public; - } - - location = / { - return 302 https://openfreemap.org; - } - - # catch-all block to deny all other requests - location / { - deny all; - - error_log /data/ofm/http_host/logs_nginx/tiles-org-error.log error; - } -}