From c097c58a3e0544d4aacf4196ec364b2e0a7bd886 Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Thu, 22 Feb 2024 20:29:58 +0100 Subject: [PATCH] mime types fix --- README.md | 48 +++++++---- ssh_lib/assets/nginx/mime.types | 139 ++++++++++++++++++++++++++++++++ ssh_lib/nginx.py | 6 +- 3 files changed, 171 insertions(+), 22 deletions(-) create mode 100644 ssh_lib/assets/nginx/mime.types diff --git a/README.md b/README.md index 9e5eab9..7bc669a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ## What is OpenFreeMap? -OpenFreeMap (OFM) provides free map hosting so you can display custom maps on your website and apps. +OpenFreeMap provides free map hosting so you can display custom maps on your website and apps. It is truly **free**: there are no limits on the number of map views or requests you can make, nor on how you use your map. There is no registration page, user database, API keys, or cookies. @@ -20,13 +20,13 @@ Currently these tools are: [OpenStreetMap](https://www.openstreetmap.org/copyrig The scope of this repo is limited (see below). Once we figure out the technical details, ideally, there should be few commits here, while everything keeps working: the map tiles are automatically generated, servers are automatically updated and load balancing takes care of failing servers. -The [styles repo](https://github.com/hyperknot/openfreemap-styles) - on the other hand - is a possibly never ending development. +The [styles repo](https://github.com/hyperknot/openfreemap-styles), on the other hand, is continuously being developed. Contributions are more than welcome! ## Limitations of this project -The only way this project can possibly work is to be super focused about what it is and what it isn't. OFM has the following limitations by design. +The only way this project can possibly work is to be super focused about what it is and what it isn't. OFM has the following limitations by design: 1. OFM is not providing: - search or geocoding @@ -51,45 +51,59 @@ The project has the following parts This sets up everything on a clean Ubuntu server. You run it locally and it sets up the server via SSH. You specify `--tile-gen` and/or `--http-host` at startup. +#### HTTP host - scripts/http_host + +Inside `http_host`, all work is done by `host_manager.py`. + +It does the following: + +- checks the most up-to-date files in the public buckets +- downloads/extracts them locally, if needed +- mounts the downloaded BTRFS images in `/mnt/ofm` +- creates the correct TileJSON file +- creates the correct nginx config +- reloads nginx + +You can run `./host_manager.py --help` to see which options are available. Some commands can be run locally, including on non-linux machines. + #### tile generation - scripts/tile_gen +*note: Tile generation is 100% optional, as we are providing the processed full planet files for public download.* + The `tile_gen` scripts downloads a full planet OSM extract and runs it through Planetiler (or soon tilemaker). Currently a run is triggered manually, by running `planetiler_{area}.sh`. The created .mbtiles file is then extracted into a BTRFS partition image using the custom [extract_mbtiles](scripts/tile_gen/extract_mbtiles) script. The partition is shrunk using the [shrink_btrfs](scripts/tile_gen/shrink_btrfs) script. Finally, it's uploaded to a public Cloudflare R2 bucket using rclone. -#### HTTP host - scripts/http_host - -Inside `http_host`, all work is done by `host_manager.py`. It checks the most up-to-date files in the public buckets and downloads/extracts them locally, if needed. - -It mounts the downloaded BTRFS images in `/mnt/ofm`, creates the correct TileJSON file and updates nginx with the correct config. - -You can run `./host_manager.py --help` to see which options are available. Some commands can be run locally, including on non-linux machines. - #### styles - [styles repo](https://github.com/hyperknot/openfreemap-styles) A very important part, probably needs the most work in the long term future. ## How to run? -Use Python 3.10/3.11. +*note: For most users, **you don't need to run anything**! The tiles are provided free of charge, without registration. Read the "How can I use it?" section on https://openfreemap.org* + +The instructions below are intended only for those who have a large server and would like to self-host. + +Use Python 3.10/3.11. Create virtualenv using: `source prepare-virtualenv.sh` It's recommended to use [direnv](https://direnv.net/), to have automatic venv activation. +*note: Currently the domains are hard coded, so you'll need to edit the nginx templates using a text editor or sed.* + ##### 1. Prepare config folder 1. copy the .sample files and change the values 2. SSH_PASSWD is only needed if you don't use SSH keys. -3. rclone.conf is only needed for uploading. For http_host there is no need for this file. +3. rclone.conf is only needed for tile generation. For http_host there is no need for this file. -4. certs - these are the certs for nginx. If you put a cert here, it'll be uploaded to `/data/nginx/certs`. +4. certs - used by nginx for HTTPS, they are uploaded to `/data/nginx/certs`. - Currently the nginx config is hard coded to use for `openfreemap.org.cert` and `openfreemap.org.key`. ##### 2. Deploy a HTTP host @@ -142,7 +156,7 @@ Replace the `20231221_134737_pt` part with any newer run, from the [index file]( ### HTTPS certs -The current HTTPS system is made to use long term Cloudflare origin certificates. The same certificates are uploaded to all the server. This is only possible because CF certs are valid for 15 years. +The current HTTPS system is made to use long term Cloudflare origin certificates. The same certificates are uploaded to all the servers. This is only possible because CF certs are valid for 15 years. Once Load Balancing on CF is working, next step will be to integrate Let's Encrypt. If you know how to do this, please comment in the Discussions. @@ -180,7 +194,7 @@ Smaller tasks: - Add tilemaker, so we see the difference between planetiler and tilemaker and they can both validate their output based on this comparison. - Figure out how to use Let's Encrypt on multiple servers with Round Robin DNS. -- Cloudflare worker for indexing the public buckets, instead of manually generating index.txt files. +- Cloudflare worker for indexing the public buckets, instead of generating index.txt files. - Some of the POI icons are missing in the styles. Bigger tasks: diff --git a/ssh_lib/assets/nginx/mime.types b/ssh_lib/assets/nginx/mime.types new file mode 100644 index 0000000..03dde44 --- /dev/null +++ b/ssh_lib/assets/nginx/mime.types @@ -0,0 +1,139 @@ +types { + + # Data interchange + + application/atom+xml atom; + application/json json map topojson; + application/ld+json jsonld; + application/rss+xml rss; + # Normalize to standard type. + # https://tools.ietf.org/html/rfc7946#section-12 + application/geo+json geojson; + application/xml xml; + # Normalize to standard type. + # https://tools.ietf.org/html/rfc3870#section-2 + application/rdf+xml rdf; + + + # JavaScript + + # Servers should use text/javascript for JavaScript resources. + # https://html.spec.whatwg.org/multipage/scripting.html#scriptingLanguages + text/javascript js mjs; + application/wasm wasm; + + + # Manifest files + + application/manifest+json webmanifest; + application/x-web-app-manifest+json webapp; + text/cache-manifest appcache; + + + # Media files + + audio/midi mid midi kar; + audio/mp4 aac f4a f4b m4a; + audio/mpeg mp3; + audio/ogg oga ogg opus; + audio/x-realaudio ra; + audio/x-wav wav; + image/apng apng; + image/avif avif avifs; + image/bmp bmp; + image/gif gif; + image/jpeg jpeg jpg; + image/jxl jxl; + image/jxr jxr hdp wdp; + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/webp webp; + image/x-jng jng; + video/3gpp 3gp 3gpp; + video/mp4 f4p f4v m4v mp4; + video/mpeg mpeg mpg; + video/ogg ogv; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-mng mng; + video/x-ms-asf asf asx; + video/x-msvideo avi; + + # Serving `.ico` image files with a different media type + # prevents Internet Explorer from displaying then as images: + # https://github.com/h5bp/html5-boilerplate/commit/37b5fec090d00f38de64b591bcddcb205aadf8ee + + image/x-icon cur ico; + + + # Microsoft Office + + application/msword doc; + application/vnd.ms-excel xls; + application/vnd.ms-powerpoint ppt; + application/vnd.openxmlformats-officedocument.wordprocessingml.document docx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx; + application/vnd.openxmlformats-officedocument.presentationml.presentation pptx; + + + # Web fonts + + font/woff woff; + font/woff2 woff2; + application/vnd.ms-fontobject eot; + font/ttf ttf; + font/collection ttc; + font/otf otf; + + + # Other + + application/java-archive ear jar war; + application/mac-binhex40 hqx; + application/octet-stream bin deb dll dmg exe img iso msi msm msp safariextz; + application/pdf pdf; + application/postscript ai eps ps; + application/rtf rtf; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/vnd.wap.wmlc wmlc; + application/x-7z-compressed 7z; + application/x-bb-appworld bbaw; + application/x-bittorrent torrent; + application/x-chrome-extension crx; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-opera-extension oex; + application/x-perl pl pm; + application/x-pilot pdb prc; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert crt der pem; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xslt+xml xsl; + application/zip zip; + text/calendar ics; + text/css css; + text/csv csv; + text/html htm html shtml; + text/markdown md markdown; + text/mathml mml; + text/plain txt; + text/vcard vcard vcf; + text/vnd.rim.location.xloc xloc; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/vtt vtt; + text/x-component htc; + +} diff --git a/ssh_lib/nginx.py b/ssh_lib/nginx.py index ebe5674..5257496 100644 --- a/ssh_lib/nginx.py +++ b/ssh_lib/nginx.py @@ -46,6 +46,7 @@ def nginx(c): ) put(c, f'{ASSETS_DIR}/nginx/nginx.conf', '/etc/nginx/') + put(c, f'{ASSETS_DIR}/nginx/mime.types', '/etc/nginx/') put(c, f'{ASSETS_DIR}/nginx/default_disable.conf', '/data/nginx/sites') put(c, f'{ASSETS_DIR}/nginx/cloudflare.conf', '/data/nginx/config') @@ -53,11 +54,6 @@ def nginx(c): c.sudo('service nginx restart') -def mime_types(c): - # TODO - pass - - def certbot(c): apt_get_install(c, 'snapd')