work

Thanks a lot, good catch!

.envrc

@@ -2,6 +2,6 @@
 # auto-activate python virtualenv
 # https://github.com/direnv/direnv
 
-source venv/bin/activate
+source .venv/bin/activate
 
 unset PS1

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [hyperknot]

.github/workflows/deploy.yml

@@ -1,40 +0,0 @@
-name: Deploy to GitHub Pages
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - 'website/**'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-concurrency:
-  group: website
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout your repository using git
-        uses: actions/checkout@v4
-      - name: Install, build, and upload your site output
-        uses: withastro/action@v2
-        with:
-          path: website
-          package-manager: pnpm@latest
-
-  deploy:
-    needs: build
-    runs-on: ubuntu-latest
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4

.gitignore

@@ -19,3 +19,6 @@ venv
 
 
 /pnpm-lock.yaml
+
+/deploy-*.sh
+tmp.txt

.prettierrc.js

@@ -1,18 +0,0 @@
-const config = {
-  printWidth: 100,
-  semi: false,
-  singleQuote: true,
-  arrowParens: 'avoid',
-
-  plugins: ['prettier-plugin-astro'],
-  overrides: [
-    {
-      files: '*.astro',
-      options: {
-        parser: 'astro',
-      },
-    },
-  ],
-}
-
-module.exports = config

.ruff.toml

@@ -1,48 +1,57 @@
-target-version = "py310"
+target-version = "py313"
+unsafe-fixes = true
 line-length = 100
-extend-exclude = ["temp"]
+extend-exclude = ["alembic", "*.ipynb", "temp"]
-
 
 
 lint.select = [
-  "E",   # pycodestyle errors
-  "W",   # pycodestyle warnings
-  "F",   # pyflakes
-  "I",   # isort
-  'UP',  # pyupgrade
   'A',   # flake8-builtins
   "C4",  # flake8-comprehensions
+  'DTZ', # flake8-datetimez
+  "E",   # pycodestyle errors
   'EXE', # flake8-executable
+  "F",   # Pyflakes
   'FA',  # flake8-future-annotations
+  "I",   # isort
   'PT',  # flake8-pytest-style
   'RSE', # flake8-raise
   'SIM', # flake8-simplify
-  'DTZ', # flake8-datetimez, https://beta.ruff.rs/docs/rules/#flake8-datetimez-dtz
+  'UP',  # pyupgrade
+  "W",   # pycodestyle warnings
 ]
 
 lint.ignore = [
   'A003',
+  # 'C408',  # keep dict() as-is
+  'DTZ007', # naive datetime.strptime() without %z
   'E501',
   'E711',
   'E712',
-  # 'E721', # type comparison
+  'E721',   # type() comparison
+  # 'E722', # bare except
   'E741',
+  'EXE003', # shebang should contain "python"
   'F401',   # unused imports
   'F841',
-  'PT004',
+  # 'PT018', # assertion should be broken into multiple parts
   'SIM102',
+  'SIM103', # return the condition directly
   'SIM105',
   'SIM108',
+  # 'SIM110', # use any() instead of a for loop
+  # 'SIM114',
   'SIM115',
+  # 'UP007', # use X | Y instead of Union[X, Y]
+  # 'UP032', # use an f-string instead of format()
+  # 'UP046', # prefer type parameters over Generic subclasses
 ]
 
 [format]
 quote-style = "single"
 
 [lint.isort]
-known-first-party = ["ssh_lib"]
+known-first-party = ["lib", "api", "deploy", "ssh_lib"]
 lines-after-imports = 2
 
 [lint.flake8-comprehensions]
 allow-dict-calls-with-keyword-arguments = true
-

README.md

@@ -1,16 +1,16 @@
-<a href="https://openfreemap.org/"><img src="website/assets/logo.jpg" alt="logo" height="200" class="logo" /></a>
+<a href="https://openfreemap.org/"><img src="https://openfreemap.org/logo.jpg" alt="logo" height="200" class="logo" /></a>
 
 # OpenFreeMap
 
-[openfreemap.org](https://openfreemap.org)
+OpenFreeMap lets you display custom maps on your website and apps for free.
 
-## What is OpenFreeMap?
+You can either [self-host](docs/self_hosting.md) or use our public instance. Everything is **open-source**, including the full production setup — there’s no 'open-core' model here. The map data comes from OpenStreetMap.
 
-OpenFreeMap provides free map hosting so you can display custom maps on your website and apps.
+Using our **public instance** is completely free: there are no limits on the number of map views or requests. There’s no registration, no user database, no API keys, and no cookies. We aim to cover the running costs of our public instance through donations.
 
-It is truly **free**: there are no limits on the number of map views or requests you can make, nor on how you use your map. There is no registration page, user database, API keys, or cookies.
+We also provide **weekly** full planet downloads both in Btrfs and MBTiles formats.
 
-It is truly **open-source**: everything, including the full production setup, is in this repo. Map data is from OpenStreetMap.
+#### Quick introduction and how to guide: [https://openfreemap.org/](https://openfreemap.org/)
 
 ## Goals of this project
 
@@ -26,12 +26,27 @@ The [styles repo](https://github.com/hyperknot/openfreemap-styles), on the other
 
 Contributions are more than welcome!
 
+## Status of this project
+
+- The tile generation works
+- The web servers work
+- Weekly auto-updates work
+- Servers in our public instance are currently:
+  - 1 server running tile generation
+  - 2 servers running web hosting
+- Web servers are in Round-Robin DNS configuration with Let's Encrypt provided certificates.
+- Load-balancer script works. Currently in monitoring-only mode, as Round-Robin DNS handles downtime.
+- The public instance has been the production basemap service of [MapHub](https://maphub.net/) since June 2024.
+
+## Sponsoring
+
+Please consider sponsoring our project on [GitHub Sponsors](https://github.com/sponsors/hyperknot).
+
 ## Limitations of this project
 
-The only way this project can possibly work is to be super focused about what it is and what it isn't. OFM has the following limitations by design:
+The only way this project can possibly work is to be super focused about what it is and what it isn't. OpenFreeMap has the following limitations by design:
-
-1. OFM is not providing:
 
+1. OpenFreeMap is not providing:
    - search or geocoding
    - route calculation, navigation or directions
    - static image generation
@@ -40,27 +55,31 @@ The only way this project can possibly work is to be super focused about what it
    - elevation lookup
    - custom tile or dataset hosting
 
-2. OFM is not something you can install on your dev machine. OFM is a deploy script specifically made to set up clean Ubuntu servers or virtual machines. It uses [Fabric](https://www.fabfile.org/) and runs commands over SSH. With a single command it can set up a production-ready OFM server, both for tile hosting and generation.
+2. OpenFreeMap is not something you can install locally. This repo is a deploy script specifically made to set up clean Ubuntu servers or virtual machines. It uses [Fabric](https://www.fabfile.org/) and runs commands over SSH. With a single command it can set up a production-ready server, both for tile hosting and generation.
 
-   This repo is also Docker free. If someone wants to make a Docker-based version of this, I'm more than happy to link it here.
+   This repo is Docker-free on purpose. If someone wants to make a Docker-based version of this, I'm more than happy to link it here.
 
-3. OFM does not promise worry-free automatic updates for self-hosters. Only use the autoupdate version of http-host if you keep a close eye on this repo.
+3. OpenFreeMap does not promise worry-free automatic updates for self-hosters. Only use the autoupdate version of http-host if you keep a close eye on this repo.
+
+## Self hosting
+
+See [self hosting docs](docs/self_hosting.md).
 
 ## What is the tech stack?
 
 There is no tile server running; only Btrfs partition images with 300 million hard-linked files. This was my idea; I haven't read about anyone else doing this in production, but it works really well.
 
-There is no cloud, just dedicated servers. The HTTPS server is nginx on Ubuntu.
+There is no cloud, just dedicated servers. The web server is nginx on Ubuntu.
 
 ## Btrfs images
 
 Production-quality hosting of 300 million tiny files is hard. The average file size is just 450 byte. Dozens of tile servers have been written to tackle this problem, but they all have their limitations.
 
-The original idea of this project is to avoid using tile servers altogether. Instead, the tiles are directly served from Btrfs partition images + hard links using an optimised nginx config. I wrote [extract_mbtiles](scripts/tile_gen/extract_mbtiles) and [shrink_btrfs](scripts/tile_gen/shrink_btrfs) scripts for this very purpose.
+The original idea of this project is to avoid using tile servers altogether. Instead, the tiles are directly served from Btrfs partition images + hard links using an optimised nginx config. I wrote [extract_mbtiles](modules/tile_gen/scripts/extract_mbtiles.py) and [shrink_btrfs](modules/tile_gen/scripts/shrink_btrfs.py) scripts for this very purpose.
 
 This replaces a running service with a pure, file-system-level implementation. Since the Linux kernel's file caching is among the highest-performing and most thoroughly tested codes ever written, it delivers serious performance.
 
-I run some [benchmarks](docs/quick_notes/http_benchmark.md) on a Hetzner server, the aim was to saturate a gigabit connection. At the end, it was able to serve 30 Gbit on loopback interface, on cold nginx cache.
+I run some [benchmarks](docs/benchmark/README.md) on a Hetzner server, the aim was to saturate a gigabit connection. At the end, it was able to serve 30 Gbit on loopback interface, on cold nginx cache.
 
 ## Code structure
 
@@ -70,69 +89,77 @@ The project has the following parts
 
 This sets up everything on a clean Ubuntu server. You run it locally and it sets up the server via SSH.
 
-#### HTTP host - scripts/http_host
+#### HTTP host - modules/http_host
 
-Inside `http_host`, all work is done by `host_manager.py`.
+Inside `http_host`, all work is done by `http_host.py`.
 
 It does the following:
 
-- checks the most up-to-date files in the public buckets
+- Downloading btrfs images
-- downloads/extracts them locally, if needed
-- mounts the downloaded Btrfs images in `/mnt/ofm`
-- creates the correct TileJSON file
-- creates the correct nginx config
-- reloads nginx
 
-You can run `./host_manager.py --help` to see which options are available. Some commands can be run locally, including on non-linux machines.
+- Downloading assets
 
-#### tile generation - scripts/tile_gen
+- Mounting downloaded btrfs images
 
-_note: Tile generation is 100% optional, as we are providing the processed full planet files for public download._
+- Fetches version files
+
+- Running the sync cron task (called every minute with http-host-autoupdate)
+
+You can run `./http_host.py --help` to see which options are available.
+
+#### tile generation - modules/tile_gen
+
+_note: Tile generation is 100% optional, as we are providing the processed full planet btrfs files for public download. You can download full planet images updated weekly, both in Btrfs and in MBTiles format._
 
 The `tile_gen` script downloads a full planet OSM extract and runs it through Planetiler.
 
-The created .mbtiles file is then extracted into a Btrfs partition image using the custom [extract_mbtiles](scripts/tile_gen/extract_mbtiles) script. The partition is shrunk using the [shrink_btrfs](scripts/tile_gen/shrink_btrfs) script.
+The created .mbtiles file is then extracted into a Btrfs partition image using the custom [extract_mbtiles](modules/tile_gen/scripts/extract_mbtiles.py) script. The partition is shrunk using the [shrink_btrfs](modules/tile_gen/scripts/shrink_btrfs.py) script.
 
 Finally, it's uploaded to a public Cloudflare R2 bucket using rclone.
 
 #### styles - [styles repo](https://github.com/hyperknot/openfreemap-styles)
 
-A very important part, probably needs the most work in the long term future.
+The default styles. I've already put countless hours into tweaking up some nice looking styles. Still, it'll take probably the most work in the long term future.
 
-#### load balancer script - scripts/loadbalancer
+Of course, you are welcome to use custom styles.
+
+#### load balancer script - modules/loadbalancer
 
 A Round Robin DNS based load balancer script for health checking and updating records. It pushes status messages to a Telegram bot.
 
-## Self hosting
-
-See [self hosting docs](docs/self_hosting.md).
-
 ## FAQ
 
 ### Full planet downloads
 
-You can directly download the processed full planet runs on the following URLs:
+Full planet runs are uploaded weekly. You can download them both in Btrfs and in MBTiles formats. The files have the following URL patterns:
 
-https://planet.openfreemap.com/20240607_232801_pt/tiles.mbtiles // 89 GB, mbtiles file
+https://btrfs.openfreemap.com/areas/planet/{version}/tiles.btrfs.gz (and .mbtiles)
-https://planet.openfreemap.com/20240607_232801_pt/tiles.btrfs.gz // 86 GB, Btrfs partition image
 
-Replace the `20240607_232801_pt` part with any newer run, from the [index file](https://planet.openfreemap.com/index.txt).
+Use the [index file](https://btrfs.openfreemap.com/files.txt) to find out about versions.
+
+_Note: MBTiles files are not required for this project. We provide them for your convenience, allowing you to use the processed planet tiles with any other tool of your choice._
 
 ### Public buckets
 
-There are three public buckets:
+There are two public buckets:
 
-- https://assets.openfreemap.com - contains fonts, sprites, styles, versions. index: [dirs](https://assets.openfreemap.com/dirs.txt), [files](https://assets.openfreemap.com/index.txt)
+- https://assets.openfreemap.com - contains fonts, sprites, styles, versions. index: [dirs](https://assets.openfreemap.com/dirs.txt), [files](https://assets.openfreemap.com/files.txt)
-- https://planet.openfreemap.com - full planet runs. index: [dirs](https://planet.openfreemap.com/dirs.txt), [files](https://planet.openfreemap.com/index.txt)
+- https://btrfs.openfreemap.com - full planet runs. index: [dirs](https://btrfs.openfreemap.com/dirs.txt), [files](https://btrfs.openfreemap.com/files.txt)
-- https://monaco.openfreemap.com - identical runs to the full planet, but only for Monaco area. Very tiny, ideal for development. index: [dirs](https://monaco.openfreemap.com/dirs.txt), [files](https://monaco.openfreemap.com/index.txt)
 
-### What about PMTiles?
+### Domains
 
-I would have loved to use PMTiles; they are a brilliant idea!
+.org - not hosted through CloudFlare \
+.com - hosted through CloudFlare - serving the public buckets
 
-Unfortunately, making range requests in 80 GB files just doesn't work in production. It is fine for files smaller than 500 MB, but it has terrible latency and caching issues for full planet datasets.
+### What about PMTiles and using the Cloud?
 
-If PMTiles implements splitting to <10 MB files, it can be a valid alternative to running servers.
+I would have loved to use PMTiles; they are a brilliant idea for serverless map hosting!
+
+Unfortunately, on Cloudflare, range requests in 90 GB files have terrible latency, and on AWS, the data transfer costs can be prohibitive.
+
+Of course, with normal usage, you might fall within cloud vendor's free tier, but the internet is full of stories about people receiving surprise bills from AWS, sometimes amounting to thousands of dollars. It only takes one bad crawling bot getting stuck in a loop on your website to trigger such a bill.
+
+In short, using cloud vendors would make it impossible for me to offer this service for free — this project simply wouldn't exist.
 
 ## Contributing
 
@@ -140,17 +167,16 @@ Contributors welcome!
 
 Smaller tasks:
 
-- Cloudflare worker for indexing the public buckets, instead of generating index.txt files.
+- Cloudflare worker for indexing the public buckets, instead of generating index files.
-- Some of the POI icons are missing in the styles.
+- [styles] Some of the POI icons are missing.
 
 Bigger tasks:
 
-- Split the styles to building blocks. For example, there should be a POI block, a label block, a road-style related block.
+- [styles] Split the styles to building blocks. For example, there should be a POI block, a label block, a road-style related block.
-- Implement automatic updates for tile gen, uploading, testing and setting versions.
 
-Tasks outside the scope of this project:
+Future:
 
-- Make a successor for the OpenMapTiles schema.
+- Migrate to [Shortbread schema](https://shortbread-tiles.org/) and possibly [VersaTiles](https://versatiles.org/)
 
 #### Dev setup
 
@@ -158,6 +184,41 @@ See [dev setup docs](docs/dev_setup.md).
 
 ## Changelog
 
+##### v0.9
+
+Updated Planetiler version to latest
+Updated OpenJDK to 24 via Temurin repo
+
+##### v0.8
+
+Lot of self-hosting related fixes.
+
+Generating the domain inside the style TileJSON files dynamically (using nginx sub_filter).
+
+Added SELF_SIGNED_CERTS variable for cases when the certificates are self-managed or self-signed is OK.
+
+##### v0.7
+
+MBTiles are now uploaded, next to the btrfs image files.
+
+##### v0.6
+
+Load-balancer implemented with new config format. Implemented relaxed mode for checking while deployments are happening.
+
+##### v0.5
+
+Using a "done" file in the R2 buckets to mark the upload as finished. All scripts are checking for this file now.
+
+Monaco is generated daily, to avoid too frequent nginx reloads, which might be bad for the in-memory cache.
+
+##### v0.4
+
+Auto-update works!
+
+Monaco is generated hourly. Set-latest runs every minute.
+
+Planet is generated weekly, every Wednesday. Set-latest runs every Saturday.
+
 ##### v0.3
 
 Lot of performance related problems with Cloudflare when using Round-Robin DNS. Works much better without any Cloudflare proxying, the browsers actually do a great job of client-side failover and selecting the best host.

TODO.md

@@ -0,0 +1,54 @@
+making web maps free and fun again
+add dark themes in website
+
+
+
+logrotate
+
+/var/log/nginx/*.log /var/log/nginx/*/*.log {
+    daily                 # rotate at least once per day
+    dateext               # add -YYYYMMDD to rotated files
+    rotate 7              # keep up to 7 rotations (about 7 days)
+    maxage 7              # hard limit: delete rotated logs older than 7 days
+    missingok
+    compress
+    delaycompress
+    notifempty
+    sharedscripts
+    create 0640 www-data adm   # adjust user:group for your distro (e.g., nginx adm)
+    postrotate
+        # Tell nginx to reopen logs without losing lines
+        [ -s /run/nginx.pid ] && kill -USR1 "$(cat /run/nginx.pid)" || true
+        # Alternatively: nginx -s reopen
+    endscript
+}
+
+---
+
+sudo systemctl enable --now logrotate.timer
+sudo systemctl status logrotate.timer
+
+/etc/cron.daily/logrotate 
+
+Dry run:
+sudo logrotate -d /etc/logrotate.d/nginx
+Force a rotation immediately:
+sudo logrotate -vf /etc/logrotate.conf
+
+---
+
+/var/log/nginx/*.log {
+    daily           # Rotate every day
+    rotate 7        # Keep only 7 rotated files
+    missingok
+    notifempty
+    compress
+    delaycompress
+    sharedscripts
+    postrotate
+        [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
+    endscript
+}
+
+---
+

biome.jsonc

@@ -0,0 +1,186 @@
+{
+  "root": true,
+  "$schema": "https://biomejs.dev/schemas/2.2.4/schema.json",
+
+  "vcs": {
+    "enabled": false,
+    "clientKind": "git",
+    "useIgnoreFile": true,
+  },
+  "files": {
+    "maxSize": 100000,
+    "ignoreUnknown": true,
+    "includes": [
+      "**",
+      //
+      "!**/.pytest_cache",
+      "!**/venv",
+      "!**/.astro",
+      "!**/.venv",
+      "!**/_astro",
+      "!**/_not_used",
+      "!**/dist",
+      "!**/dist-electron",
+      "!**/node_modules",
+      "!**/.pytest_cache",
+      //
+    ],
+  },
+  "formatter": {
+    "indentStyle": "space",
+    "lineWidth": 100,
+  },
+  "linter": {
+    "enabled": true,
+    "domains": {
+      "solid": "all",
+    },
+    "rules": {
+      "recommended": true,
+      "style": {
+        "noNonNullAssertion": "off",
+        "useConsistentArrayType": {
+          "level": "error",
+          "options": {
+            "syntax": "generic",
+          },
+        },
+        "useLiteralEnumMembers": "error",
+        "useNodejsImportProtocol": "error",
+        "useAsConstAssertion": "error",
+        "useEnumInitializers": "error",
+        "useSelfClosingElements": "error",
+        "useSingleVarDeclarator": "error",
+        "noUnusedTemplateLiteral": "error",
+        "useNumberNamespace": "error",
+        "noInferrableTypes": "error",
+        "useExponentiationOperator": "error",
+        "useTemplate": "error",
+        "noParameterAssign": "error",
+        "useDefaultParameterLast": "error",
+        "useImportType": "error",
+        "useExportType": "error",
+        "useShorthandFunctionType": "error",
+        "noUselessElse": "error",
+        "useGroupedAccessorPairs": "error",
+        "useObjectSpread": "error",
+      },
+      "security": {
+        //  "noDangerouslySetInnerHtml": "off",
+        // "noBlankTarget": "off",
+      },
+      "a11y": {
+        "useFocusableInteractive": "off", // The HTML element with the interactive role "treeitem" is not focusable.
+        "useSemanticElements": "off", // SessionTreeDay / role="group"
+        "noStaticElementInteractions": "off", // SessionTreeDay / To add interactivity such as a mouse or key event listener to a static element, give the element an appropriate role value.
+        "useAriaPropsSupportedByRole": "off", // SessionTree / The ARIA attribute 'aria-multiselectable' is not supported by this element.
+        "useKeyWithClickEvents": "off", // PasteBlock / Enforce to have the onClick mouse event with the onKeyUp, the onKeyDown, or the onKeyPress keyboard event.
+        "noNoninteractiveElementToInteractiveRole": "off", // SessionTree
+        //    "useValidAnchor": "off",
+        //    "useButtonType": "off",
+        //   "noLabelWithoutControl": "off",
+        //  "noSvgWithoutTitle": "off",
+        // "noNoninteractiveTabindex": "off",
+      },
+      "correctness": {
+        "noUnusedFunctionParameters": "off",
+        "noUnusedImports": "off",
+        "noUnusedVariables": "off",
+        "noUnusedPrivateClassMembers": "off",
+        "noUnreachable": "off",
+        "noSolidDestructuredProps": "error",
+        "noGlobalDirnameFilename": "error",
+        "useParseIntRadix": "error",
+        "useSingleJsDocAsterisk": "error",
+      },
+      "suspicious": {
+        "noExplicitAny": "off",
+        "noImplicitAnyLet": "off",
+        "noUselessEscapeInString": "error",
+        "useStaticResponseMethods": "error",
+        "useIterableCallbackReturn": "off",
+        // "noAssignInExpressions": "off",
+        //"noArrayIndexKey": "off",
+      },
+      "complexity": {
+        "noCommaOperator": "error",
+        "useNumericLiterals": "error",
+        //     "noArguments": "off",
+        "useIndexOf": "error",
+        "noImplicitCoercions": "error", // Number(), Boolean()
+        //        "noUselessFragments": "off"
+      },
+      "nursery": {
+        //        "noShadow": "error",
+        //        "useReadonlyClassProperties": "error",
+        //        "noImportCycles": "error",
+
+        //        "noFloatingPromises": "error",
+        //        "noMisusedPromises": "error",
+
+        //        "noUnassignedVariables": "error", // bug reported ref= not recognised
+        "noUselessUndefined": "error",
+      },
+      "performance": { "useSolidForComponent": "error" },
+    },
+  },
+  "javascript": {
+    "formatter": {
+      "semicolons": "asNeeded",
+      "quoteStyle": "single",
+    },
+  },
+  "json": {
+    "parser": {
+      "allowTrailingCommas": true,
+    },
+    "formatter": {
+      "trailingCommas": "none",
+    },
+  },
+  "assist": {
+    "enabled": true,
+    "actions": {
+      "source": {
+        "organizeImports": "on",
+        //        "useSortedProperties": "on" // sort CSS props
+      },
+    },
+  },
+  "overrides": [
+    {
+      "includes": ["**/*.astro"],
+      "linter": {
+        "rules": {
+          "correctness": {
+            "noUnusedImports": "off",
+          },
+        },
+      },
+    },
+    {
+      "includes": ["**/*.jsonc"],
+      "json": {
+        "formatter": {
+          "trailingCommas": "all",
+        },
+      },
+    },
+    {
+      "includes": ["**/*.css"],
+      "linter": {
+        "rules": {
+          "suspicious": {
+            "noDuplicateProperties": "off",
+            "noEmptyBlock": "off",
+            "noUnknownAtRules": "off",
+          },
+          "complexity": {
+            "noImportantStyles": "off",
+          },
+          "style": { "noDescendingSpecificity": "off" },
+        },
+      },
+    },
+  ],
+}

config/.env.sample

@@ -1,29 +0,0 @@
-# Leave it empty if you use SSH keys
-SSH_PASSWD=
-
-# Direct subdomain, using Let's Encrypt certificates
-DOMAIN_LE=
-
-# Let's Encrypt account email
-LE_EMAIL=
-
-
-
-# Skip the full planet download, useful for testing (true/false)
-SKIP_PLANET=false
-
-
-# --- Let's Encrypt DNS related variables, not needed for self-hosting
-
-DOMAIN_LEDNS=
-
-# --- host list
-
-HTTP_HOST_LIST=
-
-
-# --- Load Balancer script
-
-TELEGRAM_TOKEN=
-TELEGRAM_CHAT_ID=
-

config/certs/.gitignore

@@ -1 +0,0 @@
-*

config/cloudflare.ini.sample

@@ -1,3 +0,0 @@
-# --- Let's Encrypt DNS challange, not needed for self-hosting
-
-dns_cloudflare_api_token = xxx

config/config.schema.json

@@ -0,0 +1,90 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "$schema": {
+      "type": "string"
+    },
+    "domains": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/domain" }
+    },
+    "servers": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/server" }
+    },
+    "skip_planet": {
+      "type": "boolean",
+      "description": "Skip the full planet download, useful for testing"
+    }
+  },
+  "required": ["domains", "servers"],
+  "definitions": {
+    "cert-upload": {
+      "type": "object",
+      "description": "Upload your own certificate. Ideal for Cloudflare Origin Certificates with 15 year expiry. Steps: 1) Create an Origin Certificate on Cloudflare at SSL/TLS / Origin Server. 2) Generate private key and CSR with Cloudflare: Private key type: ECC, Certificate Validity: 15 years. 3) Key format: PEM. Save origin certificate as something.cert and private key as something.key in the same directory.",
+      "additionalProperties": false,
+      "properties": {
+        "type": { "const": "upload" },
+        "cert_path": {
+          "type": "string",
+          "minLength": 1,
+          "pattern": "^.*\\.cert$",
+          "description": "Path to your certificate file (*.cert). Both absolute and relative paths are supported. The corresponding private key (.key) should be in the same directory with the same basename."
+        }
+      },
+      "required": ["type", "cert_path"]
+    },
+    "cert-letsencrypt": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "type": { "const": "letsencrypt" },
+        "email": { "type": "string", "format": "email" }
+      },
+      "required": ["type", "email"]
+    },
+    "cert-dummy": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "type": { "const": "dummy" }
+      },
+      "required": ["type"]
+    },
+    "cert": {
+      "oneOf": [
+        { "$ref": "#/definitions/cert-upload" },
+        { "$ref": "#/definitions/cert-letsencrypt" },
+        { "$ref": "#/definitions/cert-dummy" }
+      ]
+    },
+    "domain": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "domain": { "type": "string", "format": "hostname" },
+        "cert": { "$ref": "#/definitions/cert" }
+      },
+      "required": ["domain", "cert"]
+    },
+    "server": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "hostname": {
+          "type": "string",
+          "description": "hostname used for ssh to connect. can be an IP address"
+        },
+        "server_ssh_passwd": {
+          "type": "string",
+          "description": "Leave this empty if you use SSH keys"
+        }
+      },
+      "required": ["hostname"]
+    }
+  }
+}

debug.py

@@ -0,0 +1,16 @@
+import click
+
+from ssh_lib.cli_helpers import common_options, get_connection
+from ssh_lib.tasks_http_host import upload_config_and_certs
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@common_options
+def debug(hostname, user, port, noninteractive):
+    c = get_connection(hostname, user, port)
+    upload_config_and_certs(c)

docs/benchmark/README.md

@@ -0,0 +1,43 @@
+# HTTP Hosts Benchmarking
+
+This repository contains tools and scripts for benchmarking HTTP hosts performance.
+
+## Prerequisites
+
+Before running the benchmarks, you need to create a path list (`path_list_500k.txt`). You have two options:
+
+1. Generate from real-world server logs using `nginx_to_path_list.py`
+2. Generate randomly (Note: real-world usage patterns are typically non-random, e.g., ocean tiles are rarely accessed)
+
+## Important Notes
+
+- Run the benchmarks on `localhost`, and not over the internet! Otherwise you'd be just testing your internet speed.
+- The benchmark uses [wrk](https://github.com/wg/wrk) HTTP benchmarking tool
+
+## Usage
+
+Basic command:
+
+```bash
+wrk -c10 -t4 -d10s -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost
+```
+
+### Parameters Explained
+
+- `-c10`: Number of connections to keep open
+- `-t4`: Number of threads to use
+- `-d10s`: Duration of the test (10 seconds)
+- `-s`: Script file to use
+
+### Thread Count Considerations
+
+- `-t1`: More accurate results as the URL list is loaded exactly in sequence
+- `-t4`: Better reflects real-world usage patterns
+
+## Results
+
+Benchmark results can be found in [results.md](results.md)
+
+## Contributing
+
+Feel free to submit your results including which hosts were used.

docs/benchmark/nginx_to_path_list.py

@@ -1,7 +1,11 @@
 import json
 
 
-with open('access.log') as fp:
+# This script parses a nginx server log and creates a text file
+# which can be used in the Lua script.
+# The path file is not supplied in this repo.
+
+with open('access.jsonl') as fp:
     json_lines = fp.readlines()
 
 paths = []

docs/benchmark/wrk_custom_list.lua

@@ -1,6 +1,6 @@
 local counter = 1
 local lines = {}
-local url_base = "/planet/20231221_134737_pt/"  -- trailing slash
+local url_base = "/planet/fake_version/"  -- trailing slash
 local path_list_txt = "/data/ofm/benchmark/path_list_500k.txt"
 
 for line in io.lines(path_list_txt) do

docs/debugging_names.md

@@ -0,0 +1,27 @@
+# Debugging international names
+
+If there is an issue about international names not being displayed correctly, first, we need to find **one specific example** and check at which stage does the problem appear.
+
+OpenFreeMap map data is created by the following stack:
+
+**OpenStreetMap data ➔ OpenMapTiles specification ➔ Planetiler**
+
+1. To debug OpenStreetMap data, go to OpenStreetMap.org and search for the query string. For example "Iwate Prefecture" gives these results: [nominatim](https://nominatim.openstreetmap.org/ui/details.html?osmtype=R&osmid=3792412&class=boundary) and [openstreetmap](https://www.openstreetmap.org/relation/3792412)
+
+   <img src="assets/name-osm-search.png" width=333 />
+
+2. Then we need to check what the data is in the generated vector tiles. The best way to do this is to go to [Maputnik editor](https://maputnik.github.io/editor?style=https://tiles.openfreemap.org/styles/bright) and select View / Inspect.
+
+   <img src="assets/name-maputnik-view.png" width=388 />
+
+3. Then you can search for the little red dot matching your label and make a screenshot.
+
+   <img src="assets/name-maputnik-details.png" width=240 />
+
+Now we can compare where the naming problem is coming from.
+
+In conclusion: for the **one specific example**, please link the OSM pages and add the inspector screenshot, then we can start with the debugging.
+
+## Next steps
+
+It'd be nice to compare with other OpenMapTiles implementations like [tilemaker](https://github.com/systemed/tilemaker) or the [OpenMapTiles reference](https://github.com/openmaptiles/openmaptiles). I don't have full planet datasets from these implementations currently, so if someone is willing to run one it'd be a great contribution.

docs/dev_setup.md

@@ -25,6 +25,6 @@ Host orb_my
 Then I run commands like the following:
 
 ```
-./init-server.py http-host-once orb_my
+./init-server.py http-host-static orb_my
 ./init-server.py debug orb_my
 ```

docs/self_hosting.md

@@ -1,65 +1,121 @@
 # Self-hosting Howto
 
-_note: For most users, **you don't need to run anything**! The tiles are hosted free of charge, without registration. Read the "How can I use it?" section on https://openfreemap.org_
+You can either self-host or use our public instance. Everything is **open-source**, including the full production setup — there’s no 'open-core' model here.
 
-When self-hosting, there are two tasks you can set up on a server (see details in the repo README).
+When self-hosting, there are two modules you can set up on a server (see details in the repo README).
 
 - **http-host**
 
 - **tile-gen**
 
-note: Tile generation is 100% optional, as we are providing the processed full planet files for public download. It also requires a beefy machine, see below.
+There is a 99.9% chance you only need **http-host**. Tile-gen is slow, needs a huge machine and is totally pointless, since we upload the processed files every week.
 
 ### System requirements
 
-##### Disk space
+**http-host**: 300 GB disk space for hosting a single run. SSD is recommended, but not required.
 
-- **http-host**: 300 GB for hosting a single run
+**tile-gen**: 500 GB SDD and at least 64 GB ram
 
-- **tile-gen**: 500 GB
+**Ubuntu 22** or newer
 
-##### RAM
+### Provider recommendation
 
-- **http-host**: 4 GB
+One amazing deal, which is tested and known to work well for http-host is the €4.5 / month [Contabo Storage VPS](https://contabo.com/en/storage-vps/)
-
-- **tile-gen**: 64 GB+
-
-##### OS
-
-- **Ubuntu 22+**
 
 ---
 
 ### Warning
 
-This project is made to run on clean servers or virtual machines dedicated for this project. The scripts need sudo permissions as they mount/unmount disk images. Do not run this on your dev machine without using virtual machines. If you do, please make sure you understand exactly what each script is doing.
+This project is made to run on **clean servers** or virtual machines dedicated for this project. The scripts need sudo permissions as they mount/unmount disk images. Do not run this on your dev machine without using virtual machines. If you do, please make sure you understand exactly what each script is doing.
+
+If you run it on a non-clean server, please understand that this will modify your nginx config!
 
 ---
 
 ## Instructions
 
-Create virtualenv using: `source prepare-virtualenv.sh`
+I recommend running things quickly first, with `SKIP_PLANET=true` and then once it works, running it with `SKIP_PLANET=false`.
 
-It's recommended to use [direnv](https://direnv.net/), to have automatic venv activation.
+#### 1. DNS setup
 
-#### 1. Prepare `config` folder
+Set up a server with at least 300 GB SSD space and configure the DNS for the subdomain of your choice.
+For example, make an A record for "maps.example.com" -> 185.199.110.153
 
-1. Copy `.env.sample` to `.env` and set the values.
+#### 2. Clone and prepare `config` folder
 
-   DOMAIN_LE - Use this to specify a domain to be used with Let's Encrypt.
+```
+git clone https://github.com/hyperknot/openfreemap
+```
 
-1. If you want to run tile generation and upload via rclone, you can copy the `rclone.conf.sample` file as well. For simple self-hosting there is no need for this.
+In the config folder, copy `.env.sample` to `.env` and set the values.
 
-#### 2. Deploy a http-host
+`DOMAIN_DIRECT` - Your subdomain \
+`LETSENCRYPT_EMAIL` - Your email for Let's Encrypt
 
-You run the deploy script locally. It'll connect to an SSH server, like this
+Set `SKIP_PLANET=true` first.
 
-`./init-server.py http-host-once HOSTNAME`
+#### 3. Set up Python if you don't have it yet
 
-After this, go for a walk and by the time you come back it should be up and running with the latest planet tiles deployed. Don't worry about the "Download aborted" lines in the meanwhile, it's a bug in CloudFlare.
+On Ubuntu you can get it by `sudo apt install python3-pip`
 
-#### 3. Deploy tile-gen server (optional)
+On macOS you can do `brew install python`
 
-If you have a really beefy machine (see above) and you want to generate tiles yourself, you can run `./init-server.py tile-gen HOSTNAME`.
+#### 4. Prepare the Python environment
 
-Trigger a run manually, by running `planetiler_{area}.sh`. Recommended to use tmux or similar, as it can take days to complete.
+You run the deploy script locally, and it deploys to a remote server over SSH. You can use a virtualenv if you are used to working with them, but it's not necessary.
+
+```
+cd openfreemap
+pip install -e .
+```
+
+#### 5. Deploy quick version with `SKIP_PLANET=true`
+
+Run the actual deploy command and wait a few minutes
+
+```
+./init-server.py http-host-static HOSTNAME
+```
+
+#### 5. Check
+
+If everything is OK, you'll have some curl lines printed. Run the first one locally and make sure it's showing HTTP/2 200. For example this is an OK response.
+
+```locally to test them.
+curl -sI https://test.openfreemap.org/monaco
+
+HTTP/2 200
+access-control-allow-origin: *
+cache-control: max-age=86400
+cache-control: public
+content-length: 5776
+content-type: application/json
+date: Fri, 11 Oct 2024 21:01:23 GMT
+etag: "670991d1-1690"
+expires: Sat, 12 Oct 2024 21:01:23 GMT
+last-modified: Fri, 11 Oct 2024 21:00:01 GMT
+server: nginx
+x-ofm-debug: latest JSON monaco
+```
+
+#### 6. Deploy and check with `SKIP_PLANET=false`
+
+Update your `.env` file and re-run the same `./init-server.py http-host-static HOSTNAME` as before.
+
+Go for a walk and by the time you come back it should be up and running with the latest planet tiles deployed. Don't worry about the "Download aborted" lines in the meanwhile, it's a bug in CloudFlare.
+
+If your server doesn't have an SSD, the download + uncompressing process can take hours.
+
+---
+
+#### Deploy tile-gen server (optional)
+
+If you have a really beefy machine (see above) and you really want to generate tiles yourself, you can run `./init-server.py tile-gen HOSTNAME`.
+
+Trigger a run manually, by running
+
+```
+sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-tiles planet
+```
+
+It's recommended to use tmux or similar, as it can take days to complete.

http-host.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+import json
+
+import click
+
+from modules.http_host.http_host_lib.get_version_shared import get_deployed_version
+from ssh_lib.cli_helpers import common_options, get_connection
+from ssh_lib.config import config
+from ssh_lib.pycurl import pycurl_get
+from ssh_lib.tasks_http_host import prepare_http_host, read_jsonc, run_http_host_sync
+from ssh_lib.tasks_shared import prepare_shared
+from ssh_lib.utils import (
+    get_ip_from_ssh_alias,
+    put,
+)
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@common_options
+def init_static(hostname, user, port, noninteractive):
+    if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
+        return
+
+    c = get_connection(hostname, user, port)
+
+    prepare_shared(c)
+    prepare_http_host(c)
+
+    run_http_host_sync(c)
+
+    # Check server health after deployment
+    results = check_server_health(hostname)
+    print_server_health(results)
+
+
+@cli.command()
+@common_options
+@click.option('--sync', is_flag=True, help='Run manual sync after init')
+def init_autoupdate(hostname, user, port, noninteractive, sync):
+    if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
+        return
+
+    c = get_connection(hostname, user, port)
+
+    c.sudo('rm -f /etc/cron.d/ofm_http_host')
+
+    prepare_shared(c)
+    prepare_http_host(c)
+
+    # if --sync, run manual sync
+    if sync:
+        run_http_host_sync(c)
+
+    put(c, config.local_modules_dir / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
+
+    # Check server health after deployment
+    results = check_server_health(hostname)
+    print_server_health(results)
+
+
+@cli.command()
+@common_options
+def sync(hostname, user, port, noninteractive):
+    if not noninteractive and not click.confirm(f'Run script on {hostname}?'):
+        return
+
+    c = get_connection(hostname, user, port)
+    run_http_host_sync(c)
+
+    # Check server health after sync
+    results = check_server_health(hostname)
+    print_server_health(results)
+
+
+@cli.command()
+@click.option('--hostname', help='Check only a specific server')
+def debug(hostname):
+    results = check_server_health(hostname)
+    print_server_health(results)
+
+
+def check_server_health(hostname: str = None) -> dict:
+    """
+    Check health of servers by verifying deployed version matches expected version.
+
+    Args:
+        hostname: Optional hostname to check. If None, checks all servers in config.
+
+    Returns:
+        dict: Results for each server with structure:
+            {
+                'server_hostname': {
+                    'ip': '1.2.3.4',
+                    'all_ok': True/False,
+                    'domains': {
+                        'domain.com': {'status': 'ok'/'failed', 'error': None/'error message'}
+                    }
+                }
+            }
+    """
+    config_data = read_jsonc()
+    area = 'monaco' if config_data.get('skip_planet') else 'planet'
+    version = get_deployed_version(area)['version']
+    domains = [d['domain'] for d in config_data['domains']]
+
+    servers = [
+        {'hostname': s['hostname'], 'ip': get_ip_from_ssh_alias(s['hostname'])}
+        for s in config_data['servers']
+    ]
+
+    # Filter to specific server if requested
+    if hostname:
+        servers = [s for s in servers if s['hostname'] == hostname]
+        if not servers:
+            raise ValueError(f'Server {hostname} not found in config')
+
+    results = {}
+
+    for server in servers:
+        server_hostname = server['hostname']
+        server_ip = server['ip']
+        results[server_hostname] = {'ip': server_ip, 'domains': {}, 'all_ok': True}
+
+        for domain in domains:
+            try:
+                check_host_using_tilejson(
+                    url=f'https://{domain}/{area}/{version}',
+                    ip=server_ip,
+                    version=version,
+                )
+                results[server_hostname]['domains'][domain] = {'status': 'ok', 'error': None}
+            except AssertionError:
+                results[server_hostname]['domains'][domain] = {
+                    'status': 'failed',
+                    'error': f'Version mismatch (expected {version})',
+                }
+                results[server_hostname]['all_ok'] = False
+            except Exception as e:
+                results[server_hostname]['domains'][domain] = {'status': 'failed', 'error': str(e)}
+                results[server_hostname]['all_ok'] = False
+
+    return results
+
+
+def print_server_health(results: dict) -> None:
+    """Print server health results in a human-readable format."""
+    for server_hostname, server_data in results.items():
+        status = (
+            click.style('OK', fg='green')
+            if server_data['all_ok']
+            else click.style('FAILED', fg='red')
+        )
+        server_line = f'SERVER {server_hostname} ({server_data["ip"]})'
+        print(f'{server_line:<50} {status}')
+
+        for domain, domain_data in server_data['domains'].items():
+            domain_line = f'  {domain}'
+            if domain_data['status'] == 'ok':
+                print(f'{domain_line:<50} {click.style("OK", fg="green")}')
+            else:
+                print(
+                    f'{domain_line:<50} {click.style("FAILED", fg="red")}\n  {domain_data["error"]}'
+                )
+
+        print()
+
+
+def check_host_using_tilejson(*, url: str, ip: str, version: str) -> None:
+    tilejson_str = pycurl_get(url, ip)
+    tilejson = json.loads(tilejson_str)
+    tiles_url = tilejson['tiles'][0]
+    version_in_tilejson = tiles_url.split('/')[4]
+    assert version_in_tilejson == version
+
+
+if __name__ == '__main__':
+    cli()

init-server.py

@@ -1,144 +0,0 @@
-#!/usr/bin/env python3
-
-import click
-from fabric import Config, Connection
-
-from ssh_lib import SCRIPTS_DIR, dotenv_val
-from ssh_lib.tasks import (
-    prepare_http_host,
-    prepare_shared,
-    prepare_tile_gen,
-    run_http_host_sync,
-    setup_ledns_writer,
-    setup_loadbalancer,
-    upload_http_host_config,
-)
-from ssh_lib.utils import (
-    put,
-    put_dir,
-)
-
-
-def get_connection(hostname, user, port):
-    ssh_passwd = dotenv_val('SSH_PASSWD')
-
-    if ssh_passwd:
-        print('Using SSH password')
-
-        c = Connection(
-            host=hostname,
-            user=user,
-            port=port,
-            connect_kwargs={'password': ssh_passwd},
-            config=Config(overrides={'sudo': {'password': ssh_passwd}}),
-        )
-    else:
-        c = Connection(
-            host=hostname,
-            user=user,
-            port=port,
-        )
-
-    return c
-
-
-def common_options(func):
-    """Decorator to define common options."""
-    func = click.argument('hostname')(func)
-    func = click.option('--port', type=int, help='SSH port (if not in .ssh/config)')(func)
-    func = click.option('--user', help='SSH user (if not in .ssh/config)')(func)
-    return func
-
-
-@click.group()
-def cli():
-    pass
-
-
-@cli.command()
-@common_options
-def http_host_once(hostname, user, port):
-    if not click.confirm(f'Run script on {hostname}?'):
-        return
-
-    c = get_connection(hostname, user, port)
-
-    prepare_shared(c)
-    upload_http_host_config(c)
-
-    prepare_http_host(c)
-
-    run_http_host_sync(c)
-
-
-@cli.command()
-@common_options
-def http_host_autoupdate(hostname, user, port):
-    if not click.confirm(f'Run script on {hostname}?'):
-        return
-
-    c = get_connection(hostname, user, port)
-
-    prepare_shared(c)
-    upload_http_host_config(c)
-
-    prepare_http_host(c)
-
-    put(c, SCRIPTS_DIR / 'http_host' / 'cron.d' / 'ofm_http_host', '/etc/cron.d/')
-
-
-@cli.command()
-@common_options
-def tile_gen(hostname, user, port):
-    if not click.confirm(f'Run script on {hostname}?'):
-        return
-
-    c = get_connection(hostname, user, port)
-    prepare_shared(c)
-
-    prepare_tile_gen(c)
-
-
-@cli.command()
-@common_options
-def ledns(hostname, user, port):
-    if not click.confirm(f'Run script on {hostname}?'):
-        return
-
-    c = get_connection(hostname, user, port)
-
-    setup_ledns_writer(c)
-
-
-@cli.command()
-@common_options
-def loadbalancer(hostname, user, port):
-    if not click.confirm(f'Run script on {hostname}?'):
-        return
-
-    c = get_connection(hostname, user, port)
-    prepare_shared(c)
-
-    setup_loadbalancer(c)
-
-
-@cli.command()
-@common_options
-def debug(hostname, user, port):
-    c = get_connection(hostname, user, port)
-
-    # upload_http_host_config(c)
-    # upload_http_host_files(c)
-    # sudo_cmd(c, f'{VENV_BIN}/python -u /data/ofm/http_host/bin/host_manager.py nginx-sync')
-
-    # put(c, SCRIPTS_DIR / 'tile_gen' / 'upload_manager.py', f'{TILE_GEN_BIN}')
-    put_dir(c, SCRIPTS_DIR / 'loadbalancer', '/data/ofm/loadbalancer')
-    put_dir(
-        c,
-        SCRIPTS_DIR / 'loadbalancer' / 'loadbalancer_lib',
-        '/data/ofm/loadbalancer/loadbalancer_lib',
-    )
-
-
-if __name__ == '__main__':
-    cli()

lint.sh

@@ -1,6 +1,11 @@
 #!/usr/bin/env bash
+set -e
 
-pnpm prettier -w .
+node_modules/.bin/prettier -w "**/*.md"
+
+# biome
+#pnpm biome check --write --unsafe --colors=off --log-level=info --log-kind=pretty . | grep path | sort
+pnpm biome check --write --unsafe .
 
 ruff check --fix .
 ruff format .

modules/http_host/cron.d/ofm_http_host

@@ -0,0 +1,4 @@
+# every minute sync, locking so that only one process can run at a time
+* * * * * ofm     /usr/bin/flock -n /tmp/http_host.lockfile -c 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/http_host.py sync >> /data/ofm/http_host/logs/http_host_sync.log 2>&1'
+
+

modules/http_host/http_host.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+from datetime import datetime, timezone
+
+import click
+from http_host_lib.assets import (
+    download_assets,
+)
+from http_host_lib.btrfs import (
+    download_area_version,
+)
+from http_host_lib.get_version_shared import get_versions_for_area
+from http_host_lib.mount import auto_mount
+from http_host_lib.nginx_config_gen import write_nginx_config
+from http_host_lib.sync import auto_clean_btrfs, full_sync
+from http_host_lib.versions import fetch_version_files
+
+
+now = datetime.now(timezone.utc)
+
+
+@click.group()
+def cli():
+    """
+    Manages OpenFreeMap HTTP hosts, including:\n
+    - Downloading btrfs images\n
+    - Downloading assets\n
+    - Mounting downloaded btrfs images\n
+    - Fetches version files\n
+    - Running the sync cron task (called every minute with http-host-autoupdate)
+    """
+
+
+@cli.command()
+@click.argument('area', required=False)
+@click.option(
+    '--version', default='latest', help='Optional version string, like "20231227_043106_pt"'
+)
+def download_btrfs(area: str, version: str):
+    """
+    Downloads and uncompresses tiles.btrfs files from the btrfs bucket
+    Version can be "latest" (default) or specified, like "20231227_043106_pt"
+    Use --version=1 to list all available versions
+    """
+
+    download_area_version(area, version)
+
+
+@cli.command(name='download-assets')
+def download_assets_():
+    """
+    Downloads and extracts assets
+    """
+
+    download_assets()
+
+
+@cli.command()
+def mount():
+    """
+    Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
+    When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
+    """
+
+    auto_mount()
+
+
+@cli.command(name='fetch-versions')
+def fetch_version_files_():
+    """
+    Fetches the version files from remote to local.
+    Remote versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
+    """
+
+    fetch_version_files()
+
+
+@cli.command()
+def auto_clean():
+    """
+    Cleans the old btrfs images
+    """
+
+    auto_clean_btrfs()
+
+
+@cli.command()
+def nginx_config():
+    """
+    Writes the nginx config files and reloads nginx
+    """
+
+    write_nginx_config()
+
+
+@cli.command()
+@click.option('--force', is_flag=True, help='Force nginx sync run')
+def sync(force):
+    """
+    Runs the sync task, normally called by cron every minute
+    On a new server this also takes care of everything, no need to run anything manually.
+    """
+
+    print(f'---\n{now}\nStarting sync')
+
+    full_sync(force)
+
+
+@cli.command()
+def debug():
+    versions = get_versions_for_area('monaco')
+    print(versions)
+
+
+if __name__ == '__main__':
+    cli()

modules/http_host/http_host_lib/assets.py

@@ -1,26 +1,44 @@
 import shutil
 import subprocess
-from pathlib import Path
 
 import requests
 
+from http_host_lib.config import config
 from http_host_lib.utils import download_file_aria2, download_if_size_differs
 
 
-def download_and_extract_asset_tar_gz(assets_dir, asset_kind):
+def download_assets() -> bool:
+    """
+    Downloads and extracts assets
+    """
+
+    changed = False
+
+    changed += download_and_extract_asset_tar_gz('fonts')
+    changed += download_and_extract_asset_tar_gz('styles')
+    changed += download_and_extract_asset_tar_gz('natural_earth')
+
+    changed += download_sprites()
+
+    return changed
+
+
+def download_and_extract_asset_tar_gz(asset_kind):
     """
     Download and extract asset.tgz if the file size differ or not available locally
+    Returns True if modified
     """
 
     print(f'Downloading asset {asset_kind}')
 
-    asset_dir = assets_dir / asset_kind
+    asset_dir = config.assets_dir / asset_kind
     asset_dir.mkdir(exist_ok=True, parents=True)
 
     url = f'https://assets.openfreemap.com/{asset_kind}/ofm.tar.gz'
     local_file = asset_dir / 'ofm.tar.gz'
     if not download_if_size_differs(url, local_file):
-        return
+        print(f'  skipping asset: {asset_kind}')
+        return False
 
     ofm_dir = asset_dir / 'ofm'
     ofm_dir_bak = asset_dir / 'ofm.bak'
@@ -33,24 +51,32 @@ def download_and_extract_asset_tar_gz(assets_dir, asset_kind):
         check=True,
     )
 
+    print(f'  downloaded asset: {asset_kind}')
+    return True
 
-def download_sprites(assets_dir: Path):
+
+def download_sprites() -> bool:
     """
     Sprites are special assets, as we have to keep the old versions indefinitely
     """
 
-    sprites_dir = assets_dir / 'sprites'
+    print('Downloading sprites')
+
+    sprites_dir = config.assets_dir / 'sprites'
     sprites_dir.mkdir(exist_ok=True, parents=True)
 
-    r = requests.get('https://assets.openfreemap.com/index.txt', timeout=30)
+    r = requests.get('https://assets.openfreemap.com/files.txt', timeout=30)
     r.raise_for_status()
 
     sprites_remote = [l for l in r.text.splitlines() if l.startswith('sprites/')]
 
+    changed = False
+
     for sprite in sprites_remote:
         sprite_name = sprite.split('/')[1].replace('.tar.gz', '')
 
         if (sprites_dir / sprite_name).is_dir():
+            print(f'  skipping sprite version: {sprite_name}')
             continue
 
         url = f'https://assets.openfreemap.com/sprites/{sprite_name}.tar.gz'
@@ -62,3 +88,7 @@ def download_sprites(assets_dir: Path):
             check=True,
         )
         local_file.unlink()
+        print(f'  downloaded sprite version: {sprite_name}')
+        changed = True
+
+    return changed

modules/http_host/http_host_lib/btrfs.py

@@ -0,0 +1,94 @@
+import shutil
+import subprocess
+import sys
+
+from http_host_lib.config import config
+from http_host_lib.get_version_shared import get_versions_for_area
+from http_host_lib.utils import download_file_aria2, get_remote_file_size
+
+
+def download_area_version(area: str, version: str) -> bool:
+    """
+    Downloads and uncompresses tiles.btrfs files from the btrfs bucket
+
+    "latest" version means the latest in the remote bucket
+    "deployed" version means to read the currently deployed version string from the config dir
+    """
+
+    if area not in config.areas:
+        sys.exit(f'  Please specify area: {config.areas}')
+
+    versions = get_versions_for_area(area)
+    if not versions:
+        print(f'  No versions found for {area}')
+        return False
+
+    # latest version
+    if version == 'latest':
+        selected_version = versions[-1]
+
+    # deployed version
+    elif version == 'deployed':
+        try:
+            selected_version = (config.deployed_versions_dir / f'{area}.txt').read_text().strip()
+        except Exception:
+            return False
+
+    # specific version
+    else:
+        if version not in versions:
+            available_versions_str = '\n'.join(versions)
+            print(
+                f'  Requested version is not available.\nAvailable versions for {area}:\n{available_versions_str}'
+            )
+            return False
+        selected_version = version
+
+    return download_and_extract_btrfs(area, selected_version)
+
+
+def download_and_extract_btrfs(area: str, version: str) -> bool:
+    """
+    returns True if download successful, False if skipped
+    """
+
+    print(f'Downloading btrfs: {area} {version}')
+
+    version_dir = config.runs_dir / area / version
+    btrfs_file = version_dir / 'tiles.btrfs'
+    if btrfs_file.exists():
+        print('  file exists, skipping download')
+        return False
+
+    temp_dir = config.runs_dir / '_tmp'
+    shutil.rmtree(temp_dir, ignore_errors=True)
+    temp_dir.mkdir(parents=True)
+
+    url = f'https://btrfs.openfreemap.com/areas/{area}/{version}/tiles.btrfs.gz'
+
+    # check disk space
+    disk_free = shutil.disk_usage(temp_dir).free
+    file_size = get_remote_file_size(url)
+    if not file_size:
+        print(f'  cannot get remote file size for {url}')
+        return False
+
+    needed_space = file_size * 3
+    if disk_free < needed_space:
+        print(f'  not enough disk space. Needed: {needed_space}, free space: {disk_free}')
+        return False
+
+    target_file = temp_dir / 'tiles.btrfs.gz'
+    download_file_aria2(url, target_file)
+
+    print('  uncompressing...')
+    subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
+    btrfs_src = temp_dir / 'tiles.btrfs'
+
+    shutil.rmtree(version_dir, ignore_errors=True)
+    version_dir.mkdir(parents=True)
+
+    btrfs_src.rename(btrfs_file)
+
+    shutil.rmtree(temp_dir)
+    return True

modules/http_host/http_host_lib/config.py

@@ -0,0 +1,38 @@
+import json
+import subprocess
+from pathlib import Path
+
+
+class Configuration:
+    areas = ['planet', 'monaco']
+
+    http_host_dir = Path('/data/ofm/http_host')
+
+    http_host_bin = http_host_dir / 'bin'
+    http_host_scripts_dir = http_host_bin / 'scripts'
+
+    runs_dir = http_host_dir / 'runs'
+    assets_dir = http_host_dir / 'assets'
+
+    mnt_dir = Path('/mnt/ofm')
+
+    nginx_templates = Path(__file__).parent / 'nginx_templates'
+
+    nginx_certs_dir = Path('/data/nginx/certs')
+    nginx_sites_dir = Path('/data/nginx/sites')
+
+    if Path('/data/ofm').exists():
+        ofm_config_dir = Path('/data/ofm/config')
+    else:
+        repo_root = Path(__file__).parent.parent.parent.parent
+        ofm_config_dir = repo_root / 'config'
+
+    json_config = json.loads((ofm_config_dir / 'config.json').read_text())
+
+    deployed_versions_dir = ofm_config_dir / 'deployed_versions'
+
+    rclone_config = ofm_config_dir / 'rclone.conf'
+    rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip()
+
+
+config = Configuration()

modules/http_host/http_host_lib/get_version_shared.py

@@ -0,0 +1 @@
+../../tile_gen/tile_gen_lib/get_version_shared.py

modules/http_host/http_host_lib/mount.py

@@ -0,0 +1,106 @@
+import subprocess
+import sys
+from pathlib import Path
+
+from http_host_lib.config import config
+from http_host_lib.utils import assert_linux, assert_sudo
+
+
+def auto_mount():
+    """
+    Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
+    When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
+    """
+
+    print('Running auto mount')
+
+    assert_linux()
+    assert_sudo()
+
+    if not config.runs_dir.exists():
+        sys.exit('  download-btrfs needs to be run first')
+
+    # clean_up_mounts(config.mnt_dir)  # disabling, as it can be in use before the nginx sync works
+    create_fstab()
+
+    print('  running mount -a')
+    subprocess.run(['mount', '-a'], check=True)
+
+
+def create_fstab():
+    print('  creating fstab')
+    fstab_new = []
+
+    for area in ['planet', 'monaco']:
+        area_dir = (config.runs_dir / area).resolve()
+        if not area_dir.exists():
+            continue
+
+        versions = sorted(area_dir.iterdir())
+        for version in versions:
+            version_str = version.name
+            btrfs_file = area_dir / version_str / 'tiles.btrfs'
+            if not btrfs_file.is_file():
+                print(f"  {btrfs_file} doesn't exist, skipping")
+                continue
+
+            mnt_folder = config.mnt_dir / f'{area}-{version_str}'
+            mnt_folder.mkdir(exist_ok=True, parents=True)
+
+            fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n')
+            print(f'  created fstab entry for {mnt_folder}')
+
+    with open('/etc/fstab') as fp:
+        fstab_orig = [l for l in fp.readlines() if f'{config.mnt_dir}/' not in l]
+
+    with open('/etc/fstab', 'w') as fp:
+        fp.writelines(fstab_orig + fstab_new)
+
+
+def clean_up_mounts(mnt_dir):
+    if not mnt_dir.exists():
+        return
+
+    print('Cleaning up mounts')
+
+    # handle deleted files
+    p = subprocess.run(['mount'], capture_output=True, text=True, check=True)
+    lines = [l for l in p.stdout.splitlines() if f'{mnt_dir}/' in l and '(deleted)' in l]
+
+    # Extract unique mount paths (deduplicate)
+    mount_paths = set()
+    for l in lines:
+        mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0])
+        mount_paths.add(mnt_path)
+
+    # Process each unique mount path once
+    for mnt_path in mount_paths:
+        if not mnt_path.exists():
+            print(f'  skipping {mnt_path} (already removed)')
+            continue
+
+        print(f'  removing deleted mount {mnt_path}')
+
+        # Unmount ALL instances (handle stacked mounts)
+        while subprocess.run(['mountpoint', '-q', str(mnt_path)]).returncode == 0:
+            print(f'    unmounting {mnt_path}')
+            subprocess.run(['umount', mnt_path], check=True)
+
+        mnt_path.rmdir()
+
+    # clean all mounts not in current fstab
+    with open('/etc/fstab') as fp:
+        fstab_str = fp.read()
+
+    for subdir in mnt_dir.iterdir():
+        if f'{subdir} ' in fstab_str:
+            continue
+
+        print(f'  removing old mount {subdir}')
+
+        # Unmount ALL instances here too
+        while subprocess.run(['mountpoint', '-q', str(subdir)]).returncode == 0:
+            print(f'    unmounting {subdir}')
+            subprocess.run(['umount', subdir], check=True)
+
+        subdir.rmdir()

modules/http_host/http_host_lib/nginx_config_gen.py

@@ -0,0 +1,292 @@
+import subprocess
+import sys
+from pathlib import Path
+
+from http_host_lib.config import config
+from http_host_lib.utils import python_venv_executable
+
+
+def write_nginx_config():
+    print('Writing nginx config')
+
+    if not config.mnt_dir.exists():
+        sys.exit('  mount needs to be run first')
+
+    # remove old configs
+    for file in config.nginx_sites_dir.glob('ofm-*.conf'):
+        file.unlink()
+
+    curl_help_text = ''
+
+    for domain_data in config.json_config['domains']:
+        curl_help_text += process_domain(domain_data)
+
+    subprocess.run(['nginx', '-t'], check=True)
+    subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
+
+    exclude_path = '/planet' if config.json_config.get('skip_planet') else '/monaco'
+    curl_help_lines = [l for l in curl_help_text.splitlines() if exclude_path not in l]
+
+    curl_help_joined = '\n'.join(curl_help_lines)
+    print(f'test with:\n{curl_help_joined}')
+
+
+def process_domain(domain_data) -> str:
+    if domain_data['cert']['type'] == 'upload':
+        if (
+            not Path(domain_data['cert_file']).is_file()
+            or not Path(domain_data['key_file']).is_file()
+        ):
+            sys.exit(
+                f'  cert or key file does not exist: {domain_data["cert_file"]} {domain_data["key_file"]}'
+            )
+
+        return create_nginx_conf(domain_data)
+
+    return ''
+
+
+def create_nginx_conf(domain_data: dict) -> str:
+    dynamic_block_text, curl_help_text = dynamic_blocks(domain_data)
+
+    template = (config.nginx_templates / 'common.conf').read_text()
+
+    template = template.replace('__DYNAMIC_BLOCKS__', dynamic_block_text)
+
+    template = template.replace('__DOMAIN_SLUG__', domain_data['slug'])
+    template = template.replace('__DOMAIN__', domain_data['domain'])
+
+    curl_help_text = curl_help_text.replace('__DOMAIN_SLUG__', domain_data['slug'])
+    curl_help_text = curl_help_text.replace('__DOMAIN__', domain_data['domain'])
+
+    (config.nginx_sites_dir / f'ofm-{domain_data["slug"]}.conf').write_text(template)
+    print(f'  nginx config written: {domain_data["domain"]} {domain_data["slug"]}')
+
+    return curl_help_text
+
+
+def dynamic_blocks(domain_data: dict) -> tuple[str, str]:
+    nginx_conf_text = ''
+    curl_help_text = ''
+
+    help_area = 'monaco' if config.json_config.get('skip_planet') else 'planet'
+
+    for subdir in config.mnt_dir.iterdir():
+        if not subdir.is_dir():
+            continue
+        area, version = subdir.name.split('-')
+
+        nginx_conf_text += create_version_location(
+            area=area, version=version, mnt_dir=subdir, domain_data=domain_data
+        )
+
+        if area == help_area:
+            for path in [
+                f'/{area}/{version}',
+                f'/{area}/{version}/14/8529/5974.pbf',
+                # f'/{area}/{version}/9999/9999/9999.pbf',  # empty_tile test
+            ]:
+                # curl_help_text += f'curl -H "Host: __DOMAIN_SLUG__" -I http://localhost{path}\n'
+                curl_help_text += f'curl -sI https://__DOMAIN__{path}\n'
+
+    nginx_conf_text += create_latest_locations(domain_data=domain_data)
+
+    for path in [
+        f'/{help_area}',
+        f'/{help_area}/latest',
+        f'/{help_area}/latest/14/8529/5974.pbf',
+        # f'/{help_area}/latest/9999/9999/9999.pbf',  # empty_tile test
+    ]:
+        # curl_help_text += f'curl -H "Host: __DOMAIN_SLUG__" -I http://localhost{path}\n'
+        curl_help_text += f'curl -sI https://__DOMAIN__{path}\n'
+
+    nginx_conf_text += '\n' + (config.nginx_templates / 'static_blocks.conf').read_text()
+    return nginx_conf_text, curl_help_text
+
+
+def create_version_location(*, area: str, version: str, mnt_dir: Path, domain_data: dict) -> str:
+    run_dir = config.runs_dir / area / version
+    if not run_dir.is_dir():
+        print(f"  {run_dir} doesn't exist, skipping")
+        return ''
+
+    tilejson_path = run_dir / f'tilejson-{domain_data["slug"]}.json'
+
+    metadata_path = mnt_dir / 'metadata.json'
+    if not metadata_path.is_file():
+        print(f"  {metadata_path} doesn't exist, skipping")
+        return ''
+
+    url_prefix = f'https://{domain_data["domain"]}/{area}/{version}'
+
+    subprocess.run(
+        [
+            python_venv_executable(),
+            config.http_host_scripts_dir / 'metadata_to_tilejson.py',
+            '--minify',
+            metadata_path,
+            tilejson_path,
+            url_prefix,
+        ],
+        check=True,
+    )
+
+    return f"""
+    # specific JSON {area} {version}
+    location = /{area}/{version} {{ # no trailing slash
+        alias {tilejson_path}; # no trailing slash
+
+        expires 1w;
+        default_type application/json;
+
+        add_header 'Access-Control-Allow-Origin' '*' always;
+        add_header Cache-Control public;
+        add_header X-Robots-Tag "noindex, nofollow" always;
+
+        add_header x-ofm-debug 'specific JSON {area} {version}';
+    }}
+
+    # specific PBF {area} {version}
+    location ^~ /{area}/{version}/ {{ # trailing slash
+        alias {mnt_dir}/tiles/; # trailing slash
+        try_files $uri @empty_tile;
+        add_header Content-Encoding gzip;
+
+        expires 10y;
+
+        types {{
+            application/vnd.mapbox-vector-tile pbf;
+        }}
+
+        add_header 'Access-Control-Allow-Origin' '*' always;
+        add_header Cache-Control public;
+        add_header X-Robots-Tag "noindex, nofollow" always;
+
+        add_header x-ofm-debug 'specific PBF {area} {version}';
+    }}
+    """
+
+
+def create_latest_locations(*, domain_data: dict) -> str:
+    location_str = ''
+
+    local_version_files = config.deployed_versions_dir.glob('*.txt')
+
+    for file in local_version_files:
+        area = file.stem
+        with open(file) as fp:
+            version = fp.read().strip()
+
+        print(f'  linking latest version for {area}: {version}')
+
+        # checking runs dir
+        run_dir = config.runs_dir / area / version
+        tilejson_path = run_dir / f'tilejson-{domain_data["slug"]}.json'
+        if not tilejson_path.is_file():
+            print(
+                f'    skipping latest block for {area} / {version}: {tilejson_path} does not exist'
+            )
+            continue
+
+        # checking mnt dir
+        mnt_dir = Path(f'/mnt/ofm/{area}-{version}')
+        mnt_file = mnt_dir / 'metadata.json'
+        if not mnt_file.is_file():
+            print(f'    skipping latest block for {area} / {version}: {mnt_file} does not exist')
+            continue
+
+        # latest
+        location_str += f"""
+
+        # latest JSON {area}
+        location = /{area} {{ # no trailing slash
+            alias {tilejson_path}; # no trailing slash
+
+            expires 1d;
+            default_type application/json;
+
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header Cache-Control public;
+            add_header X-Robots-Tag "noindex, nofollow" always;
+
+            add_header x-ofm-debug 'latest JSON {area}';
+        }}
+        """
+
+        # wildcard
+        # identical to create_version_location
+        location_str += f"""
+
+        # wildcard JSON {area}
+        location ~ ^/{area}/([^/]+)$ {{
+            # regex location is unreliable with alias, only root is reliable
+
+            root {run_dir}; # no trailing slash
+            try_files /tilejson-{domain_data['slug']}.json =404;
+
+            expires 1w;
+            default_type application/json;
+
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header Cache-Control public;
+            add_header X-Robots-Tag "noindex, nofollow" always;
+
+            add_header x-ofm-debug 'wildcard JSON {area}';
+        }}
+
+        # wildcard PBF {area}
+        location ~ ^/{area}/([^/]+)/(.+)$ {{
+            # regex location is unreliable with alias, only root is reliable
+
+            root {mnt_dir}/tiles/; # trailing slash
+            try_files /$2 @empty_tile;
+            add_header Content-Encoding gzip;
+
+            expires 10y;
+
+            types {{
+                application/vnd.mapbox-vector-tile pbf;
+            }}
+
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header Cache-Control public;
+            add_header X-Robots-Tag "noindex, nofollow" always;
+
+            add_header x-ofm-debug 'wildcard PBF {area}';
+        }}
+        """
+
+    return location_str
+
+
+# if not self_signed_certs:
+#     subprocess.run(
+#         [
+#             'certbot',
+#             'certonly',
+#             '--webroot',
+#             '--webroot-path=/data/nginx/acme-challenges',
+#             '--noninteractive',
+#             '-m',
+#             config.ofm_config['letsencrypt_email'],
+#             '--agree-tos',
+#             '--cert-name=ofm_direct',
+#             # '--staging',
+#             '--deploy-hook',
+#             'nginx -t && service nginx reload',
+#             '-d',
+#             domain_direct,
+#         ],
+#         check=True,
+#     )
+#
+#     # link certs to nginx dir
+#     direct_cert.unlink()
+#     direct_key.unlink()
+#
+#     etc_cert = Path('/etc/letsencrypt/live/ofm_direct/fullchain.pem')
+#     etc_key = Path('/etc/letsencrypt/live/ofm_direct/privkey.pem')
+#     assert etc_cert.is_file()
+#     assert etc_key.is_file()
+#     direct_cert.symlink_to(etc_cert)
+#     direct_key.symlink_to(etc_key)

modules/http_host/http_host_lib/nginx_templates/_letsencrypt.conf

@@ -1,5 +1,5 @@
 server {
-    server_name __LOCAL__ __DOMAIN__;
+    server_name __LOOPBACK_HOSTNAME__ __DOMAIN__;
 
     # ssl: https://ssl-config.mozilla.org / intermediate config
 
@@ -8,8 +8,8 @@ server {
     listen [::]:443 ssl;
     http2 on;
 
-    ssl_certificate /data/nginx/certs/ofm_le.cert;
+    ssl_certificate /data/nginx/certs/ofm_direct.cert;
-    ssl_certificate_key /data/nginx/certs/ofm_le.key;
+    ssl_certificate_key /data/nginx/certs/ofm_direct.key;
 
     ssl_session_timeout 1d;
     ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
@@ -22,23 +22,46 @@ server {
     ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
     ssl_prefer_server_ciphers off;
 
-    # access log disabled by default
+
-    #access_log /data/ofm/http_host/logs_nginx/le-access.log access_json buffer=32k;
+    # access log doesn't contain IP address
     access_log off;
+    #access_log /data/ofm/http_host/logs_nginx/le-access.jsonl access_json buffer=128k;
 
     error_log /data/ofm/http_host/logs_nginx/le-error.log;
 
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
+
     location ^~ /.well-known/acme-challenge/ {
         # trailing slash
         root /data/nginx/acme-challenges;
         try_files $uri =404;
     }
 
-    __LOCATION_BLOCKS__
+    __DYNAMIC_BLOCKS__
+
+    location /styles/ {
+        # trailing slash
+
+        alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash
+        try_files $uri.json =404;
+
+        expires 1d;
+        default_type application/json;
+
+        # substitute the domain in the TileJSON
+        sub_filter '__TILEJSON_DOMAIN__' '__DOMAIN__';
+        sub_filter_once off;
+        sub_filter_types '*';
+
+        add_header 'Access-Control-Allow-Origin' '*' always;
+        add_header Cache-Control public;
+        add_header X-Robots-Tag "noindex, nofollow" always;
+    }
 
     # catch-all block to deny all other requests
     location / {
         deny all;
-        error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
+        error_log /data/ofm/http_host/logs_nginx/le-deny.log error;
     }
 }

modules/http_host/http_host_lib/nginx_templates/common.conf

@@ -0,0 +1,60 @@
+server {
+    server_name __DOMAIN_SLUG__ __DOMAIN__;
+
+    # ssl: https://ssl-config.mozilla.org / intermediate config
+
+    listen 80;
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+
+    ssl_certificate /data/nginx/certs/ofm-__DOMAIN_SLUG__.cert;
+    ssl_certificate_key /data/nginx/certs/ofm-__DOMAIN_SLUG__.key;
+
+    ssl_session_timeout 1d;
+    ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
+    ssl_session_tickets off;
+
+    ssl_dhparam /etc/nginx/ffdhe2048.txt;
+
+    # intermediate configuration
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
+    ssl_prefer_server_ciphers off;
+
+    # access log doesn't contain IP address
+    #access_log off;
+    access_log /data/ofm/http_host/logs_nginx/__DOMAIN_SLUG__-access.jsonl access_json buffer=128k;
+
+    error_log /data/ofm/http_host/logs_nginx/__DOMAIN_SLUG__-error.log;
+
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
+    __DYNAMIC_BLOCKS__
+
+    location /styles/ {
+        # trailing slash
+
+        alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash
+        try_files $uri.json =404;
+
+        expires 1d;
+        default_type application/json;
+
+        # substitute the domain in the TileJSON
+        sub_filter '__TILEJSON_DOMAIN__' '__DOMAIN__';
+        sub_filter_once off;
+        sub_filter_types '*';
+
+        add_header 'Access-Control-Allow-Origin' '*' always;
+        add_header Cache-Control public;
+        add_header X-Robots-Tag "noindex, nofollow" always;
+
+    }
+
+    # catch-all block to deny all other requests
+    location / {
+        deny all;
+        error_log /data/ofm/http_host/logs_nginx/__DOMAIN_SLUG__-deny.log error;
+    }
+}

modules/http_host/http_host_lib/nginx_templates/static_blocks.conf

@@ -1,3 +1,9 @@
+# Serve robots.txt that blocks all crawlers
+#location = /robots.txt {
+#    add_header Content-Type text/plain;
+#return 200 "User-agent: *\nDisallow: /\n";
+#}
+
 location /fonts/ {
     # trailing slash
 
@@ -8,19 +14,8 @@ location /fonts/ {
 
     add_header 'Access-Control-Allow-Origin' '*' always;
     add_header Cache-Control public;
-}
+    add_header X-Robots-Tag "noindex, nofollow" always;
 
-location /styles/ {
-    # trailing slash
-
-    alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash
-    try_files $uri.json =404;
-
-    expires 1d;
-    default_type application/json;
-
-    add_header 'Access-Control-Allow-Origin' '*' always;
-    add_header Cache-Control public;
 }
 
 location /natural_earth/ {
@@ -33,6 +28,8 @@ location /natural_earth/ {
 
     add_header 'Access-Control-Allow-Origin' '*' always;
     add_header Cache-Control public;
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
 }
 
 location /sprites/ {
@@ -45,6 +42,8 @@ location /sprites/ {
 
     add_header 'Access-Control-Allow-Origin' '*' always;
     add_header Cache-Control public;
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
 }
 
 
@@ -53,10 +52,16 @@ location @empty_tile {
     return 200 '';
 
     expires 10y;
-    default_type application/vnd.mapbox-vector-tile;
+
+    types {
+        application/vnd.mapbox-vector-tile pbf;
+    }
 
     add_header 'Access-Control-Allow-Origin' '*' always;
     add_header Cache-Control public;
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
+    add_header x-ofm-debug 'empty tile';
 }
 
 location = / {

modules/http_host/http_host_lib/sync.py

@@ -0,0 +1,98 @@
+import shutil
+
+from http_host_lib.assets import download_assets
+from http_host_lib.btrfs import download_area_version
+from http_host_lib.config import config
+from http_host_lib.mount import auto_mount, clean_up_mounts
+from http_host_lib.nginx_config_gen import write_nginx_config
+from http_host_lib.utils import assert_linux, assert_sudo
+from http_host_lib.versions import fetch_version_files
+
+
+def full_sync(force=False):
+    """
+    Runs the sync task, normally called by cron every minute
+    On a new server this also takes care of everything, no need to run anything manually.
+    """
+
+    assert_linux()
+    assert_sudo()
+
+    # if it's a manual/forced run, we clean up old/deleted mounts
+    if force:
+        clean_up_mounts(config.mnt_dir)
+
+    # start
+    versions_changed = fetch_version_files()
+
+    assets_changed = download_assets()
+
+    btrfs_downloaded = False
+
+    # download latest and deployed monaco
+    btrfs_downloaded += download_area_version(area='monaco', version='latest')
+    btrfs_downloaded += download_area_version(area='monaco', version='deployed')
+
+    # download latest and deployed planet
+    if not config.json_config.get('skip_planet'):
+        btrfs_downloaded += download_area_version(area='planet', version='latest')
+        btrfs_downloaded += download_area_version(area='planet', version='deployed')
+
+    if btrfs_downloaded or versions_changed or assets_changed or force:
+        auto_clean_btrfs()
+        auto_mount()
+
+        write_nginx_config()
+
+        clean_up_mounts(config.mnt_dir)
+
+
+def auto_clean_btrfs():
+    """
+    Clean old btrfs runs
+
+    For each area we keep max two versions:
+    1. The newest one available locally
+    2. The one currently deployed, specified in /data/ofm/config/deployed_versions
+    3. If there is no deployed version, then we include the second newest one
+    """
+
+    print('Running auto clean btrfs')
+
+    for area in config.areas:
+        area_dir = config.runs_dir / area
+        if not area_dir.is_dir():
+            continue
+
+        local_versions = sorted([i.name for i in area_dir.iterdir()])
+
+        versions_to_keep = set()
+
+        # add newest version
+        if local_versions:
+            versions_to_keep.add(local_versions[-1])
+
+        # add deployed version
+        try:
+            deployed_version_file = config.deployed_versions_dir / f'{area}.txt'
+            deployed_version = deployed_version_file.read_text().strip()
+            if (config.runs_dir / area / deployed_version).exists():
+                versions_to_keep.add(deployed_version)
+        except Exception:
+            pass
+
+        # if still only one version, we include the second newest one
+        if len(versions_to_keep) == 1 and len(local_versions) >= 2:
+            versions_to_keep.add(local_versions[-2])
+
+        print(f'  keeping runs for {area}: {sorted(versions_to_keep)}')
+
+        versions_to_remove = set(local_versions).difference(versions_to_keep)
+
+        for version in versions_to_remove:
+            # Interesting bit: linux allows us to remove the disk image file for a mount
+            # while the mount is still being used.
+            # We delete the disk image, update nginx config and only then unmount the /mnt dir.
+            print(f'  removing runs for {area}: {version}')
+            version_dir = config.runs_dir / area / version
+            shutil.rmtree(version_dir)

modules/http_host/http_host_lib/utils.py

@@ -1,7 +1,6 @@
 import os
 import subprocess
 import sys
-import time
 from pathlib import Path
 
 import requests
@@ -57,3 +56,14 @@ def download_file_aria2(url: str, local_file: Path):
         ],
         check=True,
     )
+
+
+def python_venv_executable() -> Path:
+    venv_path = os.environ.get('VIRTUAL_ENV')
+
+    if venv_path:
+        return Path(venv_path) / 'bin' / 'python'
+    elif sys.prefix != sys.base_prefix:
+        return Path(sys.prefix) / 'bin' / 'python'
+    else:
+        return Path(sys.executable)

modules/http_host/http_host_lib/versions.py

@@ -0,0 +1,38 @@
+from http_host_lib.config import config
+from http_host_lib.get_version_shared import get_deployed_version
+from http_host_lib.utils import assert_linux, assert_sudo
+
+
+def fetch_version_files() -> bool:
+    """
+    Syncs the version files from remote to local.
+    Remote versions are specified by https://assets.openfreemap.com/versions/deployed_{area}.txt
+    """
+
+    print('Syncing local version files')
+
+    assert_linux()
+    assert_sudo()
+
+    need_nginx_sync = False
+
+    for area in config.areas:
+        deployed_version = get_deployed_version(area)['version']
+        if not deployed_version:
+            print(f'  deployed version not found: {area}')
+            continue
+        print(f'  deployed version {area}: {deployed_version}')
+
+        local_version_file = config.deployed_versions_dir / f'{area}.txt'
+
+        try:
+            local_version_old = local_version_file.read_text()
+        except Exception:
+            local_version_old = None
+
+        if deployed_version != local_version_old:
+            config.deployed_versions_dir.mkdir(exist_ok=True, parents=True)
+            local_version_file.write_text(deployed_version)
+            need_nginx_sync = True
+
+    return need_nginx_sync

modules/http_host/setup.py

@@ -3,9 +3,8 @@ from setuptools import find_packages, setup
 
 requirements = [
     'click',
-    'requests',
     'pycurl',
-    'python-dotenv',
+    'requests',
 ]
 
 

modules/mapterhorn_mirror/debug.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#
+# ]
+# ///
+import subprocess
+from pathlib import Path
+from urllib.parse import urlparse
+
+
+rclone_config = Path('../../config/rclone.conf')
+
+url = 'https://download.mapterhorn.com/planet.pmtiles'
+
+parsed = urlparse(url)
+base_url = f'{parsed.scheme}://{parsed.netloc}'
+path = parsed.path.lstrip('/')
+
+bucket_name = 'ofm-mapterhorn'
+remote_name = 'remote'
+destination = f'{remote_name}:{bucket_name}'
+
+common_opts = [
+    # '--verbose=10',
+    # '--dump',
+    # 'headers',
+    '--progress',
+    '--config',
+    rclone_config,
+]
+
+subprocess.run(
+    [
+        'rclone',
+        'copy',
+        '--http-url',
+        base_url,
+        f':http:{path}',
+        destination,
+        '--multi-thread-streams=8',
+        '--s3-chunk-size=100M',
+        *common_opts,
+    ]
+)

modules/tile_gen/cron.d/ofm_tile_gen

@@ -0,0 +1,21 @@
+# Define common variables
+CMD="sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py"
+LOG_DIR=/data/ofm/tile_gen/logs
+
+# every day at 23:10, make a monaco run
+10 23 * * * ofm    $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
+
+# debug monaco run, normally disabled, enable to run every minute
+#* * * * * ofm    $CMD make-tiles monaco --upload >> $LOG_DIR/monaco-make-tiles.log 2>&1
+
+# every minute, set monaco to latest
+* * * * * ofm     $CMD set-version monaco >> $LOG_DIR/monaco-set-version.log 2>&1
+
+# every Wednesday, make a planet run
+10 0 * * 3 ofm    $CMD make-tiles planet --upload >> $LOG_DIR/planet-make-tiles.log 2>&1
+
+# every Saturday, set planet to latest
+0 11 * * 6 ofm     $CMD set-version planet >> $LOG_DIR/planet-set-version.log 2>&1
+
+# once per minute, create indexes
+* * * * * ofm     $CMD make-indexes >> $LOG_DIR/make-indexes-cron.log 2>&1

modules/tile_gen/scripts/README.md

@@ -0,0 +1 @@
+These are self contained Python scripts, they can be run outside of this project's environment.

modules/tile_gen/scripts/extract_mbtiles.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 import json
-import os
 import shutil
 import sqlite3
 import sys
@@ -34,9 +33,6 @@ def cli(mbtiles_path: Path, dir_path: Path):
     write_dedupl_files(c, dir_path=dir_path)
     write_tile_files(c, dir_path=dir_path)
 
-    # planetiler has missing tiles by design, so disabling this
-    # assert_all_tiles_present(mbtiles_path, dir_path)
-
     write_metadata(c, dir_path=dir_path)
     conn.commit()
 
@@ -125,24 +121,6 @@ def write_tile_files(c, *, dir_path):
                 raise
 
 
-def assert_all_tiles_present(mbtiles_path, dir_path):
-    """
-    If it's a full planet run,
-    make sure there are exactly the right number of files generated.
-    """
-
-    if 'planet' in mbtiles_path.resolve().parent.name:
-        assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14)
-        print(f'Tile number: {calculate_tiles_sum(14)} - OK')
-
-
-def count_files(folder):
-    total = 0
-    for root, dirs, files in os.walk(folder):
-        total += len(files)
-    return
-
-
 def get_fixed_dedupl_name(bug_fix_dict, dedupl_path):
     if dedupl_path in bug_fix_dict:
         return dedupl_path.with_name(f'{dedupl_path.name}-{bug_fix_dict[dedupl_path]}')
@@ -166,16 +144,5 @@ def flip_y(zoom, y):
     return (2**zoom - 1) - y
 
 
-def calculate_tiles(zoom_level):
-    return (2**zoom_level) ** 2
-
-
-def calculate_tiles_sum(zoom_level):
-    """
-    Sum of tiles up to zoom level (geometric series)
-    """
-    return (4 ** (zoom_level + 1) - 1) // 3
-
-
 if __name__ == '__main__':
     cli()

modules/tile_gen/tile_gen.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+from datetime import datetime, timezone
+
+import click
+from tile_gen_lib.btrfs import make_btrfs
+from tile_gen_lib.get_version_shared import (
+    get_deployed_version,
+    get_versions_for_area,
+)
+from tile_gen_lib.planetiler import run_planetiler
+from tile_gen_lib.rclone import make_indexes_for_bucket, set_version_on_bucket, upload_area
+
+
+now = datetime.now(timezone.utc)
+
+
+@click.group()
+def cli():
+    """
+    Generates tiles and uploads to CloudFlare
+    """
+
+
+@cli.command()
+@click.argument('area', required=True)
+@click.option('--upload', is_flag=True, help='Upload after generation is complete')
+def make_tiles(area, upload):
+    """
+    Generate tiles for a given area, optionally upload it to the btrfs bucket
+    """
+
+    print(f'---\n{now}\nStarting make-tiles {area} upload: {upload}')
+
+    run_folder = run_planetiler(area)
+    make_btrfs(run_folder)
+
+    if upload:
+        upload_area(area)
+
+
+@cli.command(name='upload-area')
+@click.argument('area', required=True)
+def upload_area_(area):
+    """
+    Upload all runs from a given area to the btrfs bucket
+    """
+
+    print(f'---\n{now}\nStarting upload-area {area}')
+
+    upload_area(area)
+
+
+@cli.command()
+def make_indexes():
+    """
+    Make indexes for all buckets
+    """
+
+    print(f'---\n{now}\nStarting make-indexes')
+
+    for bucket in ['ofm-btrfs', 'ofm-assets']:
+        make_indexes_for_bucket(bucket)
+
+
+@cli.command()
+@click.argument('area', required=True)
+@click.option(
+    '--version', default='latest', help='Optional version string, like "20231227_043106_pt"'
+)
+def set_version(area, version):
+    """
+    Set versions for a given area
+    """
+
+    print(f'---\n{now}\nStarting set-version {area}')
+
+    if version == 'latest':
+        versions = get_versions_for_area(area)
+        if not versions:
+            print(f'  No versions found for {area}')
+            return
+
+        version = versions[-1]
+        print(f'  Latest version on bucket: {area} {version}')
+
+    try:
+        if get_deployed_version(area)['version'] == version:
+            return
+    except Exception:
+        pass
+
+    set_version_on_bucket(area, version)
+
+
+if __name__ == '__main__':
+    cli()

modules/tile_gen/tile_gen_lib/btrfs.py

@@ -0,0 +1,146 @@
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from .config import config
+from .utils import python_venv_executable
+
+
+IMAGE_SIZE = '200G'
+
+
+def make_btrfs(run_folder: Path):
+    os.chdir(run_folder)
+
+    cleanup_folder(run_folder)
+
+    # make an empty file that's definitely bigger then the current OSM output
+    for image in ['image.btrfs', 'image2.btrfs']:
+        subprocess.run(['fallocate', '-l', IMAGE_SIZE, image], check=True)
+        subprocess.run(['mkfs.btrfs', '-m', 'single', image], check=True, capture_output=True)
+
+    for image, mount in [('image.btrfs', 'mnt_rw'), ('image2.btrfs', 'mnt_rw2')]:
+        Path(mount).mkdir()
+
+        # https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
+        # compression doesn't make sense, data is already gzip compressed
+        subprocess.run(
+            [
+                'sudo',
+                'mount',
+                '-t',
+                'btrfs',
+                '-o',
+                'noacl,nobarrier,noatime,max_inline=4096',
+                image,
+                mount,
+            ],
+            check=True,
+        )
+
+        subprocess.run(['sudo', 'chown', 'ofm:ofm', '-R', mount], check=True)
+
+    # extract mbtiles
+    extract_script = config.tile_gen_scripts_dir / 'extract_mbtiles.py'
+    with open('extract_out.log', 'w') as out, open('extract_err.log', 'w') as err:
+        subprocess.run(
+            [
+                python_venv_executable(),
+                extract_script,
+                'tiles.mbtiles',
+                'mnt_rw/extract',
+            ],
+            check=True,
+            stdout=out,
+            stderr=err,
+        )
+
+    shutil.copy('mnt_rw/extract/osm_date', '.')
+
+    # process logs
+    subprocess.run('grep fixed extract_out.log > dedupl_fixed.log', shell=True)
+
+    # unfortunately, by deleting files from the btrfs partition, the partition size grows
+    # so we need to rsync onto a new partition instead of deleting
+    with open('rsync_out.log', 'w') as out, open('rsync_err.log', 'w') as err:
+        subprocess.run(
+            [
+                'rsync',
+                '-avH',
+                '--max-alloc=4294967296',
+                '--exclude',
+                'dedupl',
+                'mnt_rw/extract/',
+                'mnt_rw2/',
+            ],
+            check=True,
+            stdout=out,
+            stderr=err,
+        )
+
+    # collect stats
+    for i, mount in enumerate(['mnt_rw', 'mnt_rw2'], 1):
+        with open(f'stats{i}.txt', 'w') as f:
+            for cmd in [
+                ['df', '-h', mount],
+                ['btrfs', 'filesystem', 'df', mount],
+                ['btrfs', 'filesystem', 'show', mount],
+                ['btrfs', 'filesystem', 'usage', mount],
+            ]:
+                f.write(f'\n\n{" ".join(cmd)}\n')
+                result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True)
+                f.write(result.stdout)
+
+    # unmount and cleanup
+    for mount in ['mnt_rw', 'mnt_rw2']:
+        subprocess.run(['sudo', 'umount', mount], check=True)
+
+    shutil.rmtree('mnt_rw')
+    shutil.rmtree('mnt_rw2')
+
+    # shrink btrfs
+    shrink_script = config.tile_gen_scripts_dir / 'shrink_btrfs.py'
+    with open('shrink_out.log', 'w') as out, open('shrink_err.log', 'w') as err:
+        subprocess.run(
+            ['sudo', python_venv_executable(), shrink_script, 'image2.btrfs'],
+            check=True,
+            stdout=out,
+            stderr=err,
+        )
+
+    os.unlink('image.btrfs')
+    shutil.move('image2.btrfs', 'tiles.btrfs')
+
+    # parallel gzip (pigz)
+    subprocess.run(['pigz', 'tiles.btrfs', '--fast'], check=True)
+
+    # move logs
+    Path('logs').mkdir()
+    for pattern in ['*.log', '*.txt']:
+        for file in Path().glob(pattern):
+            shutil.move(file, 'logs')
+
+    # create a checksum file, Ubuntu style naming convention
+    with open('SHA256SUMS', 'w') as out:
+        subprocess.run(
+            ['sha256sum', 'tiles.btrfs.gz', 'tiles.mbtiles'],
+            check=True,
+            stdout=out,
+        )
+
+    print('extract_btrfs.py DONE')
+
+
+def cleanup_folder(run_folder: Path):
+    print(f'cleaning up {run_folder}')
+
+    for mount in ['mnt_rw', 'mnt_rw2']:
+        subprocess.run(['sudo', 'umount', run_folder / mount], capture_output=True)
+
+    for pattern in ['mnt_rw*', 'tmp_*', '*.btrfs', '*.gz', '*.log', '*.txt', 'logs', 'osm_date']:
+        for item in run_folder.glob(pattern):
+            if item.is_dir():
+                shutil.rmtree(item)
+            else:
+                item.unlink()

modules/tile_gen/tile_gen_lib/config.py

@@ -0,0 +1,28 @@
+import subprocess
+from pathlib import Path
+
+
+class Configuration:
+    areas = ['planet', 'monaco']
+
+    tile_gen_dir = Path('/data/ofm/tile_gen')
+
+    tile_gen_bin = tile_gen_dir / 'bin'
+    tile_gen_scripts_dir = tile_gen_bin / 'scripts'
+
+    planetiler_bin = tile_gen_dir / 'planetiler'
+    planetiler_path = planetiler_bin / 'planetiler.jar'
+
+    runs_dir = tile_gen_dir / 'runs'
+
+    if Path('/data/ofm').exists():
+        ofm_config_dir = Path('/data/ofm/config')
+    else:
+        repo_root = Path(__file__).parent.parent.parent.parent
+        ofm_config_dir = repo_root / 'config'
+
+    rclone_config = ofm_config_dir / 'rclone.conf'
+    rclone_bin = subprocess.run(['which', 'rclone'], capture_output=True, text=True).stdout.strip()
+
+
+config = Configuration()

modules/tile_gen/tile_gen_lib/get_version_shared.py

@@ -0,0 +1,48 @@
+"""
+This file is shared / symlinked between tile_gen_lib and http_host_lib
+"""
+
+from datetime import datetime, timezone
+
+import requests
+
+
+def get_versions_for_area(area: str) -> list:
+    """
+    Download the files.txt and check for the runs with the "done" file present
+    """
+    r = requests.get('https://btrfs.openfreemap.com/files.txt', timeout=30)
+    r.raise_for_status()
+
+    versions = []
+
+    files = r.text.splitlines()
+    for f in files:
+        if not f.startswith(f'areas/{area}/'):
+            continue
+        if not f.endswith('/done'):
+            continue
+        version_str = f.split('/')[2]
+        versions.append(version_str)
+
+    return sorted(versions)
+
+
+def get_deployed_version(area: str) -> dict:
+    r = requests.get(f'https://assets.openfreemap.com/deployed_versions/{area}.txt', timeout=30)
+    r.raise_for_status()
+    version = r.text.strip()
+
+    last_modified_str = r.headers.get('Last-Modified')
+    last_modified = parse_http_last_modified(last_modified_str)
+
+    return dict(
+        version=version,
+        last_modified=last_modified,
+    )
+
+
+def parse_http_last_modified(date_string) -> datetime:
+    parsed_date = datetime.strptime(date_string, '%a, %d %b %Y %H:%M:%S GMT')
+    parsed_date = parsed_date.replace(tzinfo=timezone.utc)
+    return parsed_date

modules/tile_gen/tile_gen_lib/planetiler.py

@@ -0,0 +1,70 @@
+import os
+import shutil
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .btrfs import cleanup_folder
+from .config import config
+
+
+def run_planetiler(area: str) -> Path:
+    assert area in config.areas
+
+    date = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')
+
+    area_dir = config.runs_dir / area
+
+    # delete all previous runs for the given area
+    if area_dir.is_dir():
+        for subdir in area_dir.iterdir():
+            cleanup_folder(subdir)
+
+        print('running rmtree')
+        shutil.rmtree(area_dir, ignore_errors=True)
+        print('rmtree done')
+
+    run_folder = area_dir / f'{date}_pt'
+    run_folder.mkdir(parents=True, exist_ok=True)
+
+    os.chdir(run_folder)
+
+    # link to discussion about why exactly 30 GB
+    # https://github.com/onthegomap/planetiler/discussions/690#discussioncomment-7756397
+    java_memory_gb = 30 if area == 'planet' else 1
+
+    command = [
+        'java',
+        f'-Xmx{java_memory_gb}g',
+        '-jar',
+        config.planetiler_path,
+        f'--area={area}',
+        '--download',
+        '--download-threads=10',
+        '--download-chunk-size-mb=1000',
+        '--fetch-wikidata',
+        '--output=tiles.mbtiles',
+        '--storage=mmap',
+        '--force',
+        '--languages=default,tok',
+    ]
+
+    if area == 'planet':
+        command.append('--nodemap-type=array')
+        command.append('--bounds=planet')
+
+    if area == 'monaco':
+        command.append('--nodemap-type=sortedtable')
+
+    print(command)
+
+    out_path = run_folder / 'planetiler.out'
+    err_path = run_folder / 'planetiler.err'
+
+    with out_path.open('w') as out_file, err_path.open('w') as err_file:
+        subprocess.run(command, stdout=out_file, stderr=err_file, check=True, cwd=run_folder)
+
+    shutil.rmtree(run_folder / 'data', ignore_errors=True)
+    print('planetiler.jar DONE')
+
+    return run_folder

modules/tile_gen/tile_gen_lib/rclone.py

@@ -0,0 +1,148 @@
+import subprocess
+import sys
+
+from .config import config
+
+
+def upload_area(area):
+    """
+    Uploads an area, making sure there is exactly one run present
+    """
+
+    print(f'Uploading area: {area}')
+
+    assert area in config.areas
+
+    area_dir = config.runs_dir / area
+    if not area_dir.exists():
+        return
+
+    runs = list(area_dir.iterdir())
+    if len(runs) != 1:
+        print('Error: Make sure there is only one run in the given area')
+        sys.exit(1)
+
+    run = runs[0].name
+
+    upload_area_run(area, run)
+    make_indexes_for_bucket('ofm-btrfs')
+
+
+def upload_area_run(area, run):
+    print(f'Uploading {area} {run} to btrfs bucket')
+
+    run_dir = config.runs_dir / area / run
+    assert run_dir.is_dir()
+
+    subprocess.run(
+        [
+            'rclone',
+            'sync',
+            '--verbose=1',
+            '--transfers=8',
+            '--multi-thread-streams=8',
+            '--fast-list',
+            '--stats-file-name-length=0',
+            '--stats-one-line',
+            '--log-file',
+            run_dir / 'logs' / 'rclone.log',
+            '--exclude',
+            'logs/**',
+            run_dir,
+            f'remote:ofm-btrfs/areas/{area}/{run}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+    )
+
+    # crate "done" file
+    subprocess.run(
+        [
+            'rclone',
+            'touch',
+            f'remote:ofm-btrfs/areas/{area}/{run}/done',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+    )
+
+
+def make_indexes_for_bucket(bucket):
+    print(f'Making indexes for bucket: {bucket}')
+
+    # files
+    p = subprocess.run(
+        [
+            'rclone',
+            'lsf',
+            '--recursive',
+            '--files-only',
+            '--fast-list',
+            '--exclude',
+            'dirs.txt',
+            '--exclude',
+            'files.txt',
+            f'remote:{bucket}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    index_str = p.stdout
+
+    # upload to files.txt
+    subprocess.run(
+        [
+            'rclone',
+            'rcat',
+            f'remote:{bucket}/files.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=index_str.encode(),
+    )
+
+    # directories
+    p = subprocess.run(
+        [
+            'rclone',
+            'lsf',
+            '--recursive',
+            '--dirs-only',
+            '--dir-slash=false',
+            '--fast-list',
+            f'remote:{bucket}',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    index_str = p.stdout
+
+    # upload to dirs.txt
+    subprocess.run(
+        [
+            'rclone',
+            'rcat',
+            f'remote:{bucket}/dirs.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=index_str.encode(),
+    )
+
+
+def set_version_on_bucket(area, version):
+    print(f'setting version: {area} {version}')
+    subprocess.run(
+        [
+            config.rclone_bin,
+            'rcat',
+            f'remote:ofm-assets/deployed_versions/{area}.txt',
+        ],
+        env=dict(RCLONE_CONFIG=config.rclone_config),
+        check=True,
+        input=version.strip().encode(),
+    )

modules/tile_gen/tile_gen_lib/utils.py

@@ -0,0 +1,14 @@
+import os
+import sys
+from pathlib import Path
+
+
+def python_venv_executable() -> Path:
+    venv_path = os.environ.get('VIRTUAL_ENV')
+
+    if venv_path:
+        return Path(venv_path) / 'bin' / 'python'
+    elif sys.prefix != sys.base_prefix:
+        return Path(sys.prefix) / 'bin' / 'python'
+    else:
+        return Path(sys.executable)

package.json

@@ -1,6 +1,8 @@
 {
   "dependencies": {
-    "prettier": "^3.2.4",
+    "@biomejs/biome": "^2.2.4",
+    "prettier": "^3.6.2",
     "prettier-plugin-astro": "^0.14.0"
-  }
+  },
+  "packageManager": "pnpm@10.18.2"
 }

prepare-virtualenv.sh

@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
 
-find . -name "*.egg-info" -exec rm -rf {} +
+find . -name '*.egg-info' -type d -prune -exec rm -rf {} +
-find . -name __pycache__ -exec rm -rf {} +
+find . -name '*.pyc' -delete
-
+find . -name __pycache__ -type d -prune -exec rm -rf {} +
-# deactivate
+find . -name .DS_Store -delete
-rm -rf venv
+find . -name .ipynb_checkpoints -exec rm -rf {} +
-python3 -m venv venv
+find . -name .pytest_cache -exec rm -rf {} +
-source venv/bin/activate
+find . -name .ruff_cache -exec rm -rf {} +
-
+find . -name .venv -type d -prune -exec rm -rf {} +
-pip install -U pip wheel setuptools
+find . -name venv -type d -prune -exec rm -rf {} +
-
-pip install -e .
-pip install -e scripts/http_host
-pip install -e scripts/tile_gen
-pip install -e scripts/loadbalancer
-pip install -e scripts/setversion
 
 
+uv venv --python=3.12
+source .venv/bin/activate
 
+
+uv pip install -e .
+uv pip install -e modules/http_host
+uv pip install -e modules/tile_gen

scripts/http_host/benchmark/wrk_usage.txt

@@ -1,8 +0,0 @@
-wrk -c10 -t4 -d10s -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost
-
-# -t1 => more correct, since the url list is loaded exactly in sequence
-# -t4 => reflecting real world usage
-
-
-
-

scripts/http_host/cron.d/ofm_http_host

@@ -1,4 +0,0 @@
-# every minute sync, locking so that only one process can run at a time
-* * * * * ofm     /usr/bin/flock -n /tmp/hostmanager.lockfile -c 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync >> /data/ofm/http_host/logs/host_manager_sync.log 2>&1'
-
-

scripts/http_host/cron.d/ofm_ledns_reader

@@ -1,2 +0,0 @@
-# once per day
-2 34 * * * ofm     sudo /usr/bin/bash /data/ofm/http_host/bin/ledns_reader.sh >> /data/ofm/http_host/logs/ledns_reader.log 2>&1

scripts/http_host/host_manager.py

@@ -1,202 +0,0 @@
-#!/usr/bin/env python3
-import datetime
-import subprocess
-import sys
-from pathlib import Path
-
-import click
-import requests
-from http_host_lib import DEFAULT_ASSETS_DIR, DEFAULT_RUNS_DIR, HOST_CONFIG, MNT_DIR
-from http_host_lib.download_assets import (
-    download_and_extract_asset_tar_gz,
-    download_sprites,
-)
-from http_host_lib.download_tileset import download_and_extract_tileset
-from http_host_lib.mount import clean_up_mounts, create_fstab
-from http_host_lib.nginx import write_nginx_config
-from http_host_lib.set_tileset_versions import set_tileset_versions
-from http_host_lib.utils import assert_linux, assert_sudo
-
-
-@click.group()
-def cli():
-    """
-    Manages OpenFreeMap HTTP hosts, including:\n
-    - Downloading assets\n
-    - Downloading tilesets\n
-    - Mounting directories\n
-    - Updating nginx config\n
-    - Setting the latest versions of tilesets\n
-    - Running the sync cron task (called every minute)
-    """
-
-
-@cli.command()
-@click.argument('area', required=False)
-@click.option('--version', default='latest', help='Version string, like "20231227_043106_pt"')
-@click.option(
-    '--runs-dir',
-    help='Specify runs directory',
-    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
-)
-@click.option('--list-versions', is_flag=True, help='List all versions in an area and terminate')
-def download_tileset(area: str, version: str, list_versions: bool, runs_dir: Path):
-    """
-    Downloads and extracts the latest tiles.btrfs file from the public bucket.
-    Version can also be specified.
-    """
-
-    print('running download_tileset')
-
-    if area not in {'planet', 'monaco'}:
-        sys.exit('  please specify area: "planet" or "monaco"')
-
-    r = requests.get(f'https://{area}.openfreemap.com/dirs.txt', timeout=30)
-    r.raise_for_status()
-
-    versions = sorted(r.text.splitlines())
-
-    all_versions_str = '\n'.join(versions)
-    if list_versions:
-        print(all_versions_str)
-        return
-
-    if version == 'latest':
-        selected_version = versions[-1]
-    else:
-        if version not in versions:
-            sys.exit(f'Requested version is not available. Available versions:\n{all_versions_str}')
-        selected_version = version
-
-    if not runs_dir:
-        runs_dir = DEFAULT_RUNS_DIR
-
-    if not runs_dir.parent.exists():
-        sys.exit("runs dir's parent doesn't exist")
-
-    return download_and_extract_tileset(area, selected_version, runs_dir)
-
-
-@cli.command()
-@click.option(
-    '--assets-dir',
-    help='Specify assets directory',
-    type=click.Path(dir_okay=True, file_okay=False, path_type=Path),
-)
-def download_assets(assets_dir: Path):
-    """
-    Downloads and extracts assets
-    """
-
-    print('running download_assets')
-
-    if not assets_dir:
-        assets_dir = DEFAULT_ASSETS_DIR
-
-    if not assets_dir.parent.exists():
-        sys.exit("asset dir's parent doesn't exist")
-
-    download_and_extract_asset_tar_gz(assets_dir, 'fonts')
-    download_and_extract_asset_tar_gz(assets_dir, 'styles')
-    download_and_extract_asset_tar_gz(assets_dir, 'natural_earth')
-
-    download_sprites(assets_dir)
-
-
-@cli.command()
-def mount():
-    """
-    Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
-    When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
-    """
-
-    print('running mount')
-
-    assert_linux()
-    assert_sudo()
-
-    if not DEFAULT_RUNS_DIR.exists():
-        sys.exit('  download_tileset needs to be run first')
-
-    clean_up_mounts(MNT_DIR)
-    create_fstab()
-
-    print('  running mount -a')
-    subprocess.run(['mount', '-a'], check=True)
-
-    clean_up_mounts(MNT_DIR)
-
-
-@cli.command()
-def set_latest_versions():
-    """
-    Sets the latest version of the tilesets to the version specified by
-    https://assets.openfreemap.com/versions/deployed_planet.txt
-
-    1. Checks if the given version is present on the disk and is mounted
-    2. Writes to a version file
-    """
-
-    print('running set_latest_versions')
-
-    assert_linux()
-    assert_sudo()
-
-    if not MNT_DIR.exists():
-        sys.exit('  mount needs to be run first')
-
-    return set_tileset_versions()
-
-
-@cli.command()
-def nginx_sync():
-    """
-    Syncs the nginx config to the state of the system
-    """
-
-    print('running nginx_sync')
-
-    assert_linux()
-    assert_sudo()
-
-    if not MNT_DIR.exists():
-        sys.exit('  mount needs to be run first')
-
-    write_nginx_config()
-
-
-@cli.command()
-@click.pass_context
-def sync(ctx):
-    """
-    Runs the sync task, normally called by cron every minute
-    On a new server this also takes care of everything, no need to run anything manually.
-    """
-
-    print('---')
-    print('running sync')
-    print(datetime.datetime.now(tz=datetime.timezone.utc))
-
-    assert_linux()
-    assert_sudo()
-
-    download_done = False
-    download_done += ctx.invoke(download_tileset, area='monaco')
-
-    if not HOST_CONFIG.get('skip_planet'):
-        download_done += ctx.invoke(download_tileset, area='planet')
-
-    if download_done:
-        ctx.invoke(mount)
-
-    ctx.invoke(download_assets)
-
-    deploy_done = ctx.invoke(set_latest_versions)
-
-    if download_done or deploy_done:
-        ctx.invoke(nginx_sync)
-
-
-if __name__ == '__main__':
-    # print(HOST_CONFIG)
-    cli()

scripts/http_host/http_host_lib/__init__.py

@@ -1,20 +0,0 @@
-import json
-from pathlib import Path
-
-
-NGINX_DIR = Path(__file__).parent / 'nginx'
-
-DEFAULT_RUNS_DIR = Path('/data/ofm/http_host/runs')
-DEFAULT_ASSETS_DIR = Path('/data/ofm/http_host/assets')
-
-MNT_DIR = Path('/mnt/ofm')
-OFM_CONFIG_DIR = Path('/data/ofm/config')
-HTTP_HOST_BIN_DIR = Path('/data/ofm/http_host/bin')
-
-CERTS_DIR = Path('/data/nginx/certs')
-
-try:
-    with open('/data/ofm/config/http_host.json') as fp:
-        HOST_CONFIG = json.load(fp)
-except Exception:
-    HOST_CONFIG = {}

scripts/http_host/http_host_lib/download_tileset.py

@@ -1,53 +0,0 @@
-import shutil
-import subprocess
-import sys
-from pathlib import Path
-
-import click
-
-from http_host_lib.utils import download_file_aria2, get_remote_file_size
-
-
-def download_and_extract_tileset(area: str, version: str, runs_dir: Path) -> bool:
-    """
-    returns True if downloaded something
-    """
-
-    click.echo(f'downloading {area} {version}')
-
-    version_dir = runs_dir / area / version
-    btrfs_file = version_dir / 'tiles.btrfs'
-    if btrfs_file.exists():
-        print('  file exists, skipping download')
-        return False
-
-    temp_dir = runs_dir / '_tmp'
-    shutil.rmtree(temp_dir, ignore_errors=True)
-    temp_dir.mkdir(parents=True)
-
-    url = f'https://{area}.openfreemap.com/{version}/tiles.btrfs.gz'
-
-    # check disk space
-    disk_free = shutil.disk_usage(temp_dir).free
-    file_size = get_remote_file_size(url)
-    if not file_size:
-        raise ValueError('Cannot get remote file size')
-
-    needed_space = file_size * 3
-    if disk_free < needed_space:
-        raise ValueError(f'Not enough disk space. Needed: {needed_space}, free space: {disk_free}')
-
-    target_file = temp_dir / 'tiles.btrfs.gz'
-    download_file_aria2(url, target_file)
-
-    print('Uncompressing...')
-    subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
-    btrfs_src = temp_dir / 'tiles.btrfs'
-
-    shutil.rmtree(version_dir, ignore_errors=True)
-    version_dir.mkdir(parents=True)
-
-    btrfs_src.rename(btrfs_file)
-
-    shutil.rmtree(temp_dir)
-    return True

scripts/http_host/http_host_lib/mount.py

@@ -1,62 +0,0 @@
-import subprocess
-from pathlib import Path
-
-from http_host_lib import DEFAULT_RUNS_DIR, MNT_DIR
-
-
-def create_fstab():
-    fstab_new = []
-
-    for area in ['planet', 'monaco']:
-        area_dir = (DEFAULT_RUNS_DIR / area).resolve()
-        if not area_dir.exists():
-            continue
-
-        versions = sorted(area_dir.iterdir())
-        for version in versions:
-            version_str = version.name
-            btrfs_file = area_dir / version_str / 'tiles.btrfs'
-            if not btrfs_file.is_file():
-                continue
-
-            mnt_folder = MNT_DIR / f'{area}-{version_str}'
-            mnt_folder.mkdir(exist_ok=True, parents=True)
-
-            fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n')
-            print(f'  created fstab entry for {btrfs_file} -> {mnt_folder}')
-
-    with open('/etc/fstab') as fp:
-        fstab_orig = [l for l in fp.readlines() if f'{MNT_DIR}/' not in l]
-
-    with open('/etc/fstab', 'w') as fp:
-        fp.writelines(fstab_orig + fstab_new)
-
-
-def clean_up_mounts(mnt_dir):
-    if not mnt_dir.exists():
-        return
-
-    print('  cleaning up mounts')
-
-    # handle deleted files
-    p = subprocess.run(['mount'], capture_output=True, text=True, check=True)
-    lines = [l for l in p.stdout.splitlines() if f'{mnt_dir}/' in l and '(deleted)' in l]
-
-    for l in lines:
-        mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0])
-        print(f'  removing deleted mount {mnt_path}')
-        assert mnt_path.exists()
-        subprocess.run(['umount', mnt_path], check=True)
-        mnt_path.rmdir()
-
-    # clean all mounts not in current fstab
-    with open('/etc/fstab') as fp:
-        fstab_str = fp.read()
-
-    for subdir in mnt_dir.iterdir():
-        if f'{subdir} ' in fstab_str:
-            continue
-
-        print(f'  removing old mount {subdir}')
-        subprocess.run(['umount', subdir], check=True)
-        subdir.rmdir()

scripts/http_host/http_host_lib/nginx.py

@@ -1,242 +0,0 @@
-import shutil
-import subprocess
-import sys
-from pathlib import Path
-
-from http_host_lib import (
-    CERTS_DIR,
-    DEFAULT_RUNS_DIR,
-    HOST_CONFIG,
-    HTTP_HOST_BIN_DIR,
-    MNT_DIR,
-    NGINX_DIR,
-    OFM_CONFIG_DIR,
-)
-
-
-def write_nginx_config():
-    curl_text_mix = ''
-
-    domain_le = HOST_CONFIG['domain_le']
-    domain_ledns = HOST_CONFIG['domain_ledns']
-
-    # remove old configs and certs
-    for file in Path('/data/nginx/sites').glob('ofm_*.conf'):
-        file.unlink()
-
-    for file in Path('/data/nginx/certs').glob('ofm_*'):
-        file.unlink()
-
-    # processing Round Robin DNS config
-    if domain_ledns:
-        if not (OFM_CONFIG_DIR / 'rclone.conf').is_file():
-            sys.exit('rclone.conf missing')
-
-        # download the ledns certificate from bucket using rclone
-        write_ledns_reader_script(domain_ledns)
-        subprocess.run(['bash', HTTP_HOST_BIN_DIR / 'ledns_reader.sh'], check=True)
-
-        curl_text_mix += create_nginx_conf(
-            template_path=NGINX_DIR / 'ledns.conf',
-            local='ofm_ledns',
-            domain=domain_ledns,
-        )
-
-    # processing Let's Encrypt config
-    if domain_le:
-        le_cert = CERTS_DIR / 'ofm_le.cert'
-        le_key = CERTS_DIR / 'ofm_le.key'
-
-        if not le_cert.is_file() or not le_key.is_file():
-            shutil.copyfile(Path('/etc/nginx/ssl/dummy.crt'), le_cert)
-            shutil.copyfile(Path('/etc/nginx/ssl/dummy.key'), le_key)
-
-        curl_text_mix += create_nginx_conf(
-            template_path=NGINX_DIR / 'le.conf',
-            local='ofm_le',
-            domain=domain_le,
-        )
-
-        subprocess.run(['nginx', '-t'], check=True)
-        subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
-
-        subprocess.run(
-            [
-                'certbot',
-                'certonly',
-                '--webroot',
-                '--webroot-path=/data/nginx/acme-challenges',
-                '--noninteractive',
-                '-m',
-                HOST_CONFIG['le_email'],
-                '--agree-tos',
-                '--cert-name=ofm_le',
-                # '--staging',
-                '--deploy-hook',
-                'nginx -t && service nginx reload',
-                '-d',
-                domain_le,
-            ],
-            check=True,
-        )
-
-        # link certs to nginx dir
-        le_cert.unlink()
-        le_key.unlink()
-
-        etc_cert = Path('/etc/letsencrypt/live/ofm_le/fullchain.pem')
-        etc_key = Path('/etc/letsencrypt/live/ofm_le/privkey.pem')
-        assert etc_cert.is_file()
-        assert etc_key.is_file()
-        le_cert.symlink_to(etc_cert)
-        le_key.symlink_to(etc_key)
-
-    subprocess.run(['nginx', '-t'], check=True)
-    subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
-
-    print(curl_text_mix)
-
-
-def create_nginx_conf(*, template_path, local, domain):
-    location_str, curl_text = create_location_blocks(local=local, domain=domain)
-
-    with open(template_path) as fp:
-        template = fp.read()
-
-    template = template.replace('__LOCATION_BLOCKS__', location_str)
-    template = template.replace('__LOCAL__', local)
-    template = template.replace('__DOMAIN__', domain)
-
-    curl_text = curl_text.replace('__LOCAL__', local)
-    curl_text = curl_text.replace('__DOMAIN__', domain)
-
-    with open(f'/data/nginx/sites/{local}.conf', 'w') as fp:
-        fp.write(template)
-        print(f'  nginx config written: {domain} {local}')
-
-    return curl_text
-
-
-def create_location_blocks(*, local, domain):
-    location_str = ''
-    curl_text = ''
-
-    for subdir in MNT_DIR.iterdir():
-        if not subdir.is_dir():
-            continue
-        area, version = subdir.name.split('-')
-        location_str += create_version_location(
-            area=area, version=version, subdir=subdir, local=local, domain=domain
-        )
-
-        if not curl_text:
-            curl_text = (
-                '\ntest with:\n'
-                f'curl -H "Host: __LOCAL__" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n'
-                f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf'
-            )
-
-    location_str += create_latest_locations(local=local, domain=domain)
-
-    with open(NGINX_DIR / 'location_static.conf') as fp:
-        location_str += '\n' + fp.read()
-
-    return location_str, curl_text
-
-
-def create_version_location(
-    *, area: str, version: str, subdir: Path, local: str, domain: str
-) -> str:
-    run_dir = DEFAULT_RUNS_DIR / area / version
-    if not run_dir.is_dir():
-        print(f"  {run_dir} doesn't exists, skipping")
-        return ''
-
-    tilejson_path = run_dir / f'tilejson-{local}.json'
-
-    metadata_path = subdir / 'metadata.json'
-    if not metadata_path.is_file():
-        print(f"  {metadata_path} doesn't exists, skipping")
-        return ''
-
-    url_prefix = f'https://{domain}/{area}/{version}'
-
-    subprocess.run(
-        [
-            sys.executable,
-            Path(__file__).parent.parent / 'metadata_to_tilejson.py',
-            '--minify',
-            metadata_path,
-            tilejson_path,
-            url_prefix,
-        ],
-        check=True,
-    )
-
-    return f"""
-    location = /{area}/{version} {{     # no trailing slash
-        alias {tilejson_path};          # no trailing slash
-
-        expires 1w;
-        default_type application/json;
-
-        add_header 'Access-Control-Allow-Origin' '*' always;
-        add_header Cache-Control public;
-    }}
-
-    location /{area}/{version}/ {{      # trailing slash
-        alias {subdir}/tiles/;          # trailing slash
-        try_files $uri @empty_tile;
-        add_header Content-Encoding gzip;
-
-        expires 10y;
-
-        types {{
-            application/vnd.mapbox-vector-tile pbf;
-        }}
-
-        add_header 'Access-Control-Allow-Origin' '*' always;
-        add_header Cache-Control public;
-    }}
-    """
-
-
-def create_latest_locations(*, local: str, domain: str) -> str:
-    location_str = ''
-
-    local_version_files = OFM_CONFIG_DIR.glob('tileset_version_*.txt')
-    for file in local_version_files:
-        area = file.stem.split('_')[-1]
-        with open(file) as fp:
-            version = fp.read().strip()
-        print(f'  setting latest version for {area}: {version}')
-
-        run_dir = DEFAULT_RUNS_DIR / area / version
-        tilejson_path = run_dir / f'tilejson-{local}.json'
-        assert tilejson_path.is_file()
-
-        location_str += f"""
-        location = /{area} {{          # no trailing slash
-            alias {tilejson_path};       # no trailing slash
-
-            expires 1d;
-            default_type application/json;
-
-            add_header 'Access-Control-Allow-Origin' '*' always;
-            add_header Cache-Control public;
-        }}
-        """
-
-    return location_str
-
-
-def write_ledns_reader_script(domain_ledns):
-    script = f"""
-#!/usr/bin/env bash
-export RCLONE_CONFIG=/data/ofm/config/rclone.conf
-rclone copyto -v "remote:ofm-private/ledns/{domain_ledns}/ofm_ledns.cert" /data/nginx/certs/ofm_ledns.cert
-rclone copyto -v "remote:ofm-private/ledns/{domain_ledns}/ofm_ledns.key" /data/nginx/certs/ofm_ledns.key
-    """.strip()
-
-    with open(HTTP_HOST_BIN_DIR / 'ledns_reader.sh', 'w') as fp:
-        fp.write(script)

scripts/http_host/http_host_lib/nginx/cf.bak

@@ -1,36 +0,0 @@
-server {
-    server_name __LOCAL__ __DOMAIN__;
-
-    # ssl: https://ssl-config.mozilla.org / modern config
-    # to be used with the Cloudflare proxied endpoint
-
-    listen 80;
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    http2 on;
-
-    ssl_certificate /data/nginx/certs/ofm_cf.cert;
-    ssl_certificate_key /data/nginx/certs/ofm_cf.key;
-
-    ssl_session_timeout 1d;
-    ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
-    ssl_session_tickets off;
-
-    # modern configuration
-    ssl_protocols TLSv1.3;
-    ssl_prefer_server_ciphers off;
-
-    # access log disabled by default
-    #access_log /data/ofm/http_host/logs_nginx/cf-access.log access_json buffer=32k;
-    access_log off;
-
-    error_log /data/ofm/http_host/logs_nginx/cf-error.log;
-
-    __LOCATION_BLOCKS__
-
-    # catch-all block to deny all other requests
-    location / {
-        deny all;
-        error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
-    }
-}

scripts/http_host/http_host_lib/nginx/ledns.conf

@@ -1,38 +0,0 @@
-server {
-    server_name __LOCAL__ __DOMAIN__;
-
-    # ssl: https://ssl-config.mozilla.org / intermediate config
-
-    listen 80;
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    http2 on;
-
-    ssl_certificate /data/nginx/certs/ofm_ledns.cert;
-    ssl_certificate_key /data/nginx/certs/ofm_ledns.key;
-
-    ssl_session_timeout 1d;
-    ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
-    ssl_session_tickets off;
-
-    ssl_dhparam /etc/nginx/ffdhe2048.txt;
-
-    # intermediate configuration
-    ssl_protocols TLSv1.2 TLSv1.3;
-    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
-    ssl_prefer_server_ciphers off;
-
-    # access log disabled by default
-    #access_log /data/ofm/http_host/logs_nginx/ledns-access.log access_json buffer=32k;
-    access_log off;
-
-    error_log /data/ofm/http_host/logs_nginx/ledns-error.log;
-
-    __LOCATION_BLOCKS__
-
-    # catch-all block to deny all other requests
-    location / {
-        deny all;
-        error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
-    }
-}

scripts/http_host/http_host_lib/set_tileset_versions.py

@@ -1,45 +0,0 @@
-from pathlib import Path
-
-import requests
-
-from http_host_lib import OFM_CONFIG_DIR
-
-
-def set_tileset_versions():
-    need_nginx_sync = False
-
-    for area in ['planet', 'monaco']:
-        r = requests.get(f'https://assets.openfreemap.com/versions/deployed_{area}.txt', timeout=30)
-        r.raise_for_status()
-        remote_version = r.text.strip()
-        print(f'  remote version for {area}: {remote_version}')
-
-        local_version_file = OFM_CONFIG_DIR / f'tileset_version_{area}.txt'
-
-        if not local_version_file.exists():
-            local_version_start = None
-        else:
-            with open(local_version_file) as fp:
-                local_version_start = fp.read()
-
-        if not remote_version:
-            print('    remote version not specified')
-            if local_version_start is not None:
-                local_version_file.unlink()
-                need_nginx_sync = True
-            continue
-
-        mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
-        if not mnt_file.exists():
-            print('    local version does not exist')
-            if local_version_start is not None:
-                local_version_file.unlink()
-                need_nginx_sync = True
-            continue
-
-        if remote_version != local_version_start:
-            with open(local_version_file, 'w') as fp:
-                fp.write(remote_version)
-            need_nginx_sync = True
-
-    return need_nginx_sync

scripts/ledns/rclone_write.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-#env > /data/ofm/ledns/env.txt
-#RENEWED_DOMAINS=direct.openfreemap.org
-#RENEWED_LINEAGE=/etc/letsencrypt/live/ofm_ledns
-
-export RCLONE_CONFIG=/data/ofm/config/rclone.conf
-
-rclone copyto -v --copy-links "$RENEWED_LINEAGE/fullchain.pem" "remote:ofm-private/ledns/$RENEWED_DOMAINS/ofm_ledns.cert"
-rclone copyto -v --copy-links "$RENEWED_LINEAGE/privkey.pem" "remote:ofm-private/ledns/$RENEWED_DOMAINS/ofm_ledns.key"
-

scripts/loadbalancer/cron.d/ofm_loadbalancer

@@ -1,8 +0,0 @@
-# every minute
-
-# fix
-#* * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py fix >> /data/ofm/loadbalancer/logs/run.log 2>&1
-
-
-# check
-* * * * * ofm     sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py check >> /data/ofm/loadbalancer/logs/run.log 2>&1

scripts/loadbalancer/loadbalancer.py

@@ -1,157 +0,0 @@
-#!/usr/bin/env python3
-import datetime
-import json
-
-import click
-import requests
-from dotenv import dotenv_values
-from loadbalancer_lib import OFM_CONFIG_DIR
-from loadbalancer_lib.cloudflare import get_zone_id, set_records_round_robin
-from loadbalancer_lib.curl import pycurl_get, pycurl_status
-from loadbalancer_lib.telegram_ import telegram_send_message
-
-
-AREAS = ['planet', 'monaco']
-
-
-@click.group()
-def cli():
-    """
-    Manages load-balancing of Round-Robin DNS records
-    """
-
-
-@cli.command()
-def check():
-    """
-    Runs load-balancing check (triggered by cron every minute)
-    """
-
-    print(f'starting loadbalancer check at: {datetime.datetime.now(tz=datetime.timezone.utc)}')
-    check_or_fix(fix=False)
-
-
-@cli.command()
-def fix():
-    """
-    Fixes records based on check results
-    """
-
-    print(f'starting loadbalancer fix at: {datetime.datetime.now(tz=datetime.timezone.utc)}')
-    check_or_fix(fix=True)
-
-
-def check_or_fix(fix=False):
-    with open(OFM_CONFIG_DIR / 'loadbalancer.json') as fp:
-        c = json.load(fp)
-        # print(c)
-
-    if not c['http_host_list']:
-        telegram_send_message(
-            'OFM loadbalancer no hosts found on list, terminating',
-            c['telegram_token'],
-            c['telegram_chat_id'],
-        )
-        return
-
-    try:
-        results_by_ip = {}
-        working_hosts = set()
-
-        for area in AREAS:
-            for host_ip, host_is_ok in run_area(c, area).items():
-                results_by_ip.setdefault(host_ip, True)
-                results_by_ip[host_ip] &= host_is_ok
-
-        for host_ip, host_is_ok in results_by_ip.items():
-            if not host_is_ok:
-                message = f'OFM loadbalancer ERROR with host: {host_ip}'
-                telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
-            else:
-                working_hosts.add(host_ip)
-
-    except Exception as e:
-        message = f'OFM loadbalancer ERROR with loadbalancer: {e}'
-        telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
-        return
-
-    print(f'working hosts: {sorted(working_hosts)}')
-
-    if fix:
-        # if no hosts are detected working, probably a bug in this script
-        # fail-safe to include all hosts
-        if not working_hosts:
-            working_hosts = set(c['http_host_list'])
-
-            message = 'OFM loadbalancer FIX found no working hosts, reverting to full list!'
-            telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
-
-        updated = update_records(c, working_hosts)
-        if updated:
-            message = f'OFM loadbalancer FIX modified records, new records: {working_hosts}'
-            telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
-
-
-def run_area(c, area):
-    target_version = get_target_version(area)
-
-    print(f'target version: {area}: {target_version}')
-
-    results = {}
-
-    for host_ip in c['http_host_list']:
-        try:
-            check_host(c['domain_ledns'], host_ip, area, target_version)
-            results[host_ip] = True
-        except Exception as e:
-            results[host_ip] = False
-            print(e)
-
-    return results
-
-
-def check_host(domain, host_ip, area, version):
-    # check TileJSON first
-    url = f'https://{domain}/{area}'
-    tilejson_str = pycurl_get(url, domain, host_ip)
-    tilejson = json.loads(tilejson_str)
-    tiles_url = tilejson['tiles'][0]
-    version_in_tilejson = tiles_url.split('/')[4]
-    assert version_in_tilejson == version
-
-    # check actual vector tile
-    url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
-    assert pycurl_status(url, domain, host_ip) == 200
-
-
-def get_target_version(area):
-    url = f'https://assets.openfreemap.com/versions/deployed_{area}.txt'
-    response = requests.get(url)
-    response.raise_for_status()
-    return response.text.strip()
-
-
-def update_records(c, working_hosts) -> bool:
-    config = dotenv_values(OFM_CONFIG_DIR / 'cloudflare.ini')
-    cloudflare_api_token = config['dns_cloudflare_api_token']
-
-    domain = '.'.join(c['domain_ledns'].split('.')[-2:])
-    zone_id = get_zone_id(domain, cloudflare_api_token=cloudflare_api_token)
-
-    updated = False
-
-    updated |= set_records_round_robin(
-        zone_id=zone_id,
-        name=c['domain_ledns'],
-        host_ip_set=working_hosts,
-        proxied=False,
-        ttl=300,
-        comment='domain_ledns',
-        cloudflare_api_token=cloudflare_api_token,
-    )
-
-    return updated
-
-
-if __name__ == '__main__':
-    cli()

scripts/loadbalancer/loadbalancer_lib/__init__.py

@@ -1,9 +0,0 @@
-from pathlib import Path
-
-
-if Path('/data/ofm/config').exists():
-    OFM_CONFIG_DIR = Path('/data/ofm/config')
-else:
-    OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
-
-assert OFM_CONFIG_DIR.exists()

scripts/loadbalancer/loadbalancer_lib/cloudflare.py

@@ -1,109 +0,0 @@
-from pprint import pprint
-
-import requests
-
-
-# docs: https://api.cloudflare.com/
-
-
-def cloudflare_get(path: str, params: dict, cloudflare_api_token: str):
-    headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
-    res = requests.get(
-        f'https://api.cloudflare.com/client/v4{path}', headers=headers, params=params
-    )
-    res.raise_for_status()
-    data = res.json()
-    assert data['success'] is True
-    return data
-
-
-def get_zone_id(domain, cloudflare_api_token: str):
-    data = cloudflare_get(
-        '/zones', params=dict(name=domain), cloudflare_api_token=cloudflare_api_token
-    )
-    assert len(data['result']) == 1
-    zone_info = data['result'][0]
-    return zone_info['id']
-
-
-def get_dns_records_round_robin(zone_id, cloudflare_api_token: str) -> dict:
-    data = cloudflare_get(
-        f'/zones/{zone_id}/dns_records',
-        params=dict(per_page=5000),
-        cloudflare_api_token=cloudflare_api_token,
-    )
-    records = data['result']
-
-    data = {}
-
-    for r in records:
-        if r['type'] != 'A':
-            continue
-
-        data.setdefault(r['name'], [])
-        data[r['name']].append(dict(content=r['content'], id=r['id']))
-
-    return data
-
-
-def set_records_round_robin(
-    zone_id,
-    *,
-    name: str,
-    host_ip_set: set,
-    ttl: int = 1,
-    proxied: bool,
-    comment: str = None,
-    cloudflare_api_token: str,
-) -> bool:
-    headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
-
-    dns_records = get_dns_records_round_robin(zone_id, cloudflare_api_token=cloudflare_api_token)
-    current_records = dns_records.get(name, [])
-
-    current_ips = {r['content'] for r in current_records}
-    if current_ips == host_ip_set:
-        print(f'No need to update records: {name} currently set: {sorted(current_ips)}')
-        return False
-
-    # changing records
-
-    # delete all current records first
-    for r in current_records:
-        delete_record(zone_id, id_=r['id'], cloudflare_api_token=cloudflare_api_token)
-
-    # create new records
-    for ip in host_ip_set:
-        print(f'Creating record: {name} {ip}')
-        json_data = dict(
-            type='A',
-            name=name,
-            content=ip,
-            ttl=ttl,
-            proxied=proxied,
-            comment=comment,
-        )
-        res = requests.post(
-            f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records',
-            headers=headers,
-            json=json_data,
-        )
-        res.raise_for_status()
-        data = res.json()
-        assert data['success'] is True
-
-    return True
-
-
-def delete_record(zone_id, *, id_: str, cloudflare_api_token: str):
-    headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
-
-    print(f'Deleting record: {id_}')
-    res = requests.delete(
-        f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records/{id_}',
-        headers=headers,
-        json={},
-    )
-    res.raise_for_status()
-    data = res.json()
-    assert data['success'] is True

scripts/loadbalancer/loadbalancer_lib/curl.py

@@ -1,54 +0,0 @@
-from io import BytesIO
-from pathlib import Path
-
-import pycurl
-
-
-def pycurl_status(url, domain, host_ip):
-    """
-    Uses pycurl to make a HTTPS HEAD request using custom resolving,
-    checks if the status code is 200
-    """
-
-    c = pycurl.Curl()
-    c.setopt(c.URL, url)
-
-    # linux needs CA certs specified manually
-    if Path('/etc/ssl/certs/ca-certificates.crt').exists():
-        c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
-
-    c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
-    c.setopt(c.NOBODY, True)
-    c.setopt(c.TIMEOUT, 5)
-    c.perform()
-    status_code = c.getinfo(c.RESPONSE_CODE)
-    c.close()
-
-    return status_code
-
-
-def pycurl_get(url, domain, host_ip):
-    """
-    Uses pycurl to make a HTTPS GET request using custom resolving,
-    checks if the status code is 200, and returns the content.
-    """
-
-    buffer = BytesIO()
-    c = pycurl.Curl()
-    c.setopt(c.URL, url)
-
-    # linux needs CA certs specified manually
-    if Path('/etc/ssl/certs/ca-certificates.crt').exists():
-        c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
-
-    c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
-    c.setopt(c.WRITEDATA, buffer)
-    c.setopt(c.TIMEOUT, 5)
-    c.perform()
-    status_code = c.getinfo(c.RESPONSE_CODE)
-    c.close()
-
-    if status_code != 200:
-        raise ValueError(f'status code: {status_code}')
-
-    return buffer.getvalue().decode('utf8')

scripts/loadbalancer/loadbalancer_lib/telegram_.py

@@ -1,16 +0,0 @@
-import requests
-
-
-def telegram_send_message(message, bot_token, chat_id):
-    print(message)
-
-    url = f'https://api.telegram.org/bot{bot_token}/sendMessage'
-
-    payload = {'chat_id': chat_id, 'text': message}
-
-    response = requests.post(url, data=payload)
-
-    if response.status_code == 200:
-        print('  Message sent successfully!')
-    else:
-        print('  Failed to send message:', response.text)

scripts/setversion/setup.py

@@ -1,17 +0,0 @@
-from setuptools import find_packages, setup
-
-
-requirements = [
-    'click',
-    'requests',
-    'pycurl',
-    'python-dotenv',
-    'questionary',
-]
-
-
-setup(
-    python_requires='>=3.10',
-    install_requires=requirements,
-    packages=find_packages(),
-)

scripts/setversion/setversion.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python3
-
-import subprocess
-
-import click
-import questionary
-from setversion_lib import RCLONE_BIN, RCLONE_CONF
-
-
-@click.group()
-def cli():
-    """
-    Sets deployed reference versions
-    """
-
-
-@cli.command()
-@click.argument('area', required=True)
-def interactive(area):
-    versions = get_available_versions(area)[::-1]
-
-    choices = [questionary.Choice(title=r, value=i) for i, r in enumerate(versions)]
-    answer = questionary.select(f'Select version for: {area}', choices=choices).ask()
-
-    selected = versions[answer]
-
-    set_version(area, selected)
-
-
-def get_available_versions(area):
-    p = subprocess.run(
-        [
-            RCLONE_BIN,
-            'cat',
-            f'remote:ofm-{area}/dirs.txt',
-        ],
-        env=dict(RCLONE_CONFIG=RCLONE_CONF),
-        check=True,
-        capture_output=True,
-        text=True,
-    )
-    versions = [l.strip() for l in p.stdout.strip().splitlines()]
-    versions.sort()
-
-    return versions
-
-
-def set_version(area, version):
-    subprocess.run(
-        [
-            RCLONE_BIN,
-            'rcat',
-            f'remote:ofm-assets/versions/deployed_{area}.txt',
-        ],
-        env=dict(RCLONE_CONFIG=RCLONE_CONF),
-        check=True,
-        input=version.encode(),
-    )
-
-
-if __name__ == '__main__':
-    cli()

scripts/setversion/setversion_lib/__init__.py

@@ -1,16 +0,0 @@
-from pathlib import Path
-
-
-if Path('/data/ofm/config').exists():
-    OFM_CONFIG_DIR = Path('/data/ofm/config')
-else:
-    OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
-
-assert OFM_CONFIG_DIR.exists()
-
-RCLONE_CONF = OFM_CONFIG_DIR / 'rclone.conf'
-
-if Path('/opt/homebrew/bin/rclone').exists():
-    RCLONE_BIN = '/opt/homebrew/bin/rclone'
-else:
-    RCLONE_BIN = 'rclone'

scripts/tile_gen/extract_btrfs.sh

@@ -1,102 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-TILE_GEN_BIN=/data/ofm/tile_gen/bin
-VENV_PYTHON=/data/ofm/venv/bin/python
-
-sudo umount mnt_rw 2> /dev/null || true
-sudo umount mnt_rw2 2> /dev/null || true
-rm -rf mnt_rw* tmp_*
-rm -f -- *.btrfs *.gz
-rm -rf -- *.log *.txt logs
-
-# make an empty file that's definitely bigger then the current OSM output
-fallocate -l 200G image.btrfs
-fallocate -l 200G image2.btrfs
-
-
-# metadata: single needed as default is now DUP
-mkfs.btrfs \
-  -m single \
-  image.btrfs > /dev/null
-
-mkfs.btrfs \
-  -m single \
-  image2.btrfs > /dev/null
-
-# https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
-# compression doesn't make sense, data is already gzip compressed
-mkdir -p mnt_rw mnt_rw2
-
-sudo mount \
-  -t btrfs \
-  -o noacl,nobarrier,noatime,max_inline=4096 \
-  image.btrfs mnt_rw
-
-sudo mount \
-  -t btrfs \
-  -o noacl,nobarrier,noatime,max_inline=4096 \
-  image2.btrfs mnt_rw2
-
-sudo chown ofm:ofm -R mnt_rw mnt_rw2
-
-$VENV_PYTHON $TILE_GEN_BIN/extract_mbtiles/extract_mbtiles.py \
-  tiles.mbtiles mnt_rw/extract \
-  > extract_out.log 2> extract_err.log
-
-cp mnt_rw/extract/osm_date .
-
-grep fixed extract_out.log > dedupl_fixed.log || true
-
-# Unfortunately, by deleting files from the btrfs partition, the size _grows_.
-# So we need to rsync onto a new partition.
-rsync -avH \
-  --max-alloc=4294967296 \
-  --exclude dedupl \
-  mnt_rw/extract/ mnt_rw2/ \
-  > rsync_out.log 2> rsync_err.log
-
-
-# collect stats
-{
-echo -e "df -h"
-sudo df -h mnt_rw
-echo -e "\n\nbtrfs filesystem df"
-sudo btrfs filesystem df mnt_rw
-echo -e "\n\nbtrfs filesystem show"
-sudo btrfs filesystem show mnt_rw
-echo -e "\n\nbtrfs filesystem usage"
-sudo btrfs filesystem usage mnt_rw
-} > stats1.txt
-
-{
-echo -e "df -h"
-sudo df -h mnt_rw2
-echo -e "\n\nbtrfs filesystem df"
-sudo btrfs filesystem df mnt_rw2
-echo -e "\n\nbtrfs filesystem show"
-sudo btrfs filesystem show mnt_rw2
-echo -e "\n\nbtrfs filesystem usage"
-sudo btrfs filesystem usage mnt_rw2
-} > stats2.txt
-
-
-
-sudo umount mnt_rw
-sudo umount mnt_rw2
-rm -r mnt_rw*
-
-sudo $VENV_PYTHON $TILE_GEN_BIN/shrink_btrfs/shrink_btrfs.py image2.btrfs \
-  > shrink_out.log 2> shrink_err.log
-
-
-rm image.btrfs
-mv image2.btrfs tiles.btrfs
-
-pigz tiles.btrfs --fast
-
-mkdir -p logs
-mv -- *.log logs
-mv -- *.txt logs
-
-echo extract_btrfs.sh DONE

scripts/tile_gen/planetiler_monaco.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-TILE_GEN_BIN=/data/ofm/tile_gen/bin
-
-AREA=monaco
-DATE=$(date +"%Y%m%d_%H%M%S")
-RUN_FOLDER="/data/ofm/tile_gen/runs/$AREA/${DATE}_pt"
-
-
-mkdir -p "$RUN_FOLDER"
-cd "$RUN_FOLDER" || exit
-
-java -Xmx1g \
-  -jar $TILE_GEN_BIN/planetiler.jar \
-  `# Download the latest osm.pbf from s3://osm-pds bucket` \
-  --area=$AREA --download \
-  `# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \
-  --download-threads=10 --download-chunk-size-mb=1000 \
-  `# Also download name translations from wikidata` \
-  --fetch-wikidata \
-  --output=tiles.mbtiles \
-  `# Store temporary node locations at fixed positions in a memory-mapped file` \
-  --nodemap-type=array --storage=mmap \
-  --force \
-  > planetiler.out 2> planetiler.err
-
-rm -r data
-echo planetiler.jar DONE
-
-$TILE_GEN_BIN/extract_btrfs.sh

scripts/tile_gen/planetiler_planet.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-TILE_GEN_BIN=/data/ofm/tile_gen/bin
-
-AREA=planet
-DATE=$(date +"%Y%m%d_%H%M%S")
-RUN_FOLDER="/data/ofm/tile_gen/runs/$AREA/${DATE}_pt"
-
-
-mkdir -p "$RUN_FOLDER"
-cd "$RUN_FOLDER" || exit
-
-# the Xmx value below the most important parameter here
-# 30 GB works well
-java -Xmx30g \
-  -jar $TILE_GEN_BIN/planetiler.jar \
-  `# Download the latest planet.osm.pbf from s3://osm-pds bucket` \
-  --area=planet --bounds=planet --download \
-  `# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \
-  --download-threads=10 --download-chunk-size-mb=1000 \
-  `# Also download name translations from wikidata` \
-  --fetch-wikidata \
-  --output=tiles.mbtiles \
-  `# Store temporary node locations at fixed positions in a memory-mapped file` \
-  --nodemap-type=array --storage=mmap \
-  --force \
-  > planetiler.out 2> planetiler.err
-
-rm -r data
-echo planetiler.jar DONE
-
-$TILE_GEN_BIN/extract_btrfs.sh

scripts/tile_gen/setup.py

@@ -1,13 +0,0 @@
-from setuptools import find_packages, setup
-
-
-requirements = [
-    'click',
-]
-
-
-setup(
-    python_requires='>=3.10',
-    install_requires=requirements,
-    packages=find_packages(),
-)

scripts/tile_gen/upload_manager.py

@@ -1,159 +0,0 @@
-#!/usr/bin/env python3
-import json
-import pathlib
-import shutil
-import subprocess
-
-import click
-
-
-AREAS = ['planet', 'monaco']
-
-RUNS_DIR = pathlib.Path('/data/ofm/tile_gen/runs')
-
-
-def upload_rclone(area, run):
-    subprocess.run(
-        [
-            'rclone',
-            'sync',
-            '--transfers=8',
-            '--multi-thread-streams=8',
-            '--fast-list',
-            '-v',
-            '--stats-file-name-length',
-            '0',
-            '--stats-one-line',
-            '--log-file',
-            RUNS_DIR / area / run / 'logs' / 'rclone.log',
-            '--exclude',
-            'logs/**',
-            RUNS_DIR / area / run,
-            f'remote:ofm-{area}/{run}',
-        ],
-        env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-        check=True,
-    )
-
-
-def make_indexes():
-    for area in AREAS:
-        print(f'creating index {area}')
-
-        # files
-        p = subprocess.run(
-            [
-                'rclone',
-                'lsf',
-                '-R',
-                '--files-only',
-                '--fast-list',
-                '--exclude',
-                'dirs.txt',
-                '--exclude',
-                'index.txt',
-                f'remote:ofm-{area}',
-            ],
-            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-            check=True,
-            capture_output=True,
-            text=True,
-        )
-        index_str = p.stdout
-
-        subprocess.run(
-            [
-                'rclone',
-                'rcat',
-                f'remote:ofm-{area}/index.txt',
-            ],
-            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-            check=True,
-            input=index_str.encode(),
-        )
-
-        # directories
-        p = subprocess.run(
-            [
-                'rclone',
-                'lsf',
-                '-R',
-                '--dirs-only',
-                '--dir-slash=false',
-                '--fast-list',
-                f'remote:ofm-{area}',
-            ],
-            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-            check=True,
-            capture_output=True,
-            text=True,
-        )
-        index_str = p.stdout
-
-        subprocess.run(
-            [
-                'rclone',
-                'rcat',
-                f'remote:ofm-{area}/dirs.txt',
-            ],
-            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-            check=True,
-            input=index_str.encode(),
-        )
-
-
-@click.group()
-def cli():
-    """
-    Uploads runs to Cloudflare
-    """
-
-
-@cli.command()
-def upload_runs():
-    """
-    Upload all runs present in system
-    """
-
-    print('running upload_runs')
-
-    for area in AREAS:
-        if not (RUNS_DIR / area).exists():
-            continue
-
-        p = subprocess.run(
-            [
-                'rclone',
-                'lsjson',
-                '--dirs-only',
-                '--fast-list',
-                f'remote:ofm-{area}',
-            ],
-            text=True,
-            capture_output=True,
-            check=True,
-            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
-        )
-        rclone_json = json.loads(p.stdout)
-        runs_remote = {p['Path'] for p in rclone_json}
-        runs_local = {p.name for p in (RUNS_DIR / area).iterdir()}
-
-        runs_to_upload = runs_local - runs_remote
-        for run in runs_to_upload:
-            print(f'uploading {area} {run}')
-            upload_rclone(area, run)
-
-    make_indexes()
-
-
-@cli.command()
-def index():
-    """
-    Run index on Cloudflare buckets
-    """
-
-    make_indexes()
-
-
-if __name__ == '__main__':
-    cli()