scripts -> modules

This commit is contained in:
Zsolt Ero
2024-08-29 16:33:59 +02:00
parent 7196e15837
commit 66d0bdc515
54 changed files with 65 additions and 52 deletions

172
modules/debug_proxy/.gitignore vendored Normal file
View File

@@ -0,0 +1,172 @@
# Logs
logs
_.log
npm-debug.log_
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
# Runtime data
pids
_.pid
_.seed
\*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
\*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
\*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
\*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.\*
# wrangler project
.dev.vars
.wrangler/

View File

@@ -0,0 +1,14 @@
{
"name": "cf-debug-proxy",
"version": "0.0.0",
"private": true,
"scripts": {
"deploy": "wrangler deploy",
"dev": "wrangler dev",
"start": "wrangler dev"
},
"devDependencies": {
"itty-router": "^3.0.12",
"wrangler": "^3.60.3"
}
}

822
modules/debug_proxy/pnpm-lock.yaml generated Normal file
View File

@@ -0,0 +1,822 @@
lockfileVersion: '6.0'
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
devDependencies:
itty-router:
specifier: ^3.0.12
version: 3.0.12
wrangler:
specifier: ^3.60.3
version: 3.62.0
packages:
/@cloudflare/kv-asset-handler@0.3.4:
resolution: {integrity: sha512-YLPHc8yASwjNkmcDMQMY35yiWjoKAKnhUbPRszBRS0YgH+IXtsMp61j+yTcnCE3oO2DgP0U3iejLC8FTtKDC8Q==}
engines: {node: '>=16.13'}
dependencies:
mime: 3.0.0
dev: true
/@cloudflare/workerd-darwin-64@1.20240620.1:
resolution: {integrity: sha512-YWeS2aE8jAzDefuus/3GmZcFGu3Ef94uCAoxsQuaEXNsiGM9NeAhPpKC1BJAlcv168U/Q1J+6hckcGtipf6ZcQ==}
engines: {node: '>=16'}
cpu: [x64]
os: [darwin]
requiresBuild: true
dev: true
optional: true
/@cloudflare/workerd-darwin-arm64@1.20240620.1:
resolution: {integrity: sha512-3rdND+EHpmCrwYX6hvxIBSBJ0f40tRNxond1Vfw7GiR1MJVi3gragiBx75UDFHCxfRw3J0GZ1qVlkRce2/Xbsg==}
engines: {node: '>=16'}
cpu: [arm64]
os: [darwin]
requiresBuild: true
dev: true
optional: true
/@cloudflare/workerd-linux-64@1.20240620.1:
resolution: {integrity: sha512-tURcTrXGeSbYqeM5ISVcofY20StKbVIcdxjJvNYNZ+qmSV9Fvn+zr7rRE+q64pEloVZfhsEPAlUCnFso5VV4XQ==}
engines: {node: '>=16'}
cpu: [x64]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@cloudflare/workerd-linux-arm64@1.20240620.1:
resolution: {integrity: sha512-TThvkwNxaZFKhHZnNjOGqIYCOk05DDWgO+wYMuXg15ymN/KZPnCicRAkuyqiM+R1Fgc4kwe/pehjP8pbmcf6sg==}
engines: {node: '>=16'}
cpu: [arm64]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@cloudflare/workerd-windows-64@1.20240620.1:
resolution: {integrity: sha512-Y/BA9Yj0r7Al1HK3nDHcfISgFllw6NR3XMMPChev57vrVT9C9D4erBL3sUBfofHU+2U9L+ShLsl6obBpe3vvUw==}
engines: {node: '>=16'}
cpu: [x64]
os: [win32]
requiresBuild: true
dev: true
optional: true
/@cspotcode/source-map-support@0.8.1:
resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
engines: {node: '>=12'}
dependencies:
'@jridgewell/trace-mapping': 0.3.9
dev: true
/@esbuild-plugins/node-globals-polyfill@0.2.3(esbuild@0.17.19):
resolution: {integrity: sha512-r3MIryXDeXDOZh7ih1l/yE9ZLORCd5e8vWg02azWRGj5SPTuoh69A2AIyn0Z31V/kHBfZ4HgWJ+OK3GTTwLmnw==}
peerDependencies:
esbuild: '*'
dependencies:
esbuild: 0.17.19
dev: true
/@esbuild-plugins/node-modules-polyfill@0.2.2(esbuild@0.17.19):
resolution: {integrity: sha512-LXV7QsWJxRuMYvKbiznh+U1ilIop3g2TeKRzUxOG5X3YITc8JyyTa90BmLwqqv0YnX4v32CSlG+vsziZp9dMvA==}
peerDependencies:
esbuild: '*'
dependencies:
esbuild: 0.17.19
escape-string-regexp: 4.0.0
rollup-plugin-node-polyfills: 0.2.1
dev: true
/@esbuild/android-arm64@0.17.19:
resolution: {integrity: sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==}
engines: {node: '>=12'}
cpu: [arm64]
os: [android]
requiresBuild: true
dev: true
optional: true
/@esbuild/android-arm@0.17.19:
resolution: {integrity: sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A==}
engines: {node: '>=12'}
cpu: [arm]
os: [android]
requiresBuild: true
dev: true
optional: true
/@esbuild/android-x64@0.17.19:
resolution: {integrity: sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww==}
engines: {node: '>=12'}
cpu: [x64]
os: [android]
requiresBuild: true
dev: true
optional: true
/@esbuild/darwin-arm64@0.17.19:
resolution: {integrity: sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg==}
engines: {node: '>=12'}
cpu: [arm64]
os: [darwin]
requiresBuild: true
dev: true
optional: true
/@esbuild/darwin-x64@0.17.19:
resolution: {integrity: sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw==}
engines: {node: '>=12'}
cpu: [x64]
os: [darwin]
requiresBuild: true
dev: true
optional: true
/@esbuild/freebsd-arm64@0.17.19:
resolution: {integrity: sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ==}
engines: {node: '>=12'}
cpu: [arm64]
os: [freebsd]
requiresBuild: true
dev: true
optional: true
/@esbuild/freebsd-x64@0.17.19:
resolution: {integrity: sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ==}
engines: {node: '>=12'}
cpu: [x64]
os: [freebsd]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-arm64@0.17.19:
resolution: {integrity: sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg==}
engines: {node: '>=12'}
cpu: [arm64]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-arm@0.17.19:
resolution: {integrity: sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA==}
engines: {node: '>=12'}
cpu: [arm]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-ia32@0.17.19:
resolution: {integrity: sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ==}
engines: {node: '>=12'}
cpu: [ia32]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-loong64@0.17.19:
resolution: {integrity: sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ==}
engines: {node: '>=12'}
cpu: [loong64]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-mips64el@0.17.19:
resolution: {integrity: sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A==}
engines: {node: '>=12'}
cpu: [mips64el]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-ppc64@0.17.19:
resolution: {integrity: sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg==}
engines: {node: '>=12'}
cpu: [ppc64]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-riscv64@0.17.19:
resolution: {integrity: sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA==}
engines: {node: '>=12'}
cpu: [riscv64]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-s390x@0.17.19:
resolution: {integrity: sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q==}
engines: {node: '>=12'}
cpu: [s390x]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/linux-x64@0.17.19:
resolution: {integrity: sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw==}
engines: {node: '>=12'}
cpu: [x64]
os: [linux]
requiresBuild: true
dev: true
optional: true
/@esbuild/netbsd-x64@0.17.19:
resolution: {integrity: sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q==}
engines: {node: '>=12'}
cpu: [x64]
os: [netbsd]
requiresBuild: true
dev: true
optional: true
/@esbuild/openbsd-x64@0.17.19:
resolution: {integrity: sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g==}
engines: {node: '>=12'}
cpu: [x64]
os: [openbsd]
requiresBuild: true
dev: true
optional: true
/@esbuild/sunos-x64@0.17.19:
resolution: {integrity: sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg==}
engines: {node: '>=12'}
cpu: [x64]
os: [sunos]
requiresBuild: true
dev: true
optional: true
/@esbuild/win32-arm64@0.17.19:
resolution: {integrity: sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag==}
engines: {node: '>=12'}
cpu: [arm64]
os: [win32]
requiresBuild: true
dev: true
optional: true
/@esbuild/win32-ia32@0.17.19:
resolution: {integrity: sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw==}
engines: {node: '>=12'}
cpu: [ia32]
os: [win32]
requiresBuild: true
dev: true
optional: true
/@esbuild/win32-x64@0.17.19:
resolution: {integrity: sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA==}
engines: {node: '>=12'}
cpu: [x64]
os: [win32]
requiresBuild: true
dev: true
optional: true
/@fastify/busboy@2.1.1:
resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==}
engines: {node: '>=14'}
dev: true
/@jridgewell/resolve-uri@3.1.2:
resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==}
engines: {node: '>=6.0.0'}
dev: true
/@jridgewell/sourcemap-codec@1.4.15:
resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==}
dev: true
/@jridgewell/trace-mapping@0.3.9:
resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==}
dependencies:
'@jridgewell/resolve-uri': 3.1.2
'@jridgewell/sourcemap-codec': 1.4.15
dev: true
/@types/node-forge@1.3.11:
resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==}
dependencies:
'@types/node': 20.14.9
dev: true
/@types/node@20.14.9:
resolution: {integrity: sha512-06OCtnTXtWOZBJlRApleWndH4JsRVs1pDCc8dLSQp+7PpUpX3ePdHyeNSFTeSe7FtKyQkrlPvHwJOW3SLd8Oyg==}
dependencies:
undici-types: 5.26.5
dev: true
/acorn-walk@8.3.3:
resolution: {integrity: sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==}
engines: {node: '>=0.4.0'}
dependencies:
acorn: 8.12.0
dev: true
/acorn@8.12.0:
resolution: {integrity: sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==}
engines: {node: '>=0.4.0'}
hasBin: true
dev: true
/anymatch@3.1.3:
resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==}
engines: {node: '>= 8'}
dependencies:
normalize-path: 3.0.0
picomatch: 2.3.1
dev: true
/as-table@1.0.55:
resolution: {integrity: sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ==}
dependencies:
printable-characters: 1.0.42
dev: true
/binary-extensions@2.3.0:
resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
engines: {node: '>=8'}
dev: true
/blake3-wasm@2.1.5:
resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==}
dev: true
/braces@3.0.3:
resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}
engines: {node: '>=8'}
dependencies:
fill-range: 7.1.1
dev: true
/capnp-ts@0.7.0:
resolution: {integrity: sha512-XKxXAC3HVPv7r674zP0VC3RTXz+/JKhfyw94ljvF80yynK6VkTnqE3jMuN8b3dUVmmc43TjyxjW4KTsmB3c86g==}
dependencies:
debug: 4.3.5
tslib: 2.6.3
transitivePeerDependencies:
- supports-color
dev: true
/chokidar@3.6.0:
resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}
engines: {node: '>= 8.10.0'}
dependencies:
anymatch: 3.1.3
braces: 3.0.3
glob-parent: 5.1.2
is-binary-path: 2.1.0
is-glob: 4.0.3
normalize-path: 3.0.0
readdirp: 3.6.0
optionalDependencies:
fsevents: 2.3.3
dev: true
/consola@3.2.3:
resolution: {integrity: sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==}
engines: {node: ^14.18.0 || >=16.10.0}
dev: true
/cookie@0.5.0:
resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==}
engines: {node: '>= 0.6'}
dev: true
/data-uri-to-buffer@2.0.2:
resolution: {integrity: sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==}
dev: true
/date-fns@3.6.0:
resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==}
dev: true
/debug@4.3.5:
resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==}
engines: {node: '>=6.0'}
peerDependencies:
supports-color: '*'
peerDependenciesMeta:
supports-color:
optional: true
dependencies:
ms: 2.1.2
dev: true
/defu@6.1.4:
resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==}
dev: true
/esbuild@0.17.19:
resolution: {integrity: sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw==}
engines: {node: '>=12'}
hasBin: true
requiresBuild: true
optionalDependencies:
'@esbuild/android-arm': 0.17.19
'@esbuild/android-arm64': 0.17.19
'@esbuild/android-x64': 0.17.19
'@esbuild/darwin-arm64': 0.17.19
'@esbuild/darwin-x64': 0.17.19
'@esbuild/freebsd-arm64': 0.17.19
'@esbuild/freebsd-x64': 0.17.19
'@esbuild/linux-arm': 0.17.19
'@esbuild/linux-arm64': 0.17.19
'@esbuild/linux-ia32': 0.17.19
'@esbuild/linux-loong64': 0.17.19
'@esbuild/linux-mips64el': 0.17.19
'@esbuild/linux-ppc64': 0.17.19
'@esbuild/linux-riscv64': 0.17.19
'@esbuild/linux-s390x': 0.17.19
'@esbuild/linux-x64': 0.17.19
'@esbuild/netbsd-x64': 0.17.19
'@esbuild/openbsd-x64': 0.17.19
'@esbuild/sunos-x64': 0.17.19
'@esbuild/win32-arm64': 0.17.19
'@esbuild/win32-ia32': 0.17.19
'@esbuild/win32-x64': 0.17.19
dev: true
/escape-string-regexp@4.0.0:
resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
engines: {node: '>=10'}
dev: true
/estree-walker@0.6.1:
resolution: {integrity: sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==}
dev: true
/exit-hook@2.2.1:
resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==}
engines: {node: '>=6'}
dev: true
/fill-range@7.1.1:
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
engines: {node: '>=8'}
dependencies:
to-regex-range: 5.0.1
dev: true
/fsevents@2.3.3:
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
os: [darwin]
requiresBuild: true
dev: true
optional: true
/function-bind@1.1.2:
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
dev: true
/get-source@2.0.12:
resolution: {integrity: sha512-X5+4+iD+HoSeEED+uwrQ07BOQr0kEDFMVqqpBuI+RaZBpBpHCuXxo70bjar6f0b0u/DQJsJ7ssurpP0V60Az+w==}
dependencies:
data-uri-to-buffer: 2.0.2
source-map: 0.6.1
dev: true
/glob-parent@5.1.2:
resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
engines: {node: '>= 6'}
dependencies:
is-glob: 4.0.3
dev: true
/glob-to-regexp@0.4.1:
resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==}
dev: true
/hasown@2.0.2:
resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
engines: {node: '>= 0.4'}
dependencies:
function-bind: 1.1.2
dev: true
/is-binary-path@2.1.0:
resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
engines: {node: '>=8'}
dependencies:
binary-extensions: 2.3.0
dev: true
/is-core-module@2.14.0:
resolution: {integrity: sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==}
engines: {node: '>= 0.4'}
dependencies:
hasown: 2.0.2
dev: true
/is-extglob@2.1.1:
resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
engines: {node: '>=0.10.0'}
dev: true
/is-glob@4.0.3:
resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
engines: {node: '>=0.10.0'}
dependencies:
is-extglob: 2.1.1
dev: true
/is-number@7.0.0:
resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
engines: {node: '>=0.12.0'}
dev: true
/itty-router@3.0.12:
resolution: {integrity: sha512-s98XTPhle6GGbaFf0kYrOD3Q8gyhnqvOqkwYijC3AmkceNKqWUp13YHg6dWmqmVv4pP7l7c94XI92I0EXVGO0w==}
dev: true
/magic-string@0.25.9:
resolution: {integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==}
dependencies:
sourcemap-codec: 1.4.8
dev: true
/mime@3.0.0:
resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==}
engines: {node: '>=10.0.0'}
hasBin: true
dev: true
/miniflare@3.20240620.0:
resolution: {integrity: sha512-NBMzqUE2mMlh/hIdt6U5MP+aFhEjKDq3l8CAajXAQa1WkndJdciWvzB2mfLETwoVFhMl/lphaVzyEN2AgwJpbQ==}
engines: {node: '>=16.13'}
hasBin: true
dependencies:
'@cspotcode/source-map-support': 0.8.1
acorn: 8.12.0
acorn-walk: 8.3.3
capnp-ts: 0.7.0
exit-hook: 2.2.1
glob-to-regexp: 0.4.1
stoppable: 1.1.0
undici: 5.28.4
workerd: 1.20240620.1
ws: 8.17.1
youch: 3.3.3
zod: 3.23.8
transitivePeerDependencies:
- bufferutil
- supports-color
- utf-8-validate
dev: true
/ms@2.1.2:
resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
dev: true
/mustache@4.2.0:
resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==}
hasBin: true
dev: true
/nanoid@3.3.7:
resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==}
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
hasBin: true
dev: true
/node-fetch-native@1.6.4:
resolution: {integrity: sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==}
dev: true
/node-forge@1.3.1:
resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==}
engines: {node: '>= 6.13.0'}
dev: true
/normalize-path@3.0.0:
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
engines: {node: '>=0.10.0'}
dev: true
/path-parse@1.0.7:
resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
dev: true
/path-to-regexp@6.2.2:
resolution: {integrity: sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==}
dev: true
/pathe@1.1.2:
resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==}
dev: true
/picomatch@2.3.1:
resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}
engines: {node: '>=8.6'}
dev: true
/printable-characters@1.0.42:
resolution: {integrity: sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==}
dev: true
/readdirp@3.6.0:
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
engines: {node: '>=8.10.0'}
dependencies:
picomatch: 2.3.1
dev: true
/resolve.exports@2.0.2:
resolution: {integrity: sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==}
engines: {node: '>=10'}
dev: true
/resolve@1.22.8:
resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}
hasBin: true
dependencies:
is-core-module: 2.14.0
path-parse: 1.0.7
supports-preserve-symlinks-flag: 1.0.0
dev: true
/rollup-plugin-inject@3.0.2:
resolution: {integrity: sha512-ptg9PQwzs3orn4jkgXJ74bfs5vYz1NCZlSQMBUA0wKcGp5i5pA1AO3fOUEte8enhGUC+iapTCzEWw2jEFFUO/w==}
deprecated: This package has been deprecated and is no longer maintained. Please use @rollup/plugin-inject.
dependencies:
estree-walker: 0.6.1
magic-string: 0.25.9
rollup-pluginutils: 2.8.2
dev: true
/rollup-plugin-node-polyfills@0.2.1:
resolution: {integrity: sha512-4kCrKPTJ6sK4/gLL/U5QzVT8cxJcofO0OU74tnB19F40cmuAKSzH5/siithxlofFEjwvw1YAhPmbvGNA6jEroA==}
dependencies:
rollup-plugin-inject: 3.0.2
dev: true
/rollup-pluginutils@2.8.2:
resolution: {integrity: sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==}
dependencies:
estree-walker: 0.6.1
dev: true
/selfsigned@2.4.1:
resolution: {integrity: sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==}
engines: {node: '>=10'}
dependencies:
'@types/node-forge': 1.3.11
node-forge: 1.3.1
dev: true
/source-map@0.6.1:
resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
engines: {node: '>=0.10.0'}
dev: true
/sourcemap-codec@1.4.8:
resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==}
deprecated: Please use @jridgewell/sourcemap-codec instead
dev: true
/stacktracey@2.1.8:
resolution: {integrity: sha512-Kpij9riA+UNg7TnphqjH7/CzctQ/owJGNbFkfEeve4Z4uxT5+JapVLFXcsurIfN34gnTWZNJ/f7NMG0E8JDzTw==}
dependencies:
as-table: 1.0.55
get-source: 2.0.12
dev: true
/stoppable@1.1.0:
resolution: {integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==}
engines: {node: '>=4', npm: '>=6'}
dev: true
/supports-preserve-symlinks-flag@1.0.0:
resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
engines: {node: '>= 0.4'}
dev: true
/to-regex-range@5.0.1:
resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==}
engines: {node: '>=8.0'}
dependencies:
is-number: 7.0.0
dev: true
/tslib@2.6.3:
resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==}
dev: true
/ufo@1.5.3:
resolution: {integrity: sha512-Y7HYmWaFwPUmkoQCUIAYpKqkOf+SbVj/2fJJZ4RJMCfZp0rTGwRbzQD+HghfnhKOjL9E01okqz+ncJskGYfBNw==}
dev: true
/undici-types@5.26.5:
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
dev: true
/undici@5.28.4:
resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==}
engines: {node: '>=14.0'}
dependencies:
'@fastify/busboy': 2.1.1
dev: true
/unenv-nightly@1.10.0-1717606461.a117952:
resolution: {integrity: sha512-u3TfBX02WzbHTpaEfWEKwDijDSFAHcgXkayUZ+MVDrjhLFvgAJzFGTSTmwlEhwWi2exyRQey23ah9wELMM6etg==}
dependencies:
consola: 3.2.3
defu: 6.1.4
mime: 3.0.0
node-fetch-native: 1.6.4
pathe: 1.1.2
ufo: 1.5.3
dev: true
/workerd@1.20240620.1:
resolution: {integrity: sha512-Qoq+RrFNk4pvEO+kpJVn8uJ5TRE9YJx5jX5pC5LjdKlw1XeD8EdXt5k0TbByvWunZ4qgYIcF9lnVxhcDFo203g==}
engines: {node: '>=16'}
hasBin: true
requiresBuild: true
optionalDependencies:
'@cloudflare/workerd-darwin-64': 1.20240620.1
'@cloudflare/workerd-darwin-arm64': 1.20240620.1
'@cloudflare/workerd-linux-64': 1.20240620.1
'@cloudflare/workerd-linux-arm64': 1.20240620.1
'@cloudflare/workerd-windows-64': 1.20240620.1
dev: true
/wrangler@3.62.0:
resolution: {integrity: sha512-TM1Bd8+GzxFw/JzwsC3i/Oss4LTWvIEWXXo1vZhx+7PHcsxdbnQGBBwPurHNJDSu2Pw22+2pCZiUGKexmgJksw==}
engines: {node: '>=16.17.0'}
hasBin: true
peerDependencies:
'@cloudflare/workers-types': ^4.20240620.0
peerDependenciesMeta:
'@cloudflare/workers-types':
optional: true
dependencies:
'@cloudflare/kv-asset-handler': 0.3.4
'@esbuild-plugins/node-globals-polyfill': 0.2.3(esbuild@0.17.19)
'@esbuild-plugins/node-modules-polyfill': 0.2.2(esbuild@0.17.19)
blake3-wasm: 2.1.5
chokidar: 3.6.0
date-fns: 3.6.0
esbuild: 0.17.19
miniflare: 3.20240620.0
nanoid: 3.3.7
path-to-regexp: 6.2.2
resolve: 1.22.8
resolve.exports: 2.0.2
selfsigned: 2.4.1
source-map: 0.6.1
unenv: /unenv-nightly@1.10.0-1717606461.a117952
xxhash-wasm: 1.0.2
optionalDependencies:
fsevents: 2.3.3
transitivePeerDependencies:
- bufferutil
- supports-color
- utf-8-validate
dev: true
/ws@8.17.1:
resolution: {integrity: sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==}
engines: {node: '>=10.0.0'}
peerDependencies:
bufferutil: ^4.0.1
utf-8-validate: '>=5.0.2'
peerDependenciesMeta:
bufferutil:
optional: true
utf-8-validate:
optional: true
dev: true
/xxhash-wasm@1.0.2:
resolution: {integrity: sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A==}
dev: true
/youch@3.3.3:
resolution: {integrity: sha512-qSFXUk3UZBLfggAW3dJKg0BMblG5biqSF8M34E06o5CSsZtH92u9Hqmj2RzGiHDi64fhe83+4tENFP2DB6t6ZA==}
dependencies:
cookie: 0.5.0
mustache: 4.2.0
stacktracey: 2.1.8
dev: true
/zod@3.23.8:
resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==}
dev: true

View File

@@ -0,0 +1,66 @@
async function sendTelegramMessage(message, botToken, chatId) {
const url = `https://api.telegram.org/bot${botToken}/sendMessage`
const payload = {
chat_id: chatId,
text: message,
}
try {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(payload),
})
if (!response.ok) {
console.error('Failed to send message:', await response.text())
}
} catch (error) {
console.error('Error sending Telegram message:', error)
}
}
export default {
async fetch(request, env, ctx) {
const url = new URL(request.url)
const userIP = request.headers.get('CF-Connecting-IP')
if (url.pathname === '/b') {
url.pathname = '/styles/bright'
}
// // no failure, just warning
// if (request.method !== 'GET') {
// const warningMessage = `Non-GET request ${request.method} ${url.pathname} ${userIP}`
// console.error(warningMessage)
// await sendTelegramMessage(warningMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
// }
if (!url.pathname.startsWith('/styles')) {
const errorMessage = 'Bad path'
return new Response(errorMessage, { status: 500 })
}
const proxyUrl = new URL(url.pathname, 'https://tiles.openfreemap.org')
try {
const response = await fetch(proxyUrl)
if (response.status !== 200) {
const errorMessage = `Proxy error: Bad status ${response.status} ${url.pathname} ${userIP}`
console.error(errorMessage)
await sendTelegramMessage(errorMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
return new Response('Proxy error: Bad status', { status: 500 })
}
return response
} catch (error) {
const errorMessage = `Proxy error: ${error.message} ${url.pathname} ${userIP}`
console.error(errorMessage)
await sendTelegramMessage(errorMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
return new Response('Proxy error: Fetch failed', { status: 500 })
}
},
}

View File

@@ -0,0 +1,107 @@
#:schema node_modules/wrangler/config-schema.json
name = "cf-debug-proxy"
main = "src/index.js"
compatibility_date = "2024-06-20"
# Automatically place your workloads in an optimal location to minimize latency.
# If you are running back-end logic in a Worker, running it closer to your back-end infrastructure
# rather than the end user may result in better performance.
# Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement
# [placement]
# mode = "smart"
# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
# Docs:
# - https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables
# Note: Use secrets to store sensitive data.
# - https://developers.cloudflare.com/workers/configuration/secrets/
# [vars]
# MY_VARIABLE = "production_value"
# Bind the Workers AI model catalog. Run machine learning models, powered by serverless GPUs, on Cloudflares global network
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#workers-ai
# [ai]
# binding = "AI"
# Bind an Analytics Engine dataset. Use Analytics Engine to write analytics within your Pages Function.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#analytics-engine-datasets
# [[analytics_engine_datasets]]
# binding = "MY_DATASET"
# Bind a headless browser instance running on Cloudflare's global network.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#browser-rendering
# [browser]
# binding = "MY_BROWSER"
# Bind a D1 database. D1 is Cloudflares native serverless SQL database.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#d1-databases
# [[d1_databases]]
# binding = "MY_DB"
# database_name = "my-database"
# database_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
# Bind a dispatch namespace. Use Workers for Platforms to deploy serverless functions programmatically on behalf of your customers.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#dispatch-namespace-bindings-workers-for-platforms
# [[dispatch_namespaces]]
# binding = "MY_DISPATCHER"
# namespace = "my-namespace"
# Bind a Durable Object. Durable objects are a scale-to-zero compute primitive based on the actor model.
# Durable Objects can live for as long as needed. Use these when you need a long-running "server", such as in realtime apps.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#durable-objects
# [[durable_objects.bindings]]
# name = "MY_DURABLE_OBJECT"
# class_name = "MyDurableObject"
# Durable Object migrations.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#migrations
# [[migrations]]
# tag = "v1"
# new_classes = ["MyDurableObject"]
# Bind a Hyperdrive configuration. Use to accelerate access to your existing databases from Cloudflare Workers.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#hyperdrive
# [[hyperdrive]]
# binding = "MY_HYPERDRIVE"
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
# Bind a KV Namespace. Use KV as persistent storage for small key-value pairs.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#kv-namespaces
# [[kv_namespaces]]
# binding = "MY_KV_NAMESPACE"
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
# Bind an mTLS certificate. Use to present a client certificate when communicating with another service.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#mtls-certificates
# [[mtls_certificates]]
# binding = "MY_CERTIFICATE"
# certificate_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
# Bind a Queue producer. Use this binding to schedule an arbitrary task that may be processed later by a Queue consumer.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
# [[queues.producers]]
# binding = "MY_QUEUE"
# queue = "my-queue"
# Bind a Queue consumer. Queue Consumers can retrieve tasks scheduled by Producers to act on them.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
# [[queues.consumers]]
# queue = "my-queue"
# Bind an R2 Bucket. Use R2 to store arbitrarily large blobs of data, such as files.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#r2-buckets
# [[r2_buckets]]
# binding = "MY_BUCKET"
# bucket_name = "my-bucket"
# Bind another Worker service. Use this binding to call another Worker without network overhead.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings
# [[services]]
# binding = "MY_SERVICE"
# service = "my-service"
# Bind a Vectorize index. Use to store and query vector embeddings for semantic search, classification and other vector search use-cases.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#vectorize-indexes
# [[vectorize]]
# binding = "MY_INDEX"
# index_name = "my-index"

View File

@@ -0,0 +1,27 @@
import json
with open('access.jsonl') as fp:
json_lines = fp.readlines()
paths = []
for i, line in enumerate(json_lines):
log_data = json.loads(line)
if log_data['status'] != 200:
continue
if log_data['request_method'] != 'GET':
continue
uri = log_data['uri']
if 'tiles/' not in uri or not uri.endswith('.pbf'):
continue
path = log_data['uri'].split('tiles/')[1]
paths.append(path + '\n')
print(f'{i / len(json_lines) * 100:.1f}%')
with open('path_list.txt', 'w') as fp:
fp.writelines(paths)

View File

@@ -0,0 +1,39 @@
local counter = 1
local lines = {}
local url_base = "/planet/20231221_134737_pt/" -- trailing slash
local path_list_txt = "/data/ofm/benchmark/path_list_500k.txt"
for line in io.lines(path_list_txt) do
table.insert(lines, url_base .. line)
end
local function getNextUrl()
-- Get the next URL from the list
local url_path = lines[counter]
counter = counter + 1
-- If we've gone past the end of the list, wrap around to the start
if counter > #lines then
counter = 1
end
return url_path
end
request = function()
-- Return the request object with the current URL path
path = getNextUrl()
local headers = {}
headers["Host"] = "ofm"
return wrk.format('GET', path, headers, nil)
end
response = function(status)
if status ~= 200 then
print("Non-200 response")
print("Status: ", status)
-- this only works in single threaded mode (-t1)
print("Request path: ", path)
end
end

View File

@@ -0,0 +1,8 @@
wrk -c10 -t4 -d10s -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost
# -t1 => more correct, since the url list is loaded exactly in sequence
# -t4 => reflecting real world usage

View File

@@ -0,0 +1,4 @@
# every minute sync, locking so that only one process can run at a time
* * * * * ofm /usr/bin/flock -n /tmp/hostmanager.lockfile -c 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync >> /data/ofm/http_host/logs/host_manager_sync.log 2>&1'

View File

@@ -0,0 +1,2 @@
# once per day
2 34 * * * ofm sudo /usr/bin/bash /data/ofm/http_host/bin/ledns_reader.sh >> /data/ofm/http_host/logs/ledns_reader.log 2>&1

147
modules/http_host/http_host.py Executable file
View File

@@ -0,0 +1,147 @@
#!/usr/bin/env python3
import datetime
import sys
import click
from http_host_lib.assets import (
download_assets,
)
from http_host_lib.btrfs import (
download_area_version,
get_versions_for_area,
)
from http_host_lib.config import config
from http_host_lib.mount import auto_mount_unmount
from http_host_lib.nginx import write_nginx_config
from http_host_lib.set_tileset_versions import set_tileset_versions
from http_host_lib.utils import assert_linux, assert_sudo
@click.group()
def cli():
"""
Manages OpenFreeMap HTTP hosts, including:\n
- Downloading btrfs images\n
- Downloading assets\n
- Mounting directories\n
- Updating nginx config\n
- Getting the deployed versions of tilesets\n
- Running the sync cron task (called every minute)
"""
@cli.command()
@click.argument('area', required=False)
@click.option(
'--version', default='latest', help='Optional version string, like "20231227_043106_pt"'
)
def download_btrfs(area: str, version: str):
"""
Downloads and uncompresses tiles.btrfs files from the btrfs bucket
Version can be "latest" (default) or specified, like "20231227_043106_pt"
Use --version=1 to list all available versions
"""
return download_area_version(area, version)
@cli.command(name='download-assets')
def download_assets_():
"""
Downloads and extracts assets
"""
download_assets()
@cli.command()
def mount():
"""
Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
"""
auto_mount_unmount()
@cli.command()
def set_latest_versions():
"""
Sets the latest version of the tilesets to the version specified by
https://assets.openfreemap.com/versions/deployed_planet.txt
1. Checks if the given version is present on the disk and is mounted
2. Writes to a version file
"""
print('running set_latest_versions')
assert_linux()
assert_sudo()
if not config.mnt_dir.exists():
sys.exit(' mount needs to be run first')
return set_tileset_versions()
@cli.command()
def nginx_sync():
"""
Syncs the nginx config to the state of the system
"""
print('running nginx_sync')
assert_linux()
assert_sudo()
if not config.mnt_dir.exists():
sys.exit(' mount needs to be run first')
write_nginx_config()
@cli.command()
@click.option('--force', is_flag=True, help='Force nginx sync run')
@click.pass_context
def sync(ctx, force):
"""
Runs the sync task, normally called by cron every minute
On a new server this also takes care of everything, no need to run anything manually.
"""
print('---')
print('running sync')
print(datetime.datetime.now(tz=datetime.timezone.utc))
assert_linux()
assert_sudo()
download_done = False
download_done += ctx.invoke(download_btrfs, area='monaco')
if not config.host_config.get('skip_planet'):
download_done += ctx.invoke(download_btrfs, area='planet')
if download_done:
ctx.invoke(mount)
ctx.invoke(download_assets)
deploy_done = ctx.invoke(set_latest_versions)
if download_done or deploy_done or force:
ctx.invoke(nginx_sync)
@cli.command()
def debug():
versions = get_versions_for_area('monaco')
print(versions)
if __name__ == '__main__':
# print(config.host_config)
cli()

View File

@@ -0,0 +1,83 @@
import shutil
import subprocess
import requests
from http_host_lib.config import config
from http_host_lib.utils import download_file_aria2, download_if_size_differs
def download_assets():
"""
Downloads and extracts assets
"""
download_and_extract_asset_tar_gz('fonts')
download_and_extract_asset_tar_gz('styles')
download_and_extract_asset_tar_gz('natural_earth')
download_sprites()
def download_and_extract_asset_tar_gz(asset_kind):
"""
Download and extract asset.tgz if the file size differ or not available locally
"""
print(f'Downloading asset {asset_kind}')
asset_dir = config.assets_dir / asset_kind
asset_dir.mkdir(exist_ok=True, parents=True)
url = f'https://assets.openfreemap.com/{asset_kind}/ofm.tar.gz'
local_file = asset_dir / 'ofm.tar.gz'
if not download_if_size_differs(url, local_file):
print(f' skipping asset: {asset_kind}')
return
ofm_dir = asset_dir / 'ofm'
ofm_dir_bak = asset_dir / 'ofm.bak'
shutil.rmtree(ofm_dir_bak, ignore_errors=True)
if ofm_dir.exists():
ofm_dir.rename(ofm_dir_bak)
subprocess.run(
['tar', '-xzf', local_file, '-C', asset_dir],
check=True,
)
print(f' downloaded asset: {asset_kind}')
def download_sprites():
"""
Sprites are special assets, as we have to keep the old versions indefinitely
"""
print('Downloading sprites')
sprites_dir = config.assets_dir / 'sprites'
sprites_dir.mkdir(exist_ok=True, parents=True)
r = requests.get('https://assets.openfreemap.com/files.txt', timeout=30)
r.raise_for_status()
sprites_remote = [l for l in r.text.splitlines() if l.startswith('sprites/')]
for sprite in sprites_remote:
sprite_name = sprite.split('/')[1].replace('.tar.gz', '')
if (sprites_dir / sprite_name).is_dir():
print(f' skipping sprite version: {sprite_name}')
continue
url = f'https://assets.openfreemap.com/sprites/{sprite_name}.tar.gz'
local_file = sprites_dir / 'temp.tar.gz'
download_file_aria2(url, local_file)
subprocess.run(
['tar', '-xzf', local_file, '-C', sprites_dir],
check=True,
)
local_file.unlink()
print(f' downloaded sprite version: {sprite_name}')

View File

@@ -0,0 +1,84 @@
import shutil
import subprocess
import sys
import requests
from http_host_lib.config import config
from http_host_lib.utils import download_file_aria2, get_remote_file_size
def download_area_version(area: str, version: str):
"""
Downloads and uncompresses tiles.btrfs files from the btrfs bucket
"""
if area not in config.areas:
sys.exit(f' please specify area: {config.areas}')
versions = get_versions_for_area(area)
if version == 'latest':
selected_version = versions[-1]
else:
if version not in versions:
available_versions_str = '\n'.join(versions)
sys.exit(
f'Requested version is not available.\nAvailable versions for {area}:\n{available_versions_str}'
)
selected_version = version
return download_and_extract_btrfs(area, selected_version)
def get_versions_for_area(area: str) -> list:
r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30)
r.raise_for_status()
versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')]
return sorted(versions)
def download_and_extract_btrfs(area: str, version: str) -> bool:
"""
returns True if download successful, False if skipped
"""
print(f'downloading and extracting btrfs for: {area} {version}')
version_dir = config.runs_dir / area / version
btrfs_file = version_dir / 'tiles.btrfs'
if btrfs_file.exists():
print(' file exists, skipping download')
return False
temp_dir = config.runs_dir / '_tmp'
shutil.rmtree(temp_dir, ignore_errors=True)
temp_dir.mkdir(parents=True)
url = f'https://btrfs.openfreemap.com/areas/{area}/{version}/tiles.btrfs.gz'
# check disk space
disk_free = shutil.disk_usage(temp_dir).free
file_size = get_remote_file_size(url)
if not file_size:
raise ValueError('Cannot get remote file size')
needed_space = file_size * 3
if disk_free < needed_space:
raise ValueError(f'Not enough disk space. Needed: {needed_space}, free space: {disk_free}')
target_file = temp_dir / 'tiles.btrfs.gz'
download_file_aria2(url, target_file)
print('Uncompressing...')
subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
btrfs_src = temp_dir / 'tiles.btrfs'
shutil.rmtree(version_dir, ignore_errors=True)
version_dir.mkdir(parents=True)
btrfs_src.rename(btrfs_file)
shutil.rmtree(temp_dir)
return True

View File

@@ -0,0 +1,29 @@
import json
from pathlib import Path
class Configuration:
areas = ['planet', 'monaco']
http_host_dir = Path('/data/ofm/http_host')
http_host_bin = http_host_dir / 'bin'
http_host_scripts_dir = http_host_bin / 'scripts'
runs_dir = http_host_dir / 'runs'
assets_dir = http_host_dir / 'assets'
mnt_dir = Path('/mnt/ofm')
ofm_config_dir = Path('/data/ofm/config')
certs_dir = Path('/data/nginx/certs')
nginx_confs = Path(__file__).parent / 'nginx_confs'
try:
with open(ofm_config_dir / 'http_host.json') as fp:
host_config = json.load(fp)
except Exception:
host_config = {}
config = Configuration()

View File

@@ -0,0 +1,87 @@
import subprocess
import sys
from pathlib import Path
from http_host_lib.config import config
from http_host_lib.utils import assert_linux, assert_sudo
def auto_mount_unmount():
"""
Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
"""
print('running mount')
assert_linux()
assert_sudo()
if not config.runs_dir.exists():
sys.exit(' download-btrfs needs to be run first')
clean_up_mounts(config.mnt_dir)
create_fstab()
print(' running mount -a')
subprocess.run(['mount', '-a'], check=True)
clean_up_mounts(config.mnt_dir)
def create_fstab():
fstab_new = []
for area in ['planet', 'monaco']:
area_dir = (config.runs_dir / area).resolve()
if not area_dir.exists():
continue
versions = sorted(area_dir.iterdir())
for version in versions:
version_str = version.name
btrfs_file = area_dir / version_str / 'tiles.btrfs'
if not btrfs_file.is_file():
continue
mnt_folder = config.mnt_dir / f'{area}-{version_str}'
mnt_folder.mkdir(exist_ok=True, parents=True)
fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n')
print(f' created fstab entry for {btrfs_file} -> {mnt_folder}')
with open('/etc/fstab') as fp:
fstab_orig = [l for l in fp.readlines() if f'{config.mnt_dir}/' not in l]
with open('/etc/fstab', 'w') as fp:
fp.writelines(fstab_orig + fstab_new)
def clean_up_mounts(mnt_dir):
if not mnt_dir.exists():
return
print(' cleaning up mounts')
# handle deleted files
p = subprocess.run(['mount'], capture_output=True, text=True, check=True)
lines = [l for l in p.stdout.splitlines() if f'{mnt_dir}/' in l and '(deleted)' in l]
for l in lines:
mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0])
print(f' removing deleted mount {mnt_path}')
assert mnt_path.exists()
subprocess.run(['umount', mnt_path], check=True)
mnt_path.rmdir()
# clean all mounts not in current fstab
with open('/etc/fstab') as fp:
fstab_str = fp.read()
for subdir in mnt_dir.iterdir():
if f'{subdir} ' in fstab_str:
continue
print(f' removing old mount {subdir}')
subprocess.run(['umount', subdir], check=True)
subdir.rmdir()

View File

@@ -0,0 +1,235 @@
import shutil
import subprocess
import sys
from pathlib import Path
from http_host_lib.config import config
from http_host_lib.utils import python_venv_executable
def write_nginx_config():
curl_text_mix = ''
domain_le = config.host_config['domain_le']
domain_ledns = config.host_config['domain_ledns']
# remove old configs and certs
for file in Path('/data/nginx/sites').glob('ofm_*.conf'):
file.unlink()
for file in Path('/data/nginx/certs').glob('ofm_*'):
file.unlink()
# processing Round Robin DNS config
if domain_ledns:
if not (config.ofm_config_dir / 'rclone.conf').is_file():
sys.exit('rclone.conf missing')
# download the ledns certificate from bucket using rclone
write_ledns_reader_script(domain_ledns)
subprocess.run(['bash', config.http_host_bin / 'ledns_reader.sh'], check=True)
curl_text_mix += create_nginx_conf(
template_path=config.nginx_confs / 'ledns.conf',
local='ofm_ledns',
domain=domain_ledns,
)
# processing Let's Encrypt config
if domain_le:
le_cert = config.certs_dir / 'ofm_le.cert'
le_key = config.certs_dir / 'ofm_le.key'
if not le_cert.is_file() or not le_key.is_file():
shutil.copyfile(Path('/etc/nginx/ssl/dummy.crt'), le_cert)
shutil.copyfile(Path('/etc/nginx/ssl/dummy.key'), le_key)
curl_text_mix += create_nginx_conf(
template_path=config.nginx_confs / 'le.conf',
local='ofm_le',
domain=domain_le,
)
subprocess.run(['nginx', '-t'], check=True)
subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
subprocess.run(
[
'certbot',
'certonly',
'--webroot',
'--webroot-path=/data/nginx/acme-challenges',
'--noninteractive',
'-m',
config.host_config['le_email'],
'--agree-tos',
'--cert-name=ofm_le',
# '--staging',
'--deploy-hook',
'nginx -t && service nginx reload',
'-d',
domain_le,
],
check=True,
)
# link certs to nginx dir
le_cert.unlink()
le_key.unlink()
etc_cert = Path('/etc/letsencrypt/live/ofm_le/fullchain.pem')
etc_key = Path('/etc/letsencrypt/live/ofm_le/privkey.pem')
assert etc_cert.is_file()
assert etc_key.is_file()
le_cert.symlink_to(etc_cert)
le_key.symlink_to(etc_key)
subprocess.run(['nginx', '-t'], check=True)
subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
print(curl_text_mix)
def create_nginx_conf(*, template_path, local, domain):
location_str, curl_text = create_location_blocks(local=local, domain=domain)
with open(template_path) as fp:
template = fp.read()
template = template.replace('__LOCATION_BLOCKS__', location_str)
template = template.replace('__LOCAL__', local)
template = template.replace('__DOMAIN__', domain)
curl_text = curl_text.replace('__LOCAL__', local)
curl_text = curl_text.replace('__DOMAIN__', domain)
with open(f'/data/nginx/sites/{local}.conf', 'w') as fp:
fp.write(template)
print(f' nginx config written: {domain} {local}')
return curl_text
def create_location_blocks(*, local, domain):
location_str = ''
curl_text = ''
for subdir in config.mnt_dir.iterdir():
if not subdir.is_dir():
continue
area, version = subdir.name.split('-')
location_str += create_version_location(
area=area, version=version, subdir=subdir, local=local, domain=domain
)
if not curl_text:
curl_text = (
'\ntest with:\n'
f'curl -H "Host: __LOCAL__" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n'
f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf'
)
location_str += create_latest_locations(local=local, domain=domain)
with open(config.nginx_confs / 'location_static.conf') as fp:
location_str += '\n' + fp.read()
return location_str, curl_text
def create_version_location(
*, area: str, version: str, subdir: Path, local: str, domain: str
) -> str:
run_dir = config.runs_dir / area / version
if not run_dir.is_dir():
print(f" {run_dir} doesn't exists, skipping")
return ''
tilejson_path = run_dir / f'tilejson-{local}.json'
metadata_path = subdir / 'metadata.json'
if not metadata_path.is_file():
print(f" {metadata_path} doesn't exists, skipping")
return ''
url_prefix = f'https://{domain}/{area}/{version}'
subprocess.run(
[
python_venv_executable(),
config.http_host_scripts_dir / 'metadata_to_tilejson.py',
'--minify',
metadata_path,
tilejson_path,
url_prefix,
],
check=True,
)
return f"""
location = /{area}/{version} {{ # no trailing slash
alias {tilejson_path}; # no trailing slash
expires 1w;
default_type application/json;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public;
}}
location /{area}/{version}/ {{ # trailing slash
alias {subdir}/tiles/; # trailing slash
try_files $uri @empty_tile;
add_header Content-Encoding gzip;
expires 10y;
types {{
application/vnd.mapbox-vector-tile pbf;
}}
add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public;
}}
"""
def create_latest_locations(*, local: str, domain: str) -> str:
location_str = ''
local_version_files = config.ofm_config_dir.glob('tileset_version_*.txt')
for file in local_version_files:
area = file.stem.split('_')[-1]
with open(file) as fp:
version = fp.read().strip()
print(f' setting latest version for {area}: {version}')
run_dir = config.runs_dir / area / version
tilejson_path = run_dir / f'tilejson-{local}.json'
assert tilejson_path.is_file()
location_str += f"""
location = /{area} {{ # no trailing slash
alias {tilejson_path}; # no trailing slash
expires 1d;
default_type application/json;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public;
}}
"""
return location_str
def write_ledns_reader_script(domain_ledns):
script = f"""
#!/usr/bin/env bash
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
rclone copyto -v "remote:ofm-private/ledns/{domain_ledns}/ofm_ledns.cert" /data/nginx/certs/ofm_ledns.cert
rclone copyto -v "remote:ofm-private/ledns/{domain_ledns}/ofm_ledns.key" /data/nginx/certs/ofm_ledns.key
""".strip()
with open(config.http_host_bin / 'ledns_reader.sh', 'w') as fp:
fp.write(script)

View File

@@ -0,0 +1,44 @@
server {
server_name __LOCAL__ __DOMAIN__;
# ssl: https://ssl-config.mozilla.org / intermediate config
listen 80;
listen 443 ssl;
listen [::]:443 ssl;
http2 on;
ssl_certificate /data/nginx/certs/ofm_le.cert;
ssl_certificate_key /data/nginx/certs/ofm_le.key;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
ssl_dhparam /etc/nginx/ffdhe2048.txt;
# intermediate configuration
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
ssl_prefer_server_ciphers off;
# access log disabled by default
access_log /data/ofm/http_host/logs_nginx/le-access.jsonl access_json buffer=32k;
#access_log off;
error_log /data/ofm/http_host/logs_nginx/le-error.log;
location ^~ /.well-known/acme-challenge/ {
# trailing slash
root /data/nginx/acme-challenges;
try_files $uri =404;
}
__LOCATION_BLOCKS__
# catch-all block to deny all other requests
location / {
deny all;
error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
}
}

View File

@@ -0,0 +1,38 @@
server {
server_name __LOCAL__ __DOMAIN__;
# ssl: https://ssl-config.mozilla.org / intermediate config
listen 80;
listen 443 ssl;
listen [::]:443 ssl;
http2 on;
ssl_certificate /data/nginx/certs/ofm_ledns.cert;
ssl_certificate_key /data/nginx/certs/ofm_ledns.key;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
ssl_dhparam /etc/nginx/ffdhe2048.txt;
# intermediate configuration
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
ssl_prefer_server_ciphers off;
# access log disabled by default
access_log /data/ofm/http_host/logs_nginx/ledns-access.jsonl access_json buffer=32k;
#access_log off;
error_log /data/ofm/http_host/logs_nginx/ledns-error.log;
__LOCATION_BLOCKS__
# catch-all block to deny all other requests
location / {
deny all;
error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
}
}

View File

@@ -0,0 +1,64 @@
location /fonts/ {
# trailing slash
alias /data/ofm/http_host/assets/fonts/ofm/; # trailing slash
try_files $uri =404;
expires 1w;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public;
}
location /styles/ {
# trailing slash
alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash
try_files $uri.json =404;
expires 1d;
default_type application/json;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public;
}
location /natural_earth/ {
# trailing slash
alias /data/ofm/http_host/assets/natural_earth/ofm/; # trailing slash
try_files $uri =404;
expires 10y;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public;
}
location /sprites/ {
# trailing slash
alias /data/ofm/http_host/assets/sprites/; # trailing slash
try_files $uri =404;
expires 10y;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public;
}
# we need to handle missing tiles as valid request returning empty string
location @empty_tile {
return 200 '';
expires 10y;
default_type application/vnd.mapbox-vector-tile;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public;
}
location = / {
return 302 https://openfreemap.org;
}

View File

@@ -0,0 +1,45 @@
from pathlib import Path
import requests
from http_host_lib.config import config
def set_tileset_versions():
need_nginx_sync = False
for area in ['planet', 'monaco']:
r = requests.get(f'https://assets.openfreemap.com/versions/deployed_{area}.txt', timeout=30)
r.raise_for_status()
remote_version = r.text.strip()
print(f' remote version for {area}: {remote_version}')
local_version_file = config.ofm_config_dir / f'tileset_version_{area}.txt'
if not local_version_file.exists():
local_version_start = None
else:
with open(local_version_file) as fp:
local_version_start = fp.read()
if not remote_version:
print(' remote version not specified')
if local_version_start is not None:
local_version_file.unlink()
need_nginx_sync = True
continue
mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
if not mnt_file.exists():
print(' local version does not exist')
if local_version_start is not None:
local_version_file.unlink()
need_nginx_sync = True
continue
if remote_version != local_version_start:
with open(local_version_file, 'w') as fp:
fp.write(remote_version)
need_nginx_sync = True
return need_nginx_sync

View File

@@ -0,0 +1,69 @@
import os
import subprocess
import sys
from pathlib import Path
import requests
def assert_sudo():
if os.geteuid() != 0:
sys.exit(' needs sudo')
def assert_linux():
if not Path('/etc/fstab').exists():
sys.exit(' needs to be run on Linux')
def assert_single_process():
p = subprocess.run(['pgrep', '-fl', sys.argv[0]], capture_output=True, text=True)
lines = [l for l in p.stdout.splitlines() if 'python' in l]
if len(lines) >= 2:
sys.exit(' detected multiple processes, terminating')
def download_if_size_differs(url: str, local_file: Path) -> bool:
if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url):
download_file_aria2(url, local_file)
return True
return False
def get_remote_file_size(url: str) -> int | None:
r = requests.head(url, timeout=30)
size = r.headers.get('Content-Length')
return int(size) if size else None
def download_file_aria2(url: str, local_file: Path):
print(f' downloading {url} into {local_file}')
local_file.unlink(missing_ok=True)
subprocess.run(
[
'aria2c',
'--split=8',
'--max-connection-per-server=8',
'--file-allocation=none',
'--min-split-size=1M',
'-d',
local_file.parent,
'-o',
local_file.name,
url,
],
check=True,
)
def python_venv_executable() -> Path:
venv_path = os.environ.get('VIRTUAL_ENV')
if venv_path:
return Path(venv_path) / 'bin' / 'python'
elif sys.prefix != sys.base_prefix:
return Path(sys.prefix) / 'bin' / 'python'
else:
return Path(sys.executable)

View File

@@ -0,0 +1,66 @@
#!/usr/bin/env python3
import json
from pathlib import Path
import click
@click.command()
@click.argument(
'metadata_path', type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path)
)
@click.argument('tilejson_path', type=click.Path(path_type=Path))
@click.argument('url_prefix')
@click.option('--minify', is_flag=True, help='Minify the generated JSON')
def cli(metadata_path: Path, tilejson_path: Path, url_prefix: str, minify: bool):
"""
Takes a MBTiles metadata.json and generates a TileJSON 3.0.0 file
URL_PREFIX: Base URL to use as a prefix for tiles in the generated TileJSON.
Reference: https://github.com/mapbox/tilejson-spec/tree/master/3.0.0
"""
tilejson = dict(tilejson='3.0.0')
with open(metadata_path) as fp:
metadata = json.load(fp)
metadata_json_key = json.loads(metadata.pop('json'))
tilejson['tiles'] = [url_prefix.rstrip('/') + '/{z}/{x}/{y}.pbf']
''
tilejson['vector_layers'] = metadata_json_key.pop('vector_layers')
assert not metadata_json_key # check that no more keys are left
tilejson['attribution'] = metadata.pop('attribution')
# overwriting new style OSM license, until fixed in tile_gen
tilejson['attribution'] = (
'<a href="https://openfreemap.org" target="_blank">OpenFreeMap</a> '
'<a href="https://www.openmaptiles.org/" target="_blank">&copy; OpenMapTiles</a> '
'Data from <a href="https://www.openstreetmap.org/copyright" target="_blank">OpenStreetMap</a>'
)
tilejson['bounds'] = [float(n) for n in metadata.pop('bounds').split(',')]
tilejson['center'] = [float(n) for n in metadata.pop('center').split(',')]
tilejson['center'][2] = 1
tilejson['description'] = metadata.pop('description')
tilejson['maxzoom'] = int(metadata.pop('maxzoom'))
tilejson['minzoom'] = int(metadata.pop('minzoom'))
tilejson['name'] = metadata.pop('name')
tilejson['version'] = metadata.pop('version')
with open(tilejson_path, 'w') as fp:
if minify:
json.dump(tilejson, fp, ensure_ascii=False, separators=(',', ':'))
else:
json.dump(tilejson, fp, ensure_ascii=False, indent=2)
if __name__ == '__main__':
cli()

View File

@@ -0,0 +1,14 @@
from setuptools import find_packages, setup
requirements = [
'click',
'requests',
]
setup(
python_requires='>=3.10',
install_requires=requirements,
packages=find_packages(),
)

View File

@@ -0,0 +1,11 @@
#!/usr/bin/env bash
#env > /data/ofm/ledns/env.txt
#RENEWED_DOMAINS=direct.openfreemap.org
#RENEWED_LINEAGE=/etc/letsencrypt/live/ofm_ledns
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
rclone copyto -v --copy-links "$RENEWED_LINEAGE/fullchain.pem" "remote:ofm-private/ledns/$RENEWED_DOMAINS/ofm_ledns.cert"
rclone copyto -v --copy-links "$RENEWED_LINEAGE/privkey.pem" "remote:ofm-private/ledns/$RENEWED_DOMAINS/ofm_ledns.key"

View File

@@ -0,0 +1,8 @@
# every minute
# fix
#* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py fix >> /data/ofm/loadbalancer/logs/run.log 2>&1
# check
* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py check >> /data/ofm/loadbalancer/logs/run.log 2>&1

View File

@@ -0,0 +1,162 @@
#!/usr/bin/env python3
import datetime
import json
import click
import requests
from dotenv import dotenv_values
from loadbalancer_lib import OFM_CONFIG_DIR
from loadbalancer_lib.cloudflare import get_zone_id, set_records_round_robin
from loadbalancer_lib.curl import pycurl_get, pycurl_status
from loadbalancer_lib.telegram_ import telegram_send_message
AREAS = ['planet', 'monaco']
@click.group()
def cli():
"""
Manages load-balancing of Round-Robin DNS records
"""
@cli.command()
def check():
"""
Runs load-balancing check (triggered by cron every minute)
"""
print(f'starting loadbalancer check at: {datetime.datetime.now(tz=datetime.timezone.utc)}')
check_or_fix(fix=False)
@cli.command()
def fix():
"""
Fixes records based on check results
"""
print(f'starting loadbalancer fix at: {datetime.datetime.now(tz=datetime.timezone.utc)}')
check_or_fix(fix=True)
def check_or_fix(fix=False):
with open(OFM_CONFIG_DIR / 'loadbalancer.json') as fp:
c = json.load(fp)
# print(c)
if not c['http_host_list']:
telegram_send_message(
'OFM loadbalancer no hosts found on list, terminating',
c['telegram_token'],
c['telegram_chat_id'],
)
return
try:
results_by_ip = {}
working_hosts = set()
for area in AREAS:
results = run_area(c, area)
for host_ip, host_is_ok in results.items():
results_by_ip.setdefault(host_ip, True)
results_by_ip[host_ip] &= host_is_ok
for host_ip, host_is_ok in results_by_ip.items():
if not host_is_ok:
message = f'OFM loadbalancer ERROR with host: {host_ip}'
telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
else:
working_hosts.add(host_ip)
except Exception as e:
message = f'OFM loadbalancer ERROR with loadbalancer: {e}'
telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
return
print(f'working hosts: {sorted(working_hosts)}')
if fix:
# if no hosts are detected working, probably a bug in this script
# fail-safe to include all hosts
if not working_hosts:
working_hosts = set(c['http_host_list'])
message = 'OFM loadbalancer FIX found no working hosts, reverting to full list!'
telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
updated = update_records(c, working_hosts)
if updated:
message = f'OFM loadbalancer FIX modified records, new records: {working_hosts}'
telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
def run_area(c, area):
target_version = get_target_version(area)
print(f'target version: {area}: {target_version}')
results = {}
for host_ip in c['http_host_list']:
try:
check_host(c['domain_ledns'], host_ip, area, target_version)
results[host_ip] = True
except Exception as e:
results[host_ip] = False
print(e)
return results
def check_host(domain, host_ip, area, version):
# check TileJSON first
url = f'https://{domain}/{area}'
tilejson_str = pycurl_get(url, domain, host_ip)
tilejson = json.loads(tilejson_str)
tiles_url = tilejson['tiles'][0]
version_in_tilejson = tiles_url.split('/')[4]
assert version_in_tilejson == version
# check actual vector tile
url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
assert pycurl_status(url, domain, host_ip) == 200
# check style
url = f'https://{domain}/styles/bright'
assert pycurl_status(url, domain, host_ip) == 200
def get_target_version(area):
url = f'https://assets.openfreemap.com/versions/deployed_{area}.txt'
response = requests.get(url)
response.raise_for_status()
return response.text.strip()
def update_records(c, working_hosts) -> bool:
config = dotenv_values(OFM_CONFIG_DIR / 'cloudflare.ini')
cloudflare_api_token = config['dns_cloudflare_api_token']
domain = '.'.join(c['domain_ledns'].split('.')[-2:])
zone_id = get_zone_id(domain, cloudflare_api_token=cloudflare_api_token)
updated = False
updated |= set_records_round_robin(
zone_id=zone_id,
name=c['domain_ledns'],
host_ip_set=working_hosts,
proxied=False,
ttl=300,
comment='domain_ledns',
cloudflare_api_token=cloudflare_api_token,
)
return updated
if __name__ == '__main__':
cli()

View File

@@ -0,0 +1,9 @@
from pathlib import Path
if Path('/data/ofm/config').exists():
OFM_CONFIG_DIR = Path('/data/ofm/config')
else:
OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
assert OFM_CONFIG_DIR.exists()

View File

@@ -0,0 +1,109 @@
from pprint import pprint
import requests
# docs: https://api.cloudflare.com/
def cloudflare_get(path: str, params: dict, cloudflare_api_token: str):
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
res = requests.get(
f'https://api.cloudflare.com/client/v4{path}', headers=headers, params=params
)
res.raise_for_status()
data = res.json()
assert data['success'] is True
return data
def get_zone_id(domain, cloudflare_api_token: str):
data = cloudflare_get(
'/zones', params=dict(name=domain), cloudflare_api_token=cloudflare_api_token
)
assert len(data['result']) == 1
zone_info = data['result'][0]
return zone_info['id']
def get_dns_records_round_robin(zone_id, cloudflare_api_token: str) -> dict:
data = cloudflare_get(
f'/zones/{zone_id}/dns_records',
params=dict(per_page=5000),
cloudflare_api_token=cloudflare_api_token,
)
records = data['result']
data = {}
for r in records:
if r['type'] != 'A':
continue
data.setdefault(r['name'], [])
data[r['name']].append(dict(content=r['content'], id=r['id']))
return data
def set_records_round_robin(
zone_id,
*,
name: str,
host_ip_set: set,
ttl: int = 1,
proxied: bool,
comment: str = None,
cloudflare_api_token: str,
) -> bool:
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
dns_records = get_dns_records_round_robin(zone_id, cloudflare_api_token=cloudflare_api_token)
current_records = dns_records.get(name, [])
current_ips = {r['content'] for r in current_records}
if current_ips == host_ip_set:
print(f'No need to update records: {name} currently set: {sorted(current_ips)}')
return False
# changing records
# delete all current records first
for r in current_records:
delete_record(zone_id, id_=r['id'], cloudflare_api_token=cloudflare_api_token)
# create new records
for ip in host_ip_set:
print(f'Creating record: {name} {ip}')
json_data = dict(
type='A',
name=name,
content=ip,
ttl=ttl,
proxied=proxied,
comment=comment,
)
res = requests.post(
f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records',
headers=headers,
json=json_data,
)
res.raise_for_status()
data = res.json()
assert data['success'] is True
return True
def delete_record(zone_id, *, id_: str, cloudflare_api_token: str):
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
print(f'Deleting record: {id_}')
res = requests.delete(
f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records/{id_}',
headers=headers,
json={},
)
res.raise_for_status()
data = res.json()
assert data['success'] is True

View File

@@ -0,0 +1,54 @@
from io import BytesIO
from pathlib import Path
import pycurl
def pycurl_status(url, domain, host_ip):
"""
Uses pycurl to make a HTTPS HEAD request using custom resolving,
checks if the status code is 200
"""
c = pycurl.Curl()
c.setopt(c.URL, url)
# linux needs CA certs specified manually
if Path('/etc/ssl/certs/ca-certificates.crt').exists():
c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
c.setopt(c.NOBODY, True)
c.setopt(c.TIMEOUT, 5)
c.perform()
status_code = c.getinfo(c.RESPONSE_CODE)
c.close()
return status_code
def pycurl_get(url, domain, host_ip):
"""
Uses pycurl to make a HTTPS GET request using custom resolving,
checks if the status code is 200, and returns the content.
"""
buffer = BytesIO()
c = pycurl.Curl()
c.setopt(c.URL, url)
# linux needs CA certs specified manually
if Path('/etc/ssl/certs/ca-certificates.crt').exists():
c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.TIMEOUT, 5)
c.perform()
status_code = c.getinfo(c.RESPONSE_CODE)
c.close()
if status_code != 200:
raise ValueError(f'status code: {status_code}')
return buffer.getvalue().decode('utf8')

View File

@@ -0,0 +1,16 @@
import requests
def telegram_send_message(message, bot_token, chat_id):
print(message)
url = f'https://api.telegram.org/bot{bot_token}/sendMessage'
payload = {'chat_id': chat_id, 'text': message}
response = requests.post(url, data=payload)
if response.status_code == 200:
print(' Message sent successfully!')
else:
print(' Failed to send message:', response.text)

View File

@@ -0,0 +1,16 @@
from setuptools import find_packages, setup
requirements = [
'click',
'requests',
'pycurl',
'python-dotenv',
]
setup(
python_requires='>=3.10',
install_requires=requirements,
packages=find_packages(),
)

15
modules/prepare-virtualenv.sh Executable file
View File

@@ -0,0 +1,15 @@
#!/usr/bin/env bash
find . -name "*.egg-info" -exec rm -rf {} +
find . -name __pycache__ -exec rm -rf {} +
# deactivate
rm -rf venv
python3 -m venv venv
venv/bin/pip -V
venv/bin/pip install -U pip wheel setuptools

View File

@@ -0,0 +1,17 @@
from setuptools import find_packages, setup
requirements = [
'click',
'requests',
'pycurl',
'python-dotenv',
'questionary',
]
setup(
python_requires='>=3.10',
install_requires=requirements,
packages=find_packages(),
)

View File

@@ -0,0 +1,62 @@
#!/usr/bin/env python3
import subprocess
import click
import questionary
from setversion_lib import RCLONE_BIN, RCLONE_CONF
@click.group()
def cli():
"""
Sets deployed reference versions
"""
@cli.command()
@click.argument('area', required=True)
def interactive(area):
versions = get_available_versions(area)[::-1]
choices = [questionary.Choice(title=r, value=i) for i, r in enumerate(versions)]
answer = questionary.select(f'Select version for: {area}', choices=choices).ask()
selected = versions[answer]
set_version(area, selected)
def get_available_versions(area):
p = subprocess.run(
[
RCLONE_BIN,
'cat',
f'remote:ofm-{area}/dirs.txt',
],
env=dict(RCLONE_CONFIG=RCLONE_CONF),
check=True,
capture_output=True,
text=True,
)
versions = [l.strip() for l in p.stdout.strip().splitlines()]
versions.sort()
return versions
def set_version(area, version):
subprocess.run(
[
RCLONE_BIN,
'rcat',
f'remote:ofm-assets/versions/deployed_{area}.txt',
],
env=dict(RCLONE_CONFIG=RCLONE_CONF),
check=True,
input=version.encode(),
)
if __name__ == '__main__':
cli()

View File

@@ -0,0 +1,16 @@
from pathlib import Path
if Path('/data/ofm/config').exists():
OFM_CONFIG_DIR = Path('/data/ofm/config')
else:
OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
assert OFM_CONFIG_DIR.exists()
RCLONE_CONF = OFM_CONFIG_DIR / 'rclone.conf'
if Path('/opt/homebrew/bin/rclone').exists():
RCLONE_BIN = '/opt/homebrew/bin/rclone'
else:
RCLONE_BIN = 'rclone'

View File

@@ -0,0 +1,8 @@
# every hour, make a monaco run
10 * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-tiles monaco --upload >> /data/ofm/tile_gen/logs/monaco-cron.log 2>&1
# once per minute, create indexes
* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-indexes >> /data/ofm/tile_gen/logs/make-indexes-cron.log 2>&1

View File

@@ -0,0 +1,2 @@
These are self contained Python scripts, they can be run outside of this project's environment.

View File

@@ -0,0 +1,181 @@
#!/usr/bin/env python3
import json
import os
import shutil
import sqlite3
import sys
from pathlib import Path
import click
@click.command()
@click.argument(
'mbtiles_path',
type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path),
)
@click.argument('dir_path', type=click.Path(dir_okay=True, file_okay=False, path_type=Path))
def cli(mbtiles_path: Path, dir_path: Path):
"""
Extracts a mbtiles sqlite to a folder
Deduplicating identical tiles as hard-links
used for reference: https://github.com/mapbox/mbutil
"""
if dir_path.exists() and any(dir_path.iterdir()):
sys.exit(' dir not empty')
dir_path.mkdir(exist_ok=True)
conn = sqlite3.connect(mbtiles_path)
c = conn.cursor()
write_dedupl_files(c, dir_path=dir_path)
write_tile_files(c, dir_path=dir_path)
# planetiler has missing tiles by design, so disabling this
# assert_all_tiles_present(mbtiles_path, dir_path)
write_metadata(c, dir_path=dir_path)
conn.commit()
print('extract_mbtiles.py DONE')
def write_metadata(c, *, dir_path):
metadata = dict(c.execute('select name, value from metadata').fetchall())
c.execute("update metadata set value='OpenFreeMap' where name='name'")
c.execute("update metadata set value='https://openfreemap.org' where name='description'")
if 'openfreemap' not in metadata['attribution']:
attr_str = (
'<a href="https://openfreemap.org" target="_blank">OpenFreeMap</a> '
+ metadata['attribution']
)
c.execute("UPDATE metadata SET value = ? WHERE name = 'attribution'", (attr_str,))
if 'osm_date' not in metadata:
if 'planetiler:osm:osmosisreplicationtime' in metadata:
osm_date = metadata['planetiler:osm:osmosisreplicationtime'][:10]
c.execute('INSERT INTO metadata (name, value) VALUES (?, ?)', ('osm_date', osm_date))
metadata = dict(c.execute('select name, value from metadata').fetchall())
with open(dir_path / 'metadata.json', 'w') as fp:
json.dump(metadata, fp, indent=2)
with open(dir_path / 'osm_date', 'w') as fp:
fp.write(metadata['osm_date'])
def write_dedupl_files(c, *, dir_path):
"""
dedupl files
write out the tiles_data files into a multi-level folder
"""
total = c.execute('select count(*) from tiles_data').fetchone()[0]
c.execute('select tile_data_id, tile_data from tiles_data')
for i, row in enumerate(c, start=1):
dedupl_id = row[0]
dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
dedupl_path.parent.mkdir(parents=True, exist_ok=True)
with open(dedupl_path, 'wb') as fp:
fp.write(row[1])
print(f'written dedupl file {i}/{total}')
def write_tile_files(c, *, dir_path):
total = c.execute('select count(*) from tiles_shallow').fetchone()[0]
bug_fix_dict = {}
c.execute('select zoom_level, tile_column, tile_row, tile_data_id from tiles_shallow')
for i, row in enumerate(c, start=1):
z = row[0]
x = row[1]
y = flip_y(z, row[2])
dedupl_id = row[3]
dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
tile_path = dir_path / 'tiles' / str(z) / str(x) / f'{y}.pbf'
tile_path.parent.mkdir(parents=True, exist_ok=True)
if tile_path.is_file():
continue
# create the hard link
try:
tile_path.hardlink_to(dedupl_path_fixed)
print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
except OSError as e:
# fixing Btrfs's 64k max link limit
if e.errno == 31:
bug_fix_dict.setdefault(dedupl_path, 0)
bug_fix_dict[dedupl_path] += 1
dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
shutil.copyfile(dedupl_path, dedupl_path_fixed)
print(f'Created fixed dedupl file: {dedupl_path_fixed}')
tile_path.hardlink_to(dedupl_path_fixed)
print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
else:
raise
def assert_all_tiles_present(mbtiles_path, dir_path):
"""
If it's a full planet run,
make sure there are exactly the right number of files generated.
"""
if 'planet' in mbtiles_path.resolve().parent.name:
assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14)
print(f'Tile number: {calculate_tiles_sum(14)} - OK')
def count_files(folder):
total = 0
for root, dirs, files in os.walk(folder):
total += len(files)
return
def get_fixed_dedupl_name(bug_fix_dict, dedupl_path):
if dedupl_path in bug_fix_dict:
return dedupl_path.with_name(f'{dedupl_path.name}-{bug_fix_dict[dedupl_path]}')
else:
return dedupl_path
def dedupl_helper_path(dedupl_id: int) -> Path:
"""
Naming 200 million files such that each subdir has max 1000 children
"""
str_num = f'{dedupl_id:09}'
l1 = str_num[:3]
l2 = str_num[3:6]
l3 = str_num[6:]
return Path(l1) / l2 / f'{l3}.pbf'
def flip_y(zoom, y):
return (2**zoom - 1) - y
def calculate_tiles(zoom_level):
return (2**zoom_level) ** 2
def calculate_tiles_sum(zoom_level):
"""
Sum of tiles up to zoom level (geometric series)
"""
return (4 ** (zoom_level + 1) - 1) // 3
if __name__ == '__main__':
cli()

View File

@@ -0,0 +1,105 @@
#!/usr/bin/env python3
import os
import subprocess
import sys
import tempfile
from pathlib import Path
import click
# btrfs cannot shrink smaller than 256 MiB
SMALLEST_SIZE = 256 * 1024 * 1024
@click.command()
@click.argument(
'btrfs_img',
type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path),
)
def cli(btrfs_img: Path):
"""
Shrinks a Btrfs image
// I cannot believe that Btrfs is over 15 years old,
// yet there is no resize2fs tool which can shrink a disk image
// to minimum size.
// It cannot even tell you how much should be the right size,
// it just randomly fails after which you have to umount and mount again.
// So we have to make a loop which tries to shrink it until it fails.
// Also, WONTFIX bugs like how instead of telling you that
// minimum fs size is 256 MB, it says "ERROR: unable to resize - Invalid argument"
// https://bugzilla.kernel.org/show_bug.cgi?id=118111
"""
if os.geteuid() != 0:
sys.exit(' needs sudo')
current_dir = Path.cwd()
mnt_dir = Path(tempfile.mkdtemp(dir=current_dir, prefix='tmp_shrink_'))
subprocess.run(['mount', '-t', 'btrfs', btrfs_img, mnt_dir], check=True)
# shink until max. 10 MB left or reached SMALLEST_SIZE or failure
while True:
# needs to start with a balancing
# https://btrfs.readthedocs.io/en/latest/Balance.html
# https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html
do_balancing(mnt_dir)
free_bytes = get_usage(mnt_dir, 'Device unallocated')
device_size = get_usage(mnt_dir, 'Device size')
shrink_idea = free_bytes * 0.7
# workaround for the SMALLEST_SIZE limit
if device_size - free_bytes < SMALLEST_SIZE:
shrink_idea = (device_size - SMALLEST_SIZE) * 0.7
# stop if 10 MB left
if shrink_idea < 10_000_000:
break
# stop if process error
if not do_shrink(mnt_dir, shrink_idea):
break
total_size = get_usage(mnt_dir, 'Device size')
subprocess.run(['umount', mnt_dir])
mnt_dir.rmdir()
subprocess.run(['truncate', '-s', str(total_size), btrfs_img])
print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size')
print('shrink_btrfs.py DONE')
def get_usage(mnt: Path, key: str):
p = subprocess.run(
['btrfs', 'filesystem', 'usage', '-b', mnt], text=True, capture_output=True, check=True
)
for line in p.stdout.splitlines():
if f'{key}:' not in line:
continue
free = int(line.split(':')[1])
return free
def do_shrink(mnt: Path, delta_size: float):
delta_size = int(delta_size)
print(f'Trying to shrink by {delta_size//1_000_000} MB')
p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt])
return p.returncode == 0
def do_balancing(mnt: Path):
print('Starting btrfs balancing')
p = subprocess.run(
['btrfs', 'balance', 'start', '-dusage=100', mnt], capture_output=True, text=True
)
if p.returncode:
print(f'Balance error: {p.stdout} {p.stderr}')
print('Balancing done')
if __name__ == '__main__':
cli()

13
modules/tile_gen/setup.py Normal file
View File

@@ -0,0 +1,13 @@
from setuptools import find_packages, setup
requirements = [
'click',
]
setup(
python_requires='>=3.10',
install_requires=requirements,
packages=find_packages(),
)

52
modules/tile_gen/tile_gen.py Executable file
View File

@@ -0,0 +1,52 @@
#!/usr/bin/env python3
import click
from tile_gen_lib.btrfs import make_btrfs
from tile_gen_lib.planetiler import run_planetiler
from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
@click.group()
def cli():
"""
Generates tiles and uploads to CloudFlare
"""
@cli.command()
@click.argument('area', required=True)
@click.option('--upload', is_flag=True, help='Upload after generation is complete')
def make_tiles(area, upload):
"""
Generate tiles for a given area, optionally upload it to the btrfs bucket
"""
run_folder = run_planetiler(area)
make_btrfs(run_folder)
if upload:
upload_area(area)
@cli.command(name='upload-area')
@click.argument('area', required=True)
def upload_area_(area):
"""
Upload all runs from a given area to the btrfs bucket
"""
upload_area(area)
@cli.command()
def make_indexes():
"""
Make indexes for all buckets
"""
for bucket in ['ofm-btrfs', 'ofm-assets']:
make_indexes_for_bucket(bucket)
if __name__ == '__main__':
cli()

View File

@@ -0,0 +1,140 @@
import os
import shutil
import subprocess
from pathlib import Path
from tile_gen_lib.config import config
from tile_gen_lib.utils import python_venv_executable
IMAGE_SIZE = '200G'
def make_btrfs(run_folder: Path):
os.chdir(run_folder)
cleanup_folder(run_folder)
# make an empty file that's definitely bigger then the current OSM output
for image in ['image.btrfs', 'image2.btrfs']:
subprocess.run(['fallocate', '-l', IMAGE_SIZE, image], check=True)
subprocess.run(['mkfs.btrfs', '-m', 'single', image], check=True, capture_output=True)
for image, mount in [('image.btrfs', 'mnt_rw'), ('image2.btrfs', 'mnt_rw2')]:
Path(mount).mkdir()
# https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
# compression doesn't make sense, data is already gzip compressed
subprocess.run(
[
'sudo',
'mount',
'-t',
'btrfs',
'-o',
'noacl,nobarrier,noatime,max_inline=4096',
image,
mount,
],
check=True,
)
subprocess.run(['sudo', 'chown', 'ofm:ofm', '-R', mount], check=True)
# extract mbtiles
extract_script = config.tile_gen_scripts_dir / 'extract_mbtiles.py'
with open('extract_out.log', 'w') as out, open('extract_err.log', 'w') as err:
subprocess.run(
[
python_venv_executable(),
extract_script,
'tiles.mbtiles',
'mnt_rw/extract',
],
check=True,
stdout=out,
stderr=err,
)
os.unlink('tiles.mbtiles')
shutil.copy('mnt_rw/extract/osm_date', '.')
# process logs
subprocess.run('grep fixed extract_out.log > dedupl_fixed.log', shell=True)
# unfortunately, by deleting files from the btrfs partition, the partition size grows
# so we need to rsync onto a new partition instead of deleting
with open('rsync_out.log', 'w') as out, open('rsync_err.log', 'w') as err:
subprocess.run(
[
'rsync',
'-avH',
'--max-alloc=4294967296',
'--exclude',
'dedupl',
'mnt_rw/extract/',
'mnt_rw2/',
],
check=True,
stdout=out,
stderr=err,
)
# collect stats
for i, mount in enumerate(['mnt_rw', 'mnt_rw2'], 1):
with open(f'stats{i}.txt', 'w') as f:
for cmd in [
['df', '-h', mount],
['btrfs', 'filesystem', 'df', mount],
['btrfs', 'filesystem', 'show', mount],
['btrfs', 'filesystem', 'usage', mount],
]:
f.write(f"\n\n{' '.join(cmd)}\n")
result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True)
f.write(result.stdout)
# unmount and cleanup
for mount in ['mnt_rw', 'mnt_rw2']:
subprocess.run(['sudo', 'umount', mount], check=True)
shutil.rmtree('mnt_rw')
shutil.rmtree('mnt_rw2')
# shrink btrfs
shrink_script = config.tile_gen_scripts_dir / 'shrink_btrfs.py'
with open('shrink_out.log', 'w') as out, open('shrink_err.log', 'w') as err:
subprocess.run(
['sudo', python_venv_executable(), shrink_script, 'image2.btrfs'],
check=True,
stdout=out,
stderr=err,
)
os.unlink('image.btrfs')
shutil.move('image2.btrfs', 'tiles.btrfs')
# parallel gzip (pigz)
subprocess.run(['pigz', 'tiles.btrfs', '--fast'], check=True)
# move logs
Path('logs').mkdir()
for pattern in ['*.log', '*.txt']:
for file in Path().glob(pattern):
shutil.move(file, 'logs')
print('extract_btrfs.py DONE')
def cleanup_folder(run_folder: Path):
print(f'cleaning up {run_folder}')
for mount in ['mnt_rw', 'mnt_rw2']:
subprocess.run(['sudo', 'umount', run_folder / mount], capture_output=True)
for pattern in ['mnt_rw*', 'tmp_*', '*.btrfs', '*.gz', '*.log', '*.txt', 'logs', 'osm_date']:
for item in run_folder.glob(pattern):
if item.is_dir():
shutil.rmtree(item)
else:
item.unlink()

View File

@@ -0,0 +1,21 @@
from pathlib import Path
class Configuration:
areas = ['planet', 'monaco']
tile_gen_dir = Path('/data/ofm/tile_gen')
tile_gen_bin = tile_gen_dir / 'bin'
tile_gen_scripts_dir = tile_gen_bin / 'scripts'
planetiler_bin = tile_gen_dir / 'planetiler'
planetiler_path = planetiler_bin / 'planetiler.jar'
runs_dir = tile_gen_dir / 'runs'
ofm_config_dir = Path('/data/ofm/config')
rclone_config = ofm_config_dir / 'rclone.conf'
config = Configuration()

View File

@@ -0,0 +1,65 @@
import os
import shutil
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from tile_gen_lib.config import config
from tile_gen_lib.btrfs import cleanup_folder
def run_planetiler(area: str) -> Path:
assert area in config.areas
date = datetime.now(tz=timezone.utc).strftime('%Y%m%d_%H%M%S')
area_dir = config.runs_dir / area
# delete all previous runs for the given area
for subdir in area_dir.iterdir():
cleanup_folder(subdir)
print('running rmtree')
shutil.rmtree(area_dir, ignore_errors=True)
print('rmtree done')
run_folder = area_dir / f'{date}_pt'
run_folder.mkdir(parents=True, exist_ok=True)
os.chdir(run_folder)
# link to discussion about why exactly 30 GB
# https://github.com/onthegomap/planetiler/discussions/690#discussioncomment-7756397
java_memory_gb = 30 if area == 'planet' else 1
command = [
'java',
f'-Xmx{java_memory_gb}g',
'-jar',
config.planetiler_path,
f'--area={area}',
'--download',
'--download-threads=10',
'--download-chunk-size-mb=1000',
'--fetch-wikidata',
'--output=tiles.mbtiles',
'--nodemap-type=array',
'--storage=mmap',
'--force',
]
if area == 'planet':
command.append('--bounds=planet')
print(command)
out_path = run_folder / 'planetiler.out'
err_path = run_folder / 'planetiler.err'
with out_path.open('w') as out_file, err_path.open('w') as err_file:
subprocess.run(command, stdout=out_file, stderr=err_file, check=True, cwd=run_folder)
shutil.rmtree(run_folder / 'data', ignore_errors=True)
print('planetiler.jar DONE')
return run_folder

View File

@@ -0,0 +1,123 @@
import subprocess
import sys
from tile_gen_lib.config import config
def upload_area(area):
"""
Uploads an area, making sure there is exactly one run present
"""
print(f'Uploading area: {area}')
assert area in config.areas
area_dir = config.runs_dir / area
if not area_dir.exists():
return
runs = list(area_dir.iterdir())
if len(runs) != 1:
print('Error: Make sure there is only one run in the given area')
sys.exit(1)
run = runs[0].name
upload_area_run(area, run)
make_indexes_for_bucket('ofm-btrfs')
def upload_area_run(area, run):
print(f'Uploading {area} {run} to btrfs bucket')
run_dir = config.runs_dir / area / run
assert run_dir.is_dir()
subprocess.run(
[
'rclone',
'sync',
'--verbose=1',
'--transfers=8',
'--multi-thread-streams=8',
'--fast-list',
'--stats-file-name-length=0',
'--stats-one-line',
'--log-file',
run_dir / 'logs' / 'rclone.log',
'--exclude',
'logs/**',
run_dir,
f'remote:ofm-btrfs/areas/{area}/{run}',
],
env=dict(RCLONE_CONFIG=config.rclone_config),
check=True,
)
def make_indexes_for_bucket(bucket):
print(f'Making indexes for bucket: {bucket}')
# files
p = subprocess.run(
[
'rclone',
'lsf',
'--recursive',
'--files-only',
'--fast-list',
'--exclude',
'dirs.txt',
'--exclude',
'files.txt',
f'remote:{bucket}',
],
env=dict(RCLONE_CONFIG=config.rclone_config),
check=True,
capture_output=True,
text=True,
)
index_str = p.stdout
# upload to files.txt
subprocess.run(
[
'rclone',
'rcat',
f'remote:{bucket}/files.txt',
],
env=dict(RCLONE_CONFIG=config.rclone_config),
check=True,
input=index_str.encode(),
)
# directories
p = subprocess.run(
[
'rclone',
'lsf',
'--recursive',
'--dirs-only',
'--dir-slash=false',
'--fast-list',
f'remote:{bucket}',
],
env=dict(RCLONE_CONFIG=config.rclone_config),
check=True,
capture_output=True,
text=True,
)
index_str = p.stdout
# upload to dirs.txt
subprocess.run(
[
'rclone',
'rcat',
f'remote:{bucket}/dirs.txt',
],
env=dict(RCLONE_CONFIG=config.rclone_config),
check=True,
input=index_str.encode(),
)

View File

@@ -0,0 +1,14 @@
import os
import sys
from pathlib import Path
def python_venv_executable() -> Path:
venv_path = os.environ.get('VIRTUAL_ENV')
if venv_path:
return Path(venv_path) / 'bin' / 'python'
elif sys.prefix != sys.base_prefix:
return Path(sys.prefix) / 'bin' / 'python'
else:
return Path(sys.executable)