mirror of
https://github.com/hyperknot/openfreemap.git
synced 2026-05-21 14:02:15 +00:00
scripts -> modules
This commit is contained in:
172
modules/debug_proxy/.gitignore
vendored
Normal file
172
modules/debug_proxy/.gitignore
vendored
Normal file
@@ -0,0 +1,172 @@
|
||||
# Logs
|
||||
|
||||
logs
|
||||
_.log
|
||||
npm-debug.log_
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
# Runtime data
|
||||
|
||||
pids
|
||||
_.pid
|
||||
_.seed
|
||||
\*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
|
||||
coverage
|
||||
\*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
|
||||
\*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
|
||||
.stylelintcache
|
||||
|
||||
# Microbundle cache
|
||||
|
||||
.rpt2_cache/
|
||||
.rts2_cache_cjs/
|
||||
.rts2_cache_es/
|
||||
.rts2_cache_umd/
|
||||
|
||||
# Optional REPL history
|
||||
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
|
||||
\*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variable files
|
||||
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
|
||||
.cache
|
||||
.parcel-cache
|
||||
|
||||
# Next.js build output
|
||||
|
||||
.next
|
||||
out
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
|
||||
.nuxt
|
||||
dist
|
||||
|
||||
# Gatsby files
|
||||
|
||||
.cache/
|
||||
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
|
||||
.temp
|
||||
.cache
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
|
||||
.dynamodb/
|
||||
|
||||
# TernJS port file
|
||||
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
|
||||
.vscode-test
|
||||
|
||||
# yarn v2
|
||||
|
||||
.yarn/cache
|
||||
.yarn/unplugged
|
||||
.yarn/build-state.yml
|
||||
.yarn/install-state.gz
|
||||
.pnp.\*
|
||||
|
||||
# wrangler project
|
||||
|
||||
.dev.vars
|
||||
.wrangler/
|
||||
14
modules/debug_proxy/package.json
Normal file
14
modules/debug_proxy/package.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "cf-debug-proxy",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"deploy": "wrangler deploy",
|
||||
"dev": "wrangler dev",
|
||||
"start": "wrangler dev"
|
||||
},
|
||||
"devDependencies": {
|
||||
"itty-router": "^3.0.12",
|
||||
"wrangler": "^3.60.3"
|
||||
}
|
||||
}
|
||||
822
modules/debug_proxy/pnpm-lock.yaml
generated
Normal file
822
modules/debug_proxy/pnpm-lock.yaml
generated
Normal file
@@ -0,0 +1,822 @@
|
||||
lockfileVersion: '6.0'
|
||||
|
||||
settings:
|
||||
autoInstallPeers: true
|
||||
excludeLinksFromLockfile: false
|
||||
|
||||
devDependencies:
|
||||
itty-router:
|
||||
specifier: ^3.0.12
|
||||
version: 3.0.12
|
||||
wrangler:
|
||||
specifier: ^3.60.3
|
||||
version: 3.62.0
|
||||
|
||||
packages:
|
||||
|
||||
/@cloudflare/kv-asset-handler@0.3.4:
|
||||
resolution: {integrity: sha512-YLPHc8yASwjNkmcDMQMY35yiWjoKAKnhUbPRszBRS0YgH+IXtsMp61j+yTcnCE3oO2DgP0U3iejLC8FTtKDC8Q==}
|
||||
engines: {node: '>=16.13'}
|
||||
dependencies:
|
||||
mime: 3.0.0
|
||||
dev: true
|
||||
|
||||
/@cloudflare/workerd-darwin-64@1.20240620.1:
|
||||
resolution: {integrity: sha512-YWeS2aE8jAzDefuus/3GmZcFGu3Ef94uCAoxsQuaEXNsiGM9NeAhPpKC1BJAlcv168U/Q1J+6hckcGtipf6ZcQ==}
|
||||
engines: {node: '>=16'}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@cloudflare/workerd-darwin-arm64@1.20240620.1:
|
||||
resolution: {integrity: sha512-3rdND+EHpmCrwYX6hvxIBSBJ0f40tRNxond1Vfw7GiR1MJVi3gragiBx75UDFHCxfRw3J0GZ1qVlkRce2/Xbsg==}
|
||||
engines: {node: '>=16'}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@cloudflare/workerd-linux-64@1.20240620.1:
|
||||
resolution: {integrity: sha512-tURcTrXGeSbYqeM5ISVcofY20StKbVIcdxjJvNYNZ+qmSV9Fvn+zr7rRE+q64pEloVZfhsEPAlUCnFso5VV4XQ==}
|
||||
engines: {node: '>=16'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@cloudflare/workerd-linux-arm64@1.20240620.1:
|
||||
resolution: {integrity: sha512-TThvkwNxaZFKhHZnNjOGqIYCOk05DDWgO+wYMuXg15ymN/KZPnCicRAkuyqiM+R1Fgc4kwe/pehjP8pbmcf6sg==}
|
||||
engines: {node: '>=16'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@cloudflare/workerd-windows-64@1.20240620.1:
|
||||
resolution: {integrity: sha512-Y/BA9Yj0r7Al1HK3nDHcfISgFllw6NR3XMMPChev57vrVT9C9D4erBL3sUBfofHU+2U9L+ShLsl6obBpe3vvUw==}
|
||||
engines: {node: '>=16'}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@cspotcode/source-map-support@0.8.1:
|
||||
resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
|
||||
engines: {node: '>=12'}
|
||||
dependencies:
|
||||
'@jridgewell/trace-mapping': 0.3.9
|
||||
dev: true
|
||||
|
||||
/@esbuild-plugins/node-globals-polyfill@0.2.3(esbuild@0.17.19):
|
||||
resolution: {integrity: sha512-r3MIryXDeXDOZh7ih1l/yE9ZLORCd5e8vWg02azWRGj5SPTuoh69A2AIyn0Z31V/kHBfZ4HgWJ+OK3GTTwLmnw==}
|
||||
peerDependencies:
|
||||
esbuild: '*'
|
||||
dependencies:
|
||||
esbuild: 0.17.19
|
||||
dev: true
|
||||
|
||||
/@esbuild-plugins/node-modules-polyfill@0.2.2(esbuild@0.17.19):
|
||||
resolution: {integrity: sha512-LXV7QsWJxRuMYvKbiznh+U1ilIop3g2TeKRzUxOG5X3YITc8JyyTa90BmLwqqv0YnX4v32CSlG+vsziZp9dMvA==}
|
||||
peerDependencies:
|
||||
esbuild: '*'
|
||||
dependencies:
|
||||
esbuild: 0.17.19
|
||||
escape-string-regexp: 4.0.0
|
||||
rollup-plugin-node-polyfills: 0.2.1
|
||||
dev: true
|
||||
|
||||
/@esbuild/android-arm64@0.17.19:
|
||||
resolution: {integrity: sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [android]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/android-arm@0.17.19:
|
||||
resolution: {integrity: sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm]
|
||||
os: [android]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/android-x64@0.17.19:
|
||||
resolution: {integrity: sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [android]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/darwin-arm64@0.17.19:
|
||||
resolution: {integrity: sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/darwin-x64@0.17.19:
|
||||
resolution: {integrity: sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/freebsd-arm64@0.17.19:
|
||||
resolution: {integrity: sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [freebsd]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/freebsd-x64@0.17.19:
|
||||
resolution: {integrity: sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [freebsd]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-arm64@0.17.19:
|
||||
resolution: {integrity: sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-arm@0.17.19:
|
||||
resolution: {integrity: sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-ia32@0.17.19:
|
||||
resolution: {integrity: sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [ia32]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-loong64@0.17.19:
|
||||
resolution: {integrity: sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-mips64el@0.17.19:
|
||||
resolution: {integrity: sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [mips64el]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-ppc64@0.17.19:
|
||||
resolution: {integrity: sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-riscv64@0.17.19:
|
||||
resolution: {integrity: sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-s390x@0.17.19:
|
||||
resolution: {integrity: sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-x64@0.17.19:
|
||||
resolution: {integrity: sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/netbsd-x64@0.17.19:
|
||||
resolution: {integrity: sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [netbsd]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/openbsd-x64@0.17.19:
|
||||
resolution: {integrity: sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [openbsd]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/sunos-x64@0.17.19:
|
||||
resolution: {integrity: sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [sunos]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/win32-arm64@0.17.19:
|
||||
resolution: {integrity: sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/win32-ia32@0.17.19:
|
||||
resolution: {integrity: sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [ia32]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/win32-x64@0.17.19:
|
||||
resolution: {integrity: sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@fastify/busboy@2.1.1:
|
||||
resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==}
|
||||
engines: {node: '>=14'}
|
||||
dev: true
|
||||
|
||||
/@jridgewell/resolve-uri@3.1.2:
|
||||
resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==}
|
||||
engines: {node: '>=6.0.0'}
|
||||
dev: true
|
||||
|
||||
/@jridgewell/sourcemap-codec@1.4.15:
|
||||
resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==}
|
||||
dev: true
|
||||
|
||||
/@jridgewell/trace-mapping@0.3.9:
|
||||
resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==}
|
||||
dependencies:
|
||||
'@jridgewell/resolve-uri': 3.1.2
|
||||
'@jridgewell/sourcemap-codec': 1.4.15
|
||||
dev: true
|
||||
|
||||
/@types/node-forge@1.3.11:
|
||||
resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==}
|
||||
dependencies:
|
||||
'@types/node': 20.14.9
|
||||
dev: true
|
||||
|
||||
/@types/node@20.14.9:
|
||||
resolution: {integrity: sha512-06OCtnTXtWOZBJlRApleWndH4JsRVs1pDCc8dLSQp+7PpUpX3ePdHyeNSFTeSe7FtKyQkrlPvHwJOW3SLd8Oyg==}
|
||||
dependencies:
|
||||
undici-types: 5.26.5
|
||||
dev: true
|
||||
|
||||
/acorn-walk@8.3.3:
|
||||
resolution: {integrity: sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==}
|
||||
engines: {node: '>=0.4.0'}
|
||||
dependencies:
|
||||
acorn: 8.12.0
|
||||
dev: true
|
||||
|
||||
/acorn@8.12.0:
|
||||
resolution: {integrity: sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==}
|
||||
engines: {node: '>=0.4.0'}
|
||||
hasBin: true
|
||||
dev: true
|
||||
|
||||
/anymatch@3.1.3:
|
||||
resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==}
|
||||
engines: {node: '>= 8'}
|
||||
dependencies:
|
||||
normalize-path: 3.0.0
|
||||
picomatch: 2.3.1
|
||||
dev: true
|
||||
|
||||
/as-table@1.0.55:
|
||||
resolution: {integrity: sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ==}
|
||||
dependencies:
|
||||
printable-characters: 1.0.42
|
||||
dev: true
|
||||
|
||||
/binary-extensions@2.3.0:
|
||||
resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
|
||||
engines: {node: '>=8'}
|
||||
dev: true
|
||||
|
||||
/blake3-wasm@2.1.5:
|
||||
resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==}
|
||||
dev: true
|
||||
|
||||
/braces@3.0.3:
|
||||
resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}
|
||||
engines: {node: '>=8'}
|
||||
dependencies:
|
||||
fill-range: 7.1.1
|
||||
dev: true
|
||||
|
||||
/capnp-ts@0.7.0:
|
||||
resolution: {integrity: sha512-XKxXAC3HVPv7r674zP0VC3RTXz+/JKhfyw94ljvF80yynK6VkTnqE3jMuN8b3dUVmmc43TjyxjW4KTsmB3c86g==}
|
||||
dependencies:
|
||||
debug: 4.3.5
|
||||
tslib: 2.6.3
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
dev: true
|
||||
|
||||
/chokidar@3.6.0:
|
||||
resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}
|
||||
engines: {node: '>= 8.10.0'}
|
||||
dependencies:
|
||||
anymatch: 3.1.3
|
||||
braces: 3.0.3
|
||||
glob-parent: 5.1.2
|
||||
is-binary-path: 2.1.0
|
||||
is-glob: 4.0.3
|
||||
normalize-path: 3.0.0
|
||||
readdirp: 3.6.0
|
||||
optionalDependencies:
|
||||
fsevents: 2.3.3
|
||||
dev: true
|
||||
|
||||
/consola@3.2.3:
|
||||
resolution: {integrity: sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==}
|
||||
engines: {node: ^14.18.0 || >=16.10.0}
|
||||
dev: true
|
||||
|
||||
/cookie@0.5.0:
|
||||
resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==}
|
||||
engines: {node: '>= 0.6'}
|
||||
dev: true
|
||||
|
||||
/data-uri-to-buffer@2.0.2:
|
||||
resolution: {integrity: sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==}
|
||||
dev: true
|
||||
|
||||
/date-fns@3.6.0:
|
||||
resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==}
|
||||
dev: true
|
||||
|
||||
/debug@4.3.5:
|
||||
resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==}
|
||||
engines: {node: '>=6.0'}
|
||||
peerDependencies:
|
||||
supports-color: '*'
|
||||
peerDependenciesMeta:
|
||||
supports-color:
|
||||
optional: true
|
||||
dependencies:
|
||||
ms: 2.1.2
|
||||
dev: true
|
||||
|
||||
/defu@6.1.4:
|
||||
resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==}
|
||||
dev: true
|
||||
|
||||
/esbuild@0.17.19:
|
||||
resolution: {integrity: sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw==}
|
||||
engines: {node: '>=12'}
|
||||
hasBin: true
|
||||
requiresBuild: true
|
||||
optionalDependencies:
|
||||
'@esbuild/android-arm': 0.17.19
|
||||
'@esbuild/android-arm64': 0.17.19
|
||||
'@esbuild/android-x64': 0.17.19
|
||||
'@esbuild/darwin-arm64': 0.17.19
|
||||
'@esbuild/darwin-x64': 0.17.19
|
||||
'@esbuild/freebsd-arm64': 0.17.19
|
||||
'@esbuild/freebsd-x64': 0.17.19
|
||||
'@esbuild/linux-arm': 0.17.19
|
||||
'@esbuild/linux-arm64': 0.17.19
|
||||
'@esbuild/linux-ia32': 0.17.19
|
||||
'@esbuild/linux-loong64': 0.17.19
|
||||
'@esbuild/linux-mips64el': 0.17.19
|
||||
'@esbuild/linux-ppc64': 0.17.19
|
||||
'@esbuild/linux-riscv64': 0.17.19
|
||||
'@esbuild/linux-s390x': 0.17.19
|
||||
'@esbuild/linux-x64': 0.17.19
|
||||
'@esbuild/netbsd-x64': 0.17.19
|
||||
'@esbuild/openbsd-x64': 0.17.19
|
||||
'@esbuild/sunos-x64': 0.17.19
|
||||
'@esbuild/win32-arm64': 0.17.19
|
||||
'@esbuild/win32-ia32': 0.17.19
|
||||
'@esbuild/win32-x64': 0.17.19
|
||||
dev: true
|
||||
|
||||
/escape-string-regexp@4.0.0:
|
||||
resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
|
||||
engines: {node: '>=10'}
|
||||
dev: true
|
||||
|
||||
/estree-walker@0.6.1:
|
||||
resolution: {integrity: sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==}
|
||||
dev: true
|
||||
|
||||
/exit-hook@2.2.1:
|
||||
resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==}
|
||||
engines: {node: '>=6'}
|
||||
dev: true
|
||||
|
||||
/fill-range@7.1.1:
|
||||
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
|
||||
engines: {node: '>=8'}
|
||||
dependencies:
|
||||
to-regex-range: 5.0.1
|
||||
dev: true
|
||||
|
||||
/fsevents@2.3.3:
|
||||
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
|
||||
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/function-bind@1.1.2:
|
||||
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
|
||||
dev: true
|
||||
|
||||
/get-source@2.0.12:
|
||||
resolution: {integrity: sha512-X5+4+iD+HoSeEED+uwrQ07BOQr0kEDFMVqqpBuI+RaZBpBpHCuXxo70bjar6f0b0u/DQJsJ7ssurpP0V60Az+w==}
|
||||
dependencies:
|
||||
data-uri-to-buffer: 2.0.2
|
||||
source-map: 0.6.1
|
||||
dev: true
|
||||
|
||||
/glob-parent@5.1.2:
|
||||
resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
|
||||
engines: {node: '>= 6'}
|
||||
dependencies:
|
||||
is-glob: 4.0.3
|
||||
dev: true
|
||||
|
||||
/glob-to-regexp@0.4.1:
|
||||
resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==}
|
||||
dev: true
|
||||
|
||||
/hasown@2.0.2:
|
||||
resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
|
||||
engines: {node: '>= 0.4'}
|
||||
dependencies:
|
||||
function-bind: 1.1.2
|
||||
dev: true
|
||||
|
||||
/is-binary-path@2.1.0:
|
||||
resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
|
||||
engines: {node: '>=8'}
|
||||
dependencies:
|
||||
binary-extensions: 2.3.0
|
||||
dev: true
|
||||
|
||||
/is-core-module@2.14.0:
|
||||
resolution: {integrity: sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==}
|
||||
engines: {node: '>= 0.4'}
|
||||
dependencies:
|
||||
hasown: 2.0.2
|
||||
dev: true
|
||||
|
||||
/is-extglob@2.1.1:
|
||||
resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
dev: true
|
||||
|
||||
/is-glob@4.0.3:
|
||||
resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
dependencies:
|
||||
is-extglob: 2.1.1
|
||||
dev: true
|
||||
|
||||
/is-number@7.0.0:
|
||||
resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
|
||||
engines: {node: '>=0.12.0'}
|
||||
dev: true
|
||||
|
||||
/itty-router@3.0.12:
|
||||
resolution: {integrity: sha512-s98XTPhle6GGbaFf0kYrOD3Q8gyhnqvOqkwYijC3AmkceNKqWUp13YHg6dWmqmVv4pP7l7c94XI92I0EXVGO0w==}
|
||||
dev: true
|
||||
|
||||
/magic-string@0.25.9:
|
||||
resolution: {integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==}
|
||||
dependencies:
|
||||
sourcemap-codec: 1.4.8
|
||||
dev: true
|
||||
|
||||
/mime@3.0.0:
|
||||
resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
hasBin: true
|
||||
dev: true
|
||||
|
||||
/miniflare@3.20240620.0:
|
||||
resolution: {integrity: sha512-NBMzqUE2mMlh/hIdt6U5MP+aFhEjKDq3l8CAajXAQa1WkndJdciWvzB2mfLETwoVFhMl/lphaVzyEN2AgwJpbQ==}
|
||||
engines: {node: '>=16.13'}
|
||||
hasBin: true
|
||||
dependencies:
|
||||
'@cspotcode/source-map-support': 0.8.1
|
||||
acorn: 8.12.0
|
||||
acorn-walk: 8.3.3
|
||||
capnp-ts: 0.7.0
|
||||
exit-hook: 2.2.1
|
||||
glob-to-regexp: 0.4.1
|
||||
stoppable: 1.1.0
|
||||
undici: 5.28.4
|
||||
workerd: 1.20240620.1
|
||||
ws: 8.17.1
|
||||
youch: 3.3.3
|
||||
zod: 3.23.8
|
||||
transitivePeerDependencies:
|
||||
- bufferutil
|
||||
- supports-color
|
||||
- utf-8-validate
|
||||
dev: true
|
||||
|
||||
/ms@2.1.2:
|
||||
resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
|
||||
dev: true
|
||||
|
||||
/mustache@4.2.0:
|
||||
resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==}
|
||||
hasBin: true
|
||||
dev: true
|
||||
|
||||
/nanoid@3.3.7:
|
||||
resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==}
|
||||
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
|
||||
hasBin: true
|
||||
dev: true
|
||||
|
||||
/node-fetch-native@1.6.4:
|
||||
resolution: {integrity: sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==}
|
||||
dev: true
|
||||
|
||||
/node-forge@1.3.1:
|
||||
resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==}
|
||||
engines: {node: '>= 6.13.0'}
|
||||
dev: true
|
||||
|
||||
/normalize-path@3.0.0:
|
||||
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
dev: true
|
||||
|
||||
/path-parse@1.0.7:
|
||||
resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
|
||||
dev: true
|
||||
|
||||
/path-to-regexp@6.2.2:
|
||||
resolution: {integrity: sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==}
|
||||
dev: true
|
||||
|
||||
/pathe@1.1.2:
|
||||
resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==}
|
||||
dev: true
|
||||
|
||||
/picomatch@2.3.1:
|
||||
resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}
|
||||
engines: {node: '>=8.6'}
|
||||
dev: true
|
||||
|
||||
/printable-characters@1.0.42:
|
||||
resolution: {integrity: sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==}
|
||||
dev: true
|
||||
|
||||
/readdirp@3.6.0:
|
||||
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
|
||||
engines: {node: '>=8.10.0'}
|
||||
dependencies:
|
||||
picomatch: 2.3.1
|
||||
dev: true
|
||||
|
||||
/resolve.exports@2.0.2:
|
||||
resolution: {integrity: sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==}
|
||||
engines: {node: '>=10'}
|
||||
dev: true
|
||||
|
||||
/resolve@1.22.8:
|
||||
resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}
|
||||
hasBin: true
|
||||
dependencies:
|
||||
is-core-module: 2.14.0
|
||||
path-parse: 1.0.7
|
||||
supports-preserve-symlinks-flag: 1.0.0
|
||||
dev: true
|
||||
|
||||
/rollup-plugin-inject@3.0.2:
|
||||
resolution: {integrity: sha512-ptg9PQwzs3orn4jkgXJ74bfs5vYz1NCZlSQMBUA0wKcGp5i5pA1AO3fOUEte8enhGUC+iapTCzEWw2jEFFUO/w==}
|
||||
deprecated: This package has been deprecated and is no longer maintained. Please use @rollup/plugin-inject.
|
||||
dependencies:
|
||||
estree-walker: 0.6.1
|
||||
magic-string: 0.25.9
|
||||
rollup-pluginutils: 2.8.2
|
||||
dev: true
|
||||
|
||||
/rollup-plugin-node-polyfills@0.2.1:
|
||||
resolution: {integrity: sha512-4kCrKPTJ6sK4/gLL/U5QzVT8cxJcofO0OU74tnB19F40cmuAKSzH5/siithxlofFEjwvw1YAhPmbvGNA6jEroA==}
|
||||
dependencies:
|
||||
rollup-plugin-inject: 3.0.2
|
||||
dev: true
|
||||
|
||||
/rollup-pluginutils@2.8.2:
|
||||
resolution: {integrity: sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==}
|
||||
dependencies:
|
||||
estree-walker: 0.6.1
|
||||
dev: true
|
||||
|
||||
/selfsigned@2.4.1:
|
||||
resolution: {integrity: sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==}
|
||||
engines: {node: '>=10'}
|
||||
dependencies:
|
||||
'@types/node-forge': 1.3.11
|
||||
node-forge: 1.3.1
|
||||
dev: true
|
||||
|
||||
/source-map@0.6.1:
|
||||
resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
dev: true
|
||||
|
||||
/sourcemap-codec@1.4.8:
|
||||
resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==}
|
||||
deprecated: Please use @jridgewell/sourcemap-codec instead
|
||||
dev: true
|
||||
|
||||
/stacktracey@2.1.8:
|
||||
resolution: {integrity: sha512-Kpij9riA+UNg7TnphqjH7/CzctQ/owJGNbFkfEeve4Z4uxT5+JapVLFXcsurIfN34gnTWZNJ/f7NMG0E8JDzTw==}
|
||||
dependencies:
|
||||
as-table: 1.0.55
|
||||
get-source: 2.0.12
|
||||
dev: true
|
||||
|
||||
/stoppable@1.1.0:
|
||||
resolution: {integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==}
|
||||
engines: {node: '>=4', npm: '>=6'}
|
||||
dev: true
|
||||
|
||||
/supports-preserve-symlinks-flag@1.0.0:
|
||||
resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
|
||||
engines: {node: '>= 0.4'}
|
||||
dev: true
|
||||
|
||||
/to-regex-range@5.0.1:
|
||||
resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==}
|
||||
engines: {node: '>=8.0'}
|
||||
dependencies:
|
||||
is-number: 7.0.0
|
||||
dev: true
|
||||
|
||||
/tslib@2.6.3:
|
||||
resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==}
|
||||
dev: true
|
||||
|
||||
/ufo@1.5.3:
|
||||
resolution: {integrity: sha512-Y7HYmWaFwPUmkoQCUIAYpKqkOf+SbVj/2fJJZ4RJMCfZp0rTGwRbzQD+HghfnhKOjL9E01okqz+ncJskGYfBNw==}
|
||||
dev: true
|
||||
|
||||
/undici-types@5.26.5:
|
||||
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
|
||||
dev: true
|
||||
|
||||
/undici@5.28.4:
|
||||
resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==}
|
||||
engines: {node: '>=14.0'}
|
||||
dependencies:
|
||||
'@fastify/busboy': 2.1.1
|
||||
dev: true
|
||||
|
||||
/unenv-nightly@1.10.0-1717606461.a117952:
|
||||
resolution: {integrity: sha512-u3TfBX02WzbHTpaEfWEKwDijDSFAHcgXkayUZ+MVDrjhLFvgAJzFGTSTmwlEhwWi2exyRQey23ah9wELMM6etg==}
|
||||
dependencies:
|
||||
consola: 3.2.3
|
||||
defu: 6.1.4
|
||||
mime: 3.0.0
|
||||
node-fetch-native: 1.6.4
|
||||
pathe: 1.1.2
|
||||
ufo: 1.5.3
|
||||
dev: true
|
||||
|
||||
/workerd@1.20240620.1:
|
||||
resolution: {integrity: sha512-Qoq+RrFNk4pvEO+kpJVn8uJ5TRE9YJx5jX5pC5LjdKlw1XeD8EdXt5k0TbByvWunZ4qgYIcF9lnVxhcDFo203g==}
|
||||
engines: {node: '>=16'}
|
||||
hasBin: true
|
||||
requiresBuild: true
|
||||
optionalDependencies:
|
||||
'@cloudflare/workerd-darwin-64': 1.20240620.1
|
||||
'@cloudflare/workerd-darwin-arm64': 1.20240620.1
|
||||
'@cloudflare/workerd-linux-64': 1.20240620.1
|
||||
'@cloudflare/workerd-linux-arm64': 1.20240620.1
|
||||
'@cloudflare/workerd-windows-64': 1.20240620.1
|
||||
dev: true
|
||||
|
||||
/wrangler@3.62.0:
|
||||
resolution: {integrity: sha512-TM1Bd8+GzxFw/JzwsC3i/Oss4LTWvIEWXXo1vZhx+7PHcsxdbnQGBBwPurHNJDSu2Pw22+2pCZiUGKexmgJksw==}
|
||||
engines: {node: '>=16.17.0'}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
'@cloudflare/workers-types': ^4.20240620.0
|
||||
peerDependenciesMeta:
|
||||
'@cloudflare/workers-types':
|
||||
optional: true
|
||||
dependencies:
|
||||
'@cloudflare/kv-asset-handler': 0.3.4
|
||||
'@esbuild-plugins/node-globals-polyfill': 0.2.3(esbuild@0.17.19)
|
||||
'@esbuild-plugins/node-modules-polyfill': 0.2.2(esbuild@0.17.19)
|
||||
blake3-wasm: 2.1.5
|
||||
chokidar: 3.6.0
|
||||
date-fns: 3.6.0
|
||||
esbuild: 0.17.19
|
||||
miniflare: 3.20240620.0
|
||||
nanoid: 3.3.7
|
||||
path-to-regexp: 6.2.2
|
||||
resolve: 1.22.8
|
||||
resolve.exports: 2.0.2
|
||||
selfsigned: 2.4.1
|
||||
source-map: 0.6.1
|
||||
unenv: /unenv-nightly@1.10.0-1717606461.a117952
|
||||
xxhash-wasm: 1.0.2
|
||||
optionalDependencies:
|
||||
fsevents: 2.3.3
|
||||
transitivePeerDependencies:
|
||||
- bufferutil
|
||||
- supports-color
|
||||
- utf-8-validate
|
||||
dev: true
|
||||
|
||||
/ws@8.17.1:
|
||||
resolution: {integrity: sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
peerDependencies:
|
||||
bufferutil: ^4.0.1
|
||||
utf-8-validate: '>=5.0.2'
|
||||
peerDependenciesMeta:
|
||||
bufferutil:
|
||||
optional: true
|
||||
utf-8-validate:
|
||||
optional: true
|
||||
dev: true
|
||||
|
||||
/xxhash-wasm@1.0.2:
|
||||
resolution: {integrity: sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A==}
|
||||
dev: true
|
||||
|
||||
/youch@3.3.3:
|
||||
resolution: {integrity: sha512-qSFXUk3UZBLfggAW3dJKg0BMblG5biqSF8M34E06o5CSsZtH92u9Hqmj2RzGiHDi64fhe83+4tENFP2DB6t6ZA==}
|
||||
dependencies:
|
||||
cookie: 0.5.0
|
||||
mustache: 4.2.0
|
||||
stacktracey: 2.1.8
|
||||
dev: true
|
||||
|
||||
/zod@3.23.8:
|
||||
resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==}
|
||||
dev: true
|
||||
66
modules/debug_proxy/src/index.js
Normal file
66
modules/debug_proxy/src/index.js
Normal file
@@ -0,0 +1,66 @@
|
||||
async function sendTelegramMessage(message, botToken, chatId) {
|
||||
const url = `https://api.telegram.org/bot${botToken}/sendMessage`
|
||||
const payload = {
|
||||
chat_id: chatId,
|
||||
text: message,
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
console.error('Failed to send message:', await response.text())
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error sending Telegram message:', error)
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
async fetch(request, env, ctx) {
|
||||
const url = new URL(request.url)
|
||||
const userIP = request.headers.get('CF-Connecting-IP')
|
||||
|
||||
if (url.pathname === '/b') {
|
||||
url.pathname = '/styles/bright'
|
||||
}
|
||||
|
||||
// // no failure, just warning
|
||||
// if (request.method !== 'GET') {
|
||||
// const warningMessage = `Non-GET request ${request.method} ${url.pathname} ${userIP}`
|
||||
// console.error(warningMessage)
|
||||
// await sendTelegramMessage(warningMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
|
||||
// }
|
||||
|
||||
if (!url.pathname.startsWith('/styles')) {
|
||||
const errorMessage = 'Bad path'
|
||||
return new Response(errorMessage, { status: 500 })
|
||||
}
|
||||
|
||||
const proxyUrl = new URL(url.pathname, 'https://tiles.openfreemap.org')
|
||||
|
||||
try {
|
||||
const response = await fetch(proxyUrl)
|
||||
|
||||
if (response.status !== 200) {
|
||||
const errorMessage = `Proxy error: Bad status ${response.status} ${url.pathname} ${userIP}`
|
||||
console.error(errorMessage)
|
||||
await sendTelegramMessage(errorMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
|
||||
return new Response('Proxy error: Bad status', { status: 500 })
|
||||
}
|
||||
|
||||
return response
|
||||
} catch (error) {
|
||||
const errorMessage = `Proxy error: ${error.message} ${url.pathname} ${userIP}`
|
||||
console.error(errorMessage)
|
||||
await sendTelegramMessage(errorMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
|
||||
return new Response('Proxy error: Fetch failed', { status: 500 })
|
||||
}
|
||||
},
|
||||
}
|
||||
107
modules/debug_proxy/wrangler.toml
Normal file
107
modules/debug_proxy/wrangler.toml
Normal file
@@ -0,0 +1,107 @@
|
||||
#:schema node_modules/wrangler/config-schema.json
|
||||
name = "cf-debug-proxy"
|
||||
main = "src/index.js"
|
||||
compatibility_date = "2024-06-20"
|
||||
|
||||
# Automatically place your workloads in an optimal location to minimize latency.
|
||||
# If you are running back-end logic in a Worker, running it closer to your back-end infrastructure
|
||||
# rather than the end user may result in better performance.
|
||||
# Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement
|
||||
# [placement]
|
||||
# mode = "smart"
|
||||
|
||||
# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
|
||||
# Docs:
|
||||
# - https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables
|
||||
# Note: Use secrets to store sensitive data.
|
||||
# - https://developers.cloudflare.com/workers/configuration/secrets/
|
||||
# [vars]
|
||||
# MY_VARIABLE = "production_value"
|
||||
|
||||
# Bind the Workers AI model catalog. Run machine learning models, powered by serverless GPUs, on Cloudflare’s global network
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#workers-ai
|
||||
# [ai]
|
||||
# binding = "AI"
|
||||
|
||||
# Bind an Analytics Engine dataset. Use Analytics Engine to write analytics within your Pages Function.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#analytics-engine-datasets
|
||||
# [[analytics_engine_datasets]]
|
||||
# binding = "MY_DATASET"
|
||||
|
||||
# Bind a headless browser instance running on Cloudflare's global network.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#browser-rendering
|
||||
# [browser]
|
||||
# binding = "MY_BROWSER"
|
||||
|
||||
# Bind a D1 database. D1 is Cloudflare’s native serverless SQL database.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#d1-databases
|
||||
# [[d1_databases]]
|
||||
# binding = "MY_DB"
|
||||
# database_name = "my-database"
|
||||
# database_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
|
||||
|
||||
# Bind a dispatch namespace. Use Workers for Platforms to deploy serverless functions programmatically on behalf of your customers.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#dispatch-namespace-bindings-workers-for-platforms
|
||||
# [[dispatch_namespaces]]
|
||||
# binding = "MY_DISPATCHER"
|
||||
# namespace = "my-namespace"
|
||||
|
||||
# Bind a Durable Object. Durable objects are a scale-to-zero compute primitive based on the actor model.
|
||||
# Durable Objects can live for as long as needed. Use these when you need a long-running "server", such as in realtime apps.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#durable-objects
|
||||
# [[durable_objects.bindings]]
|
||||
# name = "MY_DURABLE_OBJECT"
|
||||
# class_name = "MyDurableObject"
|
||||
|
||||
# Durable Object migrations.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#migrations
|
||||
# [[migrations]]
|
||||
# tag = "v1"
|
||||
# new_classes = ["MyDurableObject"]
|
||||
|
||||
# Bind a Hyperdrive configuration. Use to accelerate access to your existing databases from Cloudflare Workers.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#hyperdrive
|
||||
# [[hyperdrive]]
|
||||
# binding = "MY_HYPERDRIVE"
|
||||
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
|
||||
|
||||
# Bind a KV Namespace. Use KV as persistent storage for small key-value pairs.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#kv-namespaces
|
||||
# [[kv_namespaces]]
|
||||
# binding = "MY_KV_NAMESPACE"
|
||||
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
|
||||
|
||||
# Bind an mTLS certificate. Use to present a client certificate when communicating with another service.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#mtls-certificates
|
||||
# [[mtls_certificates]]
|
||||
# binding = "MY_CERTIFICATE"
|
||||
# certificate_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
|
||||
|
||||
# Bind a Queue producer. Use this binding to schedule an arbitrary task that may be processed later by a Queue consumer.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
|
||||
# [[queues.producers]]
|
||||
# binding = "MY_QUEUE"
|
||||
# queue = "my-queue"
|
||||
|
||||
# Bind a Queue consumer. Queue Consumers can retrieve tasks scheduled by Producers to act on them.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
|
||||
# [[queues.consumers]]
|
||||
# queue = "my-queue"
|
||||
|
||||
# Bind an R2 Bucket. Use R2 to store arbitrarily large blobs of data, such as files.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#r2-buckets
|
||||
# [[r2_buckets]]
|
||||
# binding = "MY_BUCKET"
|
||||
# bucket_name = "my-bucket"
|
||||
|
||||
# Bind another Worker service. Use this binding to call another Worker without network overhead.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings
|
||||
# [[services]]
|
||||
# binding = "MY_SERVICE"
|
||||
# service = "my-service"
|
||||
|
||||
# Bind a Vectorize index. Use to store and query vector embeddings for semantic search, classification and other vector search use-cases.
|
||||
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#vectorize-indexes
|
||||
# [[vectorize]]
|
||||
# binding = "MY_INDEX"
|
||||
# index_name = "my-index"
|
||||
27
modules/http_host/benchmark/create_path_list.py
Normal file
27
modules/http_host/benchmark/create_path_list.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import json
|
||||
|
||||
|
||||
with open('access.jsonl') as fp:
|
||||
json_lines = fp.readlines()
|
||||
|
||||
paths = []
|
||||
for i, line in enumerate(json_lines):
|
||||
log_data = json.loads(line)
|
||||
if log_data['status'] != 200:
|
||||
continue
|
||||
|
||||
if log_data['request_method'] != 'GET':
|
||||
continue
|
||||
|
||||
uri = log_data['uri']
|
||||
|
||||
if 'tiles/' not in uri or not uri.endswith('.pbf'):
|
||||
continue
|
||||
|
||||
path = log_data['uri'].split('tiles/')[1]
|
||||
paths.append(path + '\n')
|
||||
|
||||
print(f'{i / len(json_lines) * 100:.1f}%')
|
||||
|
||||
with open('path_list.txt', 'w') as fp:
|
||||
fp.writelines(paths)
|
||||
39
modules/http_host/benchmark/wrk_custom_list.lua
Normal file
39
modules/http_host/benchmark/wrk_custom_list.lua
Normal file
@@ -0,0 +1,39 @@
|
||||
local counter = 1
|
||||
local lines = {}
|
||||
local url_base = "/planet/20231221_134737_pt/" -- trailing slash
|
||||
local path_list_txt = "/data/ofm/benchmark/path_list_500k.txt"
|
||||
|
||||
for line in io.lines(path_list_txt) do
|
||||
table.insert(lines, url_base .. line)
|
||||
end
|
||||
|
||||
local function getNextUrl()
|
||||
-- Get the next URL from the list
|
||||
local url_path = lines[counter]
|
||||
counter = counter + 1
|
||||
|
||||
-- If we've gone past the end of the list, wrap around to the start
|
||||
if counter > #lines then
|
||||
counter = 1
|
||||
end
|
||||
|
||||
return url_path
|
||||
end
|
||||
|
||||
request = function()
|
||||
-- Return the request object with the current URL path
|
||||
path = getNextUrl()
|
||||
local headers = {}
|
||||
headers["Host"] = "ofm"
|
||||
return wrk.format('GET', path, headers, nil)
|
||||
end
|
||||
|
||||
response = function(status)
|
||||
if status ~= 200 then
|
||||
print("Non-200 response")
|
||||
print("Status: ", status)
|
||||
-- this only works in single threaded mode (-t1)
|
||||
print("Request path: ", path)
|
||||
end
|
||||
end
|
||||
|
||||
8
modules/http_host/benchmark/wrk_usage.txt
Normal file
8
modules/http_host/benchmark/wrk_usage.txt
Normal file
@@ -0,0 +1,8 @@
|
||||
wrk -c10 -t4 -d10s -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost
|
||||
|
||||
# -t1 => more correct, since the url list is loaded exactly in sequence
|
||||
# -t4 => reflecting real world usage
|
||||
|
||||
|
||||
|
||||
|
||||
4
modules/http_host/cron.d/ofm_http_host
Normal file
4
modules/http_host/cron.d/ofm_http_host
Normal file
@@ -0,0 +1,4 @@
|
||||
# every minute sync, locking so that only one process can run at a time
|
||||
* * * * * ofm /usr/bin/flock -n /tmp/hostmanager.lockfile -c 'sudo /data/ofm/venv/bin/python -u /data/ofm/http_host/bin/host_manager.py sync >> /data/ofm/http_host/logs/host_manager_sync.log 2>&1'
|
||||
|
||||
|
||||
2
modules/http_host/cron.d/ofm_ledns_reader
Normal file
2
modules/http_host/cron.d/ofm_ledns_reader
Normal file
@@ -0,0 +1,2 @@
|
||||
# once per day
|
||||
2 34 * * * ofm sudo /usr/bin/bash /data/ofm/http_host/bin/ledns_reader.sh >> /data/ofm/http_host/logs/ledns_reader.log 2>&1
|
||||
147
modules/http_host/http_host.py
Executable file
147
modules/http_host/http_host.py
Executable file
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import datetime
|
||||
import sys
|
||||
|
||||
import click
|
||||
from http_host_lib.assets import (
|
||||
download_assets,
|
||||
)
|
||||
from http_host_lib.btrfs import (
|
||||
download_area_version,
|
||||
get_versions_for_area,
|
||||
)
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.mount import auto_mount_unmount
|
||||
from http_host_lib.nginx import write_nginx_config
|
||||
from http_host_lib.set_tileset_versions import set_tileset_versions
|
||||
from http_host_lib.utils import assert_linux, assert_sudo
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""
|
||||
Manages OpenFreeMap HTTP hosts, including:\n
|
||||
- Downloading btrfs images\n
|
||||
- Downloading assets\n
|
||||
- Mounting directories\n
|
||||
- Updating nginx config\n
|
||||
- Getting the deployed versions of tilesets\n
|
||||
- Running the sync cron task (called every minute)
|
||||
"""
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument('area', required=False)
|
||||
@click.option(
|
||||
'--version', default='latest', help='Optional version string, like "20231227_043106_pt"'
|
||||
)
|
||||
def download_btrfs(area: str, version: str):
|
||||
"""
|
||||
Downloads and uncompresses tiles.btrfs files from the btrfs bucket
|
||||
Version can be "latest" (default) or specified, like "20231227_043106_pt"
|
||||
Use --version=1 to list all available versions
|
||||
"""
|
||||
|
||||
return download_area_version(area, version)
|
||||
|
||||
|
||||
@cli.command(name='download-assets')
|
||||
def download_assets_():
|
||||
"""
|
||||
Downloads and extracts assets
|
||||
"""
|
||||
|
||||
download_assets()
|
||||
|
||||
|
||||
@cli.command()
|
||||
def mount():
|
||||
"""
|
||||
Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
|
||||
When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
|
||||
"""
|
||||
|
||||
auto_mount_unmount()
|
||||
|
||||
|
||||
@cli.command()
|
||||
def set_latest_versions():
|
||||
"""
|
||||
Sets the latest version of the tilesets to the version specified by
|
||||
https://assets.openfreemap.com/versions/deployed_planet.txt
|
||||
|
||||
1. Checks if the given version is present on the disk and is mounted
|
||||
2. Writes to a version file
|
||||
"""
|
||||
|
||||
print('running set_latest_versions')
|
||||
|
||||
assert_linux()
|
||||
assert_sudo()
|
||||
|
||||
if not config.mnt_dir.exists():
|
||||
sys.exit(' mount needs to be run first')
|
||||
|
||||
return set_tileset_versions()
|
||||
|
||||
|
||||
@cli.command()
|
||||
def nginx_sync():
|
||||
"""
|
||||
Syncs the nginx config to the state of the system
|
||||
"""
|
||||
|
||||
print('running nginx_sync')
|
||||
|
||||
assert_linux()
|
||||
assert_sudo()
|
||||
|
||||
if not config.mnt_dir.exists():
|
||||
sys.exit(' mount needs to be run first')
|
||||
|
||||
write_nginx_config()
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option('--force', is_flag=True, help='Force nginx sync run')
|
||||
@click.pass_context
|
||||
def sync(ctx, force):
|
||||
"""
|
||||
Runs the sync task, normally called by cron every minute
|
||||
On a new server this also takes care of everything, no need to run anything manually.
|
||||
"""
|
||||
|
||||
print('---')
|
||||
print('running sync')
|
||||
print(datetime.datetime.now(tz=datetime.timezone.utc))
|
||||
|
||||
assert_linux()
|
||||
assert_sudo()
|
||||
|
||||
download_done = False
|
||||
download_done += ctx.invoke(download_btrfs, area='monaco')
|
||||
|
||||
if not config.host_config.get('skip_planet'):
|
||||
download_done += ctx.invoke(download_btrfs, area='planet')
|
||||
|
||||
if download_done:
|
||||
ctx.invoke(mount)
|
||||
|
||||
ctx.invoke(download_assets)
|
||||
|
||||
deploy_done = ctx.invoke(set_latest_versions)
|
||||
|
||||
if download_done or deploy_done or force:
|
||||
ctx.invoke(nginx_sync)
|
||||
|
||||
|
||||
@cli.command()
|
||||
def debug():
|
||||
versions = get_versions_for_area('monaco')
|
||||
print(versions)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# print(config.host_config)
|
||||
cli()
|
||||
0
modules/http_host/http_host_lib/__init__.py
Normal file
0
modules/http_host/http_host_lib/__init__.py
Normal file
83
modules/http_host/http_host_lib/assets.py
Normal file
83
modules/http_host/http_host_lib/assets.py
Normal file
@@ -0,0 +1,83 @@
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import requests
|
||||
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.utils import download_file_aria2, download_if_size_differs
|
||||
|
||||
|
||||
def download_assets():
|
||||
"""
|
||||
Downloads and extracts assets
|
||||
"""
|
||||
|
||||
download_and_extract_asset_tar_gz('fonts')
|
||||
download_and_extract_asset_tar_gz('styles')
|
||||
download_and_extract_asset_tar_gz('natural_earth')
|
||||
|
||||
download_sprites()
|
||||
|
||||
|
||||
def download_and_extract_asset_tar_gz(asset_kind):
|
||||
"""
|
||||
Download and extract asset.tgz if the file size differ or not available locally
|
||||
"""
|
||||
|
||||
print(f'Downloading asset {asset_kind}')
|
||||
|
||||
asset_dir = config.assets_dir / asset_kind
|
||||
asset_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
url = f'https://assets.openfreemap.com/{asset_kind}/ofm.tar.gz'
|
||||
local_file = asset_dir / 'ofm.tar.gz'
|
||||
if not download_if_size_differs(url, local_file):
|
||||
print(f' skipping asset: {asset_kind}')
|
||||
return
|
||||
|
||||
ofm_dir = asset_dir / 'ofm'
|
||||
ofm_dir_bak = asset_dir / 'ofm.bak'
|
||||
shutil.rmtree(ofm_dir_bak, ignore_errors=True)
|
||||
if ofm_dir.exists():
|
||||
ofm_dir.rename(ofm_dir_bak)
|
||||
|
||||
subprocess.run(
|
||||
['tar', '-xzf', local_file, '-C', asset_dir],
|
||||
check=True,
|
||||
)
|
||||
|
||||
print(f' downloaded asset: {asset_kind}')
|
||||
|
||||
|
||||
def download_sprites():
|
||||
"""
|
||||
Sprites are special assets, as we have to keep the old versions indefinitely
|
||||
"""
|
||||
|
||||
print('Downloading sprites')
|
||||
|
||||
sprites_dir = config.assets_dir / 'sprites'
|
||||
sprites_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
r = requests.get('https://assets.openfreemap.com/files.txt', timeout=30)
|
||||
r.raise_for_status()
|
||||
|
||||
sprites_remote = [l for l in r.text.splitlines() if l.startswith('sprites/')]
|
||||
|
||||
for sprite in sprites_remote:
|
||||
sprite_name = sprite.split('/')[1].replace('.tar.gz', '')
|
||||
|
||||
if (sprites_dir / sprite_name).is_dir():
|
||||
print(f' skipping sprite version: {sprite_name}')
|
||||
continue
|
||||
|
||||
url = f'https://assets.openfreemap.com/sprites/{sprite_name}.tar.gz'
|
||||
local_file = sprites_dir / 'temp.tar.gz'
|
||||
download_file_aria2(url, local_file)
|
||||
|
||||
subprocess.run(
|
||||
['tar', '-xzf', local_file, '-C', sprites_dir],
|
||||
check=True,
|
||||
)
|
||||
local_file.unlink()
|
||||
print(f' downloaded sprite version: {sprite_name}')
|
||||
84
modules/http_host/http_host_lib/btrfs.py
Normal file
84
modules/http_host/http_host_lib/btrfs.py
Normal file
@@ -0,0 +1,84 @@
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.utils import download_file_aria2, get_remote_file_size
|
||||
|
||||
|
||||
def download_area_version(area: str, version: str):
|
||||
"""
|
||||
Downloads and uncompresses tiles.btrfs files from the btrfs bucket
|
||||
"""
|
||||
|
||||
if area not in config.areas:
|
||||
sys.exit(f' please specify area: {config.areas}')
|
||||
|
||||
versions = get_versions_for_area(area)
|
||||
|
||||
if version == 'latest':
|
||||
selected_version = versions[-1]
|
||||
else:
|
||||
if version not in versions:
|
||||
available_versions_str = '\n'.join(versions)
|
||||
sys.exit(
|
||||
f'Requested version is not available.\nAvailable versions for {area}:\n{available_versions_str}'
|
||||
)
|
||||
selected_version = version
|
||||
|
||||
return download_and_extract_btrfs(area, selected_version)
|
||||
|
||||
|
||||
def get_versions_for_area(area: str) -> list:
|
||||
r = requests.get('https://btrfs.openfreemap.com/dirs.txt', timeout=30)
|
||||
r.raise_for_status()
|
||||
|
||||
versions = [v.split('/')[2] for v in r.text.splitlines() if v.startswith(f'areas/{area}/')]
|
||||
return sorted(versions)
|
||||
|
||||
|
||||
def download_and_extract_btrfs(area: str, version: str) -> bool:
|
||||
"""
|
||||
returns True if download successful, False if skipped
|
||||
"""
|
||||
|
||||
print(f'downloading and extracting btrfs for: {area} {version}')
|
||||
|
||||
version_dir = config.runs_dir / area / version
|
||||
btrfs_file = version_dir / 'tiles.btrfs'
|
||||
if btrfs_file.exists():
|
||||
print(' file exists, skipping download')
|
||||
return False
|
||||
|
||||
temp_dir = config.runs_dir / '_tmp'
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
temp_dir.mkdir(parents=True)
|
||||
|
||||
url = f'https://btrfs.openfreemap.com/areas/{area}/{version}/tiles.btrfs.gz'
|
||||
|
||||
# check disk space
|
||||
disk_free = shutil.disk_usage(temp_dir).free
|
||||
file_size = get_remote_file_size(url)
|
||||
if not file_size:
|
||||
raise ValueError('Cannot get remote file size')
|
||||
|
||||
needed_space = file_size * 3
|
||||
if disk_free < needed_space:
|
||||
raise ValueError(f'Not enough disk space. Needed: {needed_space}, free space: {disk_free}')
|
||||
|
||||
target_file = temp_dir / 'tiles.btrfs.gz'
|
||||
download_file_aria2(url, target_file)
|
||||
|
||||
print('Uncompressing...')
|
||||
subprocess.run(['unpigz', temp_dir / 'tiles.btrfs.gz'], check=True)
|
||||
btrfs_src = temp_dir / 'tiles.btrfs'
|
||||
|
||||
shutil.rmtree(version_dir, ignore_errors=True)
|
||||
version_dir.mkdir(parents=True)
|
||||
|
||||
btrfs_src.rename(btrfs_file)
|
||||
|
||||
shutil.rmtree(temp_dir)
|
||||
return True
|
||||
29
modules/http_host/http_host_lib/config.py
Normal file
29
modules/http_host/http_host_lib/config.py
Normal file
@@ -0,0 +1,29 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Configuration:
|
||||
areas = ['planet', 'monaco']
|
||||
|
||||
http_host_dir = Path('/data/ofm/http_host')
|
||||
|
||||
http_host_bin = http_host_dir / 'bin'
|
||||
http_host_scripts_dir = http_host_bin / 'scripts'
|
||||
|
||||
runs_dir = http_host_dir / 'runs'
|
||||
assets_dir = http_host_dir / 'assets'
|
||||
|
||||
mnt_dir = Path('/mnt/ofm')
|
||||
ofm_config_dir = Path('/data/ofm/config')
|
||||
|
||||
certs_dir = Path('/data/nginx/certs')
|
||||
nginx_confs = Path(__file__).parent / 'nginx_confs'
|
||||
|
||||
try:
|
||||
with open(ofm_config_dir / 'http_host.json') as fp:
|
||||
host_config = json.load(fp)
|
||||
except Exception:
|
||||
host_config = {}
|
||||
|
||||
|
||||
config = Configuration()
|
||||
87
modules/http_host/http_host_lib/mount.py
Normal file
87
modules/http_host/http_host_lib/mount.py
Normal file
@@ -0,0 +1,87 @@
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.utils import assert_linux, assert_sudo
|
||||
|
||||
|
||||
def auto_mount_unmount():
|
||||
"""
|
||||
Mounts/unmounts the btrfs images from /data/ofm/http_host/runs automatically.
|
||||
When finished, /mnt/ofm dir will have all the present tiles.btrfs files mounted in a read-only way.
|
||||
"""
|
||||
|
||||
print('running mount')
|
||||
|
||||
assert_linux()
|
||||
assert_sudo()
|
||||
|
||||
if not config.runs_dir.exists():
|
||||
sys.exit(' download-btrfs needs to be run first')
|
||||
|
||||
clean_up_mounts(config.mnt_dir)
|
||||
create_fstab()
|
||||
|
||||
print(' running mount -a')
|
||||
subprocess.run(['mount', '-a'], check=True)
|
||||
|
||||
clean_up_mounts(config.mnt_dir)
|
||||
|
||||
|
||||
def create_fstab():
|
||||
fstab_new = []
|
||||
|
||||
for area in ['planet', 'monaco']:
|
||||
area_dir = (config.runs_dir / area).resolve()
|
||||
if not area_dir.exists():
|
||||
continue
|
||||
|
||||
versions = sorted(area_dir.iterdir())
|
||||
for version in versions:
|
||||
version_str = version.name
|
||||
btrfs_file = area_dir / version_str / 'tiles.btrfs'
|
||||
if not btrfs_file.is_file():
|
||||
continue
|
||||
|
||||
mnt_folder = config.mnt_dir / f'{area}-{version_str}'
|
||||
mnt_folder.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
fstab_new.append(f'{btrfs_file} {mnt_folder} btrfs loop,ro 0 0\n')
|
||||
print(f' created fstab entry for {btrfs_file} -> {mnt_folder}')
|
||||
|
||||
with open('/etc/fstab') as fp:
|
||||
fstab_orig = [l for l in fp.readlines() if f'{config.mnt_dir}/' not in l]
|
||||
|
||||
with open('/etc/fstab', 'w') as fp:
|
||||
fp.writelines(fstab_orig + fstab_new)
|
||||
|
||||
|
||||
def clean_up_mounts(mnt_dir):
|
||||
if not mnt_dir.exists():
|
||||
return
|
||||
|
||||
print(' cleaning up mounts')
|
||||
|
||||
# handle deleted files
|
||||
p = subprocess.run(['mount'], capture_output=True, text=True, check=True)
|
||||
lines = [l for l in p.stdout.splitlines() if f'{mnt_dir}/' in l and '(deleted)' in l]
|
||||
|
||||
for l in lines:
|
||||
mnt_path = Path(l.split('(deleted) on ')[1].split(' type btrfs')[0])
|
||||
print(f' removing deleted mount {mnt_path}')
|
||||
assert mnt_path.exists()
|
||||
subprocess.run(['umount', mnt_path], check=True)
|
||||
mnt_path.rmdir()
|
||||
|
||||
# clean all mounts not in current fstab
|
||||
with open('/etc/fstab') as fp:
|
||||
fstab_str = fp.read()
|
||||
|
||||
for subdir in mnt_dir.iterdir():
|
||||
if f'{subdir} ' in fstab_str:
|
||||
continue
|
||||
|
||||
print(f' removing old mount {subdir}')
|
||||
subprocess.run(['umount', subdir], check=True)
|
||||
subdir.rmdir()
|
||||
235
modules/http_host/http_host_lib/nginx.py
Normal file
235
modules/http_host/http_host_lib/nginx.py
Normal file
@@ -0,0 +1,235 @@
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from http_host_lib.config import config
|
||||
from http_host_lib.utils import python_venv_executable
|
||||
|
||||
|
||||
def write_nginx_config():
|
||||
curl_text_mix = ''
|
||||
|
||||
domain_le = config.host_config['domain_le']
|
||||
domain_ledns = config.host_config['domain_ledns']
|
||||
|
||||
# remove old configs and certs
|
||||
for file in Path('/data/nginx/sites').glob('ofm_*.conf'):
|
||||
file.unlink()
|
||||
|
||||
for file in Path('/data/nginx/certs').glob('ofm_*'):
|
||||
file.unlink()
|
||||
|
||||
# processing Round Robin DNS config
|
||||
if domain_ledns:
|
||||
if not (config.ofm_config_dir / 'rclone.conf').is_file():
|
||||
sys.exit('rclone.conf missing')
|
||||
|
||||
# download the ledns certificate from bucket using rclone
|
||||
write_ledns_reader_script(domain_ledns)
|
||||
subprocess.run(['bash', config.http_host_bin / 'ledns_reader.sh'], check=True)
|
||||
|
||||
curl_text_mix += create_nginx_conf(
|
||||
template_path=config.nginx_confs / 'ledns.conf',
|
||||
local='ofm_ledns',
|
||||
domain=domain_ledns,
|
||||
)
|
||||
|
||||
# processing Let's Encrypt config
|
||||
if domain_le:
|
||||
le_cert = config.certs_dir / 'ofm_le.cert'
|
||||
le_key = config.certs_dir / 'ofm_le.key'
|
||||
|
||||
if not le_cert.is_file() or not le_key.is_file():
|
||||
shutil.copyfile(Path('/etc/nginx/ssl/dummy.crt'), le_cert)
|
||||
shutil.copyfile(Path('/etc/nginx/ssl/dummy.key'), le_key)
|
||||
|
||||
curl_text_mix += create_nginx_conf(
|
||||
template_path=config.nginx_confs / 'le.conf',
|
||||
local='ofm_le',
|
||||
domain=domain_le,
|
||||
)
|
||||
|
||||
subprocess.run(['nginx', '-t'], check=True)
|
||||
subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'certbot',
|
||||
'certonly',
|
||||
'--webroot',
|
||||
'--webroot-path=/data/nginx/acme-challenges',
|
||||
'--noninteractive',
|
||||
'-m',
|
||||
config.host_config['le_email'],
|
||||
'--agree-tos',
|
||||
'--cert-name=ofm_le',
|
||||
# '--staging',
|
||||
'--deploy-hook',
|
||||
'nginx -t && service nginx reload',
|
||||
'-d',
|
||||
domain_le,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
# link certs to nginx dir
|
||||
le_cert.unlink()
|
||||
le_key.unlink()
|
||||
|
||||
etc_cert = Path('/etc/letsencrypt/live/ofm_le/fullchain.pem')
|
||||
etc_key = Path('/etc/letsencrypt/live/ofm_le/privkey.pem')
|
||||
assert etc_cert.is_file()
|
||||
assert etc_key.is_file()
|
||||
le_cert.symlink_to(etc_cert)
|
||||
le_key.symlink_to(etc_key)
|
||||
|
||||
subprocess.run(['nginx', '-t'], check=True)
|
||||
subprocess.run(['systemctl', 'reload', 'nginx'], check=True)
|
||||
|
||||
print(curl_text_mix)
|
||||
|
||||
|
||||
def create_nginx_conf(*, template_path, local, domain):
|
||||
location_str, curl_text = create_location_blocks(local=local, domain=domain)
|
||||
|
||||
with open(template_path) as fp:
|
||||
template = fp.read()
|
||||
|
||||
template = template.replace('__LOCATION_BLOCKS__', location_str)
|
||||
template = template.replace('__LOCAL__', local)
|
||||
template = template.replace('__DOMAIN__', domain)
|
||||
|
||||
curl_text = curl_text.replace('__LOCAL__', local)
|
||||
curl_text = curl_text.replace('__DOMAIN__', domain)
|
||||
|
||||
with open(f'/data/nginx/sites/{local}.conf', 'w') as fp:
|
||||
fp.write(template)
|
||||
print(f' nginx config written: {domain} {local}')
|
||||
|
||||
return curl_text
|
||||
|
||||
|
||||
def create_location_blocks(*, local, domain):
|
||||
location_str = ''
|
||||
curl_text = ''
|
||||
|
||||
for subdir in config.mnt_dir.iterdir():
|
||||
if not subdir.is_dir():
|
||||
continue
|
||||
area, version = subdir.name.split('-')
|
||||
location_str += create_version_location(
|
||||
area=area, version=version, subdir=subdir, local=local, domain=domain
|
||||
)
|
||||
|
||||
if not curl_text:
|
||||
curl_text = (
|
||||
'\ntest with:\n'
|
||||
f'curl -H "Host: __LOCAL__" -I http://localhost/{area}/{version}/14/8529/5975.pbf\n'
|
||||
f'curl -I https://__DOMAIN__/{area}/{version}/14/8529/5975.pbf'
|
||||
)
|
||||
|
||||
location_str += create_latest_locations(local=local, domain=domain)
|
||||
|
||||
with open(config.nginx_confs / 'location_static.conf') as fp:
|
||||
location_str += '\n' + fp.read()
|
||||
|
||||
return location_str, curl_text
|
||||
|
||||
|
||||
def create_version_location(
|
||||
*, area: str, version: str, subdir: Path, local: str, domain: str
|
||||
) -> str:
|
||||
run_dir = config.runs_dir / area / version
|
||||
if not run_dir.is_dir():
|
||||
print(f" {run_dir} doesn't exists, skipping")
|
||||
return ''
|
||||
|
||||
tilejson_path = run_dir / f'tilejson-{local}.json'
|
||||
|
||||
metadata_path = subdir / 'metadata.json'
|
||||
if not metadata_path.is_file():
|
||||
print(f" {metadata_path} doesn't exists, skipping")
|
||||
return ''
|
||||
|
||||
url_prefix = f'https://{domain}/{area}/{version}'
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
python_venv_executable(),
|
||||
config.http_host_scripts_dir / 'metadata_to_tilejson.py',
|
||||
'--minify',
|
||||
metadata_path,
|
||||
tilejson_path,
|
||||
url_prefix,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
return f"""
|
||||
location = /{area}/{version} {{ # no trailing slash
|
||||
alias {tilejson_path}; # no trailing slash
|
||||
|
||||
expires 1w;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
}}
|
||||
|
||||
location /{area}/{version}/ {{ # trailing slash
|
||||
alias {subdir}/tiles/; # trailing slash
|
||||
try_files $uri @empty_tile;
|
||||
add_header Content-Encoding gzip;
|
||||
|
||||
expires 10y;
|
||||
|
||||
types {{
|
||||
application/vnd.mapbox-vector-tile pbf;
|
||||
}}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
def create_latest_locations(*, local: str, domain: str) -> str:
|
||||
location_str = ''
|
||||
|
||||
local_version_files = config.ofm_config_dir.glob('tileset_version_*.txt')
|
||||
for file in local_version_files:
|
||||
area = file.stem.split('_')[-1]
|
||||
with open(file) as fp:
|
||||
version = fp.read().strip()
|
||||
print(f' setting latest version for {area}: {version}')
|
||||
|
||||
run_dir = config.runs_dir / area / version
|
||||
tilejson_path = run_dir / f'tilejson-{local}.json'
|
||||
assert tilejson_path.is_file()
|
||||
|
||||
location_str += f"""
|
||||
location = /{area} {{ # no trailing slash
|
||||
alias {tilejson_path}; # no trailing slash
|
||||
|
||||
expires 1d;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
}}
|
||||
"""
|
||||
|
||||
return location_str
|
||||
|
||||
|
||||
def write_ledns_reader_script(domain_ledns):
|
||||
script = f"""
|
||||
#!/usr/bin/env bash
|
||||
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
|
||||
rclone copyto -v "remote:ofm-private/ledns/{domain_ledns}/ofm_ledns.cert" /data/nginx/certs/ofm_ledns.cert
|
||||
rclone copyto -v "remote:ofm-private/ledns/{domain_ledns}/ofm_ledns.key" /data/nginx/certs/ofm_ledns.key
|
||||
""".strip()
|
||||
|
||||
with open(config.http_host_bin / 'ledns_reader.sh', 'w') as fp:
|
||||
fp.write(script)
|
||||
44
modules/http_host/http_host_lib/nginx_confs/le.conf
Normal file
44
modules/http_host/http_host_lib/nginx_confs/le.conf
Normal file
@@ -0,0 +1,44 @@
|
||||
server {
|
||||
server_name __LOCAL__ __DOMAIN__;
|
||||
|
||||
# ssl: https://ssl-config.mozilla.org / intermediate config
|
||||
|
||||
listen 80;
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
http2 on;
|
||||
|
||||
ssl_certificate /data/nginx/certs/ofm_le.cert;
|
||||
ssl_certificate_key /data/nginx/certs/ofm_le.key;
|
||||
|
||||
ssl_session_timeout 1d;
|
||||
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
|
||||
ssl_session_tickets off;
|
||||
|
||||
ssl_dhparam /etc/nginx/ffdhe2048.txt;
|
||||
|
||||
# intermediate configuration
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
|
||||
ssl_prefer_server_ciphers off;
|
||||
|
||||
# access log disabled by default
|
||||
access_log /data/ofm/http_host/logs_nginx/le-access.jsonl access_json buffer=32k;
|
||||
#access_log off;
|
||||
|
||||
error_log /data/ofm/http_host/logs_nginx/le-error.log;
|
||||
|
||||
location ^~ /.well-known/acme-challenge/ {
|
||||
# trailing slash
|
||||
root /data/nginx/acme-challenges;
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
__LOCATION_BLOCKS__
|
||||
|
||||
# catch-all block to deny all other requests
|
||||
location / {
|
||||
deny all;
|
||||
error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
|
||||
}
|
||||
}
|
||||
38
modules/http_host/http_host_lib/nginx_confs/ledns.conf
Normal file
38
modules/http_host/http_host_lib/nginx_confs/ledns.conf
Normal file
@@ -0,0 +1,38 @@
|
||||
server {
|
||||
server_name __LOCAL__ __DOMAIN__;
|
||||
|
||||
# ssl: https://ssl-config.mozilla.org / intermediate config
|
||||
|
||||
listen 80;
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
http2 on;
|
||||
|
||||
ssl_certificate /data/nginx/certs/ofm_ledns.cert;
|
||||
ssl_certificate_key /data/nginx/certs/ofm_ledns.key;
|
||||
|
||||
ssl_session_timeout 1d;
|
||||
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
|
||||
ssl_session_tickets off;
|
||||
|
||||
ssl_dhparam /etc/nginx/ffdhe2048.txt;
|
||||
|
||||
# intermediate configuration
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
|
||||
ssl_prefer_server_ciphers off;
|
||||
|
||||
# access log disabled by default
|
||||
access_log /data/ofm/http_host/logs_nginx/ledns-access.jsonl access_json buffer=32k;
|
||||
#access_log off;
|
||||
|
||||
error_log /data/ofm/http_host/logs_nginx/ledns-error.log;
|
||||
|
||||
__LOCATION_BLOCKS__
|
||||
|
||||
# catch-all block to deny all other requests
|
||||
location / {
|
||||
deny all;
|
||||
error_log /data/ofm/http_host/logs_nginx/__LOCAL__-error.log error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
location /fonts/ {
|
||||
# trailing slash
|
||||
|
||||
alias /data/ofm/http_host/assets/fonts/ofm/; # trailing slash
|
||||
try_files $uri =404;
|
||||
|
||||
expires 1w;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
}
|
||||
|
||||
location /styles/ {
|
||||
# trailing slash
|
||||
|
||||
alias /data/ofm/http_host/assets/styles/ofm/; # trailing slash
|
||||
try_files $uri.json =404;
|
||||
|
||||
expires 1d;
|
||||
default_type application/json;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
}
|
||||
|
||||
location /natural_earth/ {
|
||||
# trailing slash
|
||||
|
||||
alias /data/ofm/http_host/assets/natural_earth/ofm/; # trailing slash
|
||||
try_files $uri =404;
|
||||
|
||||
expires 10y;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
}
|
||||
|
||||
location /sprites/ {
|
||||
# trailing slash
|
||||
|
||||
alias /data/ofm/http_host/assets/sprites/; # trailing slash
|
||||
try_files $uri =404;
|
||||
|
||||
expires 10y;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
}
|
||||
|
||||
|
||||
# we need to handle missing tiles as valid request returning empty string
|
||||
location @empty_tile {
|
||||
return 200 '';
|
||||
|
||||
expires 10y;
|
||||
default_type application/vnd.mapbox-vector-tile;
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' '*' always;
|
||||
add_header Cache-Control public;
|
||||
}
|
||||
|
||||
location = / {
|
||||
return 302 https://openfreemap.org;
|
||||
}
|
||||
45
modules/http_host/http_host_lib/set_tileset_versions.py
Normal file
45
modules/http_host/http_host_lib/set_tileset_versions.py
Normal file
@@ -0,0 +1,45 @@
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from http_host_lib.config import config
|
||||
|
||||
|
||||
def set_tileset_versions():
|
||||
need_nginx_sync = False
|
||||
|
||||
for area in ['planet', 'monaco']:
|
||||
r = requests.get(f'https://assets.openfreemap.com/versions/deployed_{area}.txt', timeout=30)
|
||||
r.raise_for_status()
|
||||
remote_version = r.text.strip()
|
||||
print(f' remote version for {area}: {remote_version}')
|
||||
|
||||
local_version_file = config.ofm_config_dir / f'tileset_version_{area}.txt'
|
||||
|
||||
if not local_version_file.exists():
|
||||
local_version_start = None
|
||||
else:
|
||||
with open(local_version_file) as fp:
|
||||
local_version_start = fp.read()
|
||||
|
||||
if not remote_version:
|
||||
print(' remote version not specified')
|
||||
if local_version_start is not None:
|
||||
local_version_file.unlink()
|
||||
need_nginx_sync = True
|
||||
continue
|
||||
|
||||
mnt_file = Path(f'/mnt/ofm/{area}-{remote_version}/metadata.json')
|
||||
if not mnt_file.exists():
|
||||
print(' local version does not exist')
|
||||
if local_version_start is not None:
|
||||
local_version_file.unlink()
|
||||
need_nginx_sync = True
|
||||
continue
|
||||
|
||||
if remote_version != local_version_start:
|
||||
with open(local_version_file, 'w') as fp:
|
||||
fp.write(remote_version)
|
||||
need_nginx_sync = True
|
||||
|
||||
return need_nginx_sync
|
||||
69
modules/http_host/http_host_lib/utils.py
Normal file
69
modules/http_host/http_host_lib/utils.py
Normal file
@@ -0,0 +1,69 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def assert_sudo():
|
||||
if os.geteuid() != 0:
|
||||
sys.exit(' needs sudo')
|
||||
|
||||
|
||||
def assert_linux():
|
||||
if not Path('/etc/fstab').exists():
|
||||
sys.exit(' needs to be run on Linux')
|
||||
|
||||
|
||||
def assert_single_process():
|
||||
p = subprocess.run(['pgrep', '-fl', sys.argv[0]], capture_output=True, text=True)
|
||||
lines = [l for l in p.stdout.splitlines() if 'python' in l]
|
||||
if len(lines) >= 2:
|
||||
sys.exit(' detected multiple processes, terminating')
|
||||
|
||||
|
||||
def download_if_size_differs(url: str, local_file: Path) -> bool:
|
||||
if not local_file.exists() or local_file.stat().st_size != get_remote_file_size(url):
|
||||
download_file_aria2(url, local_file)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_remote_file_size(url: str) -> int | None:
|
||||
r = requests.head(url, timeout=30)
|
||||
size = r.headers.get('Content-Length')
|
||||
return int(size) if size else None
|
||||
|
||||
|
||||
def download_file_aria2(url: str, local_file: Path):
|
||||
print(f' downloading {url} into {local_file}')
|
||||
local_file.unlink(missing_ok=True)
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'aria2c',
|
||||
'--split=8',
|
||||
'--max-connection-per-server=8',
|
||||
'--file-allocation=none',
|
||||
'--min-split-size=1M',
|
||||
'-d',
|
||||
local_file.parent,
|
||||
'-o',
|
||||
local_file.name,
|
||||
url,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def python_venv_executable() -> Path:
|
||||
venv_path = os.environ.get('VIRTUAL_ENV')
|
||||
|
||||
if venv_path:
|
||||
return Path(venv_path) / 'bin' / 'python'
|
||||
elif sys.prefix != sys.base_prefix:
|
||||
return Path(sys.prefix) / 'bin' / 'python'
|
||||
else:
|
||||
return Path(sys.executable)
|
||||
66
modules/http_host/scripts/metadata_to_tilejson.py
Executable file
66
modules/http_host/scripts/metadata_to_tilejson.py
Executable file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument(
|
||||
'metadata_path', type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path)
|
||||
)
|
||||
@click.argument('tilejson_path', type=click.Path(path_type=Path))
|
||||
@click.argument('url_prefix')
|
||||
@click.option('--minify', is_flag=True, help='Minify the generated JSON')
|
||||
def cli(metadata_path: Path, tilejson_path: Path, url_prefix: str, minify: bool):
|
||||
"""
|
||||
Takes a MBTiles metadata.json and generates a TileJSON 3.0.0 file
|
||||
|
||||
URL_PREFIX: Base URL to use as a prefix for tiles in the generated TileJSON.
|
||||
|
||||
Reference: https://github.com/mapbox/tilejson-spec/tree/master/3.0.0
|
||||
"""
|
||||
|
||||
tilejson = dict(tilejson='3.0.0')
|
||||
|
||||
with open(metadata_path) as fp:
|
||||
metadata = json.load(fp)
|
||||
|
||||
metadata_json_key = json.loads(metadata.pop('json'))
|
||||
|
||||
tilejson['tiles'] = [url_prefix.rstrip('/') + '/{z}/{x}/{y}.pbf']
|
||||
|
||||
''
|
||||
tilejson['vector_layers'] = metadata_json_key.pop('vector_layers')
|
||||
assert not metadata_json_key # check that no more keys are left
|
||||
|
||||
tilejson['attribution'] = metadata.pop('attribution')
|
||||
|
||||
# overwriting new style OSM license, until fixed in tile_gen
|
||||
tilejson['attribution'] = (
|
||||
'<a href="https://openfreemap.org" target="_blank">OpenFreeMap</a> '
|
||||
'<a href="https://www.openmaptiles.org/" target="_blank">© OpenMapTiles</a> '
|
||||
'Data from <a href="https://www.openstreetmap.org/copyright" target="_blank">OpenStreetMap</a>'
|
||||
)
|
||||
|
||||
tilejson['bounds'] = [float(n) for n in metadata.pop('bounds').split(',')]
|
||||
tilejson['center'] = [float(n) for n in metadata.pop('center').split(',')]
|
||||
tilejson['center'][2] = 1
|
||||
|
||||
tilejson['description'] = metadata.pop('description')
|
||||
|
||||
tilejson['maxzoom'] = int(metadata.pop('maxzoom'))
|
||||
tilejson['minzoom'] = int(metadata.pop('minzoom'))
|
||||
|
||||
tilejson['name'] = metadata.pop('name')
|
||||
tilejson['version'] = metadata.pop('version')
|
||||
|
||||
with open(tilejson_path, 'w') as fp:
|
||||
if minify:
|
||||
json.dump(tilejson, fp, ensure_ascii=False, separators=(',', ':'))
|
||||
else:
|
||||
json.dump(tilejson, fp, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
14
modules/http_host/setup.py
Normal file
14
modules/http_host/setup.py
Normal file
@@ -0,0 +1,14 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
||||
requirements = [
|
||||
'click',
|
||||
'requests',
|
||||
]
|
||||
|
||||
|
||||
setup(
|
||||
python_requires='>=3.10',
|
||||
install_requires=requirements,
|
||||
packages=find_packages(),
|
||||
)
|
||||
11
modules/ledns/rclone_write.sh
Normal file
11
modules/ledns/rclone_write.sh
Normal file
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#env > /data/ofm/ledns/env.txt
|
||||
#RENEWED_DOMAINS=direct.openfreemap.org
|
||||
#RENEWED_LINEAGE=/etc/letsencrypt/live/ofm_ledns
|
||||
|
||||
export RCLONE_CONFIG=/data/ofm/config/rclone.conf
|
||||
|
||||
rclone copyto -v --copy-links "$RENEWED_LINEAGE/fullchain.pem" "remote:ofm-private/ledns/$RENEWED_DOMAINS/ofm_ledns.cert"
|
||||
rclone copyto -v --copy-links "$RENEWED_LINEAGE/privkey.pem" "remote:ofm-private/ledns/$RENEWED_DOMAINS/ofm_ledns.key"
|
||||
|
||||
8
modules/loadbalancer/cron.d/ofm_loadbalancer
Normal file
8
modules/loadbalancer/cron.d/ofm_loadbalancer
Normal file
@@ -0,0 +1,8 @@
|
||||
# every minute
|
||||
|
||||
# fix
|
||||
#* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py fix >> /data/ofm/loadbalancer/logs/run.log 2>&1
|
||||
|
||||
|
||||
# check
|
||||
* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/loadbalancer/loadbalancer.py check >> /data/ofm/loadbalancer/logs/run.log 2>&1
|
||||
162
modules/loadbalancer/loadbalancer.py
Executable file
162
modules/loadbalancer/loadbalancer.py
Executable file
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
import datetime
|
||||
import json
|
||||
|
||||
import click
|
||||
import requests
|
||||
from dotenv import dotenv_values
|
||||
from loadbalancer_lib import OFM_CONFIG_DIR
|
||||
from loadbalancer_lib.cloudflare import get_zone_id, set_records_round_robin
|
||||
from loadbalancer_lib.curl import pycurl_get, pycurl_status
|
||||
from loadbalancer_lib.telegram_ import telegram_send_message
|
||||
|
||||
|
||||
AREAS = ['planet', 'monaco']
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""
|
||||
Manages load-balancing of Round-Robin DNS records
|
||||
"""
|
||||
|
||||
|
||||
@cli.command()
|
||||
def check():
|
||||
"""
|
||||
Runs load-balancing check (triggered by cron every minute)
|
||||
"""
|
||||
|
||||
print(f'starting loadbalancer check at: {datetime.datetime.now(tz=datetime.timezone.utc)}')
|
||||
check_or_fix(fix=False)
|
||||
|
||||
|
||||
@cli.command()
|
||||
def fix():
|
||||
"""
|
||||
Fixes records based on check results
|
||||
"""
|
||||
|
||||
print(f'starting loadbalancer fix at: {datetime.datetime.now(tz=datetime.timezone.utc)}')
|
||||
check_or_fix(fix=True)
|
||||
|
||||
|
||||
def check_or_fix(fix=False):
|
||||
with open(OFM_CONFIG_DIR / 'loadbalancer.json') as fp:
|
||||
c = json.load(fp)
|
||||
# print(c)
|
||||
|
||||
if not c['http_host_list']:
|
||||
telegram_send_message(
|
||||
'OFM loadbalancer no hosts found on list, terminating',
|
||||
c['telegram_token'],
|
||||
c['telegram_chat_id'],
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
results_by_ip = {}
|
||||
working_hosts = set()
|
||||
|
||||
for area in AREAS:
|
||||
results = run_area(c, area)
|
||||
for host_ip, host_is_ok in results.items():
|
||||
results_by_ip.setdefault(host_ip, True)
|
||||
results_by_ip[host_ip] &= host_is_ok
|
||||
|
||||
for host_ip, host_is_ok in results_by_ip.items():
|
||||
if not host_is_ok:
|
||||
message = f'OFM loadbalancer ERROR with host: {host_ip}'
|
||||
telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
|
||||
else:
|
||||
working_hosts.add(host_ip)
|
||||
|
||||
except Exception as e:
|
||||
message = f'OFM loadbalancer ERROR with loadbalancer: {e}'
|
||||
telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
|
||||
return
|
||||
|
||||
print(f'working hosts: {sorted(working_hosts)}')
|
||||
|
||||
if fix:
|
||||
# if no hosts are detected working, probably a bug in this script
|
||||
# fail-safe to include all hosts
|
||||
if not working_hosts:
|
||||
working_hosts = set(c['http_host_list'])
|
||||
|
||||
message = 'OFM loadbalancer FIX found no working hosts, reverting to full list!'
|
||||
telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
|
||||
|
||||
updated = update_records(c, working_hosts)
|
||||
if updated:
|
||||
message = f'OFM loadbalancer FIX modified records, new records: {working_hosts}'
|
||||
telegram_send_message(message, c['telegram_token'], c['telegram_chat_id'])
|
||||
|
||||
|
||||
def run_area(c, area):
|
||||
target_version = get_target_version(area)
|
||||
|
||||
print(f'target version: {area}: {target_version}')
|
||||
|
||||
results = {}
|
||||
|
||||
for host_ip in c['http_host_list']:
|
||||
try:
|
||||
check_host(c['domain_ledns'], host_ip, area, target_version)
|
||||
results[host_ip] = True
|
||||
except Exception as e:
|
||||
results[host_ip] = False
|
||||
print(e)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def check_host(domain, host_ip, area, version):
|
||||
# check TileJSON first
|
||||
url = f'https://{domain}/{area}'
|
||||
tilejson_str = pycurl_get(url, domain, host_ip)
|
||||
tilejson = json.loads(tilejson_str)
|
||||
tiles_url = tilejson['tiles'][0]
|
||||
version_in_tilejson = tiles_url.split('/')[4]
|
||||
assert version_in_tilejson == version
|
||||
|
||||
# check actual vector tile
|
||||
url = f'https://{domain}/{area}/{version}/14/8529/5975.pbf'
|
||||
assert pycurl_status(url, domain, host_ip) == 200
|
||||
|
||||
# check style
|
||||
url = f'https://{domain}/styles/bright'
|
||||
assert pycurl_status(url, domain, host_ip) == 200
|
||||
|
||||
|
||||
def get_target_version(area):
|
||||
url = f'https://assets.openfreemap.com/versions/deployed_{area}.txt'
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
return response.text.strip()
|
||||
|
||||
|
||||
def update_records(c, working_hosts) -> bool:
|
||||
config = dotenv_values(OFM_CONFIG_DIR / 'cloudflare.ini')
|
||||
cloudflare_api_token = config['dns_cloudflare_api_token']
|
||||
|
||||
domain = '.'.join(c['domain_ledns'].split('.')[-2:])
|
||||
zone_id = get_zone_id(domain, cloudflare_api_token=cloudflare_api_token)
|
||||
|
||||
updated = False
|
||||
|
||||
updated |= set_records_round_robin(
|
||||
zone_id=zone_id,
|
||||
name=c['domain_ledns'],
|
||||
host_ip_set=working_hosts,
|
||||
proxied=False,
|
||||
ttl=300,
|
||||
comment='domain_ledns',
|
||||
cloudflare_api_token=cloudflare_api_token,
|
||||
)
|
||||
|
||||
return updated
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
9
modules/loadbalancer/loadbalancer_lib/__init__.py
Normal file
9
modules/loadbalancer/loadbalancer_lib/__init__.py
Normal file
@@ -0,0 +1,9 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
if Path('/data/ofm/config').exists():
|
||||
OFM_CONFIG_DIR = Path('/data/ofm/config')
|
||||
else:
|
||||
OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
|
||||
|
||||
assert OFM_CONFIG_DIR.exists()
|
||||
109
modules/loadbalancer/loadbalancer_lib/cloudflare.py
Normal file
109
modules/loadbalancer/loadbalancer_lib/cloudflare.py
Normal file
@@ -0,0 +1,109 @@
|
||||
from pprint import pprint
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
# docs: https://api.cloudflare.com/
|
||||
|
||||
|
||||
def cloudflare_get(path: str, params: dict, cloudflare_api_token: str):
|
||||
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
|
||||
res = requests.get(
|
||||
f'https://api.cloudflare.com/client/v4{path}', headers=headers, params=params
|
||||
)
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
assert data['success'] is True
|
||||
return data
|
||||
|
||||
|
||||
def get_zone_id(domain, cloudflare_api_token: str):
|
||||
data = cloudflare_get(
|
||||
'/zones', params=dict(name=domain), cloudflare_api_token=cloudflare_api_token
|
||||
)
|
||||
assert len(data['result']) == 1
|
||||
zone_info = data['result'][0]
|
||||
return zone_info['id']
|
||||
|
||||
|
||||
def get_dns_records_round_robin(zone_id, cloudflare_api_token: str) -> dict:
|
||||
data = cloudflare_get(
|
||||
f'/zones/{zone_id}/dns_records',
|
||||
params=dict(per_page=5000),
|
||||
cloudflare_api_token=cloudflare_api_token,
|
||||
)
|
||||
records = data['result']
|
||||
|
||||
data = {}
|
||||
|
||||
for r in records:
|
||||
if r['type'] != 'A':
|
||||
continue
|
||||
|
||||
data.setdefault(r['name'], [])
|
||||
data[r['name']].append(dict(content=r['content'], id=r['id']))
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def set_records_round_robin(
|
||||
zone_id,
|
||||
*,
|
||||
name: str,
|
||||
host_ip_set: set,
|
||||
ttl: int = 1,
|
||||
proxied: bool,
|
||||
comment: str = None,
|
||||
cloudflare_api_token: str,
|
||||
) -> bool:
|
||||
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
|
||||
|
||||
dns_records = get_dns_records_round_robin(zone_id, cloudflare_api_token=cloudflare_api_token)
|
||||
current_records = dns_records.get(name, [])
|
||||
|
||||
current_ips = {r['content'] for r in current_records}
|
||||
if current_ips == host_ip_set:
|
||||
print(f'No need to update records: {name} currently set: {sorted(current_ips)}')
|
||||
return False
|
||||
|
||||
# changing records
|
||||
|
||||
# delete all current records first
|
||||
for r in current_records:
|
||||
delete_record(zone_id, id_=r['id'], cloudflare_api_token=cloudflare_api_token)
|
||||
|
||||
# create new records
|
||||
for ip in host_ip_set:
|
||||
print(f'Creating record: {name} {ip}')
|
||||
json_data = dict(
|
||||
type='A',
|
||||
name=name,
|
||||
content=ip,
|
||||
ttl=ttl,
|
||||
proxied=proxied,
|
||||
comment=comment,
|
||||
)
|
||||
res = requests.post(
|
||||
f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records',
|
||||
headers=headers,
|
||||
json=json_data,
|
||||
)
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
assert data['success'] is True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def delete_record(zone_id, *, id_: str, cloudflare_api_token: str):
|
||||
headers = {'Authorization': f'Bearer {cloudflare_api_token}'}
|
||||
|
||||
print(f'Deleting record: {id_}')
|
||||
res = requests.delete(
|
||||
f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records/{id_}',
|
||||
headers=headers,
|
||||
json={},
|
||||
)
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
assert data['success'] is True
|
||||
54
modules/loadbalancer/loadbalancer_lib/curl.py
Normal file
54
modules/loadbalancer/loadbalancer_lib/curl.py
Normal file
@@ -0,0 +1,54 @@
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pycurl
|
||||
|
||||
|
||||
def pycurl_status(url, domain, host_ip):
|
||||
"""
|
||||
Uses pycurl to make a HTTPS HEAD request using custom resolving,
|
||||
checks if the status code is 200
|
||||
"""
|
||||
|
||||
c = pycurl.Curl()
|
||||
c.setopt(c.URL, url)
|
||||
|
||||
# linux needs CA certs specified manually
|
||||
if Path('/etc/ssl/certs/ca-certificates.crt').exists():
|
||||
c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
|
||||
|
||||
c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
|
||||
c.setopt(c.NOBODY, True)
|
||||
c.setopt(c.TIMEOUT, 5)
|
||||
c.perform()
|
||||
status_code = c.getinfo(c.RESPONSE_CODE)
|
||||
c.close()
|
||||
|
||||
return status_code
|
||||
|
||||
|
||||
def pycurl_get(url, domain, host_ip):
|
||||
"""
|
||||
Uses pycurl to make a HTTPS GET request using custom resolving,
|
||||
checks if the status code is 200, and returns the content.
|
||||
"""
|
||||
|
||||
buffer = BytesIO()
|
||||
c = pycurl.Curl()
|
||||
c.setopt(c.URL, url)
|
||||
|
||||
# linux needs CA certs specified manually
|
||||
if Path('/etc/ssl/certs/ca-certificates.crt').exists():
|
||||
c.setopt(c.CAINFO, '/etc/ssl/certs/ca-certificates.crt')
|
||||
|
||||
c.setopt(c.RESOLVE, [f'{domain}:443:{host_ip}'])
|
||||
c.setopt(c.WRITEDATA, buffer)
|
||||
c.setopt(c.TIMEOUT, 5)
|
||||
c.perform()
|
||||
status_code = c.getinfo(c.RESPONSE_CODE)
|
||||
c.close()
|
||||
|
||||
if status_code != 200:
|
||||
raise ValueError(f'status code: {status_code}')
|
||||
|
||||
return buffer.getvalue().decode('utf8')
|
||||
16
modules/loadbalancer/loadbalancer_lib/telegram_.py
Normal file
16
modules/loadbalancer/loadbalancer_lib/telegram_.py
Normal file
@@ -0,0 +1,16 @@
|
||||
import requests
|
||||
|
||||
|
||||
def telegram_send_message(message, bot_token, chat_id):
|
||||
print(message)
|
||||
|
||||
url = f'https://api.telegram.org/bot{bot_token}/sendMessage'
|
||||
|
||||
payload = {'chat_id': chat_id, 'text': message}
|
||||
|
||||
response = requests.post(url, data=payload)
|
||||
|
||||
if response.status_code == 200:
|
||||
print(' Message sent successfully!')
|
||||
else:
|
||||
print(' Failed to send message:', response.text)
|
||||
16
modules/loadbalancer/setup.py
Normal file
16
modules/loadbalancer/setup.py
Normal file
@@ -0,0 +1,16 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
||||
requirements = [
|
||||
'click',
|
||||
'requests',
|
||||
'pycurl',
|
||||
'python-dotenv',
|
||||
]
|
||||
|
||||
|
||||
setup(
|
||||
python_requires='>=3.10',
|
||||
install_requires=requirements,
|
||||
packages=find_packages(),
|
||||
)
|
||||
15
modules/prepare-virtualenv.sh
Executable file
15
modules/prepare-virtualenv.sh
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
find . -name "*.egg-info" -exec rm -rf {} +
|
||||
find . -name __pycache__ -exec rm -rf {} +
|
||||
|
||||
# deactivate
|
||||
rm -rf venv
|
||||
python3 -m venv venv
|
||||
|
||||
venv/bin/pip -V
|
||||
|
||||
venv/bin/pip install -U pip wheel setuptools
|
||||
|
||||
|
||||
|
||||
17
modules/set_version/setup.py
Normal file
17
modules/set_version/setup.py
Normal file
@@ -0,0 +1,17 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
||||
requirements = [
|
||||
'click',
|
||||
'requests',
|
||||
'pycurl',
|
||||
'python-dotenv',
|
||||
'questionary',
|
||||
]
|
||||
|
||||
|
||||
setup(
|
||||
python_requires='>=3.10',
|
||||
install_requires=requirements,
|
||||
packages=find_packages(),
|
||||
)
|
||||
62
modules/set_version/setversion.py
Executable file
62
modules/set_version/setversion.py
Executable file
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import subprocess
|
||||
|
||||
import click
|
||||
import questionary
|
||||
from setversion_lib import RCLONE_BIN, RCLONE_CONF
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""
|
||||
Sets deployed reference versions
|
||||
"""
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument('area', required=True)
|
||||
def interactive(area):
|
||||
versions = get_available_versions(area)[::-1]
|
||||
|
||||
choices = [questionary.Choice(title=r, value=i) for i, r in enumerate(versions)]
|
||||
answer = questionary.select(f'Select version for: {area}', choices=choices).ask()
|
||||
|
||||
selected = versions[answer]
|
||||
|
||||
set_version(area, selected)
|
||||
|
||||
|
||||
def get_available_versions(area):
|
||||
p = subprocess.run(
|
||||
[
|
||||
RCLONE_BIN,
|
||||
'cat',
|
||||
f'remote:ofm-{area}/dirs.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=RCLONE_CONF),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
versions = [l.strip() for l in p.stdout.strip().splitlines()]
|
||||
versions.sort()
|
||||
|
||||
return versions
|
||||
|
||||
|
||||
def set_version(area, version):
|
||||
subprocess.run(
|
||||
[
|
||||
RCLONE_BIN,
|
||||
'rcat',
|
||||
f'remote:ofm-assets/versions/deployed_{area}.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=RCLONE_CONF),
|
||||
check=True,
|
||||
input=version.encode(),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
16
modules/set_version/setversion_lib/__init__.py
Normal file
16
modules/set_version/setversion_lib/__init__.py
Normal file
@@ -0,0 +1,16 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
if Path('/data/ofm/config').exists():
|
||||
OFM_CONFIG_DIR = Path('/data/ofm/config')
|
||||
else:
|
||||
OFM_CONFIG_DIR = Path(__file__).parent.parent.parent.parent / 'config'
|
||||
|
||||
assert OFM_CONFIG_DIR.exists()
|
||||
|
||||
RCLONE_CONF = OFM_CONFIG_DIR / 'rclone.conf'
|
||||
|
||||
if Path('/opt/homebrew/bin/rclone').exists():
|
||||
RCLONE_BIN = '/opt/homebrew/bin/rclone'
|
||||
else:
|
||||
RCLONE_BIN = 'rclone'
|
||||
8
modules/tile_gen/cron.d/ofm_tile_gen
Normal file
8
modules/tile_gen/cron.d/ofm_tile_gen
Normal file
@@ -0,0 +1,8 @@
|
||||
# every hour, make a monaco run
|
||||
10 * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-tiles monaco --upload >> /data/ofm/tile_gen/logs/monaco-cron.log 2>&1
|
||||
|
||||
|
||||
|
||||
# once per minute, create indexes
|
||||
* * * * * ofm sudo /data/ofm/venv/bin/python -u /data/ofm/tile_gen/bin/tile_gen.py make-indexes >> /data/ofm/tile_gen/logs/make-indexes-cron.log 2>&1
|
||||
|
||||
2
modules/tile_gen/scripts/README.md
Normal file
2
modules/tile_gen/scripts/README.md
Normal file
@@ -0,0 +1,2 @@
|
||||
These are self contained Python scripts, they can be run outside of this project's environment.
|
||||
|
||||
181
modules/tile_gen/scripts/extract_mbtiles.py
Executable file
181
modules/tile_gen/scripts/extract_mbtiles.py
Executable file
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument(
|
||||
'mbtiles_path',
|
||||
type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path),
|
||||
)
|
||||
@click.argument('dir_path', type=click.Path(dir_okay=True, file_okay=False, path_type=Path))
|
||||
def cli(mbtiles_path: Path, dir_path: Path):
|
||||
"""
|
||||
Extracts a mbtiles sqlite to a folder
|
||||
Deduplicating identical tiles as hard-links
|
||||
|
||||
used for reference: https://github.com/mapbox/mbutil
|
||||
"""
|
||||
|
||||
if dir_path.exists() and any(dir_path.iterdir()):
|
||||
sys.exit(' dir not empty')
|
||||
|
||||
dir_path.mkdir(exist_ok=True)
|
||||
|
||||
conn = sqlite3.connect(mbtiles_path)
|
||||
c = conn.cursor()
|
||||
|
||||
write_dedupl_files(c, dir_path=dir_path)
|
||||
write_tile_files(c, dir_path=dir_path)
|
||||
|
||||
# planetiler has missing tiles by design, so disabling this
|
||||
# assert_all_tiles_present(mbtiles_path, dir_path)
|
||||
|
||||
write_metadata(c, dir_path=dir_path)
|
||||
conn.commit()
|
||||
|
||||
print('extract_mbtiles.py DONE')
|
||||
|
||||
|
||||
def write_metadata(c, *, dir_path):
|
||||
metadata = dict(c.execute('select name, value from metadata').fetchall())
|
||||
c.execute("update metadata set value='OpenFreeMap' where name='name'")
|
||||
c.execute("update metadata set value='https://openfreemap.org' where name='description'")
|
||||
|
||||
if 'openfreemap' not in metadata['attribution']:
|
||||
attr_str = (
|
||||
'<a href="https://openfreemap.org" target="_blank">OpenFreeMap</a> '
|
||||
+ metadata['attribution']
|
||||
)
|
||||
c.execute("UPDATE metadata SET value = ? WHERE name = 'attribution'", (attr_str,))
|
||||
|
||||
if 'osm_date' not in metadata:
|
||||
if 'planetiler:osm:osmosisreplicationtime' in metadata:
|
||||
osm_date = metadata['planetiler:osm:osmosisreplicationtime'][:10]
|
||||
c.execute('INSERT INTO metadata (name, value) VALUES (?, ?)', ('osm_date', osm_date))
|
||||
|
||||
metadata = dict(c.execute('select name, value from metadata').fetchall())
|
||||
with open(dir_path / 'metadata.json', 'w') as fp:
|
||||
json.dump(metadata, fp, indent=2)
|
||||
|
||||
with open(dir_path / 'osm_date', 'w') as fp:
|
||||
fp.write(metadata['osm_date'])
|
||||
|
||||
|
||||
def write_dedupl_files(c, *, dir_path):
|
||||
"""
|
||||
dedupl files
|
||||
write out the tiles_data files into a multi-level folder
|
||||
"""
|
||||
|
||||
total = c.execute('select count(*) from tiles_data').fetchone()[0]
|
||||
|
||||
c.execute('select tile_data_id, tile_data from tiles_data')
|
||||
for i, row in enumerate(c, start=1):
|
||||
dedupl_id = row[0]
|
||||
dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
|
||||
dedupl_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(dedupl_path, 'wb') as fp:
|
||||
fp.write(row[1])
|
||||
print(f'written dedupl file {i}/{total}')
|
||||
|
||||
|
||||
def write_tile_files(c, *, dir_path):
|
||||
total = c.execute('select count(*) from tiles_shallow').fetchone()[0]
|
||||
|
||||
bug_fix_dict = {}
|
||||
|
||||
c.execute('select zoom_level, tile_column, tile_row, tile_data_id from tiles_shallow')
|
||||
for i, row in enumerate(c, start=1):
|
||||
z = row[0]
|
||||
x = row[1]
|
||||
y = flip_y(z, row[2])
|
||||
dedupl_id = row[3]
|
||||
|
||||
dedupl_path = dir_path / 'dedupl' / dedupl_helper_path(dedupl_id)
|
||||
dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
|
||||
|
||||
tile_path = dir_path / 'tiles' / str(z) / str(x) / f'{y}.pbf'
|
||||
tile_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if tile_path.is_file():
|
||||
continue
|
||||
|
||||
# create the hard link
|
||||
try:
|
||||
tile_path.hardlink_to(dedupl_path_fixed)
|
||||
print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
|
||||
except OSError as e:
|
||||
# fixing Btrfs's 64k max link limit
|
||||
if e.errno == 31:
|
||||
bug_fix_dict.setdefault(dedupl_path, 0)
|
||||
bug_fix_dict[dedupl_path] += 1
|
||||
dedupl_path_fixed = get_fixed_dedupl_name(bug_fix_dict, dedupl_path)
|
||||
shutil.copyfile(dedupl_path, dedupl_path_fixed)
|
||||
print(f'Created fixed dedupl file: {dedupl_path_fixed}')
|
||||
tile_path.hardlink_to(dedupl_path_fixed)
|
||||
print(f'hard link created {i}/{total} {i / total * 100:.1f}%: {tile_path}')
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def assert_all_tiles_present(mbtiles_path, dir_path):
|
||||
"""
|
||||
If it's a full planet run,
|
||||
make sure there are exactly the right number of files generated.
|
||||
"""
|
||||
|
||||
if 'planet' in mbtiles_path.resolve().parent.name:
|
||||
assert count_files(dir_path / 'tiles') == calculate_tiles_sum(14)
|
||||
print(f'Tile number: {calculate_tiles_sum(14)} - OK')
|
||||
|
||||
|
||||
def count_files(folder):
|
||||
total = 0
|
||||
for root, dirs, files in os.walk(folder):
|
||||
total += len(files)
|
||||
return
|
||||
|
||||
|
||||
def get_fixed_dedupl_name(bug_fix_dict, dedupl_path):
|
||||
if dedupl_path in bug_fix_dict:
|
||||
return dedupl_path.with_name(f'{dedupl_path.name}-{bug_fix_dict[dedupl_path]}')
|
||||
else:
|
||||
return dedupl_path
|
||||
|
||||
|
||||
def dedupl_helper_path(dedupl_id: int) -> Path:
|
||||
"""
|
||||
Naming 200 million files such that each subdir has max 1000 children
|
||||
"""
|
||||
|
||||
str_num = f'{dedupl_id:09}'
|
||||
l1 = str_num[:3]
|
||||
l2 = str_num[3:6]
|
||||
l3 = str_num[6:]
|
||||
return Path(l1) / l2 / f'{l3}.pbf'
|
||||
|
||||
|
||||
def flip_y(zoom, y):
|
||||
return (2**zoom - 1) - y
|
||||
|
||||
|
||||
def calculate_tiles(zoom_level):
|
||||
return (2**zoom_level) ** 2
|
||||
|
||||
|
||||
def calculate_tiles_sum(zoom_level):
|
||||
"""
|
||||
Sum of tiles up to zoom level (geometric series)
|
||||
"""
|
||||
return (4 ** (zoom_level + 1) - 1) // 3
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
105
modules/tile_gen/scripts/shrink_btrfs.py
Executable file
105
modules/tile_gen/scripts/shrink_btrfs.py
Executable file
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
|
||||
# btrfs cannot shrink smaller than 256 MiB
|
||||
SMALLEST_SIZE = 256 * 1024 * 1024
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument(
|
||||
'btrfs_img',
|
||||
type=click.Path(exists=True, dir_okay=False, file_okay=True, path_type=Path),
|
||||
)
|
||||
def cli(btrfs_img: Path):
|
||||
"""
|
||||
Shrinks a Btrfs image
|
||||
// I cannot believe that Btrfs is over 15 years old,
|
||||
// yet there is no resize2fs tool which can shrink a disk image
|
||||
// to minimum size.
|
||||
// It cannot even tell you how much should be the right size,
|
||||
// it just randomly fails after which you have to umount and mount again.
|
||||
// So we have to make a loop which tries to shrink it until it fails.
|
||||
|
||||
// Also, WONTFIX bugs like how instead of telling you that
|
||||
// minimum fs size is 256 MB, it says "ERROR: unable to resize - Invalid argument"
|
||||
// https://bugzilla.kernel.org/show_bug.cgi?id=118111
|
||||
"""
|
||||
|
||||
if os.geteuid() != 0:
|
||||
sys.exit(' needs sudo')
|
||||
|
||||
current_dir = Path.cwd()
|
||||
|
||||
mnt_dir = Path(tempfile.mkdtemp(dir=current_dir, prefix='tmp_shrink_'))
|
||||
subprocess.run(['mount', '-t', 'btrfs', btrfs_img, mnt_dir], check=True)
|
||||
|
||||
# shink until max. 10 MB left or reached SMALLEST_SIZE or failure
|
||||
while True:
|
||||
# needs to start with a balancing
|
||||
# https://btrfs.readthedocs.io/en/latest/Balance.html
|
||||
# https://marc.merlins.org/perso/btrfs/post_2014-05-04_Fixing-Btrfs-Filesystem-Full-Problems.html
|
||||
do_balancing(mnt_dir)
|
||||
|
||||
free_bytes = get_usage(mnt_dir, 'Device unallocated')
|
||||
device_size = get_usage(mnt_dir, 'Device size')
|
||||
shrink_idea = free_bytes * 0.7
|
||||
|
||||
# workaround for the SMALLEST_SIZE limit
|
||||
if device_size - free_bytes < SMALLEST_SIZE:
|
||||
shrink_idea = (device_size - SMALLEST_SIZE) * 0.7
|
||||
|
||||
# stop if 10 MB left
|
||||
if shrink_idea < 10_000_000:
|
||||
break
|
||||
|
||||
# stop if process error
|
||||
if not do_shrink(mnt_dir, shrink_idea):
|
||||
break
|
||||
|
||||
total_size = get_usage(mnt_dir, 'Device size')
|
||||
|
||||
subprocess.run(['umount', mnt_dir])
|
||||
mnt_dir.rmdir()
|
||||
|
||||
subprocess.run(['truncate', '-s', str(total_size), btrfs_img])
|
||||
print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size')
|
||||
print('shrink_btrfs.py DONE')
|
||||
|
||||
|
||||
def get_usage(mnt: Path, key: str):
|
||||
p = subprocess.run(
|
||||
['btrfs', 'filesystem', 'usage', '-b', mnt], text=True, capture_output=True, check=True
|
||||
)
|
||||
for line in p.stdout.splitlines():
|
||||
if f'{key}:' not in line:
|
||||
continue
|
||||
free = int(line.split(':')[1])
|
||||
return free
|
||||
|
||||
|
||||
def do_shrink(mnt: Path, delta_size: float):
|
||||
delta_size = int(delta_size)
|
||||
print(f'Trying to shrink by {delta_size//1_000_000} MB')
|
||||
p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt])
|
||||
return p.returncode == 0
|
||||
|
||||
|
||||
def do_balancing(mnt: Path):
|
||||
print('Starting btrfs balancing')
|
||||
p = subprocess.run(
|
||||
['btrfs', 'balance', 'start', '-dusage=100', mnt], capture_output=True, text=True
|
||||
)
|
||||
if p.returncode:
|
||||
print(f'Balance error: {p.stdout} {p.stderr}')
|
||||
print('Balancing done')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
13
modules/tile_gen/setup.py
Normal file
13
modules/tile_gen/setup.py
Normal file
@@ -0,0 +1,13 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
||||
requirements = [
|
||||
'click',
|
||||
]
|
||||
|
||||
|
||||
setup(
|
||||
python_requires='>=3.10',
|
||||
install_requires=requirements,
|
||||
packages=find_packages(),
|
||||
)
|
||||
52
modules/tile_gen/tile_gen.py
Executable file
52
modules/tile_gen/tile_gen.py
Executable file
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import click
|
||||
from tile_gen_lib.btrfs import make_btrfs
|
||||
from tile_gen_lib.planetiler import run_planetiler
|
||||
from tile_gen_lib.rclone import make_indexes_for_bucket, upload_area
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""
|
||||
Generates tiles and uploads to CloudFlare
|
||||
"""
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument('area', required=True)
|
||||
@click.option('--upload', is_flag=True, help='Upload after generation is complete')
|
||||
def make_tiles(area, upload):
|
||||
"""
|
||||
Generate tiles for a given area, optionally upload it to the btrfs bucket
|
||||
"""
|
||||
|
||||
run_folder = run_planetiler(area)
|
||||
make_btrfs(run_folder)
|
||||
|
||||
if upload:
|
||||
upload_area(area)
|
||||
|
||||
|
||||
@cli.command(name='upload-area')
|
||||
@click.argument('area', required=True)
|
||||
def upload_area_(area):
|
||||
"""
|
||||
Upload all runs from a given area to the btrfs bucket
|
||||
"""
|
||||
|
||||
upload_area(area)
|
||||
|
||||
|
||||
@cli.command()
|
||||
def make_indexes():
|
||||
"""
|
||||
Make indexes for all buckets
|
||||
"""
|
||||
|
||||
for bucket in ['ofm-btrfs', 'ofm-assets']:
|
||||
make_indexes_for_bucket(bucket)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli()
|
||||
0
modules/tile_gen/tile_gen_lib/__init__.py
Normal file
0
modules/tile_gen/tile_gen_lib/__init__.py
Normal file
140
modules/tile_gen/tile_gen_lib/btrfs.py
Normal file
140
modules/tile_gen/tile_gen_lib/btrfs.py
Normal file
@@ -0,0 +1,140 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
from tile_gen_lib.utils import python_venv_executable
|
||||
|
||||
|
||||
IMAGE_SIZE = '200G'
|
||||
|
||||
|
||||
def make_btrfs(run_folder: Path):
|
||||
os.chdir(run_folder)
|
||||
|
||||
cleanup_folder(run_folder)
|
||||
|
||||
# make an empty file that's definitely bigger then the current OSM output
|
||||
for image in ['image.btrfs', 'image2.btrfs']:
|
||||
subprocess.run(['fallocate', '-l', IMAGE_SIZE, image], check=True)
|
||||
subprocess.run(['mkfs.btrfs', '-m', 'single', image], check=True, capture_output=True)
|
||||
|
||||
for image, mount in [('image.btrfs', 'mnt_rw'), ('image2.btrfs', 'mnt_rw2')]:
|
||||
Path(mount).mkdir()
|
||||
|
||||
# https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options
|
||||
# compression doesn't make sense, data is already gzip compressed
|
||||
subprocess.run(
|
||||
[
|
||||
'sudo',
|
||||
'mount',
|
||||
'-t',
|
||||
'btrfs',
|
||||
'-o',
|
||||
'noacl,nobarrier,noatime,max_inline=4096',
|
||||
image,
|
||||
mount,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
subprocess.run(['sudo', 'chown', 'ofm:ofm', '-R', mount], check=True)
|
||||
|
||||
# extract mbtiles
|
||||
extract_script = config.tile_gen_scripts_dir / 'extract_mbtiles.py'
|
||||
with open('extract_out.log', 'w') as out, open('extract_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
[
|
||||
python_venv_executable(),
|
||||
extract_script,
|
||||
'tiles.mbtiles',
|
||||
'mnt_rw/extract',
|
||||
],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
os.unlink('tiles.mbtiles')
|
||||
|
||||
shutil.copy('mnt_rw/extract/osm_date', '.')
|
||||
|
||||
# process logs
|
||||
subprocess.run('grep fixed extract_out.log > dedupl_fixed.log', shell=True)
|
||||
|
||||
# unfortunately, by deleting files from the btrfs partition, the partition size grows
|
||||
# so we need to rsync onto a new partition instead of deleting
|
||||
with open('rsync_out.log', 'w') as out, open('rsync_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
[
|
||||
'rsync',
|
||||
'-avH',
|
||||
'--max-alloc=4294967296',
|
||||
'--exclude',
|
||||
'dedupl',
|
||||
'mnt_rw/extract/',
|
||||
'mnt_rw2/',
|
||||
],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
# collect stats
|
||||
for i, mount in enumerate(['mnt_rw', 'mnt_rw2'], 1):
|
||||
with open(f'stats{i}.txt', 'w') as f:
|
||||
for cmd in [
|
||||
['df', '-h', mount],
|
||||
['btrfs', 'filesystem', 'df', mount],
|
||||
['btrfs', 'filesystem', 'show', mount],
|
||||
['btrfs', 'filesystem', 'usage', mount],
|
||||
]:
|
||||
f.write(f"\n\n{' '.join(cmd)}\n")
|
||||
result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True)
|
||||
f.write(result.stdout)
|
||||
|
||||
# unmount and cleanup
|
||||
for mount in ['mnt_rw', 'mnt_rw2']:
|
||||
subprocess.run(['sudo', 'umount', mount], check=True)
|
||||
|
||||
shutil.rmtree('mnt_rw')
|
||||
shutil.rmtree('mnt_rw2')
|
||||
|
||||
# shrink btrfs
|
||||
shrink_script = config.tile_gen_scripts_dir / 'shrink_btrfs.py'
|
||||
with open('shrink_out.log', 'w') as out, open('shrink_err.log', 'w') as err:
|
||||
subprocess.run(
|
||||
['sudo', python_venv_executable(), shrink_script, 'image2.btrfs'],
|
||||
check=True,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
)
|
||||
|
||||
os.unlink('image.btrfs')
|
||||
shutil.move('image2.btrfs', 'tiles.btrfs')
|
||||
|
||||
# parallel gzip (pigz)
|
||||
subprocess.run(['pigz', 'tiles.btrfs', '--fast'], check=True)
|
||||
|
||||
# move logs
|
||||
Path('logs').mkdir()
|
||||
for pattern in ['*.log', '*.txt']:
|
||||
for file in Path().glob(pattern):
|
||||
shutil.move(file, 'logs')
|
||||
|
||||
print('extract_btrfs.py DONE')
|
||||
|
||||
|
||||
def cleanup_folder(run_folder: Path):
|
||||
print(f'cleaning up {run_folder}')
|
||||
|
||||
for mount in ['mnt_rw', 'mnt_rw2']:
|
||||
subprocess.run(['sudo', 'umount', run_folder / mount], capture_output=True)
|
||||
|
||||
for pattern in ['mnt_rw*', 'tmp_*', '*.btrfs', '*.gz', '*.log', '*.txt', 'logs', 'osm_date']:
|
||||
for item in run_folder.glob(pattern):
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
21
modules/tile_gen/tile_gen_lib/config.py
Normal file
21
modules/tile_gen/tile_gen_lib/config.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Configuration:
|
||||
areas = ['planet', 'monaco']
|
||||
|
||||
tile_gen_dir = Path('/data/ofm/tile_gen')
|
||||
|
||||
tile_gen_bin = tile_gen_dir / 'bin'
|
||||
tile_gen_scripts_dir = tile_gen_bin / 'scripts'
|
||||
|
||||
planetiler_bin = tile_gen_dir / 'planetiler'
|
||||
planetiler_path = planetiler_bin / 'planetiler.jar'
|
||||
|
||||
runs_dir = tile_gen_dir / 'runs'
|
||||
|
||||
ofm_config_dir = Path('/data/ofm/config')
|
||||
rclone_config = ofm_config_dir / 'rclone.conf'
|
||||
|
||||
|
||||
config = Configuration()
|
||||
65
modules/tile_gen/tile_gen_lib/planetiler.py
Normal file
65
modules/tile_gen/tile_gen_lib/planetiler.py
Normal file
@@ -0,0 +1,65 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
from tile_gen_lib.btrfs import cleanup_folder
|
||||
|
||||
|
||||
def run_planetiler(area: str) -> Path:
|
||||
assert area in config.areas
|
||||
|
||||
date = datetime.now(tz=timezone.utc).strftime('%Y%m%d_%H%M%S')
|
||||
|
||||
area_dir = config.runs_dir / area
|
||||
|
||||
# delete all previous runs for the given area
|
||||
for subdir in area_dir.iterdir():
|
||||
cleanup_folder(subdir)
|
||||
|
||||
print('running rmtree')
|
||||
shutil.rmtree(area_dir, ignore_errors=True)
|
||||
print('rmtree done')
|
||||
|
||||
run_folder = area_dir / f'{date}_pt'
|
||||
run_folder.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
os.chdir(run_folder)
|
||||
|
||||
# link to discussion about why exactly 30 GB
|
||||
# https://github.com/onthegomap/planetiler/discussions/690#discussioncomment-7756397
|
||||
java_memory_gb = 30 if area == 'planet' else 1
|
||||
|
||||
command = [
|
||||
'java',
|
||||
f'-Xmx{java_memory_gb}g',
|
||||
'-jar',
|
||||
config.planetiler_path,
|
||||
f'--area={area}',
|
||||
'--download',
|
||||
'--download-threads=10',
|
||||
'--download-chunk-size-mb=1000',
|
||||
'--fetch-wikidata',
|
||||
'--output=tiles.mbtiles',
|
||||
'--nodemap-type=array',
|
||||
'--storage=mmap',
|
||||
'--force',
|
||||
]
|
||||
|
||||
if area == 'planet':
|
||||
command.append('--bounds=planet')
|
||||
|
||||
print(command)
|
||||
|
||||
out_path = run_folder / 'planetiler.out'
|
||||
err_path = run_folder / 'planetiler.err'
|
||||
|
||||
with out_path.open('w') as out_file, err_path.open('w') as err_file:
|
||||
subprocess.run(command, stdout=out_file, stderr=err_file, check=True, cwd=run_folder)
|
||||
|
||||
shutil.rmtree(run_folder / 'data', ignore_errors=True)
|
||||
print('planetiler.jar DONE')
|
||||
|
||||
return run_folder
|
||||
123
modules/tile_gen/tile_gen_lib/rclone.py
Normal file
123
modules/tile_gen/tile_gen_lib/rclone.py
Normal file
@@ -0,0 +1,123 @@
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from tile_gen_lib.config import config
|
||||
|
||||
|
||||
def upload_area(area):
|
||||
"""
|
||||
Uploads an area, making sure there is exactly one run present
|
||||
"""
|
||||
|
||||
print(f'Uploading area: {area}')
|
||||
|
||||
assert area in config.areas
|
||||
|
||||
area_dir = config.runs_dir / area
|
||||
if not area_dir.exists():
|
||||
return
|
||||
|
||||
runs = list(area_dir.iterdir())
|
||||
if len(runs) != 1:
|
||||
print('Error: Make sure there is only one run in the given area')
|
||||
sys.exit(1)
|
||||
|
||||
run = runs[0].name
|
||||
|
||||
upload_area_run(area, run)
|
||||
make_indexes_for_bucket('ofm-btrfs')
|
||||
|
||||
|
||||
def upload_area_run(area, run):
|
||||
print(f'Uploading {area} {run} to btrfs bucket')
|
||||
|
||||
run_dir = config.runs_dir / area / run
|
||||
assert run_dir.is_dir()
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'sync',
|
||||
'--verbose=1',
|
||||
'--transfers=8',
|
||||
'--multi-thread-streams=8',
|
||||
'--fast-list',
|
||||
'--stats-file-name-length=0',
|
||||
'--stats-one-line',
|
||||
'--log-file',
|
||||
run_dir / 'logs' / 'rclone.log',
|
||||
'--exclude',
|
||||
'logs/**',
|
||||
run_dir,
|
||||
f'remote:ofm-btrfs/areas/{area}/{run}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def make_indexes_for_bucket(bucket):
|
||||
print(f'Making indexes for bucket: {bucket}')
|
||||
|
||||
# files
|
||||
p = subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'lsf',
|
||||
'--recursive',
|
||||
'--files-only',
|
||||
'--fast-list',
|
||||
'--exclude',
|
||||
'dirs.txt',
|
||||
'--exclude',
|
||||
'files.txt',
|
||||
f'remote:{bucket}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
index_str = p.stdout
|
||||
|
||||
# upload to files.txt
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'rcat',
|
||||
f'remote:{bucket}/files.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
input=index_str.encode(),
|
||||
)
|
||||
|
||||
# directories
|
||||
p = subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'lsf',
|
||||
'--recursive',
|
||||
'--dirs-only',
|
||||
'--dir-slash=false',
|
||||
'--fast-list',
|
||||
f'remote:{bucket}',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
index_str = p.stdout
|
||||
|
||||
# upload to dirs.txt
|
||||
subprocess.run(
|
||||
[
|
||||
'rclone',
|
||||
'rcat',
|
||||
f'remote:{bucket}/dirs.txt',
|
||||
],
|
||||
env=dict(RCLONE_CONFIG=config.rclone_config),
|
||||
check=True,
|
||||
input=index_str.encode(),
|
||||
)
|
||||
14
modules/tile_gen/tile_gen_lib/utils.py
Normal file
14
modules/tile_gen/tile_gen_lib/utils.py
Normal file
@@ -0,0 +1,14 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def python_venv_executable() -> Path:
|
||||
venv_path = os.environ.get('VIRTUAL_ENV')
|
||||
|
||||
if venv_path:
|
||||
return Path(venv_path) / 'bin' / 'python'
|
||||
elif sys.prefix != sys.base_prefix:
|
||||
return Path(sys.prefix) / 'bin' / 'python'
|
||||
else:
|
||||
return Path(sys.executable)
|
||||
Reference in New Issue
Block a user