diff --git a/.ruff.toml b/.ruff.toml index 2723f2e..687871b 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -29,7 +29,6 @@ lint.ignore = [ 'E741', 'F401', # unused imports 'F841', - 'PT004', 'SIM102', 'SIM105', 'SIM108', diff --git a/modules/http_host/http_host_lib/nginx.py b/modules/http_host/http_host_lib/nginx.py index d7f3611..4ef8f09 100644 --- a/modules/http_host/http_host_lib/nginx.py +++ b/modules/http_host/http_host_lib/nginx.py @@ -206,6 +206,7 @@ def create_version_location( add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; add_header x-ofm-debug 'specific JSON {area} {version}'; }} @@ -224,6 +225,7 @@ def create_version_location( add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; add_header x-ofm-debug 'specific PBF {area} {version}'; }} @@ -268,6 +270,7 @@ def create_latest_locations(*, local: str, domain: str) -> str: add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; add_header x-ofm-debug 'latest JSON {area}'; }} @@ -289,6 +292,7 @@ def create_latest_locations(*, local: str, domain: str) -> str: add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; add_header x-ofm-debug 'wildcard JSON {area}'; }} @@ -309,6 +313,7 @@ def create_latest_locations(*, local: str, domain: str) -> str: add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; add_header x-ofm-debug 'wildcard PBF {area}'; }} diff --git a/modules/http_host/http_host_lib/nginx_confs/le.conf b/modules/http_host/http_host_lib/nginx_confs/le.conf index 700e42b..5ba04fd 100644 --- a/modules/http_host/http_host_lib/nginx_confs/le.conf +++ b/modules/http_host/http_host_lib/nginx_confs/le.conf @@ -22,12 +22,16 @@ server { ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; ssl_prefer_server_ciphers off; + # access log doesn't contain IP address access_log off; #access_log /data/ofm/http_host/logs_nginx/le-access.jsonl access_json buffer=128k; error_log /data/ofm/http_host/logs_nginx/le-error.log; + add_header X-Robots-Tag "noindex, nofollow" always; + + location ^~ /.well-known/acme-challenge/ { # trailing slash root /data/nginx/acme-challenges; @@ -52,6 +56,7 @@ server { add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; } # catch-all block to deny all other requests diff --git a/modules/http_host/http_host_lib/nginx_confs/location_static.conf b/modules/http_host/http_host_lib/nginx_confs/location_static.conf index c7075d5..9d62409 100644 --- a/modules/http_host/http_host_lib/nginx_confs/location_static.conf +++ b/modules/http_host/http_host_lib/nginx_confs/location_static.conf @@ -1,3 +1,9 @@ +# Serve robots.txt that blocks all crawlers +#location = /robots.txt { +# add_header Content-Type text/plain; +#return 200 "User-agent: *\nDisallow: /\n"; +#} + location /fonts/ { # trailing slash @@ -8,6 +14,8 @@ location /fonts/ { add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; + } location /natural_earth/ { @@ -20,6 +28,8 @@ location /natural_earth/ { add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; + } location /sprites/ { @@ -32,6 +42,8 @@ location /sprites/ { add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; + } @@ -47,6 +59,7 @@ location @empty_tile { add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; add_header x-ofm-debug 'empty tile'; } diff --git a/modules/http_host/http_host_lib/nginx_confs/roundrobin.conf b/modules/http_host/http_host_lib/nginx_confs/roundrobin.conf index 0a5afa2..30d1e45 100644 --- a/modules/http_host/http_host_lib/nginx_confs/roundrobin.conf +++ b/modules/http_host/http_host_lib/nginx_confs/roundrobin.conf @@ -28,6 +28,9 @@ server { error_log /data/ofm/http_host/logs_nginx/roundrobin-error.log; + add_header X-Robots-Tag "noindex, nofollow" always; + + __LOCATION_BLOCKS__ location /styles/ { @@ -46,6 +49,8 @@ server { add_header 'Access-Control-Allow-Origin' '*' always; add_header Cache-Control public; + add_header X-Robots-Tag "noindex, nofollow" always; + } # catch-all block to deny all other requests diff --git a/modules/tile_gen/scripts/shrink_btrfs.py b/modules/tile_gen/scripts/shrink_btrfs.py index ad99bcc..e5b91a5 100755 --- a/modules/tile_gen/scripts/shrink_btrfs.py +++ b/modules/tile_gen/scripts/shrink_btrfs.py @@ -69,7 +69,7 @@ def cli(btrfs_img: Path): mnt_dir.rmdir() subprocess.run(['truncate', '-s', str(total_size), btrfs_img]) - print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size') + print(f'Truncated {btrfs_img} to {total_size // 1_000_000} MB size') print('shrink_btrfs.py DONE') @@ -86,7 +86,7 @@ def get_usage(mnt: Path, key: str): def do_shrink(mnt: Path, delta_size: float): delta_size = int(delta_size) - print(f'Trying to shrink by {delta_size//1_000_000} MB') + print(f'Trying to shrink by {delta_size // 1_000_000} MB') p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt]) return p.returncode == 0 diff --git a/modules/tile_gen/tile_gen_lib/btrfs.py b/modules/tile_gen/tile_gen_lib/btrfs.py index 089be79..353a0a2 100644 --- a/modules/tile_gen/tile_gen_lib/btrfs.py +++ b/modules/tile_gen/tile_gen_lib/btrfs.py @@ -88,7 +88,7 @@ def make_btrfs(run_folder: Path): ['btrfs', 'filesystem', 'show', mount], ['btrfs', 'filesystem', 'usage', mount], ]: - f.write(f"\n\n{' '.join(cmd)}\n") + f.write(f'\n\n{" ".join(cmd)}\n') result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True) f.write(result.stdout) diff --git a/website/astro.config.mjs b/website/astro.config.mjs index 6852d98..59ccc67 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -1,7 +1,7 @@ // @ts-check import { defineConfig } from 'astro/config' -import sitemap from '@astrojs/sitemap'; +import sitemap from '@astrojs/sitemap' // https://astro.build/config diff --git a/website/package.json b/website/package.json index 2bf7176..7f0ae49 100644 --- a/website/package.json +++ b/website/package.json @@ -15,9 +15,6 @@ "lightningcss": "^1.29.1" }, "pnpm": { - "onlyBuiltDependencies": [ - "esbuild", - "sharp" - ] + "onlyBuiltDependencies": ["esbuild", "sharp"] } } diff --git a/website/src/layouts/Layout.astro b/website/src/layouts/Layout.astro index e4eafd3..35e83a6 100644 --- a/website/src/layouts/Layout.astro +++ b/website/src/layouts/Layout.astro @@ -1,7 +1,7 @@ --- const { frontmatter } = Astro.props || {} const { title } = frontmatter || Astro.props -const canonicalURL = new URL(Astro.url.pathname, Astro.site); +const canonicalURL = new URL(Astro.url.pathname, Astro.site) import '../styles/_style.css' ---