sysadmin/openfreemap
1
0
Fork 0
mirror of https://github.com/hyperknot/openfreemap.git synced 2026-05-21 14:02:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

X-Robots-Tag "noindex, nofollow"

Browse Source
This commit is contained in:
Zsolt Ero
2025-02-27 16:52:31 +01:00
parent d76a877f1f
commit 6687311f9a
10 changed files with 34 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.ruff.toml
View File

@@ -29,7 +29,6 @@ lint.ignore = [
'E741', 'E741',
'F401', # unused imports 'F401', # unused imports
'F841', 'F841',
'PT004',
'SIM102', 'SIM102',
'SIM105', 'SIM105',
'SIM108', 'SIM108',

5
modules/http_host/http_host_lib/nginx.py
View File

@@ -206,6 +206,7 @@ def create_version_location(
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
add_header x-ofm-debug 'specific JSON {area} {version}'; add_header x-ofm-debug 'specific JSON {area} {version}';
}} }}
@@ -224,6 +225,7 @@ def create_version_location(
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
add_header x-ofm-debug 'specific PBF {area} {version}'; add_header x-ofm-debug 'specific PBF {area} {version}';
}} }}
@@ -268,6 +270,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
add_header x-ofm-debug 'latest JSON {area}'; add_header x-ofm-debug 'latest JSON {area}';
}} }}
@@ -289,6 +292,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
add_header x-ofm-debug 'wildcard JSON {area}'; add_header x-ofm-debug 'wildcard JSON {area}';
}} }}
@@ -309,6 +313,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
add_header x-ofm-debug 'wildcard PBF {area}'; add_header x-ofm-debug 'wildcard PBF {area}';
}} }}

5
modules/http_host/http_host_lib/nginx_confs/le.conf
View File

@@ -22,12 +22,16 @@ server {
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
ssl_prefer_server_ciphers off; ssl_prefer_server_ciphers off;
# access log doesn't contain IP address # access log doesn't contain IP address
access_log off; access_log off;
#access_log /data/ofm/http_host/logs_nginx/le-access.jsonl access_json buffer=128k; #access_log /data/ofm/http_host/logs_nginx/le-access.jsonl access_json buffer=128k;
error_log /data/ofm/http_host/logs_nginx/le-error.log; error_log /data/ofm/http_host/logs_nginx/le-error.log;
add_header X-Robots-Tag "noindex, nofollow" always;
location ^~ /.well-known/acme-challenge/ { location ^~ /.well-known/acme-challenge/ {
# trailing slash # trailing slash
root /data/nginx/acme-challenges; root /data/nginx/acme-challenges;
@@ -52,6 +56,7 @@ server {
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
} }
# catch-all block to deny all other requests # catch-all block to deny all other requests

13
modules/http_host/http_host_lib/nginx_confs/location_static.conf
View File

@@ -1,3 +1,9 @@
# Serve robots.txt that blocks all crawlers
#location = /robots.txt {
# add_header Content-Type text/plain;
#return 200 "User-agent: *\nDisallow: /\n";
#}
location /fonts/ { location /fonts/ {
# trailing slash # trailing slash
@@ -8,6 +14,8 @@ location /fonts/ {
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
} }
location /natural_earth/ { location /natural_earth/ {
@@ -20,6 +28,8 @@ location /natural_earth/ {
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
} }
location /sprites/ { location /sprites/ {
@@ -32,6 +42,8 @@ location /sprites/ {
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
} }
@@ -47,6 +59,7 @@ location @empty_tile {
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
add_header x-ofm-debug 'empty tile'; add_header x-ofm-debug 'empty tile';
} }

5
modules/http_host/http_host_lib/nginx_confs/roundrobin.conf
View File

@@ -28,6 +28,9 @@ server {
error_log /data/ofm/http_host/logs_nginx/roundrobin-error.log; error_log /data/ofm/http_host/logs_nginx/roundrobin-error.log;
add_header X-Robots-Tag "noindex, nofollow" always;
__LOCATION_BLOCKS__ __LOCATION_BLOCKS__
location /styles/ { location /styles/ {
@@ -46,6 +49,8 @@ server {
add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Origin' '*' always;
add_header Cache-Control public; add_header Cache-Control public;
add_header X-Robots-Tag "noindex, nofollow" always;
} }
# catch-all block to deny all other requests # catch-all block to deny all other requests

4
modules/tile_gen/scripts/shrink_btrfs.py
View File

@@ -69,7 +69,7 @@ def cli(btrfs_img: Path):
mnt_dir.rmdir() mnt_dir.rmdir()
subprocess.run(['truncate', '-s', str(total_size), btrfs_img]) subprocess.run(['truncate', '-s', str(total_size), btrfs_img])
print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size') print(f'Truncated {btrfs_img} to {total_size // 1_000_000} MB size')
print('shrink_btrfs.py DONE') print('shrink_btrfs.py DONE')
@@ -86,7 +86,7 @@ def get_usage(mnt: Path, key: str):
def do_shrink(mnt: Path, delta_size: float): def do_shrink(mnt: Path, delta_size: float):
delta_size = int(delta_size) delta_size = int(delta_size)
print(f'Trying to shrink by {delta_size//1_000_000} MB') print(f'Trying to shrink by {delta_size // 1_000_000} MB')
p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt]) p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt])
return p.returncode == 0 return p.returncode == 0

2
modules/tile_gen/tile_gen_lib/btrfs.py
View File

@@ -88,7 +88,7 @@ def make_btrfs(run_folder: Path):
['btrfs', 'filesystem', 'show', mount], ['btrfs', 'filesystem', 'show', mount],
['btrfs', 'filesystem', 'usage', mount], ['btrfs', 'filesystem', 'usage', mount],
]: ]:
f.write(f"\n\n{' '.join(cmd)}\n") f.write(f'\n\n{" ".join(cmd)}\n')
result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True) result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True)
f.write(result.stdout) f.write(result.stdout)

2
website/astro.config.mjs
View File

@@ -1,7 +1,7 @@
// @ts-check // @ts-check
import { defineConfig } from 'astro/config' import { defineConfig } from 'astro/config'
import sitemap from '@astrojs/sitemap'; import sitemap from '@astrojs/sitemap'
// https://astro.build/config // https://astro.build/config

5
website/package.json
View File

@@ -15,9 +15,6 @@
"lightningcss": "^1.29.1" "lightningcss": "^1.29.1"
}, },
"pnpm": { "pnpm": {
"onlyBuiltDependencies": [ "onlyBuiltDependencies": ["esbuild", "sharp"]
"esbuild",
"sharp"
]
} }
} }

2
website/src/layouts/Layout.astro
View File

@@ -1,7 +1,7 @@
--- ---
const { frontmatter } = Astro.props || {} const { frontmatter } = Astro.props || {}
const { title } = frontmatter || Astro.props const { title } = frontmatter || Astro.props
const canonicalURL = new URL(Astro.url.pathname, Astro.site); const canonicalURL = new URL(Astro.url.pathname, Astro.site)
import '../styles/_style.css' import '../styles/_style.css'
--- ---