start

.envrc

@@ -0,0 +1,7 @@
+# used by direnv to
+# auto-activate python virtualenv
+# https://github.com/direnv/direnv
+
+source venv/bin/activate
+
+unset PS1

.gitignore

@@ -0,0 +1,6 @@
+*.pyc
+*.egg-info
+
+.DS_Store
+/venv
+/.idea

.ruff.toml

@@ -0,0 +1,43 @@
+target-version = "py310"
+line-length = 100
+
+
+select = [
+  "E",   # pycodestyle errors
+  "W",   # pycodestyle warnings
+  "F",   # pyflakes
+  "I",   # isort
+  'UP',  # pyupgrade
+  'A',   # flake8-builtins
+  "C4",  # flake8-comprehensions
+  'EXE', # flake8-executable
+  'FA',  # flake8-future-annotations
+  'PT',  # flake8-pytest-style
+  'RSE', # flake8-raise
+  'SIM', # flake8-simplify
+  'DTZ', # flake8-datetimez, https://beta.ruff.rs/docs/rules/#flake8-datetimez-dtz
+]
+
+ignore = [
+  'E501',
+  'E711',
+  'E712',
+  'E741',
+  'A003',
+  'PT004',
+  'SIM108',
+  'SIM102',
+  'SIM105',
+  'SIM115',
+  'F841',
+]
+
+[format]
+quote-style = "single"
+
+[isort]
+known-first-party = ["openfreemaps"]
+lines-after-imports = 2
+
+[flake8-comprehensions]
+allow-dict-calls-with-keyword-arguments = true

init-server.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+from fabric import Connection
+
+from openfreemaps.kernel import setup_kernel_settings
+from openfreemaps.nginx import certbot, nginx
+
+
+def prepare_server(c):
+    setup_kernel_settings(c)
+
+    nginx(c)
+    certbot(c)
+
+
+c = Connection(host='map128')
+
+prepare_server(c)

kernel-ideas.txt

@@ -0,0 +1,10 @@
+tcp_tw_reuse
+tcp_fin_timeout
+tcp_max_syn_backlog
+TCP max buffer size
+
+Increase File Descriptors Limit
+
+Disable Swapping
+
+

nginx-ideas.txt

@@ -0,0 +1,61 @@
+# ideas https://calomel.org/nginx.html
+
+open_file_cache
+tcp_nodelay
+
+client_body_buffer_size
+client_max_body_size
+client_header_buffer_size 1k;
+large_client_header_buffers 4 8k;
+server_tokens 1k;
+
+Disable Access Logs
+Enable HTTP/2 or HTTP/3
+SSL Session Cache
+SSL OCSP Stapling
+
+keepalive_timeout 65;
+types_hash_max_size 2048;
+
+# SSL optimizations
+ssl_session_cache shared:SSL:10m;
+ssl_session_timeout 10m;
+ssl_prefer_server_ciphers on;
+
+# Caching
+open_file_cache max=10000 inactive=20s;
+open_file_cache_valid 30s;
+open_file_cache_min_uses 2;
+open_file_cache_errors on;
+
+multi_accept on
+
+
+client_body_timeout 12;
+client_header_timeout 12;
+send_timeout 10;
+
+
+# gzip
+gzip on;
+gzip_types text/plain text/css application/javascript application/json image/svg+xml;
+gzip_vary on;
+gzip_min_length 10240;
+gzip_comp_level 5;
+gzip_proxied any;
+
+access_log /var/log/nginx/access.log main buffer=32k;
+
+# https://www.nginx.com/blog/tuning-nginx/
+net.core.somaxconn - backlog
+net.core.netdev_max_backlog
+
+sys.fs.file-max
+nofile
+
+keepalive_requests
+keepalive_timeout
+keepalive
+
+
+https://github.com/denji/nginx-tuning

openfreemaps/config.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+
+base = Path(__file__).parent.parent
+templates = base / 'templates'

openfreemaps/kernel.py

@@ -0,0 +1,22 @@
+from openfreemaps.config import templates
+from openfreemaps.utils import apt_get_install, apt_get_purge, put, put_str
+
+
+def setup_kernel_settings(c):
+    put(c, f'{templates}/sysctl/60-optim.conf', '/etc/sysctl.d/')
+
+    set_cpu_governor(c)
+
+
+def set_cpu_governor(c):
+    apt_get_install(c, 'cpufrequtils')
+    apt_get_purge(c, 'linux-tools-*')
+
+    put_str(
+        c,
+        '/etc/default/cpufrequtils',
+        'GOVERNOR="performance"',
+    )
+
+    # check after reboot
+    # cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor

openfreemaps/nginx.py

@@ -0,0 +1,61 @@
+from openfreemaps.config import templates
+from openfreemaps.utils import (
+    apt_get_install,
+    apt_get_purge,
+    apt_get_update,
+    exists,
+    put,
+    put_str,
+    sudo_cmd,
+    ubuntu_codename,
+)
+
+
+def nginx(c):
+    codename = ubuntu_codename(c)
+
+    if not exists(c, '/usr/sbin/nginx'):
+        put_str(
+            c,
+            '/etc/apt/sources.list.d/nginx.list',
+            f'deb http://nginx.org/packages/mainline/ubuntu {codename} nginx',
+        )
+        sudo_cmd(
+            c,
+            'wget --quiet -O - http://nginx.org/keys/nginx_signing.key | apt-key add -',
+        )
+        apt_get_update(c)
+        apt_get_install(c, 'nginx')
+
+    c.sudo('rm -rf /data/nginx/config')
+    c.sudo('mkdir -p /data/nginx/config')
+
+    c.sudo('rm -rf /data/nginx/logs')
+    c.sudo('mkdir -p /data/nginx/logs')
+
+    c.sudo('mkdir -p /data/nginx/sites')
+
+    if not exists(c, '/etc/nginx/ssl/dummy.crt'):
+        c.sudo('mkdir -p /etc/nginx/ssl')
+        c.sudo(
+            'openssl req -x509 -nodes -days 365 -newkey rsa:2048 '
+            '-keyout /etc/nginx/ssl/dummy.key -out /etc/nginx/ssl/dummy.crt '
+            '-subj "/C=US/ST=Dummy/L=Dummy/O=Dummy/CN=example.com"'
+        )
+
+    put(c, f'{templates}/nginx/nginx.conf', '/etc/nginx/')
+    put(c, f'{templates}/nginx/default_disable.conf', '/data/nginx/sites')
+
+    c.sudo('service nginx restart')
+
+
+def certbot(c):
+    # https://certbot.eff.org/lets-encrypt/ubuntubionic-nginx
+    apt_get_install(c, 'snapd')
+    c.run('snap install core', warn=True)
+    c.run('snap refresh core', warn=True)
+
+    apt_get_purge(c, 'certbot')
+    c.run('snap install --classic certbot', warn=True)
+    c.run('snap set certbot trust-plugin-with-root=ok')
+    c.run('snap install certbot-dns-cloudflare')

openfreemaps/utils.py

@@ -0,0 +1,103 @@
+import os
+import secrets
+import string
+
+
+def put(c, local_path, remote_path, permissions=None, owner='root', group=None):
+    tmp_path = f'/tmp/fabtmp_{random_string(8)}'
+    c.put(local_path, tmp_path)
+
+    if is_dir(c, remote_path):
+        if not remote_path.endswith('/'):
+            remote_path += '/'
+
+        filename = os.path.basename(local_path)
+        remote_path += filename
+
+    c.sudo(f"mv '{tmp_path}' '{remote_path}'")
+    c.sudo(f"rm -rf '{tmp_path}'")
+
+    set_permission(c, remote_path, permissions, owner, group)
+
+
+def put_str(c, remote_path, str_):
+    tmp_file = 'tmp.txt'
+    with open(tmp_file, 'w') as outfile:
+        outfile.write(str_ + '\n')
+    put(c, tmp_file, remote_path)
+    os.remove(tmp_file)
+
+
+def append_str(c, remote_path, str_):
+    tmp_path = f'/tmp/fabtmp_{random_string(8)}'
+    put_str(c, tmp_path, str_)
+
+    sudo_cmd(c, f"cat '{tmp_path}' >> '{remote_path}'")
+    c.sudo(f'rm -f {tmp_path}')
+
+
+def sudo_cmd(c, cmd):
+    cmd = cmd.replace('"', '\\"')
+    c.sudo(f'bash -c "{cmd}"')
+
+
+def set_permission(c, path, permissions=None, owner=None, group=None):
+    if owner:
+        if not group:
+            group = owner
+
+        c.sudo(f"chown {owner}:{group} '{path}'")
+
+    if permissions:
+        c.sudo(f"chmod {permissions} '{path}'")
+
+
+def reboot(c):
+    print('Rebooting')
+    try:
+        c.sudo('reboot')
+    except Exception:
+        pass
+
+
+def exists(c, path):
+    return c.sudo(f"test -e '{path}'", hide=True, warn=True).ok
+
+
+def is_dir(c, path):
+    return c.sudo(f"test -d '{path}'", hide=True, warn=True).ok
+
+
+def random_string(length):
+    return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(length))
+
+
+def ubuntu_release(c):
+    return c.run('lsb_release -rs').stdout.strip()[:2]
+
+
+def ubuntu_codename(c):
+    return c.run('lsb_release -cs').stdout.strip()
+
+
+def apt_get_update(c):
+    c.sudo('apt-get update')
+
+
+def apt_get_install(c, pkgs, warn=False):
+    c.sudo(
+        f'DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {pkgs}',
+        warn=warn,
+    )
+
+
+def apt_get_purge(c, pkgs):
+    c.sudo(f'DEBIAN_FRONTEND=noninteractive apt-get purge -y {pkgs}')
+
+
+def apt_get_autoremove(c):
+    c.sudo('DEBIAN_FRONTEND=noninteractive apt-get autoremove -y')
+
+
+def get_username(c):
+    return c.run('whoami').stdout.strip()

prepare-virtualenv.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+find . -name "*.egg-info" -exec rm -rf {} +
+find . -name __pycache__ -exec rm -rf {} +
+
+# deactivate
+rm -rf venv
+python3 -m venv venv
+source venv/bin/activate
+
+pip install -U pip wheel setuptools
+pip install -e .
+
+

setup.py

@@ -0,0 +1,11 @@
+from setuptools import setup
+
+
+requirements = ['fabric', 'ruff']
+
+setup(
+    python_requires='>=3.10',
+    install_requires=requirements,
+    name='openfreemaps',
+    packages=['openfreemaps'],
+)

templates/nginx/default_disable.conf

@@ -0,0 +1,20 @@
+map "" $empty {
+        default "";
+}
+
+server {
+        listen 80 default_server;
+        listen [::]:80 default_server;
+
+        listen 443 ssl default_server;
+        listen [::]:443 ssl default_server;
+        http2 on;
+
+        server_name _;
+
+        ssl_ciphers aNULL;
+        ssl_certificate /etc/nginx/ssl/dummy.crt;
+        ssl_certificate_key /etc/nginx/ssl/dummy.key;
+
+        return 444;
+}

templates/nginx/nginx.conf

@@ -0,0 +1,79 @@
+# ubuntu specific
+user nginx;
+pid /var/run/nginx.pid;
+
+# universal
+
+worker_processes auto;
+worker_rlimit_nofile 100000;
+
+error_log /data/nginx/logs/nginx-error.log warn;
+
+events {
+    worker_connections 8000;
+}
+
+http {
+    open_file_cache max=200000 inactive=20s;
+    open_file_cache_valid 30s;
+    open_file_cache_min_uses 2;
+    open_file_cache_errors on;
+
+    server_tokens off;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    charset utf-8;
+
+    sendfile on;
+    tcp_nopush on;
+
+    reset_timedout_connection on;
+    client_body_timeout 10;
+    send_timeout 2;
+    keepalive_timeout 30;
+    keepalive_requests 100000;
+
+    max_ranges 0;
+
+    gzip on;
+    gzip_comp_level 1;
+    gzip_min_length 256;
+    gzip_proxied any;
+    gzip_vary on;
+
+    gzip_types
+        text/plain;
+
+    log_format access_json '{'
+        '"time": "$time_iso8601", '
+        '"msec": "$msec", '
+        '"status": $status, '
+        '"request": "$request", '
+        '"request_method": "$request_method", '
+        '"request_time": $request_time, '
+        '"body_bytes_sent": $body_bytes_sent, '
+        '"remote_addr": "$remote_addr", '
+        '"remote_user": "$remote_user", '
+        '"http_referrer": "$http_referer", '
+        '"http_x_forwarded_for": "$http_x_forwarded_for", '
+        '"http_user_agent": "$http_user_agent", '
+        '"upstream_response_time": "$upstream_response_time", '
+       # '"upstream_connect_time": "$upstream_connect_time", '
+        '"upstream_header_time": "$upstream_header_time", '
+        '"upstream_cache_status": "$upstream_cache_status", '
+        '"host": "$host", '
+        '"uri": "$uri", '
+        '"http_cf_connecting_ip": "$http_cf_connecting_ip", '
+        '"http_cf_ray": "$http_cf_ray", '
+        '"http_cf_ipcountry": "$http_cf_ipcountry", '
+        '"scheme": "$scheme", '
+        '"http_host": "$http_host"'
+    '}';
+
+    access_log /data/nginx/logs/nginx-access.log access_json;
+
+    include /data/nginx/config/*;
+    include /data/nginx/sites/*;
+}

templates/sysctl/60-optim.conf

@@ -0,0 +1,4 @@
+vm.swappiness = 1
+
+net.core.somaxconn = 65535
+