diff --git a/README.md b/README.md index cb191bf..42c282f 100644 --- a/README.md +++ b/README.md @@ -1 +1,4 @@ # openfreemap + +# TODO +not deleted dedupl files \ No newline at end of file diff --git a/config/limits/limits-ofm.conf b/config/limits/limits-ofm.conf new file mode 100644 index 0000000..867db5f --- /dev/null +++ b/config/limits/limits-ofm.conf @@ -0,0 +1,4 @@ +* soft nofile 1048576 +* hard nofile 1048576 +root soft nofile 1048576 +root hard nofile 1048576 \ No newline at end of file diff --git a/config/nginx/nginx.conf b/config/nginx/nginx.conf index bf4711c..f28f9d2 100644 --- a/config/nginx/nginx.conf +++ b/config/nginx/nginx.conf @@ -59,10 +59,6 @@ http { '"http_referrer": "$http_referer", ' '"http_x_forwarded_for": "$http_x_forwarded_for", ' '"http_user_agent": "$http_user_agent", ' - '"upstream_response_time": "$upstream_response_time", ' - # '"upstream_connect_time": "$upstream_connect_time", ' - '"upstream_header_time": "$upstream_header_time", ' - '"upstream_cache_status": "$upstream_cache_status", ' '"host": "$host", ' '"uri": "$uri", ' '"http_cf_connecting_ip": "$http_cf_connecting_ip", ' diff --git a/config/sysctl/60-optim.conf b/config/sysctl/60-ofm.conf similarity index 68% rename from config/sysctl/60-optim.conf rename to config/sysctl/60-ofm.conf index fa3c22f..a2d46f9 100644 --- a/config/sysctl/60-optim.conf +++ b/config/sysctl/60-ofm.conf @@ -1,4 +1,4 @@ vm.swappiness = 1 net.core.somaxconn = 65535 - +fs.file-max = 100000 diff --git a/ideas.txt b/ideas.txt deleted file mode 100644 index 06a78f5..0000000 --- a/ideas.txt +++ /dev/null @@ -1 +0,0 @@ -specify UID when creating user \ No newline at end of file diff --git a/init-server.py b/init-server.py index 22e2fa8..711f243 100755 --- a/init-server.py +++ b/init-server.py @@ -5,9 +5,10 @@ import click from dotenv import dotenv_values from fabric import Config, Connection -from ssh_lib.config import scripts +from ssh_lib.benchmark import benchmark, c1000k, k6 +from ssh_lib.config import config, scripts from ssh_lib.kernel import set_cpu_governor, setup_kernel_settings -from ssh_lib.nginx import certbot, k6, nginx +from ssh_lib.nginx import certbot, nginx from ssh_lib.pkg_base import pkg_base, pkg_clean, pkg_upgrade from ssh_lib.planetiler import TILE_GEN_BIN, install_planetiler from ssh_lib.utils import add_user, enable_sudo, put, reboot, setup_time, sudo_cmd @@ -52,7 +53,18 @@ def prepare_tile_gen(c): def prepare_http_host(c): nginx(c) certbot(c) - k6(c) + c1000k(c) + + +def debug_tmp(c): + c.sudo('rm -rf /data/ofm/logs') + c.sudo('mkdir -p /data/ofm/logs') + put(c, f'{config}/nginx/nginx.conf', '/etc/nginx/') + put(c, f'{scripts}/http_host/nginx_site.conf', '/data/nginx/sites') + c.sudo('nginx -t') + c.sudo('service nginx restart') + + benchmark(c) @click.command() @@ -110,9 +122,5 @@ def main(hostname, user, port, tile_gen, http_host, skip_shared, do_reboot, debu reboot(c) -def debug_tmp(c): - k6(c) - - if __name__ == '__main__': main() diff --git a/kernel-ideas.txt b/kernel-ideas.txt index c73fe08..a32a98e 100644 --- a/kernel-ideas.txt +++ b/kernel-ideas.txt @@ -3,8 +3,19 @@ tcp_fin_timeout tcp_max_syn_backlog TCP max buffer size -Increase File Descriptors Limit - Disable Swapping +fs.file-max # not needed, recent Ubuntu has it on high by default +net.core.netdev_max_backlog = 262144 +net.core.rmem_max = 16777216 +net.core.somaxconn = 262144 +net.core.wmem_max = 16777216 +net.ipv4.tcp_max_syn_backlog = 262144 +net.ipv4.tcp_max_tw_buckets = 6000000 +net.ipv4.tcp_no_metrics_save = 1 +net.ipv4.tcp_rmem = 4096 87380 16777216 +net.ipv4.tcp_syn_retries = 2 +net.ipv4.tcp_synack_retries = 2 +net.ipv4.tcp_tw_reuse = 1 +net.ipv4.tcp_wmem = 4096 65536 16777216 diff --git a/scripts/benchmark/wrk_custom_list.lua b/scripts/benchmark/wrk_custom_list.lua new file mode 100644 index 0000000..ea27af6 --- /dev/null +++ b/scripts/benchmark/wrk_custom_list.lua @@ -0,0 +1,29 @@ +local counter = 1 +local lines = {} +local base_path = "/planet/20231208_091355/tiles/" +local file_path = "/data/ofm/benchmark/path_list_small.txt" + +for line in io.lines(file_path) do + table.insert(lines, base_path .. line) +end + +local function getNextUrl() + -- Get the next URL from the list + local url_path = lines[counter] + counter = counter + 1 + + -- If we've gone past the end of the list, wrap around to the start + if counter > #lines then + counter = 1 + end + + return url_path +end + +request = function() + -- Return the request object with the current URL path + local path = getNextUrl() + local headers = {} + headers["Host"] = "ofm" + return wrk.format('GET', path, headers, nil) +end diff --git a/scripts/mbtiles_extractor/extract.py b/scripts/extract_mbtiles/extract_mbtiles.py similarity index 100% rename from scripts/mbtiles_extractor/extract.py rename to scripts/extract_mbtiles/extract_mbtiles.py diff --git a/scripts/http_host/mount_btrfs.sh b/scripts/http_host/mount_btrfs.sh new file mode 100644 index 0000000..88883d9 --- /dev/null +++ b/scripts/http_host/mount_btrfs.sh @@ -0,0 +1,5 @@ +mkdir -p mnt_ro +sudo mount -v \ + -t btrfs \ + -o ro \ + image.btrfs mnt_ro \ No newline at end of file diff --git a/scripts/http_host/nginx_site.conf b/scripts/http_host/nginx_site.conf new file mode 100644 index 0000000..c52c297 --- /dev/null +++ b/scripts/http_host/nginx_site.conf @@ -0,0 +1,17 @@ +server { + server_name ofm tiles.openfreemaps.org; + # test with + # curl -H "Host: ofm" http://localhost/planet/20231208_091355/tiles/7/72/48.pbf + + + #access_log /data/ofm/logs/nginx-access.log access_json; + access_log off; + error_log /data/ofm/logs/nginx-error.log; + + location /planet/20231208_091355 { + gzip off; + + alias /data/ofm/runs/planet_20231208_091355/mnt_ro/extract; + autoindex on; # Enables listing of directory + } +} diff --git a/scripts/tile_gen/extract_btrfs.sh b/scripts/tile_gen/extract_btrfs.sh index 54ff426..7b2bdba 100644 --- a/scripts/tile_gen/extract_btrfs.sh +++ b/scripts/tile_gen/extract_btrfs.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -sudo umount mnt || true -rm -rf mnt +sudo umount mnt_rw || true +rm -rf mnt_rw rm -f image.btrfs @@ -16,21 +16,21 @@ mkfs.btrfs -v \ # https://btrfs.readthedocs.io/en/latest/btrfs-man5.html#mount-options # compression doesn't make sense, data is already gzip compressed -mkdir -p mnt +mkdir -p mnt_rw sudo mount -v \ -t btrfs \ -o noacl,nobarrier,noatime,max_inline=4096 \ - image.btrfs mnt + image.btrfs mnt_rw -sudo chown ofm:ofm -R mnt +sudo chown ofm:ofm -R mnt_rw -../../tile_gen/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt/extract \ +../../tile_gen/venv/bin/python ../../tile_gen/extract.py output.mbtiles mnt_rw/extract \ > "extract_out.log" 2> "extract_err.log" -sudo umount mnt +sudo umount mnt_rw ../../tile_gen/venv/bin/python ../../tile_gen/shrink_btrfs.py image.btrfs - +# pigz -k image.btrfs --fast diff --git a/ssh_lib/benchmark.py b/ssh_lib/benchmark.py new file mode 100644 index 0000000..2bb024a --- /dev/null +++ b/ssh_lib/benchmark.py @@ -0,0 +1,36 @@ +from ssh_lib.config import scripts +from ssh_lib.utils import apt_get_install, apt_get_update, put, put_str, sudo_cmd + + +def k6(c): + sudo_cmd( + c, + 'curl https://dl.k6.io/key.gpg ' + '| gpg --dearmor ' + '| tee /usr/share/keyrings/k6-archive-keyring.gpg >/dev/null', + ) + put_str( + c, + '/e' 'tc/apt/sources.list.d/k6.list', + 'deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main', + ) + apt_get_update(c) + apt_get_install(c, 'k6') + + +def c1000k(c): + c.run('wget https://github.com/ideawu/c1000k/archive/master.zip -O tmp.zip') + c.run('unzip -o tmp.zip') + c.run('rm tmp.zip') + c.run('cd c1000k-master && make') + + # usage + # ./server 7000 + # ./client 127.0.0.1 7000 + # make sure it runs till 1 million + + +def benchmark(c): + apt_get_install(c, 'wrk') + c.sudo('mkdir -p /data/ofm/benchmark') + put(c, f'{scripts}/benchmark/wrk_custom_list.lua', '/data/ofm/benchmark') diff --git a/ssh_lib/kernel.py b/ssh_lib/kernel.py index 82d0402..803c246 100644 --- a/ssh_lib/kernel.py +++ b/ssh_lib/kernel.py @@ -3,7 +3,8 @@ from ssh_lib.utils import apt_get_install, apt_get_purge, put, put_str def setup_kernel_settings(c): - put(c, f'{config}/sysctl/60-optim.conf', '/etc/sysctl.d/') + put(c, f'{config}/sysctl/60-ofm.conf', '/etc/sysctl.d/') + put(c, f'{config}/limits/limits-ofm.conf', '/etc/security/limits.d/') def set_cpu_governor(c): diff --git a/ssh_lib/nginx.py b/ssh_lib/nginx.py index b7589b1..a171656 100644 --- a/ssh_lib/nginx.py +++ b/ssh_lib/nginx.py @@ -64,19 +64,3 @@ def certbot(c): apt_get_purge(c, 'certbot') c.sudo('snap install --classic certbot', warn=True) - - -def k6(c): - sudo_cmd( - c, - 'curl https://dl.k6.io/key.gpg ' - '| gpg --dearmor ' - '| tee /usr/share/keyrings/k6-archive-keyring.gpg >/dev/null', - ) - put_str( - c, - '/etc/apt/sources.list.d/k6.list', - 'deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main', - ) - apt_get_update(c) - apt_get_install(c, 'k6') diff --git a/ssh_lib/pkg_base.py b/ssh_lib/pkg_base.py index 851fdc3..230328f 100644 --- a/ssh_lib/pkg_base.py +++ b/ssh_lib/pkg_base.py @@ -39,6 +39,8 @@ def pkg_base(c): 'lsb-release', 'wget', 'git', + 'build-essential', + 'unzip', # 'gnupg2', 'gnupg-agent',