From d5365ef15bc7b828fff75846096c3ee9b1c370ff Mon Sep 17 00:00:00 2001 From: Zsolt Ero Date: Thu, 24 Oct 2024 02:15:27 +0200 Subject: [PATCH] docs update --- .../_not_used/ext4-ideas.txt | 0 .../_not_used/extract_ext4.sh | 0 .../fs_stats/README.md | 0 .../fs_stats/btrfs_after_delete.txt | 0 .../fs_stats/btrfs_before_delete.txt | 0 .../fs_stats/btrfs_not_used.txt | 0 .../fs_stats/dedupl_fixed.log | 0 .../fs_stats/ext4.md | 0 .../fs_stats/naive_mbutil_ext4.md | 0 .../fs_stats/planet_version | 0 .../kernel-ideas.txt | 0 docs/{quick_notes => }/_not_used/loop_test.sh | 0 .../nginx-ideas.txt | 0 docs/benchmark/README.md | 43 +++++++++++++++++++ .../benchmark/nginx_to_path_list.py | 4 ++ .../results.md} | 0 .../benchmark/wrk_custom_list.lua | 2 +- init-server.py | 1 + modules/http_host/benchmark/wrk_usage.txt | 8 ---- 19 files changed, 49 insertions(+), 9 deletions(-) rename docs/{quick_notes => }/_not_used/ext4-ideas.txt (100%) rename docs/{quick_notes => }/_not_used/extract_ext4.sh (100%) rename docs/{quick_notes => _not_used}/fs_stats/README.md (100%) rename docs/{quick_notes => _not_used}/fs_stats/btrfs_after_delete.txt (100%) rename docs/{quick_notes => _not_used}/fs_stats/btrfs_before_delete.txt (100%) rename docs/{quick_notes => _not_used}/fs_stats/btrfs_not_used.txt (100%) rename docs/{quick_notes => _not_used}/fs_stats/dedupl_fixed.log (100%) rename docs/{quick_notes => _not_used}/fs_stats/ext4.md (100%) rename docs/{quick_notes => _not_used}/fs_stats/naive_mbutil_ext4.md (100%) rename docs/{quick_notes => _not_used}/fs_stats/planet_version (100%) rename docs/{quick_notes => _not_used}/kernel-ideas.txt (100%) rename docs/{quick_notes => }/_not_used/loop_test.sh (100%) rename docs/{quick_notes => _not_used}/nginx-ideas.txt (100%) create mode 100644 docs/benchmark/README.md rename modules/http_host/benchmark/create_path_list.py => docs/benchmark/nginx_to_path_list.py (79%) rename docs/{quick_notes/http_benchmark.md => benchmark/results.md} (100%) rename {modules/http_host => docs}/benchmark/wrk_custom_list.lua (93%) delete mode 100644 modules/http_host/benchmark/wrk_usage.txt diff --git a/docs/quick_notes/_not_used/ext4-ideas.txt b/docs/_not_used/ext4-ideas.txt similarity index 100% rename from docs/quick_notes/_not_used/ext4-ideas.txt rename to docs/_not_used/ext4-ideas.txt diff --git a/docs/quick_notes/_not_used/extract_ext4.sh b/docs/_not_used/extract_ext4.sh similarity index 100% rename from docs/quick_notes/_not_used/extract_ext4.sh rename to docs/_not_used/extract_ext4.sh diff --git a/docs/quick_notes/fs_stats/README.md b/docs/_not_used/fs_stats/README.md similarity index 100% rename from docs/quick_notes/fs_stats/README.md rename to docs/_not_used/fs_stats/README.md diff --git a/docs/quick_notes/fs_stats/btrfs_after_delete.txt b/docs/_not_used/fs_stats/btrfs_after_delete.txt similarity index 100% rename from docs/quick_notes/fs_stats/btrfs_after_delete.txt rename to docs/_not_used/fs_stats/btrfs_after_delete.txt diff --git a/docs/quick_notes/fs_stats/btrfs_before_delete.txt b/docs/_not_used/fs_stats/btrfs_before_delete.txt similarity index 100% rename from docs/quick_notes/fs_stats/btrfs_before_delete.txt rename to docs/_not_used/fs_stats/btrfs_before_delete.txt diff --git a/docs/quick_notes/fs_stats/btrfs_not_used.txt b/docs/_not_used/fs_stats/btrfs_not_used.txt similarity index 100% rename from docs/quick_notes/fs_stats/btrfs_not_used.txt rename to docs/_not_used/fs_stats/btrfs_not_used.txt diff --git a/docs/quick_notes/fs_stats/dedupl_fixed.log b/docs/_not_used/fs_stats/dedupl_fixed.log similarity index 100% rename from docs/quick_notes/fs_stats/dedupl_fixed.log rename to docs/_not_used/fs_stats/dedupl_fixed.log diff --git a/docs/quick_notes/fs_stats/ext4.md b/docs/_not_used/fs_stats/ext4.md similarity index 100% rename from docs/quick_notes/fs_stats/ext4.md rename to docs/_not_used/fs_stats/ext4.md diff --git a/docs/quick_notes/fs_stats/naive_mbutil_ext4.md b/docs/_not_used/fs_stats/naive_mbutil_ext4.md similarity index 100% rename from docs/quick_notes/fs_stats/naive_mbutil_ext4.md rename to docs/_not_used/fs_stats/naive_mbutil_ext4.md diff --git a/docs/quick_notes/fs_stats/planet_version b/docs/_not_used/fs_stats/planet_version similarity index 100% rename from docs/quick_notes/fs_stats/planet_version rename to docs/_not_used/fs_stats/planet_version diff --git a/docs/quick_notes/kernel-ideas.txt b/docs/_not_used/kernel-ideas.txt similarity index 100% rename from docs/quick_notes/kernel-ideas.txt rename to docs/_not_used/kernel-ideas.txt diff --git a/docs/quick_notes/_not_used/loop_test.sh b/docs/_not_used/loop_test.sh similarity index 100% rename from docs/quick_notes/_not_used/loop_test.sh rename to docs/_not_used/loop_test.sh diff --git a/docs/quick_notes/nginx-ideas.txt b/docs/_not_used/nginx-ideas.txt similarity index 100% rename from docs/quick_notes/nginx-ideas.txt rename to docs/_not_used/nginx-ideas.txt diff --git a/docs/benchmark/README.md b/docs/benchmark/README.md new file mode 100644 index 0000000..60459fb --- /dev/null +++ b/docs/benchmark/README.md @@ -0,0 +1,43 @@ +# HTTP Hosts Benchmarking + +This repository contains tools and scripts for benchmarking HTTP hosts performance. + +## Prerequisites + +Before running the benchmarks, you need to create a path list (`path_list_500k.txt`). You have two options: + +1. Generate from real-world server logs using `nginx_to_path_list.py` +2. Generate randomly (Note: real-world usage patterns are typically non-random, e.g., ocean tiles are rarely accessed) + +## Important Notes + +- Run the benchmarks on `localhost`, and not over the internet! Otherwise you'd be just testing your internet speed. +- The benchmark uses [wrk](https://github.com/wg/wrk) HTTP benchmarking tool + +## Usage + +Basic command: +```bash +wrk -c10 -t4 -d10s -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost +``` + +### Parameters Explained + +- `-c10`: Number of connections to keep open +- `-t4`: Number of threads to use +- `-d10s`: Duration of the test (10 seconds) +- `-s`: Script file to use + +### Thread Count Considerations + +- `-t1`: More accurate results as the URL list is loaded exactly in sequence +- `-t4`: Better reflects real-world usage patterns + +## Results + +Benchmark results can be found in [results.md](results.md) + +## Contributing + +Feel free to submit your results including which hosts were used. + diff --git a/modules/http_host/benchmark/create_path_list.py b/docs/benchmark/nginx_to_path_list.py similarity index 79% rename from modules/http_host/benchmark/create_path_list.py rename to docs/benchmark/nginx_to_path_list.py index 36edf65..bbab856 100644 --- a/modules/http_host/benchmark/create_path_list.py +++ b/docs/benchmark/nginx_to_path_list.py @@ -1,6 +1,10 @@ import json +# This script parses a nginx server log and creates a text file +# which can be used in the Lua script. +# The path file is not suppied in this repo. + with open('access.jsonl') as fp: json_lines = fp.readlines() diff --git a/docs/quick_notes/http_benchmark.md b/docs/benchmark/results.md similarity index 100% rename from docs/quick_notes/http_benchmark.md rename to docs/benchmark/results.md diff --git a/modules/http_host/benchmark/wrk_custom_list.lua b/docs/benchmark/wrk_custom_list.lua similarity index 93% rename from modules/http_host/benchmark/wrk_custom_list.lua rename to docs/benchmark/wrk_custom_list.lua index cb1470d..9fb1356 100644 --- a/modules/http_host/benchmark/wrk_custom_list.lua +++ b/docs/benchmark/wrk_custom_list.lua @@ -1,6 +1,6 @@ local counter = 1 local lines = {} -local url_base = "/planet/20231221_134737_pt/" -- trailing slash +local url_base = "/planet/fake_version/" -- trailing slash local path_list_txt = "/data/ofm/benchmark/path_list_500k.txt" for line in io.lines(path_list_txt) do diff --git a/init-server.py b/init-server.py index 4d43473..b186b8a 100755 --- a/init-server.py +++ b/init-server.py @@ -129,6 +129,7 @@ def loadbalancer(hostname, user, port, noninteractive): @common_options def debug(hostname, user, port, noninteractive): c = get_connection(hostname, user, port) + run_http_host_sync(c) if __name__ == '__main__': diff --git a/modules/http_host/benchmark/wrk_usage.txt b/modules/http_host/benchmark/wrk_usage.txt deleted file mode 100644 index 0e11d25..0000000 --- a/modules/http_host/benchmark/wrk_usage.txt +++ /dev/null @@ -1,8 +0,0 @@ -wrk -c10 -t4 -d10s -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost - -# -t1 => more correct, since the url list is loaded exactly in sequence -# -t4 => reflecting real world usage - - - -