work

scripts/http_host/benchmark/command.txt

@@ -0,0 +1,4 @@
+wrk -c1000 -d10s -t1 -s /data/ofm/benchmark/wrk_custom_list.lua http://localhost
+
+# -t1 - needs to be single treaded, otherwise the urls would be read not in sequence
+

scripts/http_host/benchmark/create_path_list.py

@@ -0,0 +1,27 @@
+import json
+
+
+with open('access.log') as fp:
+    json_lines = fp.readlines()
+
+paths = []
+for i, line in enumerate(json_lines):
+    log_data = json.loads(line)
+    if log_data['status'] != 200:
+        continue
+
+    if log_data['request_method'] != 'GET':
+        continue
+
+    uri = log_data['uri']
+
+    if 'tiles/' not in uri or not uri.endswith('.pbf'):
+        continue
+
+    path = log_data['uri'].split('tiles/')[1]
+    paths.append(path + '\n')
+
+    print(f'{i / len(json_lines) * 100:.1f}%')
+
+with open('path_list.txt', 'w') as fp:
+    fp.writelines(paths)

scripts/http_host/benchmark/wrk_custom_list.lua

@@ -0,0 +1,39 @@
+local counter = 1
+local lines = {}
+local base_path = "/planet/20231208_091355/tiles/"
+local file_path = "/data/ofm/benchmark/path_list_100k.txt"
+
+for line in io.lines(file_path) do
+    table.insert(lines, base_path .. line)
+end
+
+local function getNextUrl()
+    -- Get the next URL from the list
+    local url_path = lines[counter]
+    counter = counter + 1
+
+    -- If we've gone past the end of the list, wrap around to the start
+    if counter > #lines then
+        counter = 1
+    end
+
+    return url_path
+end
+
+request = function()
+    -- Return the request object with the current URL path
+    path = getNextUrl()
+    local headers = {}
+    headers["Host"] = "ofm"
+    return wrk.format('GET', path, headers, nil)
+end
+
+response = function(status)
+    if status ~= 200 then
+        print("Non-200 response")
+        print("Status: ", status)
+        -- this only works in single threaded mode (-t1)
+        print("Request path: ", path)
+    end
+end
+