From ff92846e64eac2772c66ef1e79dbae101f631a96 Mon Sep 17 00:00:00 2001
From: Wolfgang Walther <walther@technowledgy.de>
Date: Mon, 18 May 2026 22:42:42 +0200
Subject: [PATCH] nix(loadtest): report multiple percentiles at once

We can look at the data over time and see which percentile works best
for our purpose.
---
 .github/workflows/test.yaml |  8 ++++++--
 nix/tools/loadtest.nix      | 32 ++++++++++++++++++++++----------
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index d019bfffe..1940ade81 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -138,7 +138,7 @@ jobs:
         uses: ./.github/actions/setup-nix
         with:
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: loadtest.loadtestAgainst.bin loadtest.report.bin cabalTools.update.bin
+          tools: loadtest.loadtestAgainst.bin loadtest.report.bin loadtest.report-load.bin cabalTools.update.bin
 
       - run: postgrest-cabal-update
 
@@ -152,7 +152,11 @@ jobs:
             latest_tag=$(git tag --merged HEAD --sort=-creatordate "v*" | head -n1)
           fi
           postgrest-loadtest-against -k ${{ matrix.kind }} "$TARGET_BRANCH" "$latest_tag"
-          postgrest-loadtest-report -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY"
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 0 >> "$GITHUB_STEP_SUMMARY"
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 50 >> "$GITHUB_STEP_SUMMARY"
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 90 >> "$GITHUB_STEP_SUMMARY"
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 95 >> "$GITHUB_STEP_SUMMARY"
+          postgrest-loadtest-report-load -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY"
 
   flake:
     strategy:
diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix
index b0066aa88..3553044d5 100644
--- a/nix/tools/loadtest.nix
+++ b/nix/tools/loadtest.nix
@@ -254,13 +254,14 @@ let
         docs = "Create a named json report for a single result file.";
         args = [
           "ARG_POSITIONAL_SINGLE([file], [Filename of result to create report for])"
+          "ARG_OPTIONAL_SINGLE([percentile], [p], [Percentile to report latency for], 50)"
           "ARG_LEFTOVERS([additional vegeta arguments])"
         ];
         workingDir = "/";
       }
       ''
         ${vegeta}/bin/vegeta encode "$_arg_file" \
-          | ${jq}/bin/jq --slurp 'map(select(.url != "")) | group_by("\(.code) \(.method) \(.url)") | map({("\(.[0].code) \(.[0].method) \(.[0].url)" | sub("http://postgrest";"")): map(.latency) | min / 10e3 }) | .[]' \
+          | ${jq}/bin/jq --arg percentile "$_arg_percentile" --slurp 'map(select(.url != "")) | group_by("\(.code) \(.method) \(.url)") | map({("\(.[0].code) \(.[0].method) \(.[0].url)" | sub("http://postgrest";"")): map(.latency) | sort | .[(length-1) * ($percentile | tonumber) / 100 | floor] / 10e3 }) | .[]' \
           | ${jq}/bin/jq --arg branch "$(basename "$_arg_file" .bin)" '. + {branch: $branch}'
       '';
 
@@ -274,8 +275,8 @@ let
         import pandas as pd
 
         pd.read_json(sys.stdin) \
-          .rename(columns={'latency': 'min latency [μs]'}) \
-          .set_index('min latency [μs]') \
+          .rename(columns={'latency': sys.argv[1]}) \
+          .set_index(sys.argv[1]) \
           .drop(['branch']) \
           .convert_dtypes() \
           .to_markdown(sys.stdout, floatfmt='.1f')
@@ -289,20 +290,31 @@ let
         docs = "Create a report of all loadtest reports as markdown.";
         args = [
           "ARG_OPTIONAL_SINGLE([group], [g], [Marker to group results])"
+          "ARG_OPTIONAL_SINGLE([percentile], [p], [Percentile to report latency for], 50)"
         ];
         workingDir = "/";
       }
       ''
-        marker=''${_arg_group:+"($_arg_group)"}
+        echo -e "## Loadtest results $_arg_group (P$_arg_percentile)\n"
 
-        echo -e "## Loadtest results $marker\n"
-
-        find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \
+        find loadtest -type f -iname '*.bin' -exec ${reporter} -p "$_arg_percentile" {} \; \
           | ${jq}/bin/jq '[paths(scalars) as $path | {latency: $path | join("."), (.branch): getpath($path)}]' \
           | ${jq}/bin/jq --slurp 'flatten | group_by(.latency) | map(add)' \
-          | ${toMarkdown}
+          | ${toMarkdown} "P$_arg_percentile latency [μs]"
+      '';
 
-        echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM usage $marker\n"
+  report-load =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-report-load";
+        docs = "Create a report of all CPU/MEM usage as markdown.";
+        args = [
+          "ARG_OPTIONAL_SINGLE([group], [g], [Marker to group results])"
+        ];
+        workingDir = "/";
+      }
+      ''
+        echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM usage $_arg_group\n"
 
         find loadtest -type f -iname '*.csv' \
           | sort -m \
@@ -334,5 +346,5 @@ let
 in
 buildToolbox {
   name = "postgrest-loadtest";
-  tools = { inherit loadtest loadtestAgainst report; };
+  tools = { inherit loadtest loadtestAgainst report report-load; };
 }