nix: add process monitoring for loadtests

Closes https://github.com/PostgREST/postgrest/issues/4107.

Adds two python scripts:

- monitor_pid.py: monitors the postgrest process each second
  until it exits, then outputs a csv with the results. The nix wrappers
  use the `loadtest` dir for the output.
- merge_monitor_result.py: receives a list of csvs and merges them into
  a single markdown table. The nix wrappers use the `loadtest/*.csv`
  files for the input.

The nix `postgrest-with-pgrst` and `postgrest-loadtest-report` commands
use these scripts to add monitoring for `postgrest-loadtest` and
`postgrest-loadtest-against`.

nix/tools/loadtest.nix

@@ -44,6 +44,7 @@ let
           "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
           "ARG_OPTIONAL_SINGLE([jwtcache], [], [JWT Cache on/off], [on])"
           "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-rsa])"
+          "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
           "ARG_TYPE_GROUP_SET([JWTCACHE], [JWTCACHE], [jwtcache], [on,off])"
           "ARG_LEFTOVERS([additional vegeta arguments])"
         ];
@@ -76,7 +77,7 @@ let
 
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
@@ -93,7 +94,7 @@ let
 
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
@@ -103,7 +104,7 @@ let
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
             ${withTools.withSlowPg} \
-            ${withTools.withPgrst} \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
             ${withTools.withSlowPgrst} \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
@@ -139,6 +140,7 @@ let
         workingDir = "/";
       }
       ''
+        # run loadtest for every target
         for tgt in "''${_arg_target[@]}"; do
 
         cat << EOF
@@ -152,7 +154,7 @@ let
         # Save the results in the current working tree, too,
         # otherwise they'd be lost in the temporary working tree
         # created by withTools.withGit.
-        ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"
+        ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/$tgt.csv" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"
 
         cat << EOF
 
@@ -162,13 +164,15 @@ let
 
         done
 
+        # run loadtest once on HEAD
+
         cat << EOF
 
         Running loadtest on HEAD...
 
         EOF
 
-        ${loadtest} -k "$_arg_kind" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load"
+        ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/head.csv" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load"
 
         cat << EOF
 
@@ -218,10 +222,18 @@ let
         workingDir = "/";
       }
       ''
+        echo -e '## Loadtest results\n'
+
         find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \
           | ${jq}/bin/jq '[paths(scalars) as $path | {param: $path | join("."), (.branch): getpath($path)}]' \
           | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \
           | ${toMarkdown}
+
+        echo -e '\n\n## Process monitoring results\n'
+
+        find loadtest -type f -iname '*.csv' \
+          | sort -nr \
+          | ${mergeMonitorResults}
       '';
 
   genTargets = genTargetsScript:
@@ -230,6 +242,13 @@ let
         libraries = [ python3Packages.pyjwt python3Packages.jwcrypto ];
       }
       (builtins.readFile genTargetsScript);
+
+  mergeMonitorResults =
+    writers.writePython3 "postgrest-merge-monitor-results"
+      {
+        libraries = [ python3Packages.pandas python3Packages.tabulate ];
+      }
+      (builtins.readFile ./merge_monitor_result.py);
 in
 buildToolbox {
   name = "postgrest-loadtest";

nix/tools/merge_monitor_result.py

@@ -0,0 +1,28 @@
+import os
+import sys
+import pandas as pd
+
+KEY = "Elapsed seconds"
+
+merged = None
+
+paths = [p.strip() for p in sys.stdin.read().split() if p.strip()]
+
+for csv_path in paths:
+    # pfix is prefix (variable shortened to pass linter)
+    pfix = os.path.splitext(os.path.basename(csv_path))[0]
+
+    df = pd.read_csv(csv_path)
+
+    if KEY not in df.columns:
+        sys.exit(f"{csv_path} is missing the {KEY} column")
+
+    # add prefix to every metric column
+    df = df.rename(columns={c: f"{pfix} {c}" for c in df.columns if c != KEY})
+
+    # outer join so missing rows appear
+    merged = df if merged is None else merged.merge(df, on=KEY, how="outer")
+
+# replace nan with empty string
+merged = merged.fillna("")
+merged.to_markdown(sys.stdout, index=False, tablefmt="github")

nix/tools/monitor_pid.py

@@ -0,0 +1,58 @@
+# Monitor a process pid with psutil and emits a CSV.
+import sys
+import time
+import psutil
+import pandas as pd
+
+KEY = "Elapsed seconds"
+SAMPLE_INTERVAL_SECS = 1
+
+if len(sys.argv) != 2 or not sys.argv[1].isdigit():
+    sys.exit(f"Usage: {sys.argv[0]} <PID>")
+
+pid = int(sys.argv[1])
+try:
+    proc = psutil.Process(pid)
+except psutil.NoSuchProcess:
+    sys.exit(f"Error: process {pid} not found.")
+
+print(f"Starting monitoring of {pid} pid", file=sys.stderr)
+
+records = []
+start = time.time()
+# ignore first result as per docs recommendation
+# https://psutil.readthedocs.io/en/latest/#psutil.cpu_percent
+proc.cpu_percent(None)
+
+while True:
+    try:
+        if not proc.is_running():
+            break
+        time.sleep(SAMPLE_INTERVAL_SECS)
+
+        elapsed_secs = int(time.time() - start)
+        cpu = proc.cpu_percent(None)
+        mem_pct = proc.memory_percent()
+        meminfo = proc.memory_info()
+        bytes_in_MB = 1024**2
+        rss_mb = meminfo.rss / bytes_in_MB
+
+        records.append(
+            [
+                str(elapsed_secs),
+                f"{cpu:.3f}",
+                f"{mem_pct:.3f}",
+                f"{rss_mb:.3f}",
+            ]
+        )
+
+    except psutil.NoSuchProcess:
+        break
+
+end = time.time()
+total_time = end - start
+print(f"Finished {pid} pid monitoring in {total_time:.3f}", file=sys.stderr)
+
+cols = [KEY, "CPU (%)", "MEM (%)", "Real (MB)"]
+df = pd.DataFrame(records, columns=cols, dtype=str)
+df.to_csv(sys.stdout, index=False)

nix/tools/withTools.nix

@@ -5,8 +5,10 @@
 , lib
 , postgresqlVersions
 , postgrest
+, python3Packages
 , slocat
 , writeText
+, writers
 }:
 let
   withTmpDb =
@@ -336,6 +338,7 @@ let
           [
             "ARG_POSITIONAL_SINGLE([command], [Command to run])"
             "ARG_LEFTOVERS([command arguments])"
+            "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])"
           ];
         positionalCompletion = "_command";
         workingDir = "/";
@@ -377,9 +380,19 @@ let
         }
         echo "done."
 
+        if [[ -n "$_arg_monitor" ]]; then
+          ${monitorPid} "$pid" > "$_arg_monitor" &
+        fi
+
         ("$_arg_command" "''${_arg_leftovers[@]}")
       '';
 
+  monitorPid =
+    writers.writePython3 "postgrest-monitor-pid"
+      {
+        libraries = [ python3Packages.pandas python3Packages.tabulate python3Packages.psutil ];
+      }
+      (builtins.readFile ./monitor_pid.py);
 in
 buildToolbox
 {