From 47763df590c4a6b7ed3a1b000aaccd45cd7bb6b1 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Sat, 14 Jun 2025 13:43:51 -0500 Subject: [PATCH] nix: add process monitoring for loadtests Closes https://github.com/PostgREST/postgrest/issues/4107. Adds two python scripts: - monitor_pid.py: monitors the postgrest process each second until it exits, then outputs a csv with the results. The nix wrappers use the `loadtest` dir for the output. - merge_monitor_result.py: receives a list of csvs and merges them into a single markdown table. The nix wrappers use the `loadtest/*.csv` files for the input. The nix `postgrest-with-pgrst` and `postgrest-loadtest-report` commands use these scripts to add monitoring for `postgrest-loadtest` and `postgrest-loadtest-against`. --- nix/tools/loadtest.nix | 29 +++++++++++++--- nix/tools/merge_monitor_result.py | 28 +++++++++++++++ nix/tools/monitor_pid.py | 58 +++++++++++++++++++++++++++++++ nix/tools/withTools.nix | 13 +++++++ 4 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 nix/tools/merge_monitor_result.py create mode 100644 nix/tools/monitor_pid.py diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix index 17516bfa6..645189abf 100644 --- a/nix/tools/loadtest.nix +++ b/nix/tools/loadtest.nix @@ -44,6 +44,7 @@ let "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])" "ARG_OPTIONAL_SINGLE([jwtcache], [], [JWT Cache on/off], [on])" "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-rsa])" + "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])" "ARG_TYPE_GROUP_SET([JWTCACHE], [JWTCACHE], [jwtcache], [on,off])" "ARG_LEFTOVERS([additional vegeta arguments])" ]; @@ -76,7 +77,7 @@ let # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ - ${withTools.withPgrst} \ + ${withTools.withPgrst} -m "$_arg_monitor" \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; @@ -93,7 +94,7 @@ let # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ - ${withTools.withPgrst} \ + ${withTools.withPgrst} -m "$_arg_monitor" \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; @@ -103,7 +104,7 @@ let # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ ${withTools.withSlowPg} \ - ${withTools.withPgrst} \ + ${withTools.withPgrst} -m "$_arg_monitor" \ ${withTools.withSlowPgrst} \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" @@ -139,6 +140,7 @@ let workingDir = "/"; } '' + # run loadtest for every target for tgt in "''${_arg_target[@]}"; do cat << EOF @@ -152,7 +154,7 @@ let # Save the results in the current working tree, too, # otherwise they'd be lost in the temporary working tree # created by withTools.withGit. - ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load" + ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/$tgt.csv" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load" cat << EOF @@ -162,13 +164,15 @@ let done + # run loadtest once on HEAD + cat << EOF Running loadtest on HEAD... EOF - ${loadtest} -k "$_arg_kind" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load" + ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/head.csv" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load" cat << EOF @@ -218,10 +222,18 @@ let workingDir = "/"; } '' + echo -e '## Loadtest results\n' + find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \ | ${jq}/bin/jq '[paths(scalars) as $path | {param: $path | join("."), (.branch): getpath($path)}]' \ | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \ | ${toMarkdown} + + echo -e '\n\n## Process monitoring results\n' + + find loadtest -type f -iname '*.csv' \ + | sort -nr \ + | ${mergeMonitorResults} ''; genTargets = genTargetsScript: @@ -230,6 +242,13 @@ let libraries = [ python3Packages.pyjwt python3Packages.jwcrypto ]; } (builtins.readFile genTargetsScript); + + mergeMonitorResults = + writers.writePython3 "postgrest-merge-monitor-results" + { + libraries = [ python3Packages.pandas python3Packages.tabulate ]; + } + (builtins.readFile ./merge_monitor_result.py); in buildToolbox { name = "postgrest-loadtest"; diff --git a/nix/tools/merge_monitor_result.py b/nix/tools/merge_monitor_result.py new file mode 100644 index 000000000..4e5498030 --- /dev/null +++ b/nix/tools/merge_monitor_result.py @@ -0,0 +1,28 @@ +import os +import sys +import pandas as pd + +KEY = "Elapsed seconds" + +merged = None + +paths = [p.strip() for p in sys.stdin.read().split() if p.strip()] + +for csv_path in paths: + # pfix is prefix (variable shortened to pass linter) + pfix = os.path.splitext(os.path.basename(csv_path))[0] + + df = pd.read_csv(csv_path) + + if KEY not in df.columns: + sys.exit(f"{csv_path} is missing the {KEY} column") + + # add prefix to every metric column + df = df.rename(columns={c: f"{pfix} {c}" for c in df.columns if c != KEY}) + + # outer join so missing rows appear + merged = df if merged is None else merged.merge(df, on=KEY, how="outer") + +# replace nan with empty string +merged = merged.fillna("") +merged.to_markdown(sys.stdout, index=False, tablefmt="github") diff --git a/nix/tools/monitor_pid.py b/nix/tools/monitor_pid.py new file mode 100644 index 000000000..24ba91477 --- /dev/null +++ b/nix/tools/monitor_pid.py @@ -0,0 +1,58 @@ +# Monitor a process pid with psutil and emits a CSV. +import sys +import time +import psutil +import pandas as pd + +KEY = "Elapsed seconds" +SAMPLE_INTERVAL_SECS = 1 + +if len(sys.argv) != 2 or not sys.argv[1].isdigit(): + sys.exit(f"Usage: {sys.argv[0]} ") + +pid = int(sys.argv[1]) +try: + proc = psutil.Process(pid) +except psutil.NoSuchProcess: + sys.exit(f"Error: process {pid} not found.") + +print(f"Starting monitoring of {pid} pid", file=sys.stderr) + +records = [] +start = time.time() +# ignore first result as per docs recommendation +# https://psutil.readthedocs.io/en/latest/#psutil.cpu_percent +proc.cpu_percent(None) + +while True: + try: + if not proc.is_running(): + break + time.sleep(SAMPLE_INTERVAL_SECS) + + elapsed_secs = int(time.time() - start) + cpu = proc.cpu_percent(None) + mem_pct = proc.memory_percent() + meminfo = proc.memory_info() + bytes_in_MB = 1024**2 + rss_mb = meminfo.rss / bytes_in_MB + + records.append( + [ + str(elapsed_secs), + f"{cpu:.3f}", + f"{mem_pct:.3f}", + f"{rss_mb:.3f}", + ] + ) + + except psutil.NoSuchProcess: + break + +end = time.time() +total_time = end - start +print(f"Finished {pid} pid monitoring in {total_time:.3f}", file=sys.stderr) + +cols = [KEY, "CPU (%)", "MEM (%)", "Real (MB)"] +df = pd.DataFrame(records, columns=cols, dtype=str) +df.to_csv(sys.stdout, index=False) diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index 40f44ea97..206037d9f 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -5,8 +5,10 @@ , lib , postgresqlVersions , postgrest +, python3Packages , slocat , writeText +, writers }: let withTmpDb = @@ -336,6 +338,7 @@ let [ "ARG_POSITIONAL_SINGLE([command], [Command to run])" "ARG_LEFTOVERS([command arguments])" + "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])" ]; positionalCompletion = "_command"; workingDir = "/"; @@ -377,9 +380,19 @@ let } echo "done." + if [[ -n "$_arg_monitor" ]]; then + ${monitorPid} "$pid" > "$_arg_monitor" & + fi + ("$_arg_command" "''${_arg_leftovers[@]}") ''; + monitorPid = + writers.writePython3 "postgrest-monitor-pid" + { + libraries = [ python3Packages.pandas python3Packages.tabulate python3Packages.psutil ]; + } + (builtins.readFile ./monitor_pid.py); in buildToolbox {