nix: add process monitoring for loadtests
Closes https://github.com/PostgREST/postgrest/issues/4107. Adds two python scripts: - monitor_pid.py: monitors the postgrest process each second until it exits, then outputs a csv with the results. The nix wrappers use the `loadtest` dir for the output. - merge_monitor_result.py: receives a list of csvs and merges them into a single markdown table. The nix wrappers use the `loadtest/*.csv` files for the input. The nix `postgrest-with-pgrst` and `postgrest-loadtest-report` commands use these scripts to add monitoring for `postgrest-loadtest` and `postgrest-loadtest-against`.
This commit is contained in:
steve-chavez
committed by
Steve Chavez
Steve Chavez
parent
53604c9db2
commit
47763df590
+24
-5
@@ -44,6 +44,7 @@ let
|
||||
"ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
|
||||
"ARG_OPTIONAL_SINGLE([jwtcache], [], [JWT Cache on/off], [on])"
|
||||
"ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-rsa])"
|
||||
"ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
|
||||
"ARG_TYPE_GROUP_SET([JWTCACHE], [JWTCACHE], [jwtcache], [on,off])"
|
||||
"ARG_LEFTOVERS([additional vegeta arguments])"
|
||||
];
|
||||
@@ -76,7 +77,7 @@ let
|
||||
|
||||
# shellcheck disable=SC2145
|
||||
${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
|
||||
${withTools.withPgrst} \
|
||||
${withTools.withPgrst} -m "$_arg_monitor" \
|
||||
sh -c "cd \"$_arg_testdir\" && \
|
||||
${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
|
||||
;;
|
||||
@@ -93,7 +94,7 @@ let
|
||||
|
||||
# shellcheck disable=SC2145
|
||||
${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
|
||||
${withTools.withPgrst} \
|
||||
${withTools.withPgrst} -m "$_arg_monitor" \
|
||||
sh -c "cd \"$_arg_testdir\" && \
|
||||
${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
|
||||
;;
|
||||
@@ -103,7 +104,7 @@ let
|
||||
# shellcheck disable=SC2145
|
||||
${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
|
||||
${withTools.withSlowPg} \
|
||||
${withTools.withPgrst} \
|
||||
${withTools.withPgrst} -m "$_arg_monitor" \
|
||||
${withTools.withSlowPgrst} \
|
||||
sh -c "cd \"$_arg_testdir\" && \
|
||||
${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
|
||||
@@ -139,6 +140,7 @@ let
|
||||
workingDir = "/";
|
||||
}
|
||||
''
|
||||
# run loadtest for every target
|
||||
for tgt in "''${_arg_target[@]}"; do
|
||||
|
||||
cat << EOF
|
||||
@@ -152,7 +154,7 @@ let
|
||||
# Save the results in the current working tree, too,
|
||||
# otherwise they'd be lost in the temporary working tree
|
||||
# created by withTools.withGit.
|
||||
${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"
|
||||
${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/$tgt.csv" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"
|
||||
|
||||
cat << EOF
|
||||
|
||||
@@ -162,13 +164,15 @@ let
|
||||
|
||||
done
|
||||
|
||||
# run loadtest once on HEAD
|
||||
|
||||
cat << EOF
|
||||
|
||||
Running loadtest on HEAD...
|
||||
|
||||
EOF
|
||||
|
||||
${loadtest} -k "$_arg_kind" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load"
|
||||
${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/head.csv" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load"
|
||||
|
||||
cat << EOF
|
||||
|
||||
@@ -218,10 +222,18 @@ let
|
||||
workingDir = "/";
|
||||
}
|
||||
''
|
||||
echo -e '## Loadtest results\n'
|
||||
|
||||
find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \
|
||||
| ${jq}/bin/jq '[paths(scalars) as $path | {param: $path | join("."), (.branch): getpath($path)}]' \
|
||||
| ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \
|
||||
| ${toMarkdown}
|
||||
|
||||
echo -e '\n\n## Process monitoring results\n'
|
||||
|
||||
find loadtest -type f -iname '*.csv' \
|
||||
| sort -nr \
|
||||
| ${mergeMonitorResults}
|
||||
'';
|
||||
|
||||
genTargets = genTargetsScript:
|
||||
@@ -230,6 +242,13 @@ let
|
||||
libraries = [ python3Packages.pyjwt python3Packages.jwcrypto ];
|
||||
}
|
||||
(builtins.readFile genTargetsScript);
|
||||
|
||||
mergeMonitorResults =
|
||||
writers.writePython3 "postgrest-merge-monitor-results"
|
||||
{
|
||||
libraries = [ python3Packages.pandas python3Packages.tabulate ];
|
||||
}
|
||||
(builtins.readFile ./merge_monitor_result.py);
|
||||
in
|
||||
buildToolbox {
|
||||
name = "postgrest-loadtest";
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import os
|
||||
import sys
|
||||
import pandas as pd
|
||||
|
||||
KEY = "Elapsed seconds"
|
||||
|
||||
merged = None
|
||||
|
||||
paths = [p.strip() for p in sys.stdin.read().split() if p.strip()]
|
||||
|
||||
for csv_path in paths:
|
||||
# pfix is prefix (variable shortened to pass linter)
|
||||
pfix = os.path.splitext(os.path.basename(csv_path))[0]
|
||||
|
||||
df = pd.read_csv(csv_path)
|
||||
|
||||
if KEY not in df.columns:
|
||||
sys.exit(f"{csv_path} is missing the {KEY} column")
|
||||
|
||||
# add prefix to every metric column
|
||||
df = df.rename(columns={c: f"{pfix} {c}" for c in df.columns if c != KEY})
|
||||
|
||||
# outer join so missing rows appear
|
||||
merged = df if merged is None else merged.merge(df, on=KEY, how="outer")
|
||||
|
||||
# replace nan with empty string
|
||||
merged = merged.fillna("")
|
||||
merged.to_markdown(sys.stdout, index=False, tablefmt="github")
|
||||
@@ -0,0 +1,58 @@
|
||||
# Monitor a process pid with psutil and emits a CSV.
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
import pandas as pd
|
||||
|
||||
KEY = "Elapsed seconds"
|
||||
SAMPLE_INTERVAL_SECS = 1
|
||||
|
||||
if len(sys.argv) != 2 or not sys.argv[1].isdigit():
|
||||
sys.exit(f"Usage: {sys.argv[0]} <PID>")
|
||||
|
||||
pid = int(sys.argv[1])
|
||||
try:
|
||||
proc = psutil.Process(pid)
|
||||
except psutil.NoSuchProcess:
|
||||
sys.exit(f"Error: process {pid} not found.")
|
||||
|
||||
print(f"Starting monitoring of {pid} pid", file=sys.stderr)
|
||||
|
||||
records = []
|
||||
start = time.time()
|
||||
# ignore first result as per docs recommendation
|
||||
# https://psutil.readthedocs.io/en/latest/#psutil.cpu_percent
|
||||
proc.cpu_percent(None)
|
||||
|
||||
while True:
|
||||
try:
|
||||
if not proc.is_running():
|
||||
break
|
||||
time.sleep(SAMPLE_INTERVAL_SECS)
|
||||
|
||||
elapsed_secs = int(time.time() - start)
|
||||
cpu = proc.cpu_percent(None)
|
||||
mem_pct = proc.memory_percent()
|
||||
meminfo = proc.memory_info()
|
||||
bytes_in_MB = 1024**2
|
||||
rss_mb = meminfo.rss / bytes_in_MB
|
||||
|
||||
records.append(
|
||||
[
|
||||
str(elapsed_secs),
|
||||
f"{cpu:.3f}",
|
||||
f"{mem_pct:.3f}",
|
||||
f"{rss_mb:.3f}",
|
||||
]
|
||||
)
|
||||
|
||||
except psutil.NoSuchProcess:
|
||||
break
|
||||
|
||||
end = time.time()
|
||||
total_time = end - start
|
||||
print(f"Finished {pid} pid monitoring in {total_time:.3f}", file=sys.stderr)
|
||||
|
||||
cols = [KEY, "CPU (%)", "MEM (%)", "Real (MB)"]
|
||||
df = pd.DataFrame(records, columns=cols, dtype=str)
|
||||
df.to_csv(sys.stdout, index=False)
|
||||
@@ -5,8 +5,10 @@
|
||||
, lib
|
||||
, postgresqlVersions
|
||||
, postgrest
|
||||
, python3Packages
|
||||
, slocat
|
||||
, writeText
|
||||
, writers
|
||||
}:
|
||||
let
|
||||
withTmpDb =
|
||||
@@ -336,6 +338,7 @@ let
|
||||
[
|
||||
"ARG_POSITIONAL_SINGLE([command], [Command to run])"
|
||||
"ARG_LEFTOVERS([command arguments])"
|
||||
"ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])"
|
||||
];
|
||||
positionalCompletion = "_command";
|
||||
workingDir = "/";
|
||||
@@ -377,9 +380,19 @@ let
|
||||
}
|
||||
echo "done."
|
||||
|
||||
if [[ -n "$_arg_monitor" ]]; then
|
||||
${monitorPid} "$pid" > "$_arg_monitor" &
|
||||
fi
|
||||
|
||||
("$_arg_command" "''${_arg_leftovers[@]}")
|
||||
'';
|
||||
|
||||
monitorPid =
|
||||
writers.writePython3 "postgrest-monitor-pid"
|
||||
{
|
||||
libraries = [ python3Packages.pandas python3Packages.tabulate python3Packages.psutil ];
|
||||
}
|
||||
(builtins.readFile ./monitor_pid.py);
|
||||
in
|
||||
buildToolbox
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user