jimmy/postgrest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nix(loadtest): run all target branches almost at the same time

Browse Source 
Instead of building, running, building, running, ... we now build all
executables once ahead of time and then run all loadtests right after
each other. This can sometimes reduce noise when load on the GHA runner
varies over time.

Since this requires us to move building into the loadtest-against
script, it also allows to go back to have the regular postgrest-loadtest
command default to building with cabal for faster local iteration.
This commit is contained in:
Wolfgang Walther
2026-06-27 18:01:50 +00:00
parent bc5d7a0e92
commit f1649bf6ee
4 changed files with 47 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
default.nix
+1 -1
View File
@@ -130,7 +130,7 @@ rec {
# Development tools.
devTools =
pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie withTools; };
pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie; };
# Documentation tools.
docs =
nix/README.md
-3
View File
@@ -162,9 +162,6 @@ The loadtests ensure that performance doesn't drop on a change. Underlyingly the
# You can loadtest comparing to a different branch
[nix-shell]$ postgrest-loadtest-against main
# You can build postgrest directly with cabal for faster iteration
[nix-shell]$ PGRST_BUILD_CABAL=1 postgrest-loadtest
# Produce a markdown report to be used on CI
[nix-shell]$ postgrest-loadtest-report
```
nix/tools/loadtest.nix
+41 -14
View File
@@ -1,5 +1,6 @@
{ buildToolbox
, checkedShellScript
, git
, jq
, libfaketime
, python3Packages
@@ -43,7 +44,6 @@ let
docs = "Run the vegeta loadtests with PostgREST.";
args = [
"ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])"
"ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])"
"ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
"ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-cache,jwt-cache-worst])"
"ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
@@ -65,23 +65,23 @@ let
case "$_arg_kind" in
jwt-cache)
export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwks.json"
export PGRST_JWT_SECRET="@test/load/gen_jwks.json"
${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} "$_arg_testdir"
${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} test/load
# shellcheck disable=SC2145
${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
${withTools.withPg} -f test/load/fixtures.sql \
${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \
sh -c "cd \"$_arg_testdir\" && \
sh -c "cd test/load && \
${runner} -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
;;
# here we sleep purposefully to check how much memory does the schema cache consume in the final report
mixed)
# shellcheck disable=SC2145
${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
${withTools.withPg} -f test/load/fixtures.sql \
${withTools.withPgrst} --timeout 2 --sleep 5 -m "$_arg_monitor" \
sh -c "cd \"$_arg_testdir\" && \
sh -c "cd test/load && \
${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
;;
esac
@@ -146,7 +146,39 @@ let
workingDir = "/";
}
''
# run loadtest for every target and HEAD
# Build postgrest for every target and HEAD.
# Keeps a reference to the postgrest binary and faketime lib for every branch to run later.
declare -A pgrst faketime
for tgt in "''${_arg_target[@]}" HEAD; do
# not using withTmpDir here, because we don't want to keep the directory on error
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' EXIT
${git}/bin/git worktree add -f "$tmpdir" "$tgt" > /dev/null
pushd "$tmpdir" > /dev/null
build_start=$SECONDS
echo -n "${name}: Building postgrest (nix) on $tgt... "
# Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet.
nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > build.log 2>&1 || {
echo "failed, output:"
cat build.log
exit 1
}
pgrst[$tgt]="$(nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage')/bin/postgrest"
# To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch.
faketime[$tgt]="$(nix-build --no-out-link -A pkgs.libfaketime)/lib/libfaketime.so.1"
build_end=$((SECONDS - build_start))
printf "done in %ss.\n" "$build_end"
popd > /dev/null
${git}/bin/git worktree remove -f "$tmpdir" > /dev/null
rm -rf "$tmpdir"
done
# Run loadtest for every target and HEAD.
# Running the tests is separated from building them to reduce the chances of
# other processes skewing the results between two runs.
for tgt in "''${_arg_target[@]}" HEAD; do
cat << EOF
@@ -155,12 +187,7 @@ let
EOF
# Runs the test files from the current working tree
# to make sure both tests are run with the same files.
# Save the results in the current working tree, too,
# otherwise they'd be lost in the temporary working tree
# created by withTools.withGit.
${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/$tgt.csv" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"
FAKETIME_LIB="''${faketime[$tgt]}" PGRST_CMD="''${pgrst[$tgt]}" ${loadtest} -k "$_arg_kind" -m "loadtest/$tgt.csv" --output "loadtest/$tgt.bin"
cat << EOF
nix/tools/withTools.nix
+3 -62
View File
@@ -1,7 +1,6 @@
{ buildToolbox
, checkedShellScript
, curl
, git
, lib
, libfaketime
, postgresqlVersions
@@ -197,47 +196,6 @@ let
withPg = withTmpDb (builtins.head postgresqlVersions);
withGit =
let
name = "postgrest-with-git";
in
checkedShellScript
{
inherit name;
docs =
''
Create a new worktree of the postgrest repo in a temporary directory and
check out <commit>, then run <command> with arguments inside the temporary folder.
'';
args =
[
"ARG_POSITIONAL_SINGLE([commit], [Commit-ish reference to run command with])"
"ARG_POSITIONAL_SINGLE([command], [Command to run])"
"ARG_LEFTOVERS([command arguments])"
];
positionalCompletion =
''
if test "$prev" == "${name}"; then
__gitcomp_nl "$(__git_refs)"
else
_command_offset 2
fi
'';
workingDir = "/";
}
''
# not using withTmpDir here, because we don't want to keep the directory on error
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' EXIT
${git}/bin/git worktree add -f "$tmpdir" "$_arg_commit" > /dev/null
cd "$tmpdir"
("$_arg_command" "''${_arg_leftovers[@]}")
${git}/bin/git worktree remove -f "$tmpdir" > /dev/null
'';
waitForPgrstReady =
checkedShellScript
{
@@ -294,7 +252,8 @@ let
"ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])"
"ARG_OPTIONAL_SINGLE([timeout], [t], [Maximum time to wait for PostgREST to be ready], [5])"
"ARG_OPTIONAL_SINGLE([sleep], [s], [Sleep time after PostgREST is ready, this is useful for monitoring])"
"ARG_USE_ENV([PGRST_CMD], [], [PostgREST executable to run])"
"ARG_USE_ENV([FAKETIME_LIB], [${libfaketime}/lib/libfaketime.so.1], [Faketime Library to preload])"
"ARG_USE_ENV([PGRST_CMD], [postgrest-run], [PostgREST executable to run])"
];
positionalCompletion = "_command";
workingDir = "/";
@@ -304,27 +263,10 @@ let
''
export PGRST_SERVER_UNIX_SOCKET="$tmpdir"/postgrest.socket
FAKETIME_LIB="${libfaketime}/lib/libfaketime.so.1"
if [ -z "''${PGRST_CMD:-}" ]; then
rm -f result
if [ "''${PGRST_CMD}" == "postgrest-run" ]; then
build_start=$SECONDS
if [ -z "''${PGRST_BUILD_CABAL:-}" ]; then
echo -n "${commandName}: Building postgrest (nix)... "
# Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet.
nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > "$tmpdir"/build.log 2>&1 || {
echo "failed, output:"
cat "$tmpdir"/build.log
exit 1
}
PGRST_CMD="$(nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage')/bin/postgrest"
# To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch.
FAKETIME_LIB="$(nix-build --no-out-link -A pkgs.libfaketime)/lib/libfaketime.so.1"
else
echo -n "${commandName}: Building postgrest (cabal)... "
postgrest-build
PGRST_CMD=postgrest-run
fi
build_end=$((SECONDS - build_start))
printf "done in %ss.\n" "$build_end"
fi
@@ -387,7 +329,6 @@ buildToolbox
name = "postgrest-with";
tools = {
inherit
withGit
withPgAll
withPgrst;
} // builtins.listToAttrs (