nix(loadtest): run all target branches almost at the same time

Instead of building, running, building, running, ... we now build all
executables once ahead of time and then run all loadtests right after
each other. This can sometimes reduce noise when load on the GHA runner
varies over time.

Since this requires us to move building into the loadtest-against
script, it also allows to go back to have the regular postgrest-loadtest
command default to building with cabal for faster local iteration.

default.nix

@@ -130,7 +130,7 @@ rec {
 
   # Development tools.
   devTools =
-    pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie withTools; };
+    pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie; };
 
   # Documentation tools.
   docs =

nix/README.md

@@ -162,9 +162,6 @@ The loadtests ensure that performance doesn't drop on a change. Underlyingly the
 # You can loadtest comparing to a different branch
 [nix-shell]$ postgrest-loadtest-against main
 
-# You can build postgrest directly with cabal for faster iteration
-[nix-shell]$ PGRST_BUILD_CABAL=1 postgrest-loadtest
-
 # Produce a markdown report to be used on CI
 [nix-shell]$ postgrest-loadtest-report
 ```

nix/tools/loadtest.nix

@@ -1,5 +1,6 @@
 { buildToolbox
 , checkedShellScript
+, git
 , jq
 , libfaketime
 , python3Packages
@@ -43,7 +44,6 @@ let
         docs = "Run the vegeta loadtests with PostgREST.";
         args = [
           "ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])"
-          "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])"
           "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
           "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-cache,jwt-cache-worst])"
           "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
@@ -65,23 +65,23 @@ let
 
         case "$_arg_kind" in
           jwt-cache)
-            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwks.json"
+            export PGRST_JWT_SECRET="@test/load/gen_jwks.json"
 
-            ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} "$_arg_testdir"
+            ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} test/load
 
             # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPg} -f test/load/fixtures.sql \
             ${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \
-            sh -c "cd \"$_arg_testdir\" && \
+            sh -c "cd test/load && \
             ${runner} -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
 
           # here we sleep purposefully to check how much memory does the schema cache consume in the final report
           mixed)
             # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPg} -f test/load/fixtures.sql \
             ${withTools.withPgrst} --timeout 2 --sleep 5 -m "$_arg_monitor" \
-            sh -c "cd \"$_arg_testdir\" && \
+            sh -c "cd test/load && \
             ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
         esac
@@ -146,7 +146,39 @@ let
         workingDir = "/";
       }
       ''
-        # run loadtest for every target and HEAD
+        # Build postgrest for every target and HEAD.
+        # Keeps a reference to the postgrest binary and faketime lib for every branch to run later.
+        declare -A pgrst faketime
+        for tgt in "''${_arg_target[@]}" HEAD; do
+          # not using withTmpDir here, because we don't want to keep the directory on error
+          tmpdir="$(mktemp -d)"
+          trap 'rm -rf "$tmpdir"' EXIT
+
+          ${git}/bin/git worktree add -f "$tmpdir" "$tgt" > /dev/null
+          pushd "$tmpdir" > /dev/null
+
+          build_start=$SECONDS
+          echo -n "${name}: Building postgrest (nix) on $tgt... "
+          # Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet.
+          nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > build.log 2>&1 || {
+            echo "failed, output:"
+            cat build.log
+            exit 1
+          }
+          pgrst[$tgt]="$(nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage')/bin/postgrest"
+          # To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch.
+          faketime[$tgt]="$(nix-build --no-out-link -A pkgs.libfaketime)/lib/libfaketime.so.1"
+          build_end=$((SECONDS - build_start))
+          printf "done in %ss.\n" "$build_end"
+
+          popd > /dev/null
+          ${git}/bin/git worktree remove -f "$tmpdir" > /dev/null
+          rm -rf "$tmpdir"
+        done
+
+        # Run loadtest for every target and HEAD.
+        # Running the tests is separated from building them to reduce the chances of
+        # other processes skewing the results between two runs.
         for tgt in "''${_arg_target[@]}" HEAD; do
 
         cat << EOF
@@ -155,12 +187,7 @@ let
 
         EOF
 
-        # Runs the test files from the current working tree
+        FAKETIME_LIB="''${faketime[$tgt]}" PGRST_CMD="''${pgrst[$tgt]}" ${loadtest} -k "$_arg_kind" -m "loadtest/$tgt.csv" --output "loadtest/$tgt.bin"
-        # to make sure both tests are run with the same files.
-        # Save the results in the current working tree, too,
-        # otherwise they'd be lost in the temporary working tree
-        # created by withTools.withGit.
-        ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/$tgt.csv" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"
 
         cat << EOF
 

nix/tools/withTools.nix

@@ -1,7 +1,6 @@
 { buildToolbox
 , checkedShellScript
 , curl
-, git
 , lib
 , libfaketime
 , postgresqlVersions
@@ -197,47 +196,6 @@ let
 
   withPg = withTmpDb (builtins.head postgresqlVersions);
 
-  withGit =
-    let
-      name = "postgrest-with-git";
-    in
-    checkedShellScript
-      {
-        inherit name;
-        docs =
-          ''
-            Create a new worktree of the postgrest repo in a temporary directory and
-            check out <commit>, then run <command> with arguments inside the temporary folder.
-          '';
-        args =
-          [
-            "ARG_POSITIONAL_SINGLE([commit], [Commit-ish reference to run command with])"
-            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
-            "ARG_LEFTOVERS([command arguments])"
-          ];
-        positionalCompletion =
-          ''
-            if test "$prev" == "${name}"; then
-              __gitcomp_nl "$(__git_refs)"
-            else
-              _command_offset 2
-            fi
-          '';
-        workingDir = "/";
-      }
-      ''
-        # not using withTmpDir here, because we don't want to keep the directory on error
-        tmpdir="$(mktemp -d)"
-        trap 'rm -rf "$tmpdir"' EXIT
-
-        ${git}/bin/git worktree add -f "$tmpdir" "$_arg_commit" > /dev/null
-
-        cd "$tmpdir"
-        ("$_arg_command" "''${_arg_leftovers[@]}")
-
-        ${git}/bin/git worktree remove -f "$tmpdir" > /dev/null
-      '';
-
   waitForPgrstReady =
     checkedShellScript
       {
@@ -294,7 +252,8 @@ let
             "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])"
             "ARG_OPTIONAL_SINGLE([timeout], [t], [Maximum time to wait for PostgREST to be ready], [5])"
             "ARG_OPTIONAL_SINGLE([sleep], [s],   [Sleep time after PostgREST is ready, this is useful for monitoring])"
-            "ARG_USE_ENV([PGRST_CMD], [], [PostgREST executable to run])"
+            "ARG_USE_ENV([FAKETIME_LIB], [${libfaketime}/lib/libfaketime.so.1], [Faketime Library to preload])"
+            "ARG_USE_ENV([PGRST_CMD], [postgrest-run], [PostgREST executable to run])"
           ];
         positionalCompletion = "_command";
         workingDir = "/";
@@ -304,27 +263,10 @@ let
       ''
         export PGRST_SERVER_UNIX_SOCKET="$tmpdir"/postgrest.socket
 
-        FAKETIME_LIB="${libfaketime}/lib/libfaketime.so.1"
+        if [ "''${PGRST_CMD}" == "postgrest-run" ]; then
-
-        if [ -z "''${PGRST_CMD:-}" ]; then
-          rm -f result
           build_start=$SECONDS
-          if [ -z "''${PGRST_BUILD_CABAL:-}" ]; then
-            echo -n "${commandName}: Building postgrest (nix)... "
-            # Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet.
-            nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > "$tmpdir"/build.log 2>&1 || {
-              echo "failed, output:"
-              cat "$tmpdir"/build.log
-              exit 1
-            }
-            PGRST_CMD="$(nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage')/bin/postgrest"
-            # To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch.
-            FAKETIME_LIB="$(nix-build --no-out-link -A pkgs.libfaketime)/lib/libfaketime.so.1"
-          else
           echo -n "${commandName}: Building postgrest (cabal)... "
           postgrest-build
-            PGRST_CMD=postgrest-run
-          fi
           build_end=$((SECONDS - build_start))
           printf "done in %ss.\n" "$build_end"
         fi
@@ -387,7 +329,6 @@ buildToolbox
   name = "postgrest-with";
   tools = {
     inherit
-      withGit
       withPgAll
       withPgrst;
   } // builtins.listToAttrs (