nix(loadtest): prevent timing errors for worst-case JWT test

Instead of taking wild guesses at the runtime of the target generation
itself, we're just making sure to reset the system time to a fixed value
when we ultimately start PostgREST. This allows us to create the right
JWT expiry values ahead of time.

.github/workflows/test.yaml

@@ -128,8 +128,6 @@ jobs:
       fail-fast: false
       matrix:
         kind: ['mixed', 'errors', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
-    # This test currently fails repeatedly - let's ignore that failure for the moment.
-    continue-on-error: ${{ matrix.kind == 'jwt-rsa-cache-worst' }}
     name: Loadtest
     runs-on: ubuntu-24.04
     steps:

nix/tools/generate_targets.py

@@ -128,11 +128,6 @@ def main():
         required=True,
         help="HTTP method for the vegeta targets",
     )
-    parser.add_argument(
-        "command",
-        nargs=argparse.REMAINDER,
-        help="Command (and arguments) to run after generating the targets",
-    )
 
     args = parser.parse_args()
 
@@ -174,9 +169,7 @@ def main():
 
     print(f"Generating {ntargets} targets...")
 
-    start_time = time.time()
-
-    now = int(start_time)
+    now = int(time.time())
 
     lines = []
 
@@ -186,21 +179,11 @@ def main():
     # so expires will occur and postgREST needs to
     # clean cached expired JWTs
     if args.worst:
-        # estimated time takes to build and run postgrest itself
-        build_run_postgrest_time = 2
-
-        # estimated time it takes to generate the targets file
-        # the division numbers are tuned by hand
-        if is_hs:  # hs generation is much faster
-            gen_time = ntargets // 66666
-        else:  # asymmetric is slower so the time is higher
-            gen_time = ntargets // 220
-
-        # estimated exp time so some JWTs will expire
-        inc = build_run_postgrest_time + gen_time
+        # estimated time it takes to run postgrest itself
+        run_postgrest_time = 2
 
         for i in range(ntargets):
-            token = generate_jwt(now, inc + i // 1000, rsa_private_key)
+            token = generate_jwt(now, run_postgrest_time + i // 1000, rsa_private_key)
             append_targets(lines, token, http_method)
 
     else:
@@ -216,11 +199,7 @@ def main():
         print(f"Error writing to {args.targets_path}: {e}", file=sys.stderr)
         sys.exit(1)
 
-    elapsed = time.time() - start_time
     print(f"Created {ntargets} targets", end=" ")
-    print(f"in {args.targets_path} ({elapsed:.2f}s)")
-
-    run_command(args.command)
 
 
 if __name__ == "__main__":

nix/tools/loadtest.nix

@@ -1,6 +1,7 @@
 { buildToolbox
 , checkedShellScript
 , jq
+, libfaketime
 , python3Packages
 , vegeta
 , withTools
@@ -68,28 +69,31 @@ let
           jwt-hs)
             export PGRST_JWT_CACHE_MAX_ENTRIES="0"
 
+            ${genTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http
+
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
             ${withTools.withPgrst} -m "$_arg_monitor" \
-            ${withGenTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
 
           jwt-hs-cache)
+            ${genTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http
+
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
             ${withTools.withPgrst} -m "$_arg_monitor" \
-            ${withGenTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
 
           jwt-hs-cache-worst)
+            ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} --method "$_arg_method" --worst "$_arg_testdir"/gen_targets.http
+
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} -m "$_arg_monitor" \
-            ${withGenTargets} --method "$_arg_method" --worst "$_arg_testdir"/gen_targets.http \
+            ${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
@@ -100,10 +104,11 @@ let
             ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
             export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
 
+            ${genTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http
+
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
             ${withTools.withPgrst} -m "$_arg_monitor" \
-            ${withGenTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
@@ -112,10 +117,11 @@ let
             ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
             export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
 
+            ${genTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http
+
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
             ${withTools.withPgrst} -m "$_arg_monitor" \
-            ${withGenTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
@@ -124,10 +130,11 @@ let
             ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
             export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
 
+            ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} --method "$_arg_method" --worst --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http
+
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} -m "$_arg_monitor" \
-            ${withGenTargets} --method "$_arg_method" --worst --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \
+            ${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
@@ -309,8 +316,8 @@ let
           | ${mergeMonitorResults}
       '';
 
-  withGenTargets =
-    writers.writePython3 "postgrest-with-gen-loadtest-targets"
+  genTargets =
+    writers.writePython3 "postgrest-gen-loadtest-targets"
       {
         libraries = [ python3Packages.pyjwt python3Packages.jwcrypto ];
         doCheck = false; # postgrest-style conflicts with this

nix/tools/withTools.nix

@@ -3,6 +3,7 @@
 , curl
 , git
 , lib
+, libfaketime
 , postgresqlVersions
 , postgrest
 , python3Packages
@@ -288,6 +289,7 @@ let
           [
             "ARG_POSITIONAL_SINGLE([command], [Command to run])"
             "ARG_LEFTOVERS([command arguments])"
+            "ARG_OPTIONAL_SINGLE([faketime], [f], [Fake the system time when starting PostgREST. This is useful to test expiry of JWT, for example in loadtests])"
             "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])"
             "ARG_OPTIONAL_SINGLE([timeout], [t], [Maximum time to wait for PostgREST to be ready], [5])"
             "ARG_OPTIONAL_SINGLE([sleep], [s],   [Sleep time after PostgREST is ready, this is useful for monitoring])"
@@ -301,6 +303,8 @@ let
       ''
         export PGRST_SERVER_UNIX_SOCKET="$tmpdir"/postgrest.socket
 
+        FAKETIME_CMD="${libfaketime}/bin/faketime"
+
         if [ -z "''${PGRST_CMD:-}" ]; then
           rm -f result
           build_start=$SECONDS
@@ -313,6 +317,8 @@ let
               exit 1
             }
             PGRST_CMD=$(echo ./result*/bin/postgrest)
+            # To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch.
+            FAKETIME_CMD="$(nix-build -A pkgs.libfaketime)/bin/faketime"
           else
             echo -n "${commandName}: Building postgrest (cabal)... "
             postgrest-build
@@ -326,7 +332,11 @@ let
 
         echo -n "${commandName}: Starting $ver... "
 
-        $PGRST_CMD > "$tmpdir"/run.log 2>&1 &
+        if [[ -n "$_arg_faketime" ]]; then
+          $FAKETIME_CMD "$_arg_faketime" "$PGRST_CMD" > "$tmpdir"/run.log 2>&1 &
+        else
+          $PGRST_CMD > "$tmpdir"/run.log 2>&1 &
+        fi
         pid=$!
         # shellcheck disable=SC2329
         cleanup() {