From 61da5dd88b942ab45843e2468219b317b7a0b7dd Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Mon, 18 May 2026 15:21:10 +0200 Subject: [PATCH] nix(loadtest): prevent timing errors for worst-case JWT test Instead of taking wild guesses at the runtime of the target generation itself, we're just making sure to reset the system time to a fixed value when we ultimately start PostgREST. This allows us to create the right JWT expiry values ahead of time. --- .github/workflows/test.yaml | 2 -- nix/tools/generate_targets.py | 29 ++++------------------------- nix/tools/loadtest.nix | 27 +++++++++++++++++---------- nix/tools/withTools.nix | 12 +++++++++++- 4 files changed, 32 insertions(+), 38 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3e9ca4d88..ad9e7d111 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -128,8 +128,6 @@ jobs: fail-fast: false matrix: kind: ['mixed', 'errors', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst'] - # This test currently fails repeatedly - let's ignore that failure for the moment. - continue-on-error: ${{ matrix.kind == 'jwt-rsa-cache-worst' }} name: Loadtest runs-on: ubuntu-24.04 steps: diff --git a/nix/tools/generate_targets.py b/nix/tools/generate_targets.py index 417a80267..9ece166ca 100644 --- a/nix/tools/generate_targets.py +++ b/nix/tools/generate_targets.py @@ -128,11 +128,6 @@ def main(): required=True, help="HTTP method for the vegeta targets", ) - parser.add_argument( - "command", - nargs=argparse.REMAINDER, - help="Command (and arguments) to run after generating the targets", - ) args = parser.parse_args() @@ -174,9 +169,7 @@ def main(): print(f"Generating {ntargets} targets...") - start_time = time.time() - - now = int(start_time) + now = int(time.time()) lines = [] @@ -186,21 +179,11 @@ def main(): # so expires will occur and postgREST needs to # clean cached expired JWTs if args.worst: - # estimated time takes to build and run postgrest itself - build_run_postgrest_time = 2 - - # estimated time it takes to generate the targets file - # the division numbers are tuned by hand - if is_hs: # hs generation is much faster - gen_time = ntargets // 66666 - else: # asymmetric is slower so the time is higher - gen_time = ntargets // 220 - - # estimated exp time so some JWTs will expire - inc = build_run_postgrest_time + gen_time + # estimated time it takes to run postgrest itself + run_postgrest_time = 2 for i in range(ntargets): - token = generate_jwt(now, inc + i // 1000, rsa_private_key) + token = generate_jwt(now, run_postgrest_time + i // 1000, rsa_private_key) append_targets(lines, token, http_method) else: @@ -216,11 +199,7 @@ def main(): print(f"Error writing to {args.targets_path}: {e}", file=sys.stderr) sys.exit(1) - elapsed = time.time() - start_time print(f"Created {ntargets} targets", end=" ") - print(f"in {args.targets_path} ({elapsed:.2f}s)") - - run_command(args.command) if __name__ == "__main__": diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix index 1d3831878..034fb6328 100644 --- a/nix/tools/loadtest.nix +++ b/nix/tools/loadtest.nix @@ -1,6 +1,7 @@ { buildToolbox , checkedShellScript , jq +, libfaketime , python3Packages , vegeta , withTools @@ -68,28 +69,31 @@ let jwt-hs) export PGRST_JWT_CACHE_MAX_ENTRIES="0" + ${genTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http + # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ ${withTools.withPgrst} -m "$_arg_monitor" \ - ${withGenTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; jwt-hs-cache) + ${genTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http + # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ ${withTools.withPgrst} -m "$_arg_monitor" \ - ${withGenTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; jwt-hs-cache-worst) + ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} --method "$_arg_method" --worst "$_arg_testdir"/gen_targets.http + # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ - ${withTools.withPgrst} -m "$_arg_monitor" \ - ${withGenTargets} --method "$_arg_method" --worst "$_arg_testdir"/gen_targets.http \ + ${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; @@ -100,10 +104,11 @@ let ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json" + ${genTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http + # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ ${withTools.withPgrst} -m "$_arg_monitor" \ - ${withGenTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; @@ -112,10 +117,11 @@ let ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json" + ${genTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http + # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ ${withTools.withPgrst} -m "$_arg_monitor" \ - ${withGenTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; @@ -124,10 +130,11 @@ let ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json" + ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} --method "$_arg_method" --worst --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http + # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ - ${withTools.withPgrst} -m "$_arg_monitor" \ - ${withGenTargets} --method "$_arg_method" --worst --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \ + ${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \ sh -c "cd \"$_arg_testdir\" && \ ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; @@ -309,8 +316,8 @@ let | ${mergeMonitorResults} ''; - withGenTargets = - writers.writePython3 "postgrest-with-gen-loadtest-targets" + genTargets = + writers.writePython3 "postgrest-gen-loadtest-targets" { libraries = [ python3Packages.pyjwt python3Packages.jwcrypto ]; doCheck = false; # postgrest-style conflicts with this diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index f5518558c..da7d72fe0 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -3,6 +3,7 @@ , curl , git , lib +, libfaketime , postgresqlVersions , postgrest , python3Packages @@ -288,6 +289,7 @@ let [ "ARG_POSITIONAL_SINGLE([command], [Command to run])" "ARG_LEFTOVERS([command arguments])" + "ARG_OPTIONAL_SINGLE([faketime], [f], [Fake the system time when starting PostgREST. This is useful to test expiry of JWT, for example in loadtests])" "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])" "ARG_OPTIONAL_SINGLE([timeout], [t], [Maximum time to wait for PostgREST to be ready], [5])" "ARG_OPTIONAL_SINGLE([sleep], [s], [Sleep time after PostgREST is ready, this is useful for monitoring])" @@ -301,6 +303,8 @@ let '' export PGRST_SERVER_UNIX_SOCKET="$tmpdir"/postgrest.socket + FAKETIME_CMD="${libfaketime}/bin/faketime" + if [ -z "''${PGRST_CMD:-}" ]; then rm -f result build_start=$SECONDS @@ -313,6 +317,8 @@ let exit 1 } PGRST_CMD=$(echo ./result*/bin/postgrest) + # To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch. + FAKETIME_CMD="$(nix-build -A pkgs.libfaketime)/bin/faketime" else echo -n "${commandName}: Building postgrest (cabal)... " postgrest-build @@ -326,7 +332,11 @@ let echo -n "${commandName}: Starting $ver... " - $PGRST_CMD > "$tmpdir"/run.log 2>&1 & + if [[ -n "$_arg_faketime" ]]; then + $FAKETIME_CMD "$_arg_faketime" "$PGRST_CMD" > "$tmpdir"/run.log 2>&1 & + else + $PGRST_CMD > "$tmpdir"/run.log 2>&1 & + fi pid=$! # shellcheck disable=SC2329 cleanup() {