nix(loadtest): merge jwt-rsa-* and jwt-hs-* tests

Instead of creating separate test suites for the key type, the PostgREST
instance now accepts both keys via a JWKSet and the targets are
generated 50/50 for both.

The results are still reported seperately by using a different URL,
which shows up as separate rows in the results.

.github/workflows/test.yaml

@@ -131,7 +131,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kind: ['mixed', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
+        kind: ['mixed', 'jwt', 'jwt-cache', 'jwt-cache-worst']
     name: Loadtest
     runs-on: ubuntu-24.04
     steps:

.gitignore

@@ -25,7 +25,7 @@ loadtest
 .history
 .docs-build
 gen_targets.http
-gen_jwk.json
+gen_jwks.json
 gen_private.json
 .pytest_cache
 .ruff_cache

nix/tools/gen_key_materials.py

@@ -1,4 +1,4 @@
-# Generate RSA JWK/public material for loadtests.
+# Generate HS & RSA JWK/public material for loadtests.
 
 import argparse
 import sys
@@ -12,12 +12,12 @@ def main():
         description="Generate RSA JWK/private key pair for loadtests"
     )
     parser.add_argument(
-        "--rsa",
+        "--jwks",
-        dest="jwk_path",
+        dest="jwks_path",
-        metavar="JWK_PATH",
+        metavar="JWKS_PATH",
         type=Path,
         required=True,
-        help="Path to write the RSA JWK file",
+        help="Path to write the JWKS file",
     )
     parser.add_argument(
         "--private-key",
@@ -30,18 +30,25 @@ def main():
 
     args = parser.parse_args()
 
-    key = jwk.JWK.generate(kty="RSA", size=4096)
+    hs = jwk.JWK.from_password("reallyreallyreallyreallyverysafe")
-    private_jwk, public_jwk = key.export_private(), key.export_public()
+    rsa = jwk.JWK.generate(kty="RSA", size=4096)
+
+    jwks = jwk.JWKSet()
+    jwks.add(hs)
+    jwks.add(rsa)
 
     try:
-        args.jwk_path.write_text(public_jwk)
+        # Technically, this exports the private keys, because HS does not have the concept
-        print(f"Created RSA JWK on {args.jwk_path}")
+        # of a public key. This is not a problem for tests, though, PostgREST can verify
+        # tokens with the private key just as well.
+        args.jwks_path.write_text(jwks.export())
+        print(f"Created JWKSet on {args.jwks_path}")
     except OSError as e:
-        print(f"Error writing to {args.jwk_path}:{e}", file=sys.stderr)
+        print(f"Error writing to {args.jwks_path}:{e}", file=sys.stderr)
         sys.exit(1)
 
     try:
-        args.private_key_path.write_text(private_jwk)
+        args.private_key_path.write_text(rsa.export_private())
         print(f"Created private key on {args.private_key_path}")
     except OSError as e:
         print(f"Error writing to {args.private_key_path}:{e}", file=sys.stderr)

nix/tools/generate_targets.py

@@ -48,7 +48,7 @@ def generate_target(
     token = jwt.encode(payload, key, alg)
 
     return [
-        f"OPTIONS {URL}/authors_only",
+        f"OPTIONS {URL}/authors_only?{alg}",
         f"Authorization: Bearer {token}",
         "",  # blank line to separate requests
     ]
@@ -90,7 +90,6 @@ def main():
         metavar="PRIVATE_KEY_PATH",
         type=Path,
         help="Path to the RSA private key file",
-        default=None,
     )
     parser.add_argument(
         "--worst",
@@ -104,35 +103,25 @@ def main():
 
     rsa_private_key: Optional[jwt.algorithms.RSAAlgorithm] = None
 
-    is_hs = args.private_key_path is None
+    nsamples = 500  # per algorithm
+    ntargets = 100000
 
-    nsamples = 1000
+    try:
+        private_key_data = args.private_key_path.read_text()
+    except OSError as e:
+        err = (
+            f"Error reading RSA private key from {args.private_key_path}: "
+            f"{e}. Generate RSA materials first with gen_rsa_materials.py."
+        )
+        print(err, file=sys.stderr)
+        sys.exit(1)
 
-    if is_hs:
+    try:
-        ntargets = 200000
+        rsa_private_key = jwt.algorithms.RSAAlgorithm.from_jwk(private_key_data)
-    else:
+    except Exception as exc:  # broad exception to capture parsing errors
-        # The asymmetric targets take too long to compute so we reduce them
+        err = f"Error loading RSA private key from {args.private_key_path}: " f"{exc}"
-        ntargets = 50000
+        print(err, file=sys.stderr)
-
+        sys.exit(1)
-    if not is_hs:
-        try:
-            private_key_data = args.private_key_path.read_text()
-        except OSError as e:
-            err = (
-                f"Error reading RSA private key from {args.private_key_path}: "
-                f"{e}. Generate RSA materials first with gen_rsa_materials.py."
-            )
-            print(err, file=sys.stderr)
-            sys.exit(1)
-
-        try:
-            rsa_private_key = jwt.algorithms.RSAAlgorithm.from_jwk(private_key_data)
-        except Exception as exc:  # broad exception to capture parsing errors
-            err = (
-                f"Error loading RSA private key from {args.private_key_path}: " f"{exc}"
-            )
-            print(err, file=sys.stderr)
-            sys.exit(1)
 
     print(f"Generating {ntargets} targets...")
 
@@ -151,14 +140,19 @@ def main():
 
         for i in range(ntargets):
             target = generate_target(
-                now, run_postgrest_time + i // 1000, rsa_private_key
+                now,
+                run_postgrest_time + i // 1000,
+                rsa_private_key if i % 2 == 0 else None,
             )
             lines.extend(target)
 
     else:
-        targets = [generate_target(now, None, rsa_private_key) for _ in range(nsamples)]
+        hs_targets = [generate_target(now, None, None) for _ in range(nsamples)]
+        rsa_targets = [
+            generate_target(now, None, rsa_private_key) for _ in range(nsamples)
+        ]
         for i in range(ntargets):
-            target = random.choice(targets)
+            target = random.choice(hs_targets if i % 2 == 0 else rsa_targets)
             lines.extend(target)
 
     try:

nix/tools/loadtest.nix

@@ -45,7 +45,7 @@ let
           "ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])"
           "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])"
           "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
-          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
+          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt,jwt-cache,jwt-cache-worst])"
           "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
           "ARG_LEFTOVERS([additional vegeta arguments])"
         ];
@@ -64,43 +64,11 @@ let
         abs_output="$(realpath "$_arg_output")"
 
         case "$_arg_kind" in
-          jwt-hs)
+          jwt)
             export PGRST_JWT_CACHE_MAX_ENTRIES="0"
 
-            ${genTargets} "$_arg_testdir"/gen_targets.http
+            ${genKeyMaterials} --jwks="$_arg_testdir"/gen_jwks.json --private-key="$_arg_testdir"/gen_private.json
-
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwks.json"
-            # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} -m "$_arg_monitor" \
-            sh -c "cd \"$_arg_testdir\" && \
-            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
-            ;;
-
-          jwt-hs-cache)
-            ${genTargets} "$_arg_testdir"/gen_targets.http
-
-            # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} -m "$_arg_monitor" \
-            sh -c "cd \"$_arg_testdir\" && \
-            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
-            ;;
-
-          jwt-hs-cache-worst)
-            ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} --worst "$_arg_testdir"/gen_targets.http
-
-            # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \
-            sh -c "cd \"$_arg_testdir\" && \
-            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
-            ;;
-
-          jwt-rsa)
-            export PGRST_JWT_CACHE_MAX_ENTRIES="0"
-
-            ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
-            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
 
             ${genTargets} --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http
 
@@ -111,9 +79,9 @@ let
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
 
-          jwt-rsa-cache)
+          jwt-cache)
-            ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
+            ${genKeyMaterials} --jwks="$_arg_testdir"/gen_jwks.json --private-key="$_arg_testdir"/gen_private.json
-            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwks.json"
 
             ${genTargets} --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http
 
@@ -124,9 +92,9 @@ let
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
 
-          jwt-rsa-cache-worst)
+          jwt-cache-worst)
-            ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
+            ${genKeyMaterials} --jwks="$_arg_testdir"/gen_jwks.json --private-key="$_arg_testdir"/gen_private.json
-            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwks.json"
 
             ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} --worst --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http
 
@@ -349,13 +317,13 @@ let
       }
       (builtins.readFile ./generate_targets.py);
 
-  genRsaMaterials =
+  genKeyMaterials =
-    writers.writePython3 "postgrest-gen-rsa-materials"
+    writers.writePython3 "postgrest-gen-key-materials"
       {
         libraries = [ python3Packages.jwcrypto ];
         doCheck = false; # postgrest-style conflicts with this
       }
-      (builtins.readFile ./gen_rsa_materials.py);
+      (builtins.readFile ./gen_key_materials.py);
 
   mergeMonitorResults =
     writers.writePython3 "postgrest-merge-monitor-results"