nix(feat): Add postgrest-loadtest

.github/workflows/loadtest.yaml

@@ -0,0 +1,34 @@
+name: Loadtest
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  Loadtest-Nix:
+    name: Loadtest (Nix)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          fetch-depth: 0
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+        with:
+          tools: loadtest
+      - name: Run loadtest
+        run: |
+          postgrest-loadtest-against main
+          postgrest-loadtest-report > loadtest/loadtest.md
+      - name: Upload report
+        uses: actions/upload-artifact@v2.2.4
+        with:
+          name: loadtest.md
+          path: loadtest/loadtest.md
+          if-no-files-found: error

.gitignore

@@ -21,3 +21,4 @@ __pycache__
 *.tix
 coverage
 .hpc
+loadtest

default.nix

@@ -123,6 +123,10 @@ rec {
   docker =
     pkgs.callPackage nix/tools/docker { postgrest = postgrestStatic; };
 
+  # Load testing tools.
+  loadtest =
+    pkgs.callPackage nix/tools/loadtest.nix { inherit withTools; };
+
   # Script for running memory tests.
   memory =
     pkgs.callPackage nix/tools/memory.nix { inherit postgrestProfiled withTools; };

nix/tools/loadtest.nix

@@ -0,0 +1,165 @@
+{ buildToolbox
+, checkedShellScript
+, jq
+, python3Packages
+, vegeta
+, withTools
+, writers
+}:
+let
+  runner =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-runner";
+        docs = "Run vegeta. Assume PostgREST to be running.";
+        args = [
+          "ARG_LEFTOVERS([additional vegeta arguments])"
+          "ARG_USE_ENV([PGRST_SERVER_UNIX_SOCKET], [], [Unix socket to connect to running PostgREST instance])"
+        ];
+      }
+      ''
+        # ARG_USE_ENV only adds defaults or docs for environment variables
+        # We manually implement a required check here
+        # See also: https://github.com/matejak/argbash/issues/80
+        : "''${PGRST_SERVER_UNIX_SOCKET:?PGRST_SERVER_UNIX_SOCKET is required}"
+
+        ${vegeta}/bin/vegeta -cpus 1 attack \
+                                     -unix-socket "$PGRST_SERVER_UNIX_SOCKET" \
+                                     -max-workers 1 \
+                                     -workers 1 \
+                                     -rate 0 \
+                                     -duration 60s \
+                                     "''${_arg_leftovers[@]}"
+      '';
+
+  loadtest =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest";
+        docs = "Run the vegeta loadtests with PostgREST.";
+        args = [
+          "ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])"
+          "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/loadtest])"
+          "ARG_LEFTOVERS([additional vegeta arguments])"
+        ];
+        inRootDir = true;
+      }
+      ''
+        export PGRST_DB_CONFIG="false"
+        export PGRST_DB_POOL="1"
+        export PGRST_DB_TX_END="rollback-allow-override"
+        export PGRST_LOG_LEVEL="crit"
+
+        mkdir -p "$(dirname "$_arg_output")"
+
+        # shellcheck disable=SC2145
+        ${withTools.withPg} --fixtures "$_arg_testdir"/fixtures.sql \
+        ${withTools.withPgrst} \
+        sh -c "cd \"$_arg_testdir\" && ${runner} -targets targets.http \"''${_arg_leftovers[@]}\"" \
+          | tee "$_arg_output" \
+          | ${vegeta}/bin/vegeta report -type=text
+      '';
+
+  loadtestAgainst =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-against";
+        docs =
+          ''
+            Run the vegeta loadtest twice:
+            - once on the <target> branch
+            - once in the current worktree
+          '';
+        args = [
+          "ARG_POSITIONAL_SINGLE([target], [Commit-ish reference to compare with])"
+          "ARG_LEFTOVERS([additional vegeta arguments])"
+        ];
+        inRootDir = true;
+      }
+      ''
+        cat << EOF
+
+        Running loadtest on "$_arg_target"...
+
+        EOF
+
+        # Runs the test files from the current working tree
+        # to make sure both tests are run with the same files.
+        # Save the results in the current working tree, too,
+        # otherwise they'd be lost in the temporary working tree
+        # created by withTools.withGit.
+        ${withTools.withGit} "$_arg_target" ${loadtest} --output "$PWD/loadtest/$_arg_target.bin" --testdir "$PWD/test/loadtest" "''${_arg_leftovers[@]}"
+
+        cat << EOF
+
+        Done running on "$_arg_target".
+
+        EOF
+
+        cat << EOF
+
+        Running loadtest on HEAD...
+
+        EOF
+
+        ${loadtest} --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/loadtest" "''${_arg_leftovers[@]}"
+
+        cat << EOF
+
+        Done running on HEAD.
+
+        EOF
+      '';
+
+  reporter =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-reporter";
+        docs = "Create a named json report for a single result file.";
+        args = [
+          "ARG_POSITIONAL_SINGLE([file], [Filename of result to create report for])"
+          "ARG_LEFTOVERS([additional vegeta arguments])"
+        ];
+        inRootDir = true;
+      }
+      ''
+        ${vegeta}/bin/vegeta report -type=json "$_arg_file" \
+          | ${jq}/bin/jq --arg branch "$(basename "$_arg_file" .bin)" '. + {branch: $branch}'
+      '';
+
+  toMarkdown =
+    writers.writePython3 "postgrest-loadtest-to-markdown"
+      {
+        libraries = [ python3Packages.pandas python3Packages.tabulate ];
+      }
+      ''
+        import sys
+        import pandas as pd
+
+        pd.read_json(sys.stdin) \
+          .set_index('param') \
+          .drop(['branch', 'earliest', 'end', 'latest']) \
+          .convert_dtypes() \
+          .to_markdown(sys.stdout, floatfmt='.0f')
+      '';
+
+
+  report =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-report";
+        docs = "Create a report of all loadtest reports as markdown.";
+        inRootDir = true;
+      }
+      ''
+        find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \
+          | ${jq}/bin/jq '[leaf_paths as $path | {param: $path | join("."), (.branch): getpath($path)}]' \
+          | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \
+          | ${toMarkdown}
+      '';
+
+in
+buildToolbox {
+  name = "postgrest-loadtest";
+  tools = [ loadtest loadtestAgainst report ];
+}

shell.nix

@@ -21,6 +21,7 @@ let
     [
       postgrest.cabalTools
       postgrest.devTools
+      postgrest.loadtest
       postgrest.nixpkgsTools
       postgrest.style
       postgrest.tests

test/loadtest/fixtures.sql

@@ -0,0 +1,34 @@
+CREATE ROLE postgrest_test_anonymous;
+GRANT postgrest_test_anonymous TO :USER;
+CREATE SCHEMA test;
+
+-- PUT+PATCH target needs one record and column to modify
+CREATE TABLE test.actors (
+  PRIMARY KEY (actor),
+  actor         INT,
+  name          TEXT,
+  last_modified TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
+);
+INSERT INTO test.actors VALUES (1, 'John Doe');
+
+-- POST target needs generated PK
+CREATE TABLE test.films (
+  PRIMARY KEY (film),
+  film  INT GENERATED BY DEFAULT AS IDENTITY,
+  title TEXT
+);
+
+-- DELETE target remains empty
+CREATE TABLE test.roles (
+  actor     INT REFERENCES test.actors,
+  film      INT REFERENCES test.films,
+  character TEXT
+);
+
+CREATE FUNCTION test.call_me (name TEXT) RETURNS TEXT
+STABLE LANGUAGE SQL AS $$
+  SELECT 'Hello ' || name || ', how are you?';
+$$;
+
+GRANT USAGE ON SCHEMA test TO postgrest_test_anonymous;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA test TO postgrest_test_anonymous;

test/loadtest/patch.json

@@ -0,0 +1,3 @@
+{
+  "last_modified": "now"
+}

test/loadtest/post.json

@@ -0,0 +1,3 @@
+{
+  "title": "Workers Leaving The Lumière Factory In Lyon"
+}

test/loadtest/put.json

@@ -0,0 +1,5 @@
+{
+  "actor": 1,
+  "name": "John Doe",
+  "last_modified": "now"
+}

test/loadtest/rpc.json

@@ -0,0 +1,3 @@
+{
+  "name": "John"
+}

test/loadtest/targets.http

@@ -0,0 +1,30 @@
+GET http://postgrest/
+Prefer: tx=commit
+
+HEAD http://postgrest/actors?actor=eq.1
+Prefer: tx=commit
+
+GET http://postgrest/actors?select=*,roles(*,films(*))
+Prefer: tx=commit
+
+POST http://postgrest/films?columns=title
+Prefer: tx=rollback
+@post.json
+
+PUT http://postgrest/actors?actor=eq.1&columns=name
+Prefer: tx=rollback
+@put.json
+
+PATCH http://postgrest/actors?actor=eq.1
+Prefer: tx=rollback
+@patch.json
+
+DELETE http://postgrest/roles
+Prefer: tx=rollback
+
+GET http://postgrest/rpc/call_me?name=John
+
+POST http://postgrest/rpc/call_me
+@rpc.json
+
+OPTIONS http://postgrest/actors