diff --git a/.github/workflows/loadtest.yaml b/.github/workflows/loadtest.yaml new file mode 100644 index 000000000..18cba2df4 --- /dev/null +++ b/.github/workflows/loadtest.yaml @@ -0,0 +1,34 @@ +name: Loadtest + +on: + push: + branches: + - main + tags: + - v* + pull_request: + branches: + - main + +jobs: + Loadtest-Nix: + name: Loadtest (Nix) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - name: Setup Nix Environment + uses: ./.github/actions/setup-nix + with: + tools: loadtest + - name: Run loadtest + run: | + postgrest-loadtest-against main + postgrest-loadtest-report > loadtest/loadtest.md + - name: Upload report + uses: actions/upload-artifact@v2.2.4 + with: + name: loadtest.md + path: loadtest/loadtest.md + if-no-files-found: error diff --git a/.gitignore b/.gitignore index d00cdb788..b91c5996f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ __pycache__ *.tix coverage .hpc +loadtest diff --git a/default.nix b/default.nix index dcf437a66..a484819a6 100644 --- a/default.nix +++ b/default.nix @@ -123,6 +123,10 @@ rec { docker = pkgs.callPackage nix/tools/docker { postgrest = postgrestStatic; }; + # Load testing tools. + loadtest = + pkgs.callPackage nix/tools/loadtest.nix { inherit withTools; }; + # Script for running memory tests. memory = pkgs.callPackage nix/tools/memory.nix { inherit postgrestProfiled withTools; }; diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix new file mode 100644 index 000000000..adc38f2b5 --- /dev/null +++ b/nix/tools/loadtest.nix @@ -0,0 +1,165 @@ +{ buildToolbox +, checkedShellScript +, jq +, python3Packages +, vegeta +, withTools +, writers +}: +let + runner = + checkedShellScript + { + name = "postgrest-loadtest-runner"; + docs = "Run vegeta. Assume PostgREST to be running."; + args = [ + "ARG_LEFTOVERS([additional vegeta arguments])" + "ARG_USE_ENV([PGRST_SERVER_UNIX_SOCKET], [], [Unix socket to connect to running PostgREST instance])" + ]; + } + '' + # ARG_USE_ENV only adds defaults or docs for environment variables + # We manually implement a required check here + # See also: https://github.com/matejak/argbash/issues/80 + : "''${PGRST_SERVER_UNIX_SOCKET:?PGRST_SERVER_UNIX_SOCKET is required}" + + ${vegeta}/bin/vegeta -cpus 1 attack \ + -unix-socket "$PGRST_SERVER_UNIX_SOCKET" \ + -max-workers 1 \ + -workers 1 \ + -rate 0 \ + -duration 60s \ + "''${_arg_leftovers[@]}" + ''; + + loadtest = + checkedShellScript + { + name = "postgrest-loadtest"; + docs = "Run the vegeta loadtests with PostgREST."; + args = [ + "ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])" + "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/loadtest])" + "ARG_LEFTOVERS([additional vegeta arguments])" + ]; + inRootDir = true; + } + '' + export PGRST_DB_CONFIG="false" + export PGRST_DB_POOL="1" + export PGRST_DB_TX_END="rollback-allow-override" + export PGRST_LOG_LEVEL="crit" + + mkdir -p "$(dirname "$_arg_output")" + + # shellcheck disable=SC2145 + ${withTools.withPg} --fixtures "$_arg_testdir"/fixtures.sql \ + ${withTools.withPgrst} \ + sh -c "cd \"$_arg_testdir\" && ${runner} -targets targets.http \"''${_arg_leftovers[@]}\"" \ + | tee "$_arg_output" \ + | ${vegeta}/bin/vegeta report -type=text + ''; + + loadtestAgainst = + checkedShellScript + { + name = "postgrest-loadtest-against"; + docs = + '' + Run the vegeta loadtest twice: + - once on the branch + - once in the current worktree + ''; + args = [ + "ARG_POSITIONAL_SINGLE([target], [Commit-ish reference to compare with])" + "ARG_LEFTOVERS([additional vegeta arguments])" + ]; + inRootDir = true; + } + '' + cat << EOF + + Running loadtest on "$_arg_target"... + + EOF + + # Runs the test files from the current working tree + # to make sure both tests are run with the same files. + # Save the results in the current working tree, too, + # otherwise they'd be lost in the temporary working tree + # created by withTools.withGit. + ${withTools.withGit} "$_arg_target" ${loadtest} --output "$PWD/loadtest/$_arg_target.bin" --testdir "$PWD/test/loadtest" "''${_arg_leftovers[@]}" + + cat << EOF + + Done running on "$_arg_target". + + EOF + + cat << EOF + + Running loadtest on HEAD... + + EOF + + ${loadtest} --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/loadtest" "''${_arg_leftovers[@]}" + + cat << EOF + + Done running on HEAD. + + EOF + ''; + + reporter = + checkedShellScript + { + name = "postgrest-loadtest-reporter"; + docs = "Create a named json report for a single result file."; + args = [ + "ARG_POSITIONAL_SINGLE([file], [Filename of result to create report for])" + "ARG_LEFTOVERS([additional vegeta arguments])" + ]; + inRootDir = true; + } + '' + ${vegeta}/bin/vegeta report -type=json "$_arg_file" \ + | ${jq}/bin/jq --arg branch "$(basename "$_arg_file" .bin)" '. + {branch: $branch}' + ''; + + toMarkdown = + writers.writePython3 "postgrest-loadtest-to-markdown" + { + libraries = [ python3Packages.pandas python3Packages.tabulate ]; + } + '' + import sys + import pandas as pd + + pd.read_json(sys.stdin) \ + .set_index('param') \ + .drop(['branch', 'earliest', 'end', 'latest']) \ + .convert_dtypes() \ + .to_markdown(sys.stdout, floatfmt='.0f') + ''; + + + report = + checkedShellScript + { + name = "postgrest-loadtest-report"; + docs = "Create a report of all loadtest reports as markdown."; + inRootDir = true; + } + '' + find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \ + | ${jq}/bin/jq '[leaf_paths as $path | {param: $path | join("."), (.branch): getpath($path)}]' \ + | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \ + | ${toMarkdown} + ''; + +in +buildToolbox { + name = "postgrest-loadtest"; + tools = [ loadtest loadtestAgainst report ]; +} diff --git a/shell.nix b/shell.nix index d3b343732..6e6fb5f97 100644 --- a/shell.nix +++ b/shell.nix @@ -21,6 +21,7 @@ let [ postgrest.cabalTools postgrest.devTools + postgrest.loadtest postgrest.nixpkgsTools postgrest.style postgrest.tests diff --git a/test/loadtest/fixtures.sql b/test/loadtest/fixtures.sql new file mode 100644 index 000000000..aa0b43ec5 --- /dev/null +++ b/test/loadtest/fixtures.sql @@ -0,0 +1,34 @@ +CREATE ROLE postgrest_test_anonymous; +GRANT postgrest_test_anonymous TO :USER; +CREATE SCHEMA test; + +-- PUT+PATCH target needs one record and column to modify +CREATE TABLE test.actors ( + PRIMARY KEY (actor), + actor INT, + name TEXT, + last_modified TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP +); +INSERT INTO test.actors VALUES (1, 'John Doe'); + +-- POST target needs generated PK +CREATE TABLE test.films ( + PRIMARY KEY (film), + film INT GENERATED BY DEFAULT AS IDENTITY, + title TEXT +); + +-- DELETE target remains empty +CREATE TABLE test.roles ( + actor INT REFERENCES test.actors, + film INT REFERENCES test.films, + character TEXT +); + +CREATE FUNCTION test.call_me (name TEXT) RETURNS TEXT +STABLE LANGUAGE SQL AS $$ + SELECT 'Hello ' || name || ', how are you?'; +$$; + +GRANT USAGE ON SCHEMA test TO postgrest_test_anonymous; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA test TO postgrest_test_anonymous; diff --git a/test/loadtest/patch.json b/test/loadtest/patch.json new file mode 100644 index 000000000..035306dd7 --- /dev/null +++ b/test/loadtest/patch.json @@ -0,0 +1,3 @@ +{ + "last_modified": "now" +} diff --git a/test/loadtest/post.json b/test/loadtest/post.json new file mode 100644 index 000000000..2ffa497c9 --- /dev/null +++ b/test/loadtest/post.json @@ -0,0 +1,3 @@ +{ + "title": "Workers Leaving The Lumière Factory In Lyon" +} diff --git a/test/loadtest/put.json b/test/loadtest/put.json new file mode 100644 index 000000000..ccb40ba39 --- /dev/null +++ b/test/loadtest/put.json @@ -0,0 +1,5 @@ +{ + "actor": 1, + "name": "John Doe", + "last_modified": "now" +} diff --git a/test/loadtest/rpc.json b/test/loadtest/rpc.json new file mode 100644 index 000000000..52192609d --- /dev/null +++ b/test/loadtest/rpc.json @@ -0,0 +1,3 @@ +{ + "name": "John" +} diff --git a/test/loadtest/targets.http b/test/loadtest/targets.http new file mode 100644 index 000000000..dbc762dbc --- /dev/null +++ b/test/loadtest/targets.http @@ -0,0 +1,30 @@ +GET http://postgrest/ +Prefer: tx=commit + +HEAD http://postgrest/actors?actor=eq.1 +Prefer: tx=commit + +GET http://postgrest/actors?select=*,roles(*,films(*)) +Prefer: tx=commit + +POST http://postgrest/films?columns=title +Prefer: tx=rollback +@post.json + +PUT http://postgrest/actors?actor=eq.1&columns=name +Prefer: tx=rollback +@put.json + +PATCH http://postgrest/actors?actor=eq.1 +Prefer: tx=rollback +@patch.json + +DELETE http://postgrest/roles +Prefer: tx=rollback + +GET http://postgrest/rpc/call_me?name=John + +POST http://postgrest/rpc/call_me +@rpc.json + +OPTIONS http://postgrest/actors