diff --git a/default.nix b/default.nix index 2161e2624..e05bb45db 100644 --- a/default.nix +++ b/default.nix @@ -130,7 +130,7 @@ rec { # Development tools. devTools = - pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie withTools; }; + pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie; }; # Documentation tools. docs = diff --git a/nix/README.md b/nix/README.md index 10d1fb3e8..faa2fef9f 100644 --- a/nix/README.md +++ b/nix/README.md @@ -162,9 +162,6 @@ The loadtests ensure that performance doesn't drop on a change. Underlyingly the # You can loadtest comparing to a different branch [nix-shell]$ postgrest-loadtest-against main -# You can build postgrest directly with cabal for faster iteration -[nix-shell]$ PGRST_BUILD_CABAL=1 postgrest-loadtest - # Produce a markdown report to be used on CI [nix-shell]$ postgrest-loadtest-report ``` diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix index c47b54428..27fe35566 100644 --- a/nix/tools/loadtest.nix +++ b/nix/tools/loadtest.nix @@ -1,5 +1,6 @@ { buildToolbox , checkedShellScript +, git , jq , libfaketime , python3Packages @@ -43,7 +44,6 @@ let docs = "Run the vegeta loadtests with PostgREST."; args = [ "ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])" - "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])" "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])" "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-cache,jwt-cache-worst])" "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])" @@ -65,23 +65,23 @@ let case "$_arg_kind" in jwt-cache) - export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwks.json" + export PGRST_JWT_SECRET="@test/load/gen_jwks.json" - ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} "$_arg_testdir" + ${libfaketime}/bin/faketime '2000-01-01 00:00:00' ${genTargets} test/load # shellcheck disable=SC2145 - ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ + ${withTools.withPg} -f test/load/fixtures.sql \ ${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \ - sh -c "cd \"$_arg_testdir\" && \ + sh -c "cd test/load && \ ${runner} -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; # here we sleep purposefully to check how much memory does the schema cache consume in the final report mixed) # shellcheck disable=SC2145 - ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ + ${withTools.withPg} -f test/load/fixtures.sql \ ${withTools.withPgrst} --timeout 2 --sleep 5 -m "$_arg_monitor" \ - sh -c "cd \"$_arg_testdir\" && \ + sh -c "cd test/load && \ ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; esac @@ -146,7 +146,39 @@ let workingDir = "/"; } '' - # run loadtest for every target and HEAD + # Build postgrest for every target and HEAD. + # Keeps a reference to the postgrest binary and faketime lib for every branch to run later. + declare -A pgrst faketime + for tgt in "''${_arg_target[@]}" HEAD; do + # not using withTmpDir here, because we don't want to keep the directory on error + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' EXIT + + ${git}/bin/git worktree add -f "$tmpdir" "$tgt" > /dev/null + pushd "$tmpdir" > /dev/null + + build_start=$SECONDS + echo -n "${name}: Building postgrest (nix) on $tgt... " + # Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet. + nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > build.log 2>&1 || { + echo "failed, output:" + cat build.log + exit 1 + } + pgrst[$tgt]="$(nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage')/bin/postgrest" + # To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch. + faketime[$tgt]="$(nix-build --no-out-link -A pkgs.libfaketime)/lib/libfaketime.so.1" + build_end=$((SECONDS - build_start)) + printf "done in %ss.\n" "$build_end" + + popd > /dev/null + ${git}/bin/git worktree remove -f "$tmpdir" > /dev/null + rm -rf "$tmpdir" + done + + # Run loadtest for every target and HEAD. + # Running the tests is separated from building them to reduce the chances of + # other processes skewing the results between two runs. for tgt in "''${_arg_target[@]}" HEAD; do cat << EOF @@ -155,12 +187,7 @@ let EOF - # Runs the test files from the current working tree - # to make sure both tests are run with the same files. - # Save the results in the current working tree, too, - # otherwise they'd be lost in the temporary working tree - # created by withTools.withGit. - ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/$tgt.csv" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load" + FAKETIME_LIB="''${faketime[$tgt]}" PGRST_CMD="''${pgrst[$tgt]}" ${loadtest} -k "$_arg_kind" -m "loadtest/$tgt.csv" --output "loadtest/$tgt.bin" cat << EOF diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index dddac0470..3696bf478 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -1,7 +1,6 @@ { buildToolbox , checkedShellScript , curl -, git , lib , libfaketime , postgresqlVersions @@ -197,47 +196,6 @@ let withPg = withTmpDb (builtins.head postgresqlVersions); - withGit = - let - name = "postgrest-with-git"; - in - checkedShellScript - { - inherit name; - docs = - '' - Create a new worktree of the postgrest repo in a temporary directory and - check out , then run with arguments inside the temporary folder. - ''; - args = - [ - "ARG_POSITIONAL_SINGLE([commit], [Commit-ish reference to run command with])" - "ARG_POSITIONAL_SINGLE([command], [Command to run])" - "ARG_LEFTOVERS([command arguments])" - ]; - positionalCompletion = - '' - if test "$prev" == "${name}"; then - __gitcomp_nl "$(__git_refs)" - else - _command_offset 2 - fi - ''; - workingDir = "/"; - } - '' - # not using withTmpDir here, because we don't want to keep the directory on error - tmpdir="$(mktemp -d)" - trap 'rm -rf "$tmpdir"' EXIT - - ${git}/bin/git worktree add -f "$tmpdir" "$_arg_commit" > /dev/null - - cd "$tmpdir" - ("$_arg_command" "''${_arg_leftovers[@]}") - - ${git}/bin/git worktree remove -f "$tmpdir" > /dev/null - ''; - waitForPgrstReady = checkedShellScript { @@ -294,7 +252,8 @@ let "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])" "ARG_OPTIONAL_SINGLE([timeout], [t], [Maximum time to wait for PostgREST to be ready], [5])" "ARG_OPTIONAL_SINGLE([sleep], [s], [Sleep time after PostgREST is ready, this is useful for monitoring])" - "ARG_USE_ENV([PGRST_CMD], [], [PostgREST executable to run])" + "ARG_USE_ENV([FAKETIME_LIB], [${libfaketime}/lib/libfaketime.so.1], [Faketime Library to preload])" + "ARG_USE_ENV([PGRST_CMD], [postgrest-run], [PostgREST executable to run])" ]; positionalCompletion = "_command"; workingDir = "/"; @@ -304,27 +263,10 @@ let '' export PGRST_SERVER_UNIX_SOCKET="$tmpdir"/postgrest.socket - FAKETIME_LIB="${libfaketime}/lib/libfaketime.so.1" - - if [ -z "''${PGRST_CMD:-}" ]; then - rm -f result + if [ "''${PGRST_CMD}" == "postgrest-run" ]; then build_start=$SECONDS - if [ -z "''${PGRST_BUILD_CABAL:-}" ]; then - echo -n "${commandName}: Building postgrest (nix)... " - # Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet. - nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > "$tmpdir"/build.log 2>&1 || { - echo "failed, output:" - cat "$tmpdir"/build.log - exit 1 - } - PGRST_CMD="$(nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage')/bin/postgrest" - # To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch. - FAKETIME_LIB="$(nix-build --no-out-link -A pkgs.libfaketime)/lib/libfaketime.so.1" - else - echo -n "${commandName}: Building postgrest (cabal)... " - postgrest-build - PGRST_CMD=postgrest-run - fi + echo -n "${commandName}: Building postgrest (cabal)... " + postgrest-build build_end=$((SECONDS - build_start)) printf "done in %ss.\n" "$build_end" fi @@ -387,7 +329,6 @@ buildToolbox name = "postgrest-with"; tools = { inherit - withGit withPgAll withPgrst; } // builtins.listToAttrs (