diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1940ade81..4ed3cb02a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,6 +25,10 @@ on: - test/** - '!**.md' +defaults: + run: + shell: bash + concurrency: # Terminate all previous runs of the same workflow for pull requests group: test-${{ github.head_ref || github.run_id }} @@ -152,10 +156,34 @@ jobs: latest_tag=$(git tag --merged HEAD --sort=-creatordate "v*" | head -n1) fi postgrest-loadtest-against -k ${{ matrix.kind }} "$TARGET_BRANCH" "$latest_tag" + + - name: Report P50 + # This step checks whether any red cross indicators (:x:) are present in the step summary. + # The loadtest reporter writes them when any of individual steps fails the performance + # regression threshold. + run: | + postgrest-loadtest-report -g ${{ matrix.kind }} -p 50 \ + | tee "$GITHUB_STEP_SUMMARY" \ + | grep -v ':x:' + + - name: Report P0 + if: always() + run: | postgrest-loadtest-report -g ${{ matrix.kind }} -p 0 >> "$GITHUB_STEP_SUMMARY" - postgrest-loadtest-report -g ${{ matrix.kind }} -p 50 >> "$GITHUB_STEP_SUMMARY" + + - name: Report P90 + if: always() + run: | postgrest-loadtest-report -g ${{ matrix.kind }} -p 90 >> "$GITHUB_STEP_SUMMARY" + + - name: Report P95 + if: always() + run: | postgrest-loadtest-report -g ${{ matrix.kind }} -p 95 >> "$GITHUB_STEP_SUMMARY" + + - name: Report CPU/MEM + if: always() + run: | postgrest-loadtest-report-load -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY" flake: diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix index 3553044d5..2130bf1a7 100644 --- a/nix/tools/loadtest.nix +++ b/nix/tools/loadtest.nix @@ -274,12 +274,28 @@ let import sys import pandas as pd + + def evaluate_change(df): + return ((df['head'] / df['main'] - 1) * 100) \ + .map(lambda r: "{icon} {ratio:.1f} %".format( + ratio=r, + # Hardcoded failure threshold for CI is 5% here. + icon="" if r < 5 else ":x:" + )) + + pd.read_json(sys.stdin) \ .rename(columns={'latency': sys.argv[1]}) \ .set_index(sys.argv[1]) \ .drop(['branch']) \ .convert_dtypes() \ - .to_markdown(sys.stdout, floatfmt='.1f') + .assign(change=evaluate_change) \ + .to_markdown( + sys.stdout, + floatfmt='.1f', + colglobalalign='right', + colalign=('left',) + ) '';