bump version to 14.8

fix: use int32/int64 formats for integer types
Fixed integer type mapping in OpenAPI 2.0: replaced the invalid integer format with int32/int64 and added the toSwaggerFormat function to map PostgreSQL types to valid OpenAPI 2.0 formats: smallint -> int32 integer -> int32 bigint -> int64
2026-04-03 16:51:55 -05:00 · 2026-04-03 16:36:55 -05:00 · 2026-04-03 16:36:55 -05:00 · 2026-04-03 08:00:57 +05:00 · 2026-04-02 20:06:52 +00:00 · 2026-04-02 13:34:38 -05:00
184 changed files with 6312 additions and 3377 deletions
@@ -4,17 +4,18 @@ When submitting a new feature or fix:
 - Add a new entry to the CHANGELOG - https://github.com/PostgREST/postgrest/blob/main/CHANGELOG.md#unreleased
 - If relevant, update the docs
 - Use a prefix for the PR title or commits, e.g. "fix: description of the fix".
-  + `fix`, bug fixes
-  + `feat`, new features added
-  + `perf`, performance improvements
-  + `docs`, updating the documentation
-  + `nix`, related to the Nix development environment
-  + `ci`, related to the Continuous Integration modules
-  + `test`, related to the testing modules
-  + `refactor`, refactoring code
-  + `deprecate`, deprecating a feature
-  + `changelog`, updating the CHANGELOG
-  + `chore`, maintenance (build process, updating sponsors, etc.)
+  + `add`, Add a new feature
+  + `amend`, To amend an unrealease commit
+  + `change`, Breaking changes
+  + `chore`, Maintenance, update sponsors, changelog, readme etc
+  + `ci`, CI configuration files and scripts
+  + `docs`, Documentation
+  + `fix`, Bug fix
+  + `nix`, Related to Nix
+  + `perf`, Performance improvements
+  + `refactor`, Refactoring code
+  + `remove`, Remove a feature or fix
+  + `test`, Adding tests
  + Other prefixes may be used if necessary
 - If there's a breaking change, add `BREAKING CHANGE` and an explanation to your commit message
 -->
@@ -112,7 +112,7 @@ runs:
        echo "artifacts=${artifacts}" >> "$GITHUB_OUTPUT"
    - name: Save artifact to GitHub Actions
      if: steps.find-task.outputs.task_found
-      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
      with:
        name: ${{ inputs.upload }}
        path: ${{ steps.download.outputs.artifacts }}
@@ -19,14 +19,14 @@ inputs:
 runs:
  using: composite
  steps:
-    - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+    - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
      if: ${{ startsWith(github.ref, 'refs/heads/') || (inputs.save-prs && startsWith(github.ref, 'refs/pull/')) }}
      with:
        path: ${{ inputs.path }}
        key: ${{ runner.os }}-${{ inputs.prefix }}-${{ inputs.suffix }}
        restore-keys: |
          ${{ runner.os }}-${{ inputs.prefix }}-
-    - uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+    - uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
      if: ${{ !startsWith(github.ref, 'refs/heads/') && !(inputs.save-prs && startsWith(github.ref, 'refs/pull/'))  }}
      with:
        path: ${{ inputs.path }}
@@ -11,12 +11,12 @@ inputs:
 runs:
  using: composite
  steps:
-    - uses: nixbuild/nix-quick-install-action@63ca48f939ee3b8d835f4126562537df0fee5b91 # v32
+    - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
      with:
        nix_conf: |-
          always-allow-substitutes = true
          max-jobs = auto
-    - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16
+    - uses: cachix/cachix-action@1eb2ef646ac0255473d23a5907ad7b04ce94065c # v17
      with:
        name: postgrest
        authToken: ${{ inputs.authToken }}
@@ -3,7 +3,7 @@
  "extends": [
    "config:best-practices"
  ],
-  "baseBranches": [
+  "baseBranchPatterns": [
    "main",
    "/^v[0-9]+/"
  ],
@@ -13,27 +13,56 @@
  },
  "packageRules": [
    {
-      "matchBaseBranches": [ "/^v[0-9]+/" ],
-      "matchManagers": ["haskell-cabal"],
+      "matchBaseBranches": [
+        "/^v[0-9]+/"
+      ],
+      "matchManagers": [
+        "haskell-cabal"
+      ],
      "enabled": false
    },
    {
-      "matchBaseBranches": [ "/^v[0-9]+/" ],
+      "matchBaseBranches": [
+        "/^v[0-9]+/"
+      ],
      "groupName": "all dependencies"
    },
    {
-      "matchManagers": ["haskell-cabal"],
-      "matchPackageNames": ["base", "bytestring", "containers", "directory", "mtl", "parsec", "process", "text"],
+      "matchManagers": [
+        "haskell-cabal"
+      ],
+      "matchPackageNames": [
+        "base",
+        "bytestring",
+        "containers",
+        "directory",
+        "mtl",
+        "parsec",
+        "process",
+        "text"
+      ],
      "groupName": "GHC dependencies"
    },
    {
-      "matchManagers": ["haskell-cabal"],
-      "matchPackageNames": ["hasql", "hasql-dynamic-statements", "hasql-notifications", "hasql-transaction", "hasql-pool"],
+      "matchManagers": [
+        "haskell-cabal"
+      ],
+      "matchPackageNames": [
+        "hasql",
+        "hasql-dynamic-statements",
+        "hasql-notifications",
+        "hasql-transaction",
+        "hasql-pool"
+      ],
      "groupName": "hasql"
    },
    {
-      "matchManagers": ["haskell-cabal"],
-      "matchPackageNames": ["fuzzyset"],
+      "matchManagers": [
+        "haskell-cabal"
+      ],
+      "matchPackageNames": [
+        "fuzzyset"
+      ],
      "allowedVersions": "<0.3"
    }
  ]
@@ -0,0 +1,52 @@
+name: Backport
+
+on:
+  pull_request_target:
+    types:
+      - closed
+      - labeled
+
+jobs:
+  backport:
+    name: Backport
+    runs-on: ubuntu-24.04
+    # It triggers only when PR is already merged on either:
+    #
+    # - The merge event itself (action != labeled) or
+    # - A label event with the right label (backport ...).
+    #
+    # The result will be that we can add the label before or after merge,
+    # but the workflow will only run once the PR had been merged.
+    if: >
+      github.event.pull_request.merged &&
+      (
+        github.event.action != 'labeled' ||
+        startsWith(github.event.label.name, 'backport')
+      )
+    steps:
+
+      # This actions creates the github token using the postgrest app secrets
+      - name: Create Github App Token
+        id: app-token
+        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
+        with:
+          app-id: ${{ vars.POSTGREST_CI_APP_ID }}
+          private-key: ${{ secrets.POSTGREST_CI_PRIVATE_KEY }}
+          permission-contents: write
+          permission-pull-requests: write
+          permission-workflows: write # required when backporting CI changes
+
+      # This is required for backport action to cherry-pick the PR
+      - name: Fetch PR ref
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          token: ${{ steps.app-token.outputs.token }}
+
+      # Backport action that creates the PR with given settings
+      - name: Create backport PR
+        uses: korthout/backport-action@3c06f323a58619da1e8522229ebc8d5de2633e46 # v4.3.0
+        with:
+          github_token: ${{ steps.app-token.outputs.token }}
+          pull_description: 'Backport for #${pull_number}.'
+          pull_title: '${target_branch}: ${pull_title}'
@@ -33,16 +33,16 @@ jobs:
    name: Nix - Linux x86-64 static
    runs-on: ubuntu-24.04
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'

      - name: Build static executable
-        run: nix-build -A postgrestStatic
+        run: nix-build -A postgrestStatic -A postgrestStatic.tests
      - name: Save built executable as artifact
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: postgrest-linux-static-x86-64
          path: result/bin/postgrest
@@ -51,7 +51,7 @@ jobs:
      - name: Build Docker image
        run: nix-build -A docker.image --out-link postgrest-docker.tar.gz
      - name: Save built Docker image as artifact
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: postgrest-docker-x86-64
          path: postgrest-docker.tar.gz
@@ -60,9 +60,9 @@ jobs:

  macos:
    name: Nix - MacOS
-    runs-on: macos-14
+    runs-on: macos-15
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
@@ -105,15 +105,6 @@ jobs:
            artifact: postgrest-macos-aarch64
            deps: brew link --force libpq

-          - name: MacOS x86-64
-            runs-on: macos-13
-            cache: |
-              ~/.stack/pantry
-              ~/.stack/snapshots
-              ~/.stack/stack.sqlite3
-            artifact: postgrest-macos-x86-64
-            deps: brew link --force libpq
-
          - name: Windows
            runs-on: windows-2022
            cache: |
@@ -126,8 +117,8 @@ jobs:
    name: Stack - ${{ matrix.name }}
    runs-on: ${{ matrix.runs-on }}
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - uses: haskell-actions/setup@7909071ceec0344debcc968c6c7a96a52e8dd0d7 # v2.8.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: haskell-actions/setup@f9150cb1d140e9a9271700670baa38991e6fa25c # v2.10.3
        with:
          # This must match the version in stack.yaml's resolver
          ghc-version: 9.6.7
@@ -155,7 +146,7 @@ jobs:
      - name: Strip Executable
        run: strip result/postgrest*
      - name: Save built executable as artifact
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: ${{ matrix.artifact }}
          path: |
@@ -168,7 +159,7 @@ jobs:
    name: Stack - FreeBSD from CirrusCI
    runs-on: ubuntu-24.04
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: ./.github/actions/artifact-from-cirrus
        with:
          token: ${{ github.token }}
@@ -185,8 +176,8 @@ jobs:
    name: Cabal - Linux x86-64 - GHC ${{ matrix.ghc }}
    runs-on: ubuntu-24.04
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - uses: haskell-actions/setup@7909071ceec0344debcc968c6c7a96a52e8dd0d7 # v2.8.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: haskell-actions/setup@f9150cb1d140e9a9271700670baa38991e6fa25c # v2.10.3
        with:
          ghc-version: ${{ matrix.ghc }}
      - name: Cache .cabal
@@ -20,7 +20,7 @@ jobs:
    name: Lint & Style
    runs-on: ubuntu-24.04
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
@@ -30,3 +30,24 @@ jobs:
        run: postgrest-lint
      - name: Run style check (auto-format with `nix-shell --run postgrest-style`)
        run: postgrest-style-check
+
+  commit:
+    if: github.event_name != 'push'  # we don't run this on a push, a failure on push disrupts the release workflow
+    name: Commit
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 100  # fetch history (last 100 commits) instead of default shallow clone history, this is deemed enough for a PR history
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+        with:
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+          tools: gitTools.commitCheck.bin
+      - name: Run commitlint (check locally with `nix-shell --run postgrest-commitlint`)
+        run: |
+          # Fetch target branch explicitly
+          git fetch origin ${{ github.base_ref }}
+
+          # Run commitlint
+          postgrest-commitlint --from origin/${{ github.base_ref }} --to HEAD
@@ -50,14 +50,14 @@ jobs:
      - test
      - build
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ssh-key: ${{ secrets.POSTGREST_SSH_KEY }}
      - name: Tag latest commit
        run: |
          cabal_version="$(grep -oP '^version:\s*\K.*' postgrest.cabal)"

-          if [[ "$cabal_version" == *.*.* ]]; then
+          if [[ "$cabal_version" == *.* ]]; then
            git fetch --tags

            if [ -z "$(git tag --list "v$cabal_version")" ]; then
@@ -27,7 +27,7 @@ jobs:
    name: Build
    runs-on: ubuntu-24.04
    steps:
-    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Setup Nix Environment
      uses: ./.github/actions/setup-nix
      with:
@@ -41,7 +41,7 @@ jobs:
    name: Spellcheck
    runs-on: ubuntu-24.04
    steps:
-    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Setup Nix Environment
      uses: ./.github/actions/setup-nix
      with:
@@ -3,12 +3,13 @@ name: Linkcheck
 on:
  schedule:
    - cron: '1 2 * * 3'
+  workflow_dispatch:

 jobs:
  linkcheck:
    runs-on: ubuntu-24.04
    steps:
-    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Setup Nix Environment
      uses: ./.github/actions/setup-nix
      with:
@@ -26,7 +26,7 @@ jobs:
    needs:
      - build
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Check the version to be released
        run: |
          cabal_version="$(grep -oP '^version:\s*\K.*' postgrest.cabal)"
@@ -49,7 +49,7 @@ jobs:
          echo "Relevant extract from CHANGELOG.md:"
          cat CHANGES.md
      - name: Save CHANGES.md as artifact
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: release-changes
          path: CHANGES.md
@@ -64,9 +64,9 @@ jobs:
    needs:
      - prepare
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Download all artifacts
-        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: artifacts
      - name: Create release bundle with archives for all builds
@@ -81,9 +81,6 @@ jobs:
          tar cJvf "release-bundle/postgrest-${GITHUB_REF_NAME}-macos-aarch64.tar.xz" \
            -C artifacts/postgrest-macos-aarch64 postgrest

-          tar cJvf "release-bundle/postgrest-${GITHUB_REF_NAME}-macos-x86-64.tar.xz" \
-            -C artifacts/postgrest-macos-x86-64 postgrest
-
          tar cJvf "release-bundle/postgrest-${GITHUB_REF_NAME}-freebsd-x86-64.tar.xz" \
            -C artifacts/postgrest-freebsd-x86-64 postgrest

@@ -94,7 +91,7 @@ jobs:
            artifacts/postgrest-windows-x86-64/postgrest.exe

      - name: Save release bundle
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: release-bundle
          path: release-bundle
@@ -138,17 +135,17 @@ jobs:
    env:
      DOCKER_REPO: ${{ vars.DOCKER_REPO }}
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Download x86-64 Docker image
-        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: postgrest-docker-x86-64
      - name: Download aarch64 binary
-        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: postgrest-ubuntu-aarch64
-      - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
-      - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
+      - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+      - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ vars.DOCKER_USER }}
          password: ${{ secrets.DOCKER_PASS }}
@@ -194,8 +191,8 @@ jobs:
      vars.DOCKER_REPO && vars.DOCKER_USER &&
      github.ref == 'refs/tags/devel'
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0
        with:
          username: ${{ vars.DOCKER_USER }}
          password: ${{ secrets.DOCKER_PASS }}
@@ -39,17 +39,19 @@ jobs:
        # https://github.com/actions/runner/issues/241#issuecomment-842566950
        shell: script -qec "bash --noprofile --norc -eo pipefail {0}"
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: tests.coverage.bin tests.testDoctests.bin tests.testSpecIdempotence.bin
+          tools: tests.coverage.bin tests.testDoctests.bin tests.testSpecIdempotence.bin cabalTools.update.bin
+
+      - run: postgrest-cabal-update

      - name: Run coverage (IO tests and Spec tests against PostgreSQL 15)
        run: postgrest-coverage
      - name: Upload coverage to codecov
-        uses: codecov/codecov-action@fdcc8476540edceab3de004e990f80d881c6cc00 # v5.5.0
+        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
        with:
          files: ./coverage/codecov.json
          token: ${{ secrets.CODECOV_TOKEN }}
@@ -67,7 +69,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        pgVersion: [12, 13, 14, 15, 16, 17]
+        pgVersion: [13, 14, 15, 16, 17]
    name: PG ${{ matrix.pgVersion }}
    runs-on: ubuntu-24.04
    defaults:
@@ -76,36 +78,45 @@ jobs:
        # https://github.com/actions/runner/issues/241#issuecomment-842566950
        shell: script -qec "bash --noprofile --norc -eo pipefail {0}"
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: tests.testSpec.bin tests.testIO.bin tests.testBigSchema.bin withTools.postgresql-${{ matrix.pgVersion }}.bin
+          tools: tests.testSpec.bin tests.testObservability.bin tests.testIO.bin tests.testBigSchema.bin withTools.pg-${{ matrix.pgVersion }}.bin cabalTools.update.bin
+
+      - run: postgrest-cabal-update

      - name: Run spec tests
        if: always()
-        run: postgrest-with-postgresql-${{ matrix.pgVersion }} postgrest-test-spec
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-spec
+
+      - name: Run observability tests
+        if: always()
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-observability

      - name: Run IO tests
        if: always()
-        run: postgrest-with-postgresql-${{ matrix.pgVersion }} postgrest-test-io -vv
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-io -vv

      - name: Run IO tests on a big schema
        if: always()
-        run: postgrest-with-postgresql-${{ matrix.pgVersion }} postgrest-test-big-schema -vv
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-big-schema -vv


  memory:
    name: Memory
    runs-on: ubuntu-24.04
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: tests.testMemory.bin
+          tools: tests.testMemory.bin cabalTools.update.bin
+
+      - run: postgrest-cabal-update
+
      - name: Run memory tests
        run: postgrest-test-memory

@@ -113,18 +124,21 @@ jobs:
  loadtest:
    strategy:
      matrix:
-        kind: ['mixed', 'jwt']
+        kind: ['mixed', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
    name: Loadtest
    runs-on: ubuntu-24.04
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: loadtest.loadtestAgainst.bin loadtest.report.bin
+          tools: loadtest.loadtestAgainst.bin loadtest.report.bin cabalTools.update.bin
+
+      - run: postgrest-cabal-update
+
      - name: Run loadtest
        env:
          TARGET_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -135,21 +149,20 @@ jobs:
            latest_tag=$(git tag --merged HEAD --sort=-creatordate "v*" | head -n1)
          fi
          postgrest-loadtest-against -k ${{ matrix.kind }} "$TARGET_BRANCH" "$latest_tag"
-          postgrest-loadtest-report >> "$GITHUB_STEP_SUMMARY"
+          postgrest-loadtest-report -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY"

  flake:
    strategy:
      fail-fast: false
      matrix:
        runs-on:
-          - macos-13 # x86_64-darwin
          - macos-14 # aarch64-darwin
          - ubuntu-24.04 # x86_64-linux
          - ubuntu-24.04-arm # aarch64-linux
    name: Flake Check
    runs-on: ${{ matrix.runs-on }}
    steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
      - name: Setup Nix Environment
@@ -25,3 +25,4 @@ loadtest
 .history
 .docs-build
 gen_targets.http
+gen_jwk.json
@@ -13,26 +13,26 @@ PostgREST ongoing development is only possible thanks to our Sponsors and Backer
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/neon.jpg">
+        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
+          <img width="296px" src="static/supabase.svg">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://tembo.io/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/tembo.png">
+        <a href="https://www.euronodes.com/postgrest" target="_blank">
+          <img width="296px" src="static/euronodes.svg">
        </a>
      </td>
    </tr>
    <tr></tr>
    <tr>
      <td align="center" valign="middle">
-        <a href="https://www.euronodes.com/postgrest" target="_blank">
-          <img width="296px" src="static/euronodes.svg">
+        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/neon.jpg">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
-          <img width="296px" src="static/supabase.png">
+        <a href="https://www.bytebase.com/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/bytebase.svg">
        </a>
      </td>
    </tr>
@@ -93,6 +93,11 @@ PostgREST ongoing development is only possible thanks to our Sponsors and Backer
          <img width="222px" src="static/code-build.png">
        </a>
      </td>
+      <td align="center" valign="middle">
+        <a href="https://tembo.io/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/tembo.png">
+        </a>
+      </td>
    </tr>
  </tbody>
 </table>
@@ -1,15 +1,123 @@
 # Change Log

-All notable changes to this project will be documented in this file.
-This project adheres to [Semantic Versioning](http://semver.org/).
+All notable changes to this project will be documented in this file. From version `14.0` onwards PostgREST follows a `MAJOR.PATCH` two-part versioning. Only even-numbered MAJOR versions will be released, reserving odd-numbered MAJOR versions for development.

 ## Unreleased

+## [14.8] - 2026-04-03
+
+### Added
+
+- Log a `HINT` when the LISTEN channel stops working due to a PostgreSQL bug by @laurenceisla in #4581
+
+### Fixed
+
+- Fix invalid OpenAPI 2.0 format for integer types (`smallint`, `integer`, `bigint`) by @arturbent0 in #4641
+
+## [14.7] - 2026-03-20
+
+### Fixed
+
+- Fix not logging SIGTERM and SIGINT by @steve-chavez in #4728
+
+## [14.6] - 2026-03-06
+
+### Fixed
+
+- Fix leaking table and function names when calculating error hint by @taimoorzaeem in #4675
+
+## [14.5] - 2026-02-12
+
+### Fixed
+
+- Don't hide async exceptions in logs by @stevechavez in #4646
+
+## [14.4] - 2026-01-29
+
+### Fixed
+
+- Ensure Listener connections are released by @mkleczek in #4614
+- Fix incorrectly filtering the returned representation for PATCH requests when using `or/and` filters by @laurenceisla in #3707
+- Fix listener running with exception masked after first failure by @mkleczek #4615
+
+## [14.3] - 2026-01-03
+
+### Fixed
+
+- Fix performance and high memory usage of relation hint calculation by @mkleczek in #4462, #4463
+
+## [14.2] - 2025-12-18
+
+### Fixed
+
+- Fix `hasSingleUnnamedParam` incorrectly matching functions with named parameters by @joelonsql in #4553
+  + Functions with a single named parameter (e.g., `foo(data json)`) no longer incorrectly match the single-param fallback, returning a clean `PGRST202` error instead of a confusing PostgreSQL `42883` error.
+- Fix misleading logs on unsupported PostgreSQL versions by @taimoorzaeem in #4519
+- Fix regression where the `PGRST103` error response was truncated by @laurenceisla in #4455
+  + Happened when an `offset` was greater than the rows requested and `Prefer: count=exact` was sent.
+- Fix not returning `Content-Length` on empty HTTP `201` responses by @laurenceisla in #4518
+- Fix inaccurate Server-Timing header durations by @steve-chavez in #4522
+- Fix inaccurate "Schema cache queried" logs by @steve-chavez in #4522
+
+## [14.1] - 2025-11-05
+
+## Fixed
+
+- Fix `db-pre-config` function failing when function names are pg reserved words by @taimoorzaeem in #4380
+- Fix `server-host=!6` incorrectly binds to IPv4 address by @taimoorzaeem in #3202
+
+## [14.0] - 2025-10-24
+
+### Added
+
+- Bounded JWT cache using the SIEVE algorithm by @mkleczek in #4084
+  + It now uses a fixed size cache instead of arbitrary sized cache.
+- Add `--ready` flag for postgrest healthcheck by @taimoorzaeem in #4239
+
+### Fixed
+
+- Fix not logging OpenAPI queries when `log-query=main-query` is enabled by @steve-chavez in #4226
+- Fix not logging explain query when `log-query=main-query` is enabled by @steve-chavez in #4319
+- Fix not logging transaction variables and db-pre-request function when `log-query=main-query` is enabled by @steve-chavez in #3934
+- Fix not logging the JSON message to stderr on a `PGRST002` error by @laurenceisla in #4129
+- Fix reloading the Schema Cache unnecessarily on a `PGRST002` error by @laurenceisla in #4367
+- Fix schema cache loading taking a long time for large schemas by @mkleczek in #4360, #3704
+
+### Changed
+
+- Drop support for PostgreSQL EOL version 12 by @wolfgangwalther in #3865
+- From now on PostgREST will follow a `MAJOR.PATCH` two-part versioning. Only even-numbered MAJOR versions will be released, reserving odd-numbered MAJOR versions for development.
+- Replaced `jwt-cache-max-lifetime` config with `jwt-cache-max-entries` by @mkleczek in #4084
+- `log-query` config now takes a boolean instead of a string value by @steve-chavez in #3934
+
+## [13.0.8] - 2025-10-24
+
+### Fixed
+
+- Fix loading utf-8 config files with `ASCII` locale set by @taimoorzaeem in #4386
+
+## [13.0.7] - 2025-09-14
+
+### Added
+
+- Improve the `PGRST106` error when the requested schema is invalid by @laurenceisla in #4089
+  + It now shows the invalid schema in the `message` field.
+  + The exposed schemas are now listed in the `hint` instead of the `message` field.
+- Improve error details of `PGRST301` error by @taimoorzaeem in #4051
+
+## [13.0.6] - 2025-08-30
+
+### Fixed
+
+- Fix logging the Haskell type instead of the listener error message directly by @laurenceisla in #3588
+- Fix format of `IPv6` address logged at PostgREST startup by @taimoorzaeem in #4291
+- Fix empty enum in `preferParams` OpenAPI parameter by @laurenceisla in #4292
+
 ## [13.0.5] - 2025-08-24

 ### Fixed

- Fix OpenAPI broken docs link by @taimoorzaeem in #4048
+- Fix OpenAPI broken docs link by @taimoorzaeem in #4080
 - Fix OpenAPI specification incorrectly exposing GET methods for volatile functions by @joelonsql in #4174
 - Fix empty spread embeddings return unexpected SQL error by @taimoorzaeem in #3887
 - Fix `/metrics` endpoint not responding with `Content-Type` header by @taimoorzaeem in #4271
@@ -23,11 +131,11 @@ This project adheres to [Semantic Versioning](http://semver.org/).

 ## [13.0.3] - 2025-06-16

-### Fixed
-
 - Fix `max-affected` preference not failing with RPC when `handling=strict` by @taimoorzaeem in #4100
 - Fix a property definition's type in OpenAPI not showing the correct base type of a recursive domain by @laurenceisla in #4136

+### Fixed
+
 ## [13.0.2] - 2025-06-02

 ### Fixed
@@ -95,6 +203,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - #3956, Drop `/config` endpoint of admin server - @steve-chavez
   + The endpoint was at risk of being left unprotected when exposing it.
   + The accompanying `admin-server-config-enabled` config was also dropped.
+ - #3598, PostgREST now validates the `kid` parameter of the JWT - @wolfgangwalther
+   + If the JWT contains a ``kid`` parameter, then PostgREST will look for the JSON Web Key in the `jwt-secret`.
+   + If the JWT doesn't contain a `kid`, the behavior should be backwards compatible. PostgREST  will try each key in the `jwt-secret` one by one until it finds one that works.
 - #3697, #3602, Querying non-existent table now returns `PGRST205` error instead of empty json - @taimoorzaeem
 - #3600, #3926, Improve JWT errors - @taimoorzaeem
   + Return `PGRST301` error when `Bearer` in auth header is sent empty
@@ -21,7 +21,7 @@ For questions on how to use PostgREST, please use
 ### Reporting an Issue

 * Make sure you test against the latest [stable release](https://github.com/PostgREST/postgrest/releases/latest)
-  and also against the latest [nightly release](https://github.com/PostgREST/postgrest/releases/tag/nightly).
+  and also against the latest [devel release](https://github.com/PostgREST/postgrest/releases/tag/devel).
  It is possible we already fixed the bug you're experiencing.

 * Provide steps to reproduce the issue, including your OS version and
@@ -2,7 +2,7 @@
 # The x86-64 is a single-static-binary image built via Nix, see:
 # nix/tools/docker/README.md

-FROM ubuntu:noble@sha256:7c06e91f61fa88c08cc74f7e1b7c69ae24910d745357e0dfe1d2c0322aaf20f9 AS postgrest
+FROM ubuntu:noble@sha256:186072bba1b2f436cbb91ef2567abca677337cfc786c86e107d25b7072feef0c AS postgrest

 RUN apt-get update -y \
    && apt install -y --no-install-recommends libpq-dev zlib1g-dev jq gcc libnuma-dev \
@@ -22,26 +22,26 @@ API than you are likely to write from scratch.
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/neon.jpg">
+        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
+          <img width="296px" src="static/supabase.svg">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://tembo.io/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/tembo.png">
+        <a href="https://www.euronodes.com/postgrest" target="_blank">
+          <img width="296px" src="static/euronodes.svg">
        </a>
      </td>
    </tr>
    <tr></tr>
    <tr>
      <td align="center" valign="middle">
-        <a href="https://www.euronodes.com/postgrest" target="_blank">
-          <img width="296px" src="static/euronodes.svg">
+        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/neon.jpg">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
-          <img width="296px" src="static/supabase.png">
+        <a href="https://www.bytebase.com/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/bytebase.svg">
        </a>
      </td>
    </tr>
@@ -52,8 +52,8 @@ Big thanks to our sponsors! You can join them by supporting PostgREST on [Patreo

 ## Usage

-1. Download the binary ([latest release](https://github.com/PostgREST/postgrest/releases/latest))
-   for your platform.
+1. See the docs for [how to install PostgREST on your platform](https://docs.postgrest.org/en/stable/explanations/install.html). You can also [use Docker](https://docs.postgrest.org/en/stable/explanations/install.html#docker).
+
 2. Invoke for help:

    ```bash
@@ -142,6 +142,10 @@ You can help PostgREST ongoing maintenance and development by making a regular d

 Every donation will be spent on making PostgREST better for the whole community.

+## Contributing
+
+Contributions are always welcome and appreciated. Please see the [Contributing guidelines](https://github.com/PostgREST/postgrest/blob/main/CONTRIBUTING.md).
+
 ## Thanks

 The PostgREST organization is grateful to:
@@ -1,4 +1,2 @@
 packages: postgrest.cabal
 tests: true
-package *
-  ghc-options: -split-sections
@@ -1 +1 @@
-index-state: hackage.haskell.org 2025-07-08T15:50:10Z
+index-state: hackage.haskell.org 2025-10-29T04:02:18Z
@@ -43,7 +43,6 @@ let
      allOverlays.build-toolbox
      allOverlays.checked-shell-script
      allOverlays.gitignore
-      allOverlays.postgresql-libpq
      (allOverlays.haskell-packages { inherit compiler; })
      allOverlays.slocat
    ];
@@ -54,16 +53,17 @@ let

  postgresqlVersions =
    [
-      { name = "postgresql-17"; postgresql = pkgs.postgresql_17.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
-      { name = "postgresql-16"; postgresql = pkgs.postgresql_16.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
-      { name = "postgresql-15"; postgresql = pkgs.postgresql_15.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
-      { name = "postgresql-14"; postgresql = pkgs.postgresql_14.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
-      { name = "postgresql-13"; postgresql = pkgs.postgresql_13.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
-      { name = "postgresql-12"; postgresql = pkgs.postgresql_12.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
+      { name = "pg-17"; postgresql = pkgs.postgresql_17.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
+      { name = "pg-16"; postgresql = pkgs.postgresql_16.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
+      { name = "pg-15"; postgresql = pkgs.postgresql_15.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
+      { name = "pg-14"; postgresql = pkgs.postgresql_14.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
+      { name = "pg-13"; postgresql = pkgs.postgresql_13.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
    ];

+  haskellPackages = pkgs.haskell.packages."${compiler}";
+
  # Dynamic derivation for PostgREST
-  postgrest = pkgs.lib.pipe (pkgs.haskell.packages."${compiler}".callCabal2nix name src { }) [
+  postgrest = pkgs.lib.pipe (haskellPackages.callCabal2nix name src { }) [
    # To allow ghc-datasize to be used.
    lib.disableLibraryProfiling
    # We are never going to use dynamic haskell libraries anyway. "Dynamic" refers to how
@@ -75,7 +75,7 @@ let

  # Options passed to cabal in dev tools and tests
  devCabalOptions =
-    "-f dev --test-show-detail=direct --disable-shared";
+    "-f dev --test-show-detail=direct";

  inherit (pkgs.haskell) lib;
 in
@@ -84,9 +84,14 @@ rec {

  # Derivation for the PostgREST Haskell package, including the executable,
  # libraries and documentation. We disable running the test suite on Nix
-  # builds, as they require a database to be set up.
-  postgrestPackage =
-    lib.dontCheck postgrest;
+  # builds, as they require a database to be set up. We split the binary
+  # into a separate output, so that the default distribution via flake.nix
+  # has a much smaller closure size.
+  postgrestPackage = pkgs.lib.pipe postgrest [
+    lib.dontCheck
+    lib.enableSeparateBinOutput
+    (haskellPackages.generateOptparseApplicativeCompletions [ "postgrest" ])
+  ];

  # Profiled dynamic executable.
  postgrestProfiled = pkgs.lib.pipe postgrestPackage [
@@ -119,10 +124,18 @@ rec {
  docs =
    pkgs.callPackage nix/tools/docs.nix { };

+  # Git tools.
+  gitTools =
+    pkgs.callPackage nix/tools/gitTools.nix { };
+
  # Load testing tools.
  loadtest =
    pkgs.callPackage nix/tools/loadtest.nix { inherit withTools; };

+  # Utility for updating the pinned version of Nixpkgs.
+  nixpkgsTools =
+    pkgs.callPackage nix/tools/nixpkgsTools.nix { };
+
  # Scripts for publishing new releases.
  release =
    pkgs.callPackage nix/tools/release.nix { };
@@ -19,26 +19,26 @@ write from scratch.
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/neon.jpg">
+        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.svg">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://tembo.io/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/tembo.png">
+        <a href="https://www.euronodes.com/postgrest" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/euronodes.svg">
        </a>
      </td>
    </tr>
    <tr></tr>
    <tr>
      <td align="center" valign="middle">
-        <a href="https://www.euronodes.com/postgrest" target="_blank">
-          <img width="296px" src="static/euronodes.svg">
+        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/neon.jpg">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
+        <a href="https://www.bytebase.com/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/bytebase.svg">
        </a>
      </td>
    </tr>
@@ -40,9 +40,14 @@ database "PostgreSQL" {

 :user:
 hexagon Proxy
-:user: .r-> Proxy
+:user: .r-> Proxy : request with JWT
 HTTPAPI <.l- Proxy

+hexagon ExternalAuth
+ExternalAuth -u[hidden]- Proxy
+:user: .r-> ExternalAuth : login
+:user: <.r- ExternalAuth : JWT
+
 :operator: .d-> HTTPADMIN
 :operator: .d-> CLI

@@ -51,9 +56,8 @@ PostgreSQL <.developer : "\t"
 Listener -r.> "PostgreSQL"
 "Connection Pool" -r.> "PostgreSQL" : "\t\t"

-
 note bottom of Auth
-  Authenticates the user request
+  Validates the JWT
 end note

 note bottom of ApiRequest
@@ -72,6 +76,7 @@ note top of Listener
  LISTEN session
 end note

+url of ExternalAuth is [[../explanations/external_auth.html]]
 url of Admin is [[../references/admin_server.html#admin-server]]
 url of API is [[../explanations/schema_isolation.html]]
 url of Auth is [[../references/auth.html#authn]]
@@ -12,7 +12,6 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.

-import sys
 import os

 # If extensions (or modules to document with autodoc) are in another directory,
@@ -50,7 +49,7 @@ source_suffix = ".rst"
 master_doc = "index"

 # This is overriden by readthedocs with the version tag anyway
-version = "13.0"
+version = "14"
 # To avoid repetition in <title> we set this to an empty string.
 release = ""

@@ -300,8 +299,10 @@ linkcheck_ignore = [
    # 403 only in CI / GitHub Actions
    r"https://www.patreon.com/postgrest",
    r"https://blog.frankel.ch/poor-man-api",
+    r"https://www.cybertec-postgresql.com/.*",
    # Odd SSL error
    r"https://www.dripdepot.com",
+    r"https://www.euronodes.com",
    # New GitHub UI delays comment load, so anchor fails
    r"https://github.com/.*#issuecomment",
    # Random 500 Internal Server Error
@@ -34,7 +34,7 @@ Templates
 Example Apps
 ------------

-* `archtika <https://github.com/thiloho/archtika>`_ - self‑hosted CMS
+* `archtika <https://github.com/thiloho/archtika>`_ - self-hosted CMS
 * `delibrium-postgrest <https://gitlab.com/delibrium/delibrium-postgrest/>`_ - example school API and front-end in Vue.js
 * `ETH-transactions-storage <https://github.com/Adamant-im/ETH-transactions-storage>`_ - indexer for Ethereum to get transaction list by ETH address
 * `general <https://github.com/PierreRochard/general>`_ - example auth back-end
@@ -13,7 +13,7 @@ A role can be thought of as either a database user, or a group of database users
 Roles for Each Web User
 -----------------------

-PostgREST can accommodate either viewpoint. If you treat a role as a single user then the :ref:`jwt_impersonation` does most of what you need. When an authenticated user makes a request PostgREST will switch into the database role for that user, which in addition to restricting queries, is available to SQL through the :code:`current_user` variable.
+PostgREST can accommodate either viewpoint. If you treat a role as a single user then :ref:`user_impersonation` does most of what you need. When an authenticated user makes a request PostgREST will switch into the database role for that user, which in addition to restricting queries, is available to SQL through the :code:`current_user` variable.

 You can use row-level security to flexibly restrict visibility and access for the current user. Here is an `example <https://www.enterprisedb.com:443/blog/application-users-vs-row-level-security>`_ from Tomas Vondra, a chat table storing messages sent between users. Users can insert rows into it to send messages to other users, and query it to see messages sent to them by other users.

@@ -16,7 +16,7 @@ Supported PostgreSQL versions
 =============================

 =============== =================================
-**Supported**   PostgreSQL >= 12
+**Supported**   PostgreSQL >= 13
 =============== =================================

 PostgREST works with all PostgreSQL versions still `officially supported <https://www.postgresql.org/support/versioning/>`_.
@@ -75,8 +75,12 @@ You can get the `official PostgREST Docker image <https://hub.docker.com/r/postg

 .. code-block:: bash

+  # pull the latest version
  docker pull postgrest/postgrest

+  # to pull a particular version, use one of the versions on https://hub.docker.com/r/postgrest/postgrest/tags
+  docker pull postgrest/postgrest:<version>
+
 To configure the container image, use :ref:`env_variables_config`.

 There are two ways to run the PostgREST container: with an existing external database, or through docker-compose.
@@ -142,6 +146,7 @@ To avoid having to install the database at all, you can run both it and the serv
      ports:
        - "3000:3000"
      environment:
+        PGRST_SERVER_HOST: 0.0.0.0 # necessary for `postgrest --ready` flag to work
        PGRST_DB_URI: postgres://app_user:password@db:5432/app_db
        PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000
      depends_on:
@@ -43,7 +43,7 @@ As in :ref:`sql_user_management`, we create the :code:`pgcrypto` and :code:`pgjw
  CREATE EXTENSION pgcrypto WITH SCHEMA ext_pgcrypto;


-Concerning the `pgjwt extension <https://github.com/michelp/pgjwt>`_, please cf. to :ref:`client_auth`.
+Concerning the `pgjwt extension <https://github.com/michelp/pgjwt>`_, please cf. to :ref:`jwt-from-sql`.

 .. code-block:: postgres

@@ -157,7 +157,7 @@ Here we use the username instead of the email address to identify a user.
 Logins
 ~~~~~~

-As described in :ref:`client_auth`, we'll create a JWT token inside our login function. Note that you'll need to adjust the secret key which is hard-coded in this example to a secure (at least thirty-two character) secret of your choosing.
+As described in :ref:`jwt-from-sql`, we'll create a JWT token inside our login function. Note that you'll need to adjust the secret key which is hard-coded in this example to a secure (at least thirty-two character) secret of your choosing.


 .. code-block:: postgres
@@ -3,7 +3,7 @@
 SQL User Management
 ===================

-As mentioned on :ref:`jwt_generation`, an external service can provide user management and coordinate with the PostgREST server using JWT. It’s also possible to support logins entirely through SQL. It’s a fair bit of work, so get ready.
+As mentioned on :ref:`jwt_generation`, an external service can provide user management and coordinate with the PostgREST server using JWT. It's also possible to support logins entirely through SQL. It's a fair bit of work, so get ready.

 Storing Users and Passwords
 ---------------------------
@@ -110,6 +110,8 @@ Then, add ``db-anon-role`` to the configuration file to allow anonymous requests

  db-anon-role = "anon"

+.. _jwt-from-sql:
+
 JWT from SQL
 ~~~~~~~~~~~~

@@ -38,6 +38,28 @@ Sponsors
    .. image:: ../static/cybertec.svg
      :target: https://www.cybertec-postgresql.com/en/?utm_source=postgrest.org&utm_medium=referral&utm_campaign=postgrest

+  .. container:: img-dark
+
+    .. image:: ../static/supabase-dark.svg
+      :target: https://supabase.com/?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage
+
+  .. container:: img-light
+
+    .. image:: ../static/supabase.svg
+      :target: https://supabase.com/?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage
+
+  .. container:: img-dark
+
+    .. image:: ../static/euronodes.svg
+      :target: https://www.euronodes.com/postgrest
+
+  .. container:: img-light
+
+    .. image:: ../static/euronodes.svg
+      :target: https://www.euronodes.com/postgrest
+
+  |
+
  .. container:: img-dark

    .. image:: ../static/neon-dark.jpg
@@ -48,30 +70,15 @@ Sponsors
    .. image:: ../static/neon.jpg
      :target: https://neon.com/?utm_source=sponsor&utm_campaign=postgrest

-  .. image:: ../static/tembo.png
-    :target: https://www.tembo.io/?utm_source=sponsor&utm_campaign=postgrest
-
-  |
-
  .. container:: img-dark

-    .. image:: ../static/euronodes.svg
-      :target: https://www.euronodes.com/postgrest
+    .. image:: ../static/bytebase-dark.svg
+      :target: https://www.bytebase.com/?utm_source=sponsor&utm_campaign=postgrest

  .. container:: img-light

-    .. image:: ../static/euronodes.svg
-      :target: https://www.euronodes.com/postgrest
-
-  .. container:: img-dark
-
-    .. image:: ../static/supabase-dark.png
-      :target: https://supabase.com/?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage
-
-  .. container:: img-light
-
-    .. image:: ../static/supabase.png
-      :target: https://supabase.com/?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage
+    .. image:: ../static/bytebase.svg
+      :target: https://www.bytebase.com/?utm_source=sponsor&utm_campaign=postgrest

  .. The static/empty.png(created with `convert -size 320x95 xc:#fcfcfc empty.png`) is an ugly workaround
     to create space and center the logos. It's not easy to layout with restructuredText.
@@ -106,10 +113,17 @@ Getting Support

 The project has a friendly and growing community. For discussions, use the Github `discussions page <https://github.com/PostgREST/postgrest/discussions>`_. You can also report or search for bugs/features on the Github `issues <https://github.com/PostgREST/postgrest/issues>`_ page.

-Release Notes
-------------
+Releases
+--------

-The release notes are published on `PostgREST's GitHub release page <https://github.com/PostgREST/postgrest/releases>`_.
+PostgREST follows ``MAJOR.PATCH`` two-part versioning:
+
+- ``MAJOR``: feature release, may deprecate or remove things.
+- ``PATCH``: fix/security release only; no features, no behavior changes.
+
+Starting from ``v14.0``, only even-numbered MAJOR versions will be released, reserving odd-numbered MAJOR versions for development.
+
+All the releases are published on `PostgREST's GitHub release page <https://github.com/PostgREST/postgrest/releases>`_.

 Tutorials
 ---------
@@ -206,7 +220,6 @@ In Production
 Here are some companies that use PostgREST in production.

 * `Catarse <https://www.catarse.me>`_
-* `Datrium <https://www.datrium.com>`_
 * `Drip Depot <https://www.dripdepot.com>`_
 * `Image-charts <https://www.image-charts.com>`_
 * `Netwo <https://www.netwo.io>`_
@@ -258,4 +271,21 @@ Testimonials
 Contributing
 ------------

-Please see the `Contributing guidelines <https://github.com/PostgREST/postgrest/blob/main/.github/CONTRIBUTING.md>`_ in the main PostgREST repository.
+Please see the `Contributing guidelines <https://github.com/PostgREST/postgrest/blob/main/CONTRIBUTING.md>`_ in the main PostgREST repository.
+
+.. raw:: html
+
+  <script type="text/javascript">
+    let hash = window.location.hash;
+
+    const redirects = {
+      // Tables and Views
+      '#release-notes': '#releases',
+    };
+
+    let willRedirectTo = redirects[hash];
+
+    if (willRedirectTo) {
+      window.location.href = willRedirectTo;
+    }
+  </script>
@@ -194,3 +194,5 @@ Websockets
 webuser
 wfts
 www
+debouncing
+deduplicates
@@ -173,4 +173,4 @@ Domain Representations avoid all the above drawbacks. Their only drawback is tha

  Why not create a `base type <https://www.postgresql.org/docs/current/sql-createtype.html#id-1.9.3.94.5.8>`_ instead? ``CREATE TYPE app_uuid (INTERNALLENGTH = 22, INPUT = app_uuid_parser, OUTPUT = app_uuid_formatter)``.

-  Creating base types need superuser, which is restricted on cloud hosted databases. Additionally this way lets “how the data is presented” dictate “how the data is stored” which would be backwards.
+  Creating base types need superuser, which is restricted on cloud hosted databases. Additionally this way lets "how the data is presented" dictate "how the data is stored" which would be backwards.
@@ -143,7 +143,7 @@ Since the table name is plural, we can be more accurate by making it singular wi
 One-to-many relationships
 -------------------------

-The **foreign key reference** establishes the inverse one-to-many relationship. In this case, ``films`` returns as a JSON array because of the “to-many” end.
+The **foreign key reference** establishes the inverse one-to-many relationship. In this case, ``films`` returns as a JSON array because of the "to-many" end.

 .. code-block:: bash

@@ -251,6 +251,12 @@ Computed Relationships

 You can manually define relationships by using functions. This is useful for database objects that can't define foreign keys, like `Foreign Data Wrappers <https://wiki.postgresql.org/wiki/Foreign_data_wrappers>`_.

+Computed relationships have good performance as their intended design enable `function inlining <https://wiki.postgresql.org/wiki/Inlining_of_SQL_functions#Inlining_conditions_for_table_functions>`_.
+
+.. important::
+
+  - Always use ``SETOF`` when creating computed relationships. Functions can return a table without using ``SETOF``, but bear in mind that PostgreSQL will not inline them. e.g. ``RETURNS <table_name>`` is not inlinable.
+
 Assuming there's a foreign table ``premieres`` that we want to relate to ``films``.

 .. code-block:: postgres
@@ -283,6 +289,10 @@ The name of the function ``film`` is arbitrary and can be used to do the embeddi
    ".."
  ]

+.. warning::
+
+  - Make sure to correctly label the ``to-one`` part of the relationship. When using the ``ROWS 1`` estimation, PostgREST will expect a single row to be returned. If that is not the case, it will unnest the embedding and return repeated values for the top level resource.
+
 Now let's define the opposite one-to-many relationship.

 .. code-block:: postgres
@@ -331,12 +341,6 @@ Thanks to overloaded functions, you can use the same function name for different

 Computed relationships have good performance as their intended design enable `function inlining <https://wiki.postgresql.org/wiki/Inlining_of_SQL_functions#Inlining_conditions_for_table_functions>`_.

-.. warning::
-
-  - Always use ``SETOF`` when creating computed relationships. Functions can return a table without using ``SETOF``, but bear in mind that PostgreSQL will not inline them.
-
-  - Make sure to correctly label the ``to-one`` part of the relationship. When using the ``ROWS 1`` estimation, PostgREST will expect a single row to be returned. If that is not the case, it will unnest the embedding and return repeated values for the top level resource.
-
 .. _embed_disamb:
 .. _target_disamb:
 .. _hint_disamb:
@@ -1206,7 +1210,7 @@ Spread on resources forming :ref:`one-to-many <one-to-many>` and :ref:`many-to-m
     }
   ]

-Note that ``films`` is no longer an array of objects, unlike regularly embedding :ref:`one-to-many`. The embedded columns become arrays and they're correlated—in the above result, we can say that "Pulp Fiction" premiered in 1994 and "Reservoir Dogs" in 1992.
+Note that ``films`` is no longer an array of objects, unlike regularly embedding :ref:`one-to-many`. The embedded columns become arrays and they're correlated-in the above result, we can say that "Pulp Fiction" premiered in 1994 and "Reservoir Dogs" in 1992.

 Order in spread to-many
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -1302,7 +1306,7 @@ You can use multiple spreads at any level. For example, let's spread ``technical

 Note that:

- All the ``film_*`` arrays are correlated—"Reservoir Dogs" premiered in 1992, its runtime is 1:39:00 and it has the following characters: ``[ "Mr. Pink", "Mr. White" ]``.
+- All the ``film_*`` arrays are correlated-"Reservoir Dogs" premiered in 1992, its runtime is 1:39:00 and it has the following characters: ``[ "Mr. Pink", "Mr. White" ]``.
 - The ``film_*`` arrays are ordered by ``year`` (due to ``films.order=year``).
 - The bottom level array ``film_characters`` is ordered (due to ``films.roles.order=character``).

@@ -217,6 +217,15 @@ This allows using the ``fts`` operator on ``text`` and ``json`` types out of the
  curl --get "http://localhost:3000/people" \
    -d "my_json_column=not.phfts(english).The%20Fat%20Cats"

+.. important::
+
+  To ensure this operation is fast, you need to create an index on the expression:
+
+  .. code-block:: postgres
+
+    CREATE INDEX idx_people_col ON people
+    USING GIN (to_tsvector('french', my_text_column));
+
 .. _v_filter:

 Vertical Filtering
@@ -31,7 +31,7 @@ The authenticator role is used for connecting to the database and should be conf
 .. _user_impersonation:

 User Impersonation
------------------
+~~~~~~~~~~~~~~~~~~

 The picture below shows how the server handles authentication. If auth succeeds, it switches into the user role specified by the request, otherwise it switches into the anonymous role (if it's set in :ref:`db-anon-role`).

@@ -43,12 +43,13 @@ This role switching mechanism is called **user impersonation**. In PostgreSQL it

  The impersonated roles will have their settings applied. See :ref:`impersonated_settings`.

-.. _jwt_impersonation:
+.. _jwt_auth:

-JWT-Based User Impersonation
----------------------------
+JWT Authentication
+------------------

-We use `JSON Web Tokens <https://jwt.io/>`_ to authenticate API requests, this allows us to be stateless and not require database lookups for verification. As you'll recall a JWT contains a list of cryptographically signed claims. All claims are allowed but PostgREST cares specifically about a claim called role.
+We use `JSON Web Tokens <https://datatracker.ietf.org/doc/html/rfc7519/>`_ to authenticate API requests, this allows us to be stateless and not require database lookups for verification.
+As you'll recall a JWT contains a list of cryptographically signed claims. All claims are allowed but PostgREST cares specifically about a claim called role (configurable with :ref:`jwt_role_extract`).

 .. code:: json

@@ -72,17 +73,10 @@ Note that the database administrator must allow the authenticator role to switch

 If the client included no JWT (or one without a role claim) then PostgREST switches into the anonymous role. The database administrator must set the anonymous role permissions correctly to prevent anonymous users from seeing or changing things they shouldn't.

-.. _jwt_generation:
+.. _bearer_auth:

-JWT Generation
-~~~~~~~~~~~~~~
-
-You can create a valid JWT either from inside your database (see :ref:`sql_user_management`) or via an external service (see :ref:`external_auth`).
-
-.. _client_auth:
-
-Client Auth
-~~~~~~~~~~~
+Bearer Authentication
+~~~~~~~~~~~~~~~~~~~~~

 To make an authenticated request the client must include an :code:`Authorization` HTTP header with the value :code:`Bearer <jwt>`. For instance:

@@ -93,24 +87,29 @@ To make an authenticated request the client must include an :code:`Authorization

 The ``Bearer`` header value can be used with or without capitalization(``bearer``).

-.. _jwt_caching:
+.. _jwt_generation:

-JWT Caching
-----------
+JWT Generation
+~~~~~~~~~~~~~~

-PostgREST validates ``JWTs`` on every request. We can cache ``JWTs`` to avoid this performance overhead.
+You can create a valid JWT either from inside your database (see :ref:`sql_user_management`) or via an external service (see :ref:`external_auth`).

-To enable JWT caching, the config :code:`jwt-cache-max-lifetime` is to be set. It is the maximum number of seconds for which the cache stores the JWT validation results. The cache uses the :code:`exp` claim to set the cache entry lifetime. If the JWT does not have an :code:`exp` claim, it uses the config value. See :ref:`jwt-cache-max-lifetime` for more details.
+.. _jwt_signature:

-.. note::
+JWT Signature Verification
+--------------------------

-  You can use the :ref:`server-timing_header` to see the effect of JWT caching.
+PostgREST supports both symmetric and asymmetric keys for verifying the signature of the token.

 Symmetric Keys
 ~~~~~~~~~~~~~~

-Each token is cryptographically signed with a secret key. In the case of symmetric cryptography the signer and verifier share the same secret passphrase, which can be configured with :ref:`jwt-secret`.
-If it is set to a simple string value like “reallyreallyreallyreallyverysafe” then PostgREST interprets it as an HMAC-SHA256 passphrase.
+In the case of symmetric cryptography the signer and verifier share the same secret passphrase, which can be configured with :ref:`jwt-secret`.
+If it is set to a simple string then PostgREST interprets it as an HMAC-SHA256 passphrase.
+
+.. code-block:: ini
+
+  jwt-secret = "reallyreallyreallyreallyverysafe"

 .. _asym_keys:

@@ -156,39 +155,39 @@ You can specify the literal value as we saw earlier, or reference a filename to

  jwt-secret = "@rsa.jwk.pub"

-JWK ``kid`` validation
-^^^^^^^^^^^^^^^^^^^^^^
+``kid`` verification
+^^^^^^^^^^^^^^^^^^^^

-PostgREST has built-in validation of the `key ID parameter <https://www.rfc-editor.org/rfc/rfc7517#section-4.5>`_, useful when working with a JWK Set.
+PostgREST has built-in verification of the `key ID parameter <https://www.rfc-editor.org/rfc/rfc7517#section-4.5>`_, useful when working with a JSON Web Key Set.
 It goes as follows:

- If the JWT contains a ``kid`` parameter, then PostgREST will look for the JWK in the :ref:`jwt-secret`.
+- If the JWT contains a ``kid`` parameter, then PostgREST will look for the JSON Web Key in the :ref:`jwt-secret`.

-  + If no JWK matches the same ``kid`` value (or if they do not have a ``kid``), then the token will be rejected with a :ref:`401 Unauthorized <pgrst301>` error.
-  + If a JWK matches the ``kid`` value then it will validate the token against that JWK accordingly.
+  + If no key has a matching ``kid`` (or if they don't have one defined), the token will be rejected with a :ref:`401 Unauthorized <pgrst301>` error.
+  + If a key matches the ``kid`` value then it will validate the token against that key accordingly.

- If the JWT does not have a ``kid`` parameter, then PostgREST will validate the token against each JWK in the :ref:`jwt-secret`.
+- If the JWT doesn't have a ``kid``, PostgREST  will try each key in the :ref:`jwt-secret` one by one until it finds one that works.

 .. _jwt_claims_validation:

 JWT Claims Validation
-~~~~~~~~~~~~~~~~~~~~~
+---------------------

-PostgREST honors the following `JWT claims <https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.4>`_:
+Time-Based claims validation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The time-based JWT claims specified in `RFC 7519 <https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.4>`_ are validated:

 - ``exp`` Expiration Time
 - ``iat`` Issued At
 - ``nbf`` Not Before
- ``aud`` :ref:`Audience <jwt_aud_validation>`

-.. note::
-  PostgREST allows for a 30-second clock skew when validating the ``exp``, ``iat`` and ``nbf`` claims.
-  In other words, it gives an extra 30 seconds before the token is rejected if there is a slight discrepancy in the timestamps.
+We allow a 30-second clock skew when validating the above claims. In other words, we give an extra 30 seconds before the JWT is rejected if there is a slight discrepancy in the timestamps.

-.. _jwt_aud_validation:
+.. _jwt_aud:

-JWT ``aud`` Claim Validation
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+``aud`` validation
+~~~~~~~~~~~~~~~~~~

 PostgREST has built-in validation of the `JWT audience claim <https://datatracker.ietf.org/doc/html/rfc7519#section-4.1.3>`_.
 It works this way:
@@ -201,10 +200,29 @@ It works this way:
  + If the match fails or if the ``aud`` value is not a string or array of strings, then the token will be rejected with a :ref:`401 Unauthorized <pgrst303>` error.
  + If the ``aud`` key **is not present** or if its value is ``null`` or ``[]``, PostgREST will interpret this token as allowed for all audiences and will complete the request.

+.. _jwt_caching:
+
+JWT Cache
+---------
+
+JWT signature validation (specially :ref:`asym_keys` such as RSA) is slow, we can cache ``JWT`` validation results to avoid this performance overhead.
+
+The JWT cache is bounded and uses the `SIEVE algorithm <https://cachemon.github.io/SIEVE-website>`_ for efficient eviction. The cache is enabled by default and can be configured with :ref:`jwt-cache-max-entries`.
+
+It's recommended to leave the JWT cache enabled as our load tests indicate ~20% more throughput for simple GET requests when using it. This while reducing CPU utilization in exchange for a bit more memory.
+
+:ref:`jwt_cache_metrics` are available.
+
+.. note::
+
+  - If the ``jwt-secret`` is changed and the config is reloaded, the JWT cache will reset.
+  - JWTs that pass :ref:`jwt_signature` are cached, regardless if they pass :ref:`jwt_claims_validation`. We do this to ensure responses stays fast under common failure cases (such as expired JWTs).
+  - You can use the :ref:`server-timing_header` to see the peformance benefit of JWT caching.
+
 .. _jwt_role_extract:

 JWT Role Extraction
-~~~~~~~~~~~~~~~~~~~
+-------------------

 A JSPath DSL that specifies the location of the :code:`role` key in the JWT claims. It's configured by :ref:`jwt-role-claim-key`. This can be used to consume a JWT provided by a third party service like Auth0, Okta, Microsoft Entra or Keycloak.

@@ -242,7 +260,7 @@ Usage examples:
  The string comparison operators are implemented as a custom extension to the JSPath and does not strictly follow the `RFC 9535 <https://www.rfc-editor.org/rfc/rfc9535.html>`_.

 JWT Security
-~~~~~~~~~~~~
+------------

 There are at least three types of common critiques against using JWT: 1) against the standard itself, 2) against using libraries with known security vulnerabilities, and 3) against using JWT for web sessions. We'll briefly explain each critique, how PostgREST deals with it, and give recommendations for appropriate user action.

@@ -278,3 +296,23 @@ doing custom logic based on the web user info.
    END IF;
  END
  $$ LANGUAGE plpgsql;
+
+.. raw:: html
+
+  <script type="text/javascript">
+    let hash = window.location.hash;
+
+    const redirects = {
+      '#jwt-based-user-impersonation': '#jwt-authentication',
+      '#client-auth': '#bearer-authentication',
+      '#jwt-caching': '#jwt-cache',
+      '#jwk-kid-validation': '#kid-verification',
+      '#jwt-aud-claim-validation': '#aud-validation',
+    };
+
+    let willRedirectTo = redirects[hash];
+
+    if (willRedirectTo) {
+      window.location.href = willRedirectTo;
+    }
+  </script>
@@ -3,23 +3,47 @@
 CLI
 ===

-PostgREST provides a CLI with the commands listed below:
+PostgREST provides a CLI with the options listed below:
+
+.. code:: text
+
+  Usage: postgrest [-v|--version] [-e|--example] [--dump-config | --dump-schema | --ready]
+                 [FILENAME]
+
+    PostgREST / create a REST API to an existing Postgres
+    database
+
+  Available options:
+    -h,--help                Show this help text
+    -v,--version             Show the version information
+    -e,--example             Show an example configuration file
+    --dump-config            Dump loaded configuration and exit
+    --dump-schema            Dump loaded schema as JSON and exit (for debugging,
+                             output structure is unstable)
+    --ready                  Checks the health of PostgREST by doing a request on
+                             the admin server /ready endpoint
+    FILENAME                 Path to configuration file
+
+FILENAME
+--------
+
+Runs PostgREST with the given :ref:`file_config`.

 Help
 ----

 .. code:: bash

-  $ postgrest [-h|--help]
+  $ postgrest --help

-Shows all the commands available.
+Shows all the options available.

 Version
 -------

 .. code:: bash

-  $ postgrest [-v|--version]
+  $ postgrest --version

 Prints the PostgREST version.

@@ -28,16 +52,16 @@ Example

 .. code:: bash

-  $ postgrest [-e|--example]
+  $ postgrest --example

-Shows example configuration options.
+Shows example configuration settings.

 Dump Config
 -----------

 .. code:: bash

-  $ postgrest [--dump-config]
+  $ postgrest --dump-config

 Dumps the loaded :ref:`configuration` values, considering the configuration file, environment variables and :ref:`in_db_config`.

@@ -46,6 +70,20 @@ Dump Schema

 .. code:: bash

-  $ postgrest [--dump-schema]
+  $ postgrest --dump-schema

 Dumps the schema cache in JSON format.
+
+Ready Flag
+----------
+
+Makes a request to the ``/ready`` endpoint of the :ref:`admin_server`. It exits with a return code of ``0`` on success and ``1`` on failure.
+
+.. code-block:: bash
+
+  $ postgrest --ready
+  OK: http://localhost:3001/ready
+
+.. note::
+
+  The ``--ready`` flag cannot be used when :ref:`server-host` is configured with special hostnames. We suggest to change it to ``localhost``.
@@ -316,7 +316,7 @@ db-extra-search-path
  Multiple schemas can be added in a comma-separated string, e.g. ``public, extensions``.

 .. important::
-  
+
  We default this config to ``public`` because it is the most common schema used to install PostgreSQL extensions such as :ref:`PostGIS <ww_postgis>`. You can disable this by setting this config to ``""``.

 .. _db-hoisted-tx-settings:
@@ -409,7 +409,7 @@ db-pool-max-idletime
  **In-Database** `n/a`
  =============== =================================

-   *For backwards compatibility, this config parameter is also available as “db-pool-timeout”.*
+   *For backwards compatibility, this config parameter is also available as "db-pool-timeout".*

   Time in seconds to close idle pool connections.

@@ -603,7 +603,7 @@ jwt-aud
  **In-Database** pgrst.jwt_aud
  =============== =================================

-    Specifies an audience for the JWT ``aud`` claim. See :ref:`jwt_aud_validation`.
+    Specifies an audience for the JWT ``aud`` claim. See :ref:`jwt_aud`.

 .. _jwt-role-claim-key:

@@ -658,20 +658,20 @@ jwt-secret-is-base64

  When this is set to :code:`true`, the value derived from :code:`jwt-secret` will be treated as a base64 encoded secret.

-.. _jwt-cache-max-lifetime:
+.. _jwt-cache-max-entries:

-jwt-cache-max-lifetime
+jwt-cache-max-entries
 ----------------------

  =============== =================================
  **Type**        Int
-  **Default**     0
+  **Default**     1000
  **Reloadable**  Y
-  **Environment** PGRST_JWT_CACHE_MAX_LIFETIME
-  **In-Database** pgrst.jwt_cache_max_lifetime
+  **Environment** PGRST_JWT_CACHE_MAX_ENTRIES
+  **In-Database** pgrst.jwt_cache_max_entries
  =============== =================================

-  Maximum number of seconds of lifetime for cached entries. The default :code:`0` disables caching. See :ref:`jwt_caching`.
+  Maximum number of entries in JWT cache. The value :code:`0` disables JWT caching. See :ref:`jwt_caching`.

 .. _log-level:

@@ -714,23 +714,14 @@ log-query
 ---------

  =============== =================================
-  **Type**        String
-  **Default**     "disabled"
+  **Type**        Boolean
+  **Default**     False
  **Reloadable**  Y
  **Environment** PGRST_LOG_QUERY
  **In-Database** `n/a`
  =============== =================================

-  Logs the SQL query for the corresponding request at the current :ref:`log-level`.
-  See :ref:`sql_query_logs`.
-
-    .. code:: bash
-
-        # Logs the main SQL query
-        log-query = "main-query"
-
-        # Disables logging the SQL query
-        log-query = "disabled"
+  Logs the SQL query for the corresponding request at the current :ref:`log-level`. See :ref:`sql_query_logs`.

 .. _openapi-mode:

@@ -15,7 +15,7 @@ Dynamic Connection Pool

 To conserve system resources, PostgREST uses a dynamic connection pool. This enables the number of connections in the pool to increase and decrease depending on request traffic.

- If all the connections are being used, a new connection is added. The pool can grow until it reaches the :ref:`db-pool` size. Note that it’s pointless to set this higher than the ``max_connections`` setting in your database.
+- If all the connections are being used, a new connection is added. The pool can grow until it reaches the :ref:`db-pool` size. Note that it's pointless to set this higher than the ``max_connections`` setting in your database.
 - If a connection is unused for a period of time (:ref:`db-pool-max-idletime`), it will be released.
 - For connecting to the database, the :ref:`authenticator <roles>` role is used. You can configure this using :ref:`db-uri`.

@@ -106,4 +106,4 @@ Also set :ref:`db-channel-enabled` to ``false`` since ``LISTEN`` is not compatib

 .. note::

-  It’s not recommended to use an external connection pooler. `Our benchmarks <https://github.com/PostgREST/postgrest/issues/2294#issuecomment-1139148672>`_ indicate it provides much lower performance than PostgREST built-in pool.
+  It's not recommended to use an external connection pooler. `Our benchmarks <https://github.com/PostgREST/postgrest/issues/2294#issuecomment-1139148672>`_ indicate it provides much lower performance than PostgREST built-in pool.
@@ -329,7 +329,7 @@ Related to the authentication process using JWT. You can follow the :ref:`tut1`
 | PGRST301      |             |                                                             |
 +---------------+-------------+-------------------------------------------------------------+
 | .. _pgrst302: | 401         | Attempted to do a request without                           |
-|               |             | :ref:`authentication <client_auth>` when the anonymous role |
+|               |             | :ref:`bearer_auth` when the anonymous role                  |
 | PGRST302      |             | is disabled by not setting it in :ref:`db-anon-role`.       |
 +---------------+-------------+-------------------------------------------------------------+
 | .. _pgrst303: | 401         | :ref:`JWT claims validation <jwt_claims_validation>`        |
@@ -4,7 +4,7 @@ Listener
 ########

 PostgREST uses `LISTEN <https://www.postgresql.org/docs/current/sql-listen.html>`_ to reload its :ref:`Schema Cache <schema_reloading_notify>` and :ref:`Configuration <config_reloading_notify>` via `NOTIFY <https://www.postgresql.org/docs/current/sql-notify.html>`_.
-This is useful in environments where you can’t send SIGUSR1 or SIGUSR2 Unix Signals.
+This is useful in environments where you can't send SIGUSR1 or SIGUSR2 Unix Signals.
 Like on cloud managed containers or on Windows systems.

 .. code:: postgresql
@@ -52,14 +52,12 @@ For diagnostic information about the server itself, PostgREST logs to ``stderr``
 SQL Query Logs
 --------------

-To log the :ref:`main SQL query <main_query>` executed for a request, set the :ref:`log-query` to ``main-query``.
-It will be logged based on the current :ref:`log-level` setting.
-For example, with this configuration:
+To log the SQL queries executed for a request, set the :ref:`log-query` to ``true``. It will be logged based on the current :ref:`log-level` setting.

 .. code-block:: bash

  log-level = "warn"
-  log-query = "main-query"
+  log-query = "true"

 The SQL queries will only be logged on ``400`` HTTP errors and up.
 So, if the user requests a resource without sufficient privileges:
@@ -122,7 +120,7 @@ Restart the database and watch the log file in real-time to understand how HTTP
 Metrics
 =======

-The ``metrics`` endpoint on the :ref:`admin_server` endpoint provides metrics in `Prometheus text format <https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format>`_.
+The ``metrics`` endpoint on the :ref:`admin_server` endpoint provides metrics in `Prometheus text format <https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format>`_.

 .. code-block:: bash

@@ -206,6 +204,40 @@ pgrst_db_pool_max

 Max pool connections.

+.. _jwt_cache_metrics:
+
+JWT Cache Metrics
+-----------------
+
+Metrics related to the :ref:`jwt_caching`.
+
+pgrst_jwt_cache_requests_total
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+======== =======
+**Type** Counter
+======== =======
+
+The total number of JWT cache lookups.
+
+pgrst_jwt_cache_hits_total
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+======== =======
+**Type** Counter
+======== =======
+
+The total number of JWT cache hits.
+
+pgrst_jwt_cache_evictions_total
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+======== =======
+**Type** Counter
+======== =======
+
+The total number of JWT cache evictions.
+
 Traces
 ======

@@ -265,7 +297,7 @@ This header communicates metrics of the different phases in the request-response
  Server-Timing: jwt;dur=14.9, parse;dur=71.1, plan;dur=109.0, transaction;dur=353.2, response;dur=4.4

 - All the durations (``dur``) are in milliseconds.
- The ``jwt`` stage is when :ref:`jwt_impersonation` is done. This duration can be lowered with :ref:`jwt_caching`.
+- The ``jwt`` stage is when :ref:`jwt_auth` is done. This duration can be lowered with :ref:`jwt_caching`.
 - On the ``parse`` stage, the :ref:`url_grammar` is parsed.
 - On the ``plan`` stage, the :ref:`schema_cache` is used to generate the :ref:`main_query` of the transaction.
 - The ``transaction`` stage corresponds to the database transaction. See :ref:`transactions`.
@@ -53,6 +53,19 @@ To reload the schema cache from within the database, you can use the ``NOTIFY``

  NOTIFY pgrst, 'reload schema'

+Debouncing
+~~~~~~~~~~
+
+PostgREST does not reload the schema cache for each notification when several ``NOTIFY pgrst`` events are generated quickly after one another.
+
+There are two cases to consider: when notifications are sent within a single transaction and when they are sent across multiple transactions.
+
+In the first case, PostgreSQL deduplicates identical ``NOTIFY`` events within the same transaction. This means that even if multiple ``NOTIFY pgrst`` statements are executed before a ``COMMIT``, only a single notification is delivered to PostgREST.
+
+In the second case, when notifications are sent from separate transactions in a short time span, PostgREST applies a debouncing mechanism to avoid excessive schema cache reloads.
+
+Instead of reloading the schema cache for each notification, events are grouped within a small time window of 100 milliseconds. The reload function is executed once immediately when the first notification is received and once more after the burst of events settles, resulting in at most two executions within that time window.
+
 .. _auto_schema_reloading:

 Automatic Schema Cache Reloading
@@ -1,7 +1,7 @@
 # This file is auto-generated by postgrest-nixpkgs-upgrade
-sphinx==7.4.7
+sphinx==8.2.3
 sphinx-copybutton==0.5.2
 sphinx-rtd-dark-mode==1.3.0
 sphinx-rtd-theme==3.0.2
 sphinx-tabs==3.4.7
-sphinxext-opengraph==0.9.1
+sphinxext-opengraph==0.9.1
@@ -2,16 +2,16 @@
  "nodes": {
    "nixpkgs": {
      "locked": {
-        "lastModified": 1731165248,
-        "narHash": "sha256-DiHFKIdBmMx5/DUARhVqaxvEIiy4EE6Eqs9Qs4oxme8=",
+        "lastModified": 1752006229,
+        "narHash": "sha256-BeuAPwNM2RBc5bvUTb0j4GRs2yBkDeRCw/8Y3v9Xesc=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "a90280100f41a10914edfe729a4053e60c92b8e3",
+        "rev": "c80edd02003fe3d8af527215a3ac069be9cfd47f",
        "type": "github"
      },
      "original": {
        "owner": "nixos",
-        "ref": "nixpkgs-unstable",
+        "ref": "nixpkgs-25.05-darwin",
        "repo": "nixpkgs",
        "type": "github"
      }
@@ -2,7 +2,7 @@
  description = "REST API for any Postgres database";

  inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-25.05-darwin";
  };

  nixConfig = {
@@ -33,8 +33,8 @@
    in
    {
      packages = genSystems (attrs: {
-        default = attrs.postgrestPackage;
-        profiled = attrs.postgrestProfiled;
+        default = attrs.postgrestPackage.bin;
+        profiled = attrs.postgrestProfiled.bin;
      } // nixpkgs.lib.optionalAttrs (attrs ? postgrestStatic) {
        static = attrs.postgrestStatic;
      });
@@ -42,7 +42,7 @@
      apps = genSystems (attrs: {
        default = {
          type = "app";
-          program = "${attrs.postgrestStatic or attrs.postgrestPackage}/bin/postgrest";
+          program = "${attrs.postgrestStatic or attrs.postgrestPackage.bin}/bin/postgrest";
          meta.description = "REST API for any Postgres database";
        };
      });
@@ -72,9 +72,10 @@ The PostgREST utilities available in `nix-shell` all have names that begin with
 ```bash
 # Note: The utilities listed here might not be up to date.
 [nix-shell]$ postgrest-<tab>
-postgrest-build                   postgrest-profiled-run
-postgrest-check                   postgrest-push-cachix
-postgrest-clean                   postgrest-release
+postgrest-build                   postgrest-parallel-curl
+postgrest-check                   postgrest-profiled-run
+postgrest-clean                   postgrest-push-cachix
+postgrest-commitlint              postgrest-release
 postgrest-coverage                postgrest-repl
 postgrest-coverage-draft-overlay  postgrest-run
 postgrest-docs-build              postgrest-style
@@ -90,15 +91,14 @@ postgrest-gen-ctags               postgrest-watch
 postgrest-gen-jwt                 postgrest-with-all
 postgrest-gen-secret              postgrest-with-git
 postgrest-git-hooks               postgrest-with-pgrst
-postgrest-hsie-graph-modules      postgrest-with-postgresql-12
-postgrest-hsie-graph-symbols      postgrest-with-postgresql-13
-postgrest-hsie-minimal-imports    postgrest-with-postgresql-14
-postgrest-lint                    postgrest-with-postgresql-15
-postgrest-loadtest                postgrest-with-postgresql-16
-postgrest-loadtest-against        postgrest-with-postgresql-17
-postgrest-loadtest-report         postgrest-with-slow-pg
-postgrest-nixpkgs-upgrade         postgrest-with-slow-postgrest
-postgrest-parallel-curl
+postgrest-hsie-graph-modules      postgrest-with-pg-13
+postgrest-hsie-graph-symbols      postgrest-with-pg-14
+postgrest-hsie-minimal-imports    postgrest-with-pg-15
+postgrest-lint                    postgrest-with-pg-16
+postgrest-loadtest                postgrest-with-pg-17
+postgrest-loadtest-against        postgrest-with-slow-pg
+postgrest-loadtest-report         postgrest-with-slow-postgrest
+postgrest-nixpkgs-upgrade
 ...

 [nix-shell]$
@@ -174,7 +174,7 @@ $ nix-shell --run "postgrest-with-all postgrest-test-spec"

 # Run the tests against a specific version of PostgreSQL (use tab-completion in
 # nix-shell to see all available versions):
-$ nix-shell --run "postgrest-with-postgresql-13 postgrest-test-spec"
+$ nix-shell --run "postgrest-with-pg-13 postgrest-test-spec"

 ```

@@ -284,7 +284,7 @@ Tools like `postgrest-build`, `postgrest-run`, `postgrest-repl` etc. are simple
 also run in CI, with the exception of the IO and Memory checks that need to be run
 separately.

-`postgrest-with-postgresql-*` take a command as an argument and will run it
+`postgrest-with-pg-*` take a command as an argument and will run it
 with a temporary database. `postgrest-with-all` will run the command against
 all supported PostgreSQL versions. Tests run without `postgrest-with-*` are
 run against the latest PostgreSQL version by default.
@@ -380,7 +380,7 @@ that).
 We also use `default.nix` to load our pinned version of the `nixpkgs`
 repository. This set of packages will always be the same, independently from
 where or when you use it. The pinned version is taken from `flake.lock` and
-can be updated with `nix flake update`.
+can be updated with `postgrest-nixpkgs-upgrade`.

 ### `shell.nix`

@@ -1,61 +0,0 @@
-# Creating a separate libpq package is is discussed in
-# https://github.com/NixOS/nixpkgs/issues/61580, but nixpkgs has not moved
-# forward, yet.
-# This package is passed to postgresql-libpq (haskell) which needs to be
-# cross-compiled to the static build and possibly other architectures as
-# as well. To reduce the number of dependencies that need to be built with
-# it, this derivation focuses on building the client libraries only. No
-# server, no tests.
-{ stdenv
-, lib
-, openssl
-, zlib
-, postgresql
-, pkg-config
-, tzdata
-}:
-
-stdenv.mkDerivation {
-  pname = "libpq";
-  inherit (postgresql) src version patches;
-
-  __structuredAttrs = true;
-  env.CFLAGS = "-fdata-sections -ffunction-sections"
-    + (if stdenv.cc.isClang then " -flto" else " -fmerge-constants -Wl,--gc-sections");
-
-  configureFlags = [
-    "--without-gssapi"
-    "--without-icu"
-    "--without-readline"
-    "--with-openssl"
-    "--with-system-tzdata=${tzdata}/share/zoneinfo"
-    "--sysconfdir=/etc/postgresql"
-  ];
-
-  nativeBuildInputs = [ pkg-config tzdata ];
-  buildInputs = [ openssl zlib ];
-
-  buildFlags = [ "submake-libpq" "submake-libpgport" ];
-
-  installPhase = ''
-    runHook preInstall
-
-    make -C src/bin/pg_config install
-    make -C src/common install
-    make -C src/include install
-    make -C src/interfaces/libpq install
-    make -C src/port install
-
-    rm -rfv $out/share
-
-    runHook postInstall
-  '';
-
-  outputs = [ "out" ];
-
-  meta = with lib; {
-    homepage = "https://www.postgresql.org";
-    description = "Client API library for PostgreSQL";
-    license = licenses.postgresql;
-  };
-}
@@ -6,6 +6,7 @@
 , coreutils
 , git
 , lib
+, moreutils
 , runCommand
 , shellcheck
 , stdenv
@@ -56,7 +57,7 @@ let
        # Example: This way `postgrest-watch -h` will return the help output for watch, while
        # `postgrest-watch postgrest-test-spec -h` will return the help output for test-spec.
        # Taken from: https://github.com/matejak/argbash/issues/114#issuecomment-557108274
-        sed '/_positionals_count + 1/a\\t\t\t\tset -- "''${@:1:1}" "--" "''${@:2}"' -i $out
+        sed '/_positionals_count + 1/a\\t\t\t\tset -- "''${@:1:1}" "--" "''${@:2}"' $out | ${moreutils}/bin/sponge $out
      '';

  bash-completion =
@@ -66,7 +67,7 @@ let
      ''

      + lib.optionalString (positionalCompletion != "") ''
-        sed 's#COMPREPLY.*compgen -o bashdefault .*$#${escape positionalCompletion}#' -i $out
+        sed 's#COMPREPLY.*compgen -o bashdefault .*$#${escape positionalCompletion}#' $out | ${moreutils}/bin/sponge $out
      ''
    );

@@ -103,8 +104,7 @@ let
        ''

        + lib.optionalString withTmpDir ''
-          mkdir -p "''${TMPDIR:-/tmp}/postgrest"
-          tmpdir="$(${coreutils}/bin/mktemp -d --tmpdir postgrest/${name}-XXX)"
+          tmpdir="$(${coreutils}/bin/mktemp -d --tmpdir ${name}-XXX)"

          # we keep the tmpdir when an error occurs for debugging
          trap 'echo Temporary directory kept at: $tmpdir' ERR
@@ -3,6 +3,5 @@
  checked-shell-script = import ./checked-shell-script;
  gitignore = import ./gitignore.nix;
  haskell-packages = import ./haskell-packages.nix;
-  postgresql-libpq = import ./postgresql-libpq.nix;
  slocat = import ./slocat.nix;
 }
@@ -50,34 +50,34 @@ let
      # jailbreak, because hspec limit for tests
      fuzzyset = prev.fuzzyset_0_2_4;

-      hasql-pool = lib.dontCheck (prev.callHackageDirect
-        {
-          pkg = "hasql-pool";
-          ver = "1.0.1";
-          sha256 = "sha256-Hf1f7lX0LWkjrb25SDBovCYPRdmUP1H6pAxzi7kT4Gg=";
-        }
-        { });
+      # TODO: Remove once available in nixpkgs haskellPackages
+      configurator-pg =
+        prev.callHackageDirect
+          {
+            pkg = "configurator-pg";
+            ver = "0.2.11";
+            sha256 = "sha256-mtGtNawDJgz2ZIEVca+IYXVu4oNw9xsfJiYWAqAbbgc=";
+          }
+          { };

-      hasql-notifications = lib.dontCheck (prev.callHackageDirect
-        {
-          pkg = "hasql-notifications";
-          ver = "0.2.2.2";
-          sha256 = "sha256-myKwlug7OgTa/qP6mHfCD+5Q8IhM17JvpJBfSo+M01k=";
-        }
-        { });
+      # TODO: Remove once available in nixpkgs haskellPackages
+      streaming-commons =
+        prev.callHackageDirect
+          {
+            pkg = "streaming-commons";
+            ver = "0.2.3.1";
+            sha256 = "sha256-Gl2eaJcWe1sxmcE/octWlH9uSnERguf+5H66K4fV87s=";
+          }
+          { };

-      # newer nixpkgs already has 0.10., so we fallback to default for forward compat
-      jose-jwt = prev.jose-jwt_0_10_0 or prev.jose-jwt;
-
-      postgresql-libpq = lib.dontCheck (prev.callHackageDirect
-        {
-          pkg = "postgresql-libpq";
-          ver = "0.10.1.0";
-          sha256 = "sha256-tXOMqCO8opMilI9rx0D+njqjIjbZsH168Bzb8Aq8Ff4=";
-        }
-        {
-          postgresql = super.libpq;
-        });
+      # Downgrade hasql and related packages while we are still on GHC 9.4 for the static build.
+      hasql = lib.dontCheck (lib.doJailbreak prev.hasql_1_6_4_4);
+      hasql-dynamic-statements = lib.dontCheck prev.hasql-dynamic-statements_0_3_1_5;
+      hasql-implicits = lib.dontCheck prev.hasql-implicits_0_1_1_3;
+      hasql-notifications = lib.dontCheck prev.hasql-notifications_0_2_2_2;
+      hasql-pool = lib.dontCheck prev.hasql-pool_1_0_1;
+      hasql-transaction = lib.dontCheck prev.hasql-transaction_1_1_0_1;
+      postgresql-binary = lib.dontCheck (lib.doJailbreak prev.postgresql-binary_0_13_1_3);
    };
 in
 {
@@ -1,7 +0,0 @@
-_: super:
-{
-  # Depending on which nixpkgs version is pinned, libpq might either be available already - or not.
-  libpq = super.libpq or (super.callPackage ../libpq.nix {
-    postgresql = super.postgresql_16;
-  });
-}
@@ -8,65 +8,25 @@ let
  inherit (pkgs) pkgsStatic;
  inherit (pkgsStatic.haskell) lib;

-  packagesStatic =
-    pkgsStatic.haskell.packages."${compiler}".override (old: {
-      ghc = pkgsStatic.pkgsBuildHost.haskell.compiler."${compiler}".override {
-        # Using the bundled libffi generally works better for cross-compiling
-        libffi = null;
-        # Building sphinx fails on some platforms
-        enableDocs = false;
-        # Cross compiling with native bignum works better than with gmp
-        enableNativeBignum = true;
-      };
-
-      overrides = pkgs.lib.composeExtensions old.overrides (_: prev: {
-        postgresql-libpq = (lib.overrideCabal prev.postgresql-libpq {
-          # TODO: This section can be simplified when this PR has made it's way to us:
-          #  https://github.com/NixOS/nixpkgs/pull/286370
-          # Additionally, we need to use the default version in nixpkgs, otherwise the
-          # override will not be active as well.
-          # Using use-pkg-config flag, because pg_config won't work when cross-compiling
-          configureFlags = [ "-fuse-pkg-config" ];
-          # postgresql doesn't build in the fully static overlay - but the default
-          # derivation is built with static libraries anyway.
-          libraryPkgconfigDepends = [ pkgsStatic.libpq ];
-          librarySystemDepends = [ ];
-        }).overrideAttrs (_: prevAttrs: {
-          buildInputs = prevAttrs.buildInputs ++ [ pkgsStatic.openssl ];
-        });
-      });
-    });
+  packagesStatic = pkgsStatic.haskell.packages.native-bignum."${compiler}";

  makeExecutableStatic = drv: pkgs.lib.pipe drv [
    lib.compose.justStaticExecutables

    # To successfully compile a redistributable, fully static executable we need to:
-    # 1. make executable really statically linked.
-    # 2. avoid any references to /nix/store to prevent blowing up the closure size.
-    # 3. be able to run the executable.
-    # When checking for references, we ignore the following:
-    # - eeee... are removed references which don't actually exist
-    # - openssl-etc references are purposely designed to be very small
-    (lib.compose.overrideCabal (drv: {
-      postFixup = drv.postFixup + ''
-        exe="$out/bin/postgrest"
+    # 1. avoid any references to /nix/store to prevent blowing up the closure size.
+    (drv: drv.overrideAttrs {
+      allowedReferences = [
+        pkgsStatic.openssl.etc
+      ];
+    })

-        if ! (file "$exe" | grep 'statically linked') then
-          echo "not a static executable, ldd output:"
-          ldd "$exe"
-          exit 1
-        fi
-
-        echo "Checking for references to /nix/store..."
-        (${pkgsStatic.binutils}/bin/strings "$exe" \
-          | grep -v /nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee \
-          | grep -v -etc/etc/ssl \
-          | grep /nix/store || exit 0 && exit 1)
-        echo "No references to /nix/store found"
-
-        "$exe" --help
-      '';
-    }))
+    # 2. be able to run the executable.
+    (drv: drv.overrideAttrs {
+      passthru.tests.version = pkgsStatic.testers.testVersion {
+        package = drv;
+      };
+    })
  ];

 in
@@ -15,7 +15,6 @@ let
        withEnv = postgrest.env;
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        exec ${cabal-install}/bin/cabal v2-build ${devCabalOptions} "''${_arg_leftovers[@]}"
      '';

@@ -34,6 +33,17 @@ let
        exec ${cabal-install}/bin/cabal v2-clean
      '';

+  update =
+    checkedShellScript
+      {
+        name = "postgrest-cabal-update";
+        docs = "Update cabal's package list from hackage.haskell.org";
+        workingDir = "/";
+      }
+      ''
+        exec ${cabal-install}/bin/cabal v2-update
+      '';
+
  run =
    checkedShellScript
      {
@@ -45,6 +55,7 @@ let
            "ARG_USE_ENV([PGRST_DB_POOL], [1], [PostgREST pool size])"
            "ARG_USE_ENV([PGRST_DB_POOL_ACQUISITION_TIMEOUT], [1], [PostgREST pool timeout])"
            "ARG_USE_ENV([PGRST_JWT_SECRET], [reallyreallyreallyreallyverysafe], [PostgREST JWT secret])"
+            "ARG_USE_ENV([PGRST_ADMIN_SERVER_PORT], [3001], [PostgREST admin server port])"
            "ARG_LEFTOVERS([PostgREST arguments])"
          ];
        workingDir = "/";
@@ -55,6 +66,7 @@ let
        export PGRST_DB_POOL
        export PGRST_DB_POOL_ACQUISITION_TIMEOUT
        export PGRST_JWT_SECRET
+        export PGRST_ADMIN_SERVER_PORT

        exec ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
          postgrest "''${_arg_leftovers[@]}"
@@ -83,10 +95,8 @@ let
        export PGRST_DB_POOL_ACQUISITION_TIMEOUT
        export PGRST_JWT_SECRET

-        ${cabal-install}/bin/cabal v2-update
-        ${cabal-install}/bin/cabal --builddir="dist-prof" v2-build --enable-profiling --disable-shared exe:postgrest
-        ${cabal-install}/bin/cabal --builddir="dist-prof" v2-run -- \
-          postgrest +RTS -p -h -RTS "''${_arg_leftovers[@]}"
+        exec ${cabal-install}/bin/cabal --builddir="dist-prof" v2-run --enable-profiling --disable-shared exe:postgrest -- \
+          +RTS -p -h -RTS "''${_arg_leftovers[@]}"
      '';

  repl =
@@ -109,6 +119,7 @@ buildToolbox
    inherit
      build
      clean
+      update
      run
      runProfiled
      repl;
@@ -10,6 +10,7 @@
 , hsie
 , nix
 , silver-searcher
+, stdenv
 , style
 , tests
 , withTools
@@ -53,10 +54,14 @@ let

          Requires authentication with `cachix authtoken ...`.
        '';
+        args =
+          [
+            "ARG_OPTIONAL_SINGLE([system], , [System], [${stdenv.system}])"
+          ];
        workingDir = "/";
      }
      ''
-        ${nix}/bin/nix-instantiate \
+        ${nix}/bin/nix-instantiate --argstr system "$_arg_system" \
          | xargs ${nix}/bin/nix-store -qR --include-outputs \
          | ${cachix}/bin/cachix push postgrest
      '';
@@ -77,6 +82,7 @@ let
      }
      ''
        ${tests}/bin/postgrest-test-spec
+        ${tests}/bin/postgrest-test-observability
        ${tests}/bin/postgrest-test-doctests
        ${tests}/bin/postgrest-test-io
        ${tests}/bin/postgrest-test-big-schema
@@ -246,7 +252,6 @@ let
      }
      ''
        mkdir -p "$_arg_dumpdir"
-        ${cabal-install}/bin/cabal v2-update
        ${cabal-install}/bin/cabal v2-build ${devCabalOptions} \
          --builddir="$tmpdir" \
          --ghc-option=-ddump-minimal-imports \
@@ -2,7 +2,6 @@
 , aspellDicts
 , buildToolbox
 , checkedShellScript
-, fetchPypi
 , lib
 , plantuml
 , python3
@@ -11,26 +10,14 @@
 , writers
 }:
 let
-  selectPythonPackages = ps:
-    let
-      # TODO: Remove with next nixpkgs update
-      sphinx-rtd-theme = assert ps.sphinx-rtd-theme.version == "2.0.0"; ps.sphinx-rtd-theme.overrideAttrs rec {
-        version = "3.0.2";
-        src = fetchPypi {
-          pname = "sphinx_rtd_theme";
-          inherit version;
-          hash = "sha256-t0V7wl3acjsgsIamcLmVPIWeq2CioD7o6yuyPhduX4U=";
-        };
-      };
-    in
-    [
-      ps.sphinx
-      ps.sphinx-copybutton
-      (ps.sphinx-rtd-dark-mode.override { inherit sphinx-rtd-theme; })
-      sphinx-rtd-theme
-      ps.sphinx-tabs
-      ps.sphinxext-opengraph
-    ];
+  selectPythonPackages = ps: [
+    ps.sphinx
+    ps.sphinx-copybutton
+    ps.sphinx-rtd-dark-mode
+    ps.sphinx-rtd-theme
+    ps.sphinx-tabs
+    ps.sphinxext-opengraph
+  ];

  requirements = writeTextFile {
    name = "requirements.txt";
@@ -48,6 +35,9 @@ let
        workingDir = "/docs";
      }
      ''
+        # https://github.com/sphinx-doc/sphinx/issues/11739
+        export LC_ALL=C
+
        function build() {
          ${python}/bin/sphinx-build --color -W -a -n . -b "$@"
        }
@@ -132,6 +122,8 @@ let
        workingDir = "/docs";
      }
      ''
+        export LC_ALL=C
+
        FILES=$(find . -type f -iname '*.rst' | tr '\n' ' ')

        # shellcheck disable=SC2086 disable=SC2016
@@ -152,6 +144,8 @@ let
        workingDir = "/docs";
      }
      ''
+        export LC_ALL=C
+
        FILES=$(find . -type f -iname '*.rst' | tr '\n' ' ')

        tail -n+2 postgrest.dict \
@@ -170,6 +164,8 @@ let
        workingDir = "/docs";
      }
      ''
+        export LC_ALL=C
+
        ${python}/bin/sphinx-build --color -b linkcheck . ../.docs-build
      '';

@@ -1,63 +1,53 @@
 # generates a file to be used by the vegeta load testing tool
+
+# It includes a worst case scenario for the JWT cache:
+# - all requests will have a unique JWT so no cache hits
+# - all jwts have an expiration that will be long enough to be
+#   valid at time of request but short enough that already
+#   validated jwts will expire later during the loadtest run
+# - the above guarantees JWT cache purging will happen
+# - we want this to track resource consumption in the worst case
+
+# And a more normal scenario where non-expiring JWTs are picked
+# from an array
 import time
-import hmac
-import hashlib
-import base64
-import json
 import argparse
 import sys
 import random
+import jwt
+import jwcrypto.jwk as jwk
+from typing import Optional
+from pathlib import Path

-SECRET = b"reallyreallyreallyreallyverysafe"
 URL = "http://postgrest"
-JWT_DURATION = 120
-TOTAL_TARGETS = 50000  # tuned by hand to reduce result variance
+
+secret_key = b"reallyreallyreallyreallyverysafe"
+
+key = jwk.JWK.generate(kty="RSA", size=4096)
+private_key = jwt.algorithms.RSAAlgorithm.from_jwk(key.export_private())
+public_key = key.export_public()


-def base64url_encode(data: bytes) -> str:
-    """URL-safe Base64 encode without padding."""
-    return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
-
-
-def generate_jwt(exp_inc: int) -> str:
-    """Generate an HS256 JWT"""
-    # Header & payload
-    header = {"alg": "HS256", "typ": "JWT"}
-    now = int(time.time())
+def generate_jwt(now: int, exp_inc: Optional[int], is_hs: bool) -> str:
+    """Generate an HS256 or RS256 JWT"""
    payload = {
        "sub": f"user_{random.getrandbits(32)}",
        "iat": now,
-        "exp": now + exp_inc,
        "role": "postgrest_test_author",
    }

-    # Encode to JSON and then to Base64URL
-    header_b = json.dumps(header, separators=(",", ":")).encode()
-    payload_b = json.dumps(payload, separators=(",", ":")).encode()
-    header_b64 = base64url_encode(header_b)
-    payload_b64 = base64url_encode(payload_b)
+    if exp_inc is not None:
+        payload["exp"] = now + exp_inc

-    # Sign (HMAC‑SHA256) the "<header>.<payload>" string
-    signing_input = f"{header_b64}.{payload_b64}".encode()
-    signature = hmac.new(SECRET, signing_input, hashlib.sha256).digest()
-    signature_b64 = base64url_encode(signature)
-
-    return f"{header_b64}.{payload_b64}.{signature_b64}"
+    k = secret_key if is_hs else private_key
+    alg = "HS256" if is_hs else "RS256"
+    return jwt.encode(payload, k, alg)


-# We want to ensure 401 Unauthorized responses don't happen during
-# JWT validation, this can happen when the jwt `exp` is too short.
-# At the same time, we want to ensure the `exp` is not too big,
-# so expires will occur and postgREST will have to clean cached expired JWTs.
-def estimate_adequate_jwt_exp_increase(iteration: int) -> int:
-    # estimated time takes to build and run postgrest itself
-    build_run_postgrest_time = 2
-    # estimated time it takes to generate the targets file
-    file_generation_time = TOTAL_TARGETS // (10**-5)
-    # estimated exp time so some JWTs will expire
-    dynamic_exp_inc = iteration // 1000
-
-    return build_run_postgrest_time + file_generation_time + dynamic_exp_inc
+def append_targets(lines: list[str], token: str):
+    lines.append(f"OPTIONS {URL}/authors_only")
+    lines.append(f"Authorization: Bearer {token}")
+    lines.append("")  # blank line to separate requests


 def main():
@@ -68,16 +58,77 @@ def main():
        "output",
        help="Path to write the generated targets file",
    )
+    parser.add_argument(
+        "--worst",
+        dest="worst",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        help="Generate worst case targets for a JWT cache",
+    )
+    parser.add_argument(
+        "--rsa",
+        dest="jwk_path",
+        metavar="JWK_PATH",
+        type=Path,
+        default=None,
+        help="Path for generating a RSA JWK file to sign tokens with",
+    )
+
    args = parser.parse_args()

-    lines = []
+    is_hs = args.jwk_path is None
+
+    nsamples = 1000
+    if is_hs:
+        ntargets = 200000
+    else:
+        # The asymmetric targets take too long to compute so we reduce them
+        ntargets = 50000
+
+    if not is_hs:
+        try:
+            with open(args.jwk_path, "w") as jwk:
+                jwk.write(public_key)
+                print(f"Created {args.jwk_path} file containing the RSA JWK")
+        except IOError as e:
+            print(f"Error writing to {args.jwk_path}: {e}", file=sys.stderr)
+            sys.exit(1)
+
+    print(f"Generating {ntargets} targets...")
+
    start_time = time.time()

-    for i in range(TOTAL_TARGETS):
-        token = generate_jwt(estimate_adequate_jwt_exp_increase(i))
-        lines.append(f"OPTIONS {URL}/authors_only")
-        lines.append(f"Authorization: Bearer {token}")
-        lines.append("")  # blank line to separate requests
+    now = int(start_time)
+
+    lines = []
+
+    # We want to ensure 401 Unauthorized responses don't happen during
+    # JWT validation, this can happen when the jwt `exp` is too short.
+    # At the same time, we want to ensure the `exp` is not too big,
+    # so expires will occur and postgREST needs to
+    # clean cached expired JWTs
+    if args.worst:
+        # estimated time takes to build and run postgrest itself
+        build_run_postgrest_time = 2
+        # estimated time it takes to generate the targets file
+        # the division numbers are tuned by hand
+        if is_hs:  # hs generation is much faster
+            gen_time = ntargets // 66666
+        else:  # asymmetric is slower so the time is higher
+            gen_time = ntargets // 220
+
+        # estimated exp time so some JWTs will expire
+        inc = build_run_postgrest_time + gen_time
+
+        for i in range(ntargets):
+            token = generate_jwt(now, inc + i // 1000, is_hs)
+            append_targets(lines, token)
+
+    else:
+        tokens = [generate_jwt(now, None, is_hs) for _ in range(nsamples)]
+        for i in range(ntargets):
+            token = random.choice(tokens)
+            append_targets(lines, token)

    try:
        with open(args.output, "w") as f:
@@ -87,7 +138,8 @@ def main():
        sys.exit(1)

    elapsed = time.time() - start_time
-    print(f"Created {TOTAL_TARGETS} targets in {args.output} ({elapsed:.2f}s)")
+    print(f"Created {ntargets} targets", end=" ")
+    print(f"in {args.output} ({elapsed:.2f}s)")


 if __name__ == "__main__":
@@ -0,0 +1,60 @@
+{ buildToolbox
+, checkedShellScript
+, commitlint
+, writeText
+}:
+let
+  # Rules format: [<severity>, <"always"/"never">, <value>]
+  commitlintConfig = writeText "commitlint.config.mjs" ''
+    export default {
+      rules: {
+        "type-enum": [2, "always", [
+            'add',      // Add a new feature
+            'amend',    // To amend an unrealease commit
+            'change',   // Breaking changes
+            'chore',    // Update sponsors, changelog, readme etc
+            'ci',       // CI configuration files and scripts
+            'docs',     // Documentation
+            'fix',      // Bug fix
+            'nix',      // Related to Nix
+            'perf',     // Performance improvements
+            'refactor', // Refactoring code
+            'remove',   // Remove a feature or fix
+            'test',     // Adding tests
+          ]],
+
+          'subject-case':       [2, 'never', ['pascal-case', 'start-case']],
+          'subject-empty':      [2, 'never'],
+          'subject-full-stop':  [2, 'never', '.'],
+          'subject-max-length': [2, 'always', 80],
+          'subject-min-length': [2, 'always', 5],
+
+          'scope-case':         [2, 'always', 'lower-case'],
+
+          'body-leading-blank': [2, 'always'],
+      },
+    };
+  '';
+
+  commitCheck =
+    checkedShellScript
+      {
+        name = "postgrest-commitlint";
+        docs = "Script to validate commit messages";
+        workingDir = "/";
+        args = [
+          "ARG_OPTIONAL_SINGLE([from],, [commit ref start from], [main])"
+          "ARG_OPTIONAL_SINGLE([to],, [commit ref end at], [HEAD])"
+        ];
+      }
+      ''
+        # Run commitlint with the given configuration
+
+        ${commitlint}/bin/commitlint --config ${commitlintConfig} --from "$_arg_from" --to "$_arg_to"
+      '';
+in
+buildToolbox
+{
+  name = "postgrest-commitlint";
+  tools = { inherit commitCheck; };
+}
@@ -41,8 +41,9 @@ let
        args = [
          "ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])"
          "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])"
-          "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest (mixed: repeat mixed requests, jwt: run once over many requests with unique jwts)], [mixed])"
-          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt])"
+          "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
+          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
+          "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
          "ARG_LEFTOVERS([additional vegeta arguments])"
        ];
        workingDir = "/";
@@ -58,35 +59,67 @@ let
        export PGRST_DB_TX_END="rollback-allow-override"
        export PGRST_LOG_LEVEL="crit"
        export PGRST_JWT_SECRET="reallyreallyreallyreallyverysafe"
+        # set previous PGRST_JWT_CACHE_MAX_LIFETIME configuration so that
+        # load test works across branches
+        # TODO clean once PGRST_JWT_CACHE_MAX_ENTRIES merged and released
        export PGRST_JWT_CACHE_MAX_LIFETIME="86400"

        mkdir -p "$(dirname "$_arg_output")"
        abs_output="$(realpath "$_arg_output")"

        case "$_arg_kind" in
-          jwt)
+          jwt-hs)
+            ${genTargetsHS} "$_arg_testdir"/gen_targets.http
+            export PGRST_JWT_CACHE_MAX_ENTRIES="0"
+            export PGRST_JWT_CACHE_MAX_LIFETIME="0"
+            ;;

-            ${genTargets} "$_arg_testdir"/gen_targets.http
+          jwt-hs-cache)
+            ${genTargetsHS} "$_arg_testdir"/gen_targets.http
+            ;;

-            # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} \
-            sh -c "cd \"$_arg_testdir\" && ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
-            ${vegeta}/bin/vegeta report -type=text "$_arg_output"
+          jwt-hs-cache-worst)
+            ${genTargetsHS} --worst "$_arg_testdir"/gen_targets.http
+            ;;
+
+          jwt-rsa)
+            ${genTargetsHS} --rsa="$_arg_testdir"/gen_jwk.json "$_arg_testdir"/gen_targets.http
+            export PGRST_JWT_CACHE_MAX_ENTRIES="0"
+            export PGRST_JWT_CACHE_MAX_LIFETIME="0"
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
+            ;;
+
+          jwt-rsa-cache)
+            ${genTargetsHS} --rsa="$_arg_testdir"/gen_jwk.json "$_arg_testdir"/gen_targets.http
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
+            ;;
+
+          jwt-rsa-cache-worst)
+            ${genTargetsHS} --worst --rsa="$_arg_testdir"/gen_jwk.json "$_arg_testdir"/gen_targets.http
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
            ;;

          *)
-
-            # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withSlowPg} \
-            ${withTools.withPgrst} \
-            ${withTools.withSlowPgrst} \
-            sh -c "cd \"$_arg_testdir\" && ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
-            ${vegeta}/bin/vegeta report -type=text "$_arg_output"
            ;;
-
        esac
+
+        if [ "$_arg_kind" == "mixed" ]; then
+          # shellcheck disable=SC2145
+          ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+          ${withTools.withSlowPg} \
+          ${withTools.withPgrst} -m "$_arg_monitor" \
+          ${withTools.withSlowPgrst} \
+          sh -c "cd \"$_arg_testdir\" && \
+          ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+        else
+          # shellcheck disable=SC2145
+          ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+          ${withTools.withPgrst} -m "$_arg_monitor" \
+          sh -c "cd \"$_arg_testdir\" && \
+          ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+        fi
+
+        ${vegeta}/bin/vegeta report -type=text "$_arg_output"
      '';

  loadtestAgainst =
@@ -115,11 +148,12 @@ let
        workingDir = "/";
      }
      ''
+        # run loadtest for every target
        for tgt in "''${_arg_target[@]}"; do

        cat << EOF

-        Running loadtest on "$tgt"...
+        Running "$_arg_kind" loadtest on "$tgt"...

        EOF

@@ -128,7 +162,7 @@ let
        # Save the results in the current working tree, too,
        # otherwise they'd be lost in the temporary working tree
        # created by withTools.withGit.
-        ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"
+        ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/$tgt.csv" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"

        cat << EOF

@@ -138,13 +172,15 @@ let

        done

+        # run loadtest once on HEAD
+
        cat << EOF

-        Running loadtest on HEAD...
+        Running "$_arg_kind" loadtest on HEAD...

        EOF

-        ${loadtest} -k "$_arg_kind" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load"
+        ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/head.csv" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load"

        cat << EOF

@@ -181,6 +217,7 @@ let
        pd.read_json(sys.stdin) \
          .set_index('param') \
          .drop(['branch', 'earliest', 'end', 'latest']) \
+          .fillna("") \
          .convert_dtypes() \
          .to_markdown(sys.stdout, floatfmt='.0f')
      '';
@@ -191,16 +228,41 @@ let
      {
        name = "postgrest-loadtest-report";
        docs = "Create a report of all loadtest reports as markdown.";
+        args = [
+          "ARG_OPTIONAL_SINGLE([group], [g], [Marker to group results])"
+        ];
        workingDir = "/";
      }
      ''
+        marker=''${_arg_group:+"($_arg_group)"}
+
+        echo -e "## Loadtest results $marker\n"
+
        find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \
          | ${jq}/bin/jq '[paths(scalars) as $path | {param: $path | join("."), (.branch): getpath($path)}]' \
          | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \
          | ${toMarkdown}
+
+        echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM usage $marker\n"
+
+        find loadtest -type f -iname '*.csv' \
+          | sort -m \
+          | ${mergeMonitorResults}
      '';

-  genTargets = writers.writePython3 "postgrest-gen-loadtest-targets" { } (builtins.readFile ./generate_targets.py);
+  genTargetsHS =
+    writers.writePython3 "postgrest-gen-loadtest-targets-hs"
+      {
+        libraries = [ python3Packages.pyjwt python3Packages.jwcrypto ];
+      }
+      (builtins.readFile ./generate_targets.py);
+
+  mergeMonitorResults =
+    writers.writePython3 "postgrest-merge-monitor-results"
+      {
+        libraries = [ python3Packages.pandas python3Packages.tabulate ];
+      }
+      (builtins.readFile ./merge_monitor_result.py);
 in
 buildToolbox {
  name = "postgrest-loadtest";
@@ -0,0 +1,44 @@
+import os
+import sys
+import pandas as pd
+
+KEY = "Elapsed seconds"
+BASE_METRICS = ["CPU (%)", "Real (MB)"]
+branch_order = []
+merged = None
+
+paths = [p.strip() for p in sys.stdin.read().split() if p.strip()]
+
+for csv_path in paths:
+    # br is branch (variable shortened to pass linter)
+    br = os.path.splitext(os.path.basename(csv_path))[0]
+    branch_order.append(br)
+
+    df = pd.read_csv(csv_path)
+
+    if KEY not in df.columns:
+        sys.exit(f"{csv_path} is missing the {KEY} column")
+
+    for m in BASE_METRICS:
+        if m not in df.columns:
+            sys.exit(f"Error: '{csv_path}' missing required column '{m}'.")
+
+    # add branch marker to every metric column
+    df = df.rename(columns={c: f"{c} [{br}]" for c in df.columns if c != KEY})
+
+    # outer join so missing rows appear
+    merged = df if merged is None else merged.merge(df, on=KEY, how="outer")
+
+# Re-order columns so related metrics are adjacent
+ordered_cols = [KEY]
+for metric in BASE_METRICS:
+    for br in branch_order:
+        col_name = f"{metric} [{br}]"
+        if col_name in merged.columns:
+            ordered_cols.append(col_name)
+
+merged = merged[ordered_cols]
+
+# replace nan with empty string
+merged = merged.fillna("")
+merged.to_markdown(sys.stdout, index=False, tablefmt="github")
@@ -0,0 +1,57 @@
+# Monitor a process pid with psutil and emits a CSV.
+import sys
+import time
+import psutil
+import pandas as pd
+
+KEY = "Elapsed seconds"
+BASE_METRICS = ["CPU (%)", "Real (MB)"]
+SAMPLE_INTERVAL_SECS = 1
+
+if len(sys.argv) != 2 or not sys.argv[1].isdigit():
+    sys.exit(f"Usage: {sys.argv[0]} <PID>")
+
+pid = int(sys.argv[1])
+try:
+    proc = psutil.Process(pid)
+except psutil.NoSuchProcess:
+    sys.exit(f"Error: process {pid} not found.")
+
+print(f"Starting monitoring of {pid} pid", file=sys.stderr)
+
+records = []
+start = time.time()
+# ignore first result as per docs recommendation
+# https://psutil.readthedocs.io/en/latest/#psutil.cpu_percent
+proc.cpu_percent(None)
+
+while True:
+    try:
+        if not proc.is_running():
+            break
+        time.sleep(SAMPLE_INTERVAL_SECS)
+
+        elapsed_secs = int(time.time() - start)
+        cpu = proc.cpu_percent(None)
+        meminfo = proc.memory_info()
+        bytes_in_MB = 1024**2
+        rss_mb = meminfo.rss / bytes_in_MB
+
+        records.append(
+            [
+                str(elapsed_secs),
+                f"{cpu:.3f}",
+                f"{rss_mb:.3f}",
+            ]
+        )
+
+    except psutil.NoSuchProcess:
+        break
+
+end = time.time()
+total_time = end - start
+print(f"Finished {pid} pid monitoring in {total_time:.3f}", file=sys.stderr)
+
+cols = [KEY] + BASE_METRICS
+df = pd.DataFrame(records, columns=cols, dtype=str)
+df.to_csv(sys.stdout, index=False)
@@ -0,0 +1,28 @@
+{ buildToolbox
+, checkedShellScript
+}:
+# Utility script for pinning the latest stable version of Nixpkgs.
+
+# Instead of running `nix flake update` manually, we run this script
+# to also pin readthedocs dependencies at the same time.
+let
+  upgrade =
+    checkedShellScript
+      {
+        name = "postgrest-nixpkgs-upgrade";
+        docs = "Pin the newest version of Nixpkgs.";
+        workingDir = "/";
+      }
+      ''
+        nix flake update
+
+        echo "# This file is auto-generated by postgrest-nixpkgs-upgrade" > docs/requirements.txt
+        cat "$(nix-build -A docs.requirements)" >> docs/requirements.txt
+      '';
+
+in
+buildToolbox
+{
+  name = "postgrest-nixpkgs";
+  tools = { inherit upgrade; };
+}
@@ -20,27 +20,24 @@ let
        git diff --exit-code HEAD postgrest.cabal > /dev/null
        trap "" ERR

+        # TODO: Support C+D bumps when implementing hackage releases
        bump () {
          current_version="$(grep -oP '^version:\s*\K.*' postgrest.cabal)"
          # shellcheck disable=SC2034
-          IFS=. read -r major minor patch <<< "$current_version"
+          IFS=. read -r A B C D <<< "$current_version"
          echo "Current version is $current_version"

          case "$1" in
-            major)
-              new_version="$((major+1)).0.0"
-              new_docs_version="$((major+1)).0"
+            A)
+              new_version="$((A+1)).0"
+              new_docs_version="$((A+1))"
              ;;
-            minor)
-              new_version="$major.$((minor+1)).0"
-              new_docs_version="$major.$((minor+1))"
-              ;;
-            patch)
-              new_version="$major.$minor.$((patch+1))"
-              new_docs_version="$major.$minor"
+            B)
+              new_version="$A.$((B+1))"
+              new_docs_version="$A"
              ;;
            devel)
-              new_version="$major.$((minor+1))"
+              new_version="$((A+1))"
              new_docs_version="devel"
              ;;
          esac
@@ -55,13 +52,9 @@ let

        today_date_for_changelog="$(date '+%Y-%m-%d')"
        if [[ "$current_branch" == "main" ]]; then
-          if [[ "$_arg_major" == "on" ]]; then
-            bump major
-          else
-            bump minor
-          fi
+          bump A
        else
-          bump patch
+          bump B
        fi

        echo "Updating CHANGELOG.md ..."
@@ -75,10 +68,10 @@ let
          bump devel

          # The order of operations is important here:
-          # - bump devel is run and $major is upated to the new version
-          # - the branch is created with the new major, but the commit before the devel bump
+          # - bump devel is run and $A is upated to the new version
+          # - the branch is created with the new A, but the commit before the devel bump
          # - the devel bump is committed
-          git branch -f "v$major"
+          git branch "v$A"

          echo "Committing (devel bump)..."
          git commit -m "bump version to $new_version" > /dev/null
@@ -90,7 +83,7 @@ let

        if [[ "$current_branch" == "main" ]]; then
          push1="git push $remote $current_branch"
-          push2="git push $remote v$major --force"
+          push2="git push $remote v$A"
        else
          push1="git push $remote $current_branch"
          push2=""
@@ -7,9 +7,12 @@
 , hlint
 , hsie
 , nixpkgs-fmt
+, python3Packages
+, ruff
 , silver-searcher
 , statix
 , stylish-haskell
+, writeText
 }:
 let
  style =
@@ -49,6 +52,20 @@ let
        ${git}/bin/git diff-index --exit-code HEAD -- '*.hs' '*.lhs' '*.nix' '*.py'
      '';

+  hlintConfig = writeText "hlintConfig.yml" ''
+
+    # Arguments passed to hlint
+    - arguments: [-j, -XQuasiQuotes, -XNoPatternSynonyms]
+
+    # Warnings
+    - warn: { lhs: "a == a", rhs: "True",  note: "This comparison always evaluates to True" }
+    - warn: { lhs: "a /= a", rhs: "False", note: "This comparison always evaluates to False" }
+    - warn: { lhs: "a < a",  rhs: "False", note: "This comparison always evaluates to False" }
+    - warn: { lhs: "a > a",  rhs: "False", note: "This comparison always evaluates to False" }
+    - warn: { lhs: "a <= a", rhs: "True",  note: "This comparison always evaluates to True" }
+    - warn: { lhs: "a >= a", rhs: "True",  note: "This comparison always evaluates to True" }
+  '';
+
  lint =
    checkedShellScript
      {
@@ -63,13 +80,21 @@ let
        echo "Scanning nix files for unused code..."
        ${deadnix}/bin/deadnix -f

+        # ruff has gaps in scanning for unused code, so we use vulture
+        echo "Scanning python files for unused code..."
+        ${silver-searcher}/bin/ag -l --vimgrep -g '\.l?py$' . \
+          | xargs ${python3Packages.vulture}/bin/vulture --exclude docs/conf.py
+
+        echo "Linting python files..."
+        ${ruff}/bin/ruff check .
+
        echo "Checking consistency of import aliases in Haskell code..."
        ${hsie} check-aliases main src

        echo "Linting Haskell files..."
        # --vimgrep fixes a bug in ag: https://github.com/ggreer/the_silver_searcher/issues/753
        ${silver-searcher}/bin/ag -l --vimgrep -g '\.l?hs$' . \
-          | xargs ${hlint}/bin/hlint -X QuasiQuotes -X NoPatternSynonyms
+          | xargs ${hlint}/bin/hlint --hint=${hlintConfig}
      '';

 in
@@ -28,11 +28,24 @@ let
        withEnv = postgrest.env;
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        ${withTools.withPg} -f test/spec/fixtures/load.sql \
          ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec -- "''${_arg_leftovers[@]}"
      '';

+  testObservability =
+    checkedShellScript
+      {
+        name = "postgrest-test-observability";
+        docs = "Run the Haskell observability test suite.";
+        args = [ "ARG_LEFTOVERS([hspec arguments])" ];
+        workingDir = "/";
+        withEnv = postgrest.env;
+      }
+      ''
+        ${withTools.withPg} -f test/observability/fixtures/load.sql \
+          ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:observability -- "''${_arg_leftovers[@]}"
+      '';
+
  testDoctests =
    checkedShellScript
      {
@@ -42,7 +55,6 @@ let
        withEnv = postgrest.env;
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        # This makes nix-env -iA tests.doctests.bin work.
        export NIX_GHC=${postgrest.env.NIX_GHC}
        ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:doctests
@@ -57,7 +69,6 @@ let
        withEnv = postgrest.env;
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        ${withTools.withPg} -f test/spec/fixtures/load.sql \
          ${runtimeShell} -c " \
            ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec && \
@@ -85,9 +96,8 @@ let
        withEnv = postgrest.env;
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest
-        ${cabal-install}/bin/cabal v2-exec -- ${withTools.withPg} -f test/io/fixtures.sql \
+        ${cabal-install}/bin/cabal v2-exec -- ${withTools.withPg} -f test/io/fixtures/load.sql \
          ${ioTestPython}/bin/pytest --ignore=test/io/test_big_schema.py --ignore=test/io/test_replica.py -v test/io "''${_arg_leftovers[@]}"
      '';

@@ -101,9 +111,8 @@ let
        withEnv = postgrest.env;
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest
-        ${cabal-install}/bin/cabal v2-exec -- ${withTools.withPg} -f test/io/big_schema.sql \
+        ${cabal-install}/bin/cabal v2-exec -- ${withTools.withPg} -f test/io/fixtures/big_schema.sql \
          ${ioTestPython}/bin/pytest -v test/io/test_big_schema.py "''${_arg_leftovers[@]}"
      '';

@@ -117,9 +126,8 @@ let
        withEnv = postgrest.env;
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest
-        ${cabal-install}/bin/cabal v2-exec -- ${withTools.withPg} --replica -f test/io/replica.sql \
+        ${cabal-install}/bin/cabal v2-exec -- ${withTools.withPg} --replica -f test/io/fixtures/replica.sql \
          ${ioTestPython}/bin/pytest -v test/io/test_replica.py "''${_arg_leftovers[@]}"
      '';

@@ -133,7 +141,6 @@ let
        withPath = [ jq ];
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        ${withTools.withPg} -f test/spec/fixtures/load.sql \
            ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
            postgrest --dump-schema
@@ -162,8 +169,7 @@ let
          rm -rf coverage/*

          # build once before running all the tests
-          ${cabal-install}/bin/cabal v2-update
-          ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest lib:postgrest test:spec
+          ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest lib:postgrest test:spec test:observability

          (
            trap 'echo Found dead code: Check file list above.' ERR ;
@@ -172,26 +178,31 @@ let

          # collect all tests
          HPCTIXFILE="$tmpdir"/io.tix \
-            ${withTools.withPg} -f test/io/fixtures.sql \
+            ${withTools.withPg} -f test/io/fixtures/load.sql \
            ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest --ignore=test/io/test_big_schema.py --ignore=test/io/test_replica.py -v test/io

          HPCTIXFILE="$tmpdir"/big_schema.tix \
-            ${withTools.withPg} -f test/io/big_schema.sql \
+            ${withTools.withPg} -f test/io/fixtures/big_schema.sql \
            ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest -v test/io/test_big_schema.py

          HPCTIXFILE="$tmpdir"/replica.tix \
-            ${withTools.withPg} --replica -f test/io/replica.sql \
+            ${withTools.withPg} --replica -f test/io/fixtures/replica.sql \
            ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest -v test/io/test_replica.py

          HPCTIXFILE="$tmpdir"/spec.tix \
            ${withTools.withPg} -f test/spec/fixtures/load.sql \
            ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec

+          HPCTIXFILE="$tmpdir"/observability.tix \
+            ${withTools.withPg} -f test/observability/fixtures/load.sql \
+            ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:observability
+
          # Note: No coverage for doctests, as doctests leverage GHCi and GHCi does not support hpc

          # collect all the tix files
          ${ghc}/bin/hpc sum  --union --exclude=Paths_postgrest --output="$tmpdir"/tests.tix \
-            "$tmpdir"/io*.tix "$tmpdir"/big_schema*.tix "$tmpdir"/replica*.tix "$tmpdir"/spec.tix
+            "$tmpdir"/io*.tix "$tmpdir"/big_schema*.tix "$tmpdir"/replica*.tix "$tmpdir"/spec.tix \
+            "$tmpdir"/observability.tix

          # prepare the overlay
          ${ghc}/bin/hpc overlay --output="$tmpdir"/overlay.tix test/coverage.overlay
@@ -246,7 +257,6 @@ let
        withPath = [ curl ];
      }
      ''
-        ${cabal-install}/bin/cabal v2-update
        ${cabal-install}/bin/cabal --builddir="dist-prof" v2-build --enable-profiling --disable-shared exe:postgrest
        ${cabal-install}/bin/cabal --builddir="dist-prof" v2-exec -- ${withTools.withPg} -f test/spec/fixtures/load.sql \
          test/memory/memory-tests.sh
@@ -259,6 +269,7 @@ buildToolbox
  tools = {
    inherit
      testSpec
+      testObservability
      testDoctests
      testSpecIdempotence
      testIO
@@ -5,8 +5,10 @@
 , lib
 , postgresqlVersions
 , postgrest
+, python3Packages
 , slocat
 , writeText
+, writers
 }:
 let
  withTmpDb =
@@ -44,7 +46,7 @@ let
        }

        # Avoid starting multiple layers of withTmpDb, but make sure to have the last invocation
-        # load fixtures. Otherwise postgrest-with-postgresql-xx postgrest-test-io would not be possible.
+        # load fixtures. Otherwise postgrest-with-pg-xx postgrest-test-io would not be possible.
        if ! test -v PGHOST; then

          mkdir -p "$tmpdir"/{db,socket}
@@ -72,7 +74,13 @@ let
            >> "$setuplog"

          log "Starting the database cluster..."
+
          # Instead of listening on a local port, we will listen on a unix domain socket.
+          # NOTE: unix domain socket filename name must remain under max limit.
+          # On Linux, it's 108 chars (including '\0' terminator)
+          # On MacOS, it's 104 chars
+          # See: https://serverfault.com/questions/641347/check-if-a-path-exceeds-maximum-for-unix-domain-socket
+
          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
            >> "$setuplog"

@@ -327,6 +335,22 @@ let
        done
      '';

+  # Broadcast SIGINT to any running postgrest instances on the host. Uses python for cross-platform compatibility.
+  signalPostgrest =
+    writers.writePython3 "postgrest-signal-int"
+      { libraries = [ python3Packages.psutil ]; }
+      ''
+        import psutil
+        import signal
+
+        for proc in psutil.process_iter(["name"]):
+            try:
+                if proc.info["name"] == "postgrest":
+                    proc.send_signal(signal.SIGINT)
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+      '';
+
  withPgrst =
    checkedShellScript
      {
@@ -336,6 +360,7 @@ let
          [
            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
            "ARG_LEFTOVERS([command arguments])"
+            "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])"
          ];
        positionalCompletion = "_command";
        workingDir = "/";
@@ -348,12 +373,13 @@ let
        rm -f result
        if [ -z "''${PGRST_BUILD_CABAL:-}" ]; then
          echo -n "Building postgrest (nix)... "
-          nix-build -A postgrestPackage > "$tmpdir"/build.log 2>&1 || {
+          # Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet.
+          nix-build -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > "$tmpdir"/build.log 2>&1 || {
            echo "failed, output:"
            cat "$tmpdir"/build.log
            exit 1
          }
-          PGRST_CMD=./result/bin/postgrest
+          PGRST_CMD=$(echo ./result*/bin/postgrest)
        else
          echo -n "Building postgrest (cabal)... "
          postgrest-build
@@ -361,11 +387,22 @@ let
        fi
        echo "done."

-        echo -n "Starting postgrest... "
+        ver=$($PGRST_CMD ${legacyConfig} --version)
+
+        echo -n "Starting $ver... "
+
        $PGRST_CMD ${legacyConfig} > "$tmpdir"/run.log 2>&1 &
        pid=$!
        # shellcheck disable=SC2317
        cleanup() {
+          # Send INT to all postgrest processes.
+          # Workaround to trigger dumping postgrest.prof for postgrest-profiled-run
+          # Caveat: we cannot realistically limit this to the current process' tree,
+          # since pkill's --parent supports only direct children; therefore this
+          # would reap neighbor postgrest instances as well, because INT is asking
+          # the process to terminate too.
+          # TODO: consider cgroups to make this cleaner
+          ${signalPostgrest}
          kill "$pid" || true
        }
        trap cleanup EXIT
@@ -377,9 +414,19 @@ let
        }
        echo "done."

+        if [[ -n "$_arg_monitor" ]]; then
+          ${monitorPid} "$pid" > "$_arg_monitor" &
+        fi
+
        ("$_arg_command" "''${_arg_leftovers[@]}")
      '';

+  monitorPid =
+    writers.writePython3 "postgrest-monitor-pid"
+      {
+        libraries = [ python3Packages.pandas python3Packages.tabulate python3Packages.psutil ];
+      }
+      (builtins.readFile ./monitor_pid.py);
 in
 buildToolbox
 {
@@ -392,7 +439,7 @@ buildToolbox
      withSlowPg
      withSlowPgrst;
  } // builtins.listToAttrs (
-    # Create a `postgrest-with-postgresql-` for each PostgreSQL version
+    # Create a `postgrest-with-pg-` for each PostgreSQL version
    builtins.map (pg: { inherit (pg) name; value = withTmpDb pg; }) postgresqlVersions
  );
  # make latest withPg available for other nix files
@@ -1,5 +1,5 @@
 name:               postgrest
-version:            13.0.5
+version:            14.8
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                    for tables, views, and functions, supporting all HTTP methods that security
@@ -26,7 +26,7 @@ tested-with:

 source-repository head
  type:     git
-  location: git://github.com/PostgREST/postgrest.git
+  location: https://github.com/PostgREST/postgrest.git

 flag dev
  default:     False
@@ -47,9 +47,12 @@ library
                      PostgREST.App
                      PostgREST.AppState
                      PostgREST.Auth
+                      PostgREST.Auth.Jwt
                      PostgREST.Auth.JwtCache
                      PostgREST.Auth.Types
+                      PostgREST.Cache.Sieve
                      PostgREST.CLI
+                      PostgREST.Client
                      PostgREST.Config
                      PostgREST.Config.Database
                      PostgREST.Config.JSPath
@@ -65,11 +68,13 @@ library
                      PostgREST.Error
                      PostgREST.Listener
                      PostgREST.Logger
+                      PostgREST.MainTx
                      PostgREST.MediaType
                      PostgREST.Metrics
                      PostgREST.Network
                      PostgREST.Observation
                      PostgREST.Query
+                      PostgREST.Query.PreQuery
                      PostgREST.Query.QueryBuilder
                      PostgREST.Query.SqlFragment
                      PostgREST.Query.Statements
@@ -83,11 +88,13 @@ library
                      PostgREST.ApiRequest
                      PostgREST.ApiRequest.Preferences
                      PostgREST.ApiRequest.QueryParams
+                      PostgREST.ApiRequest.Payload
                      PostgREST.ApiRequest.Types
                      PostgREST.Response
                      PostgREST.Response.OpenAPI
                      PostgREST.Response.GucHeader
                      PostgREST.Response.Performance
+                      PostgREST.TimeIt
                      PostgREST.Version
  build-depends:      base                      >= 4.9 && < 4.20
                    , HTTP                      >= 4000.3.7 && < 4000.5
@@ -96,11 +103,9 @@ library
                    , auto-update               >= 0.1.4 && < 0.3
                    , base64-bytestring         >= 1 && < 1.3
                    , bytestring                >= 0.10.8 && < 0.13
-                    , cache                     >= 0.1.3 && < 0.2.0
                    , case-insensitive          >= 1.2 && < 1.3
                    , cassava                   >= 0.4.5 && < 0.6
-                    , clock                     >= 0.8.3 && < 0.9.0
-                    , configurator-pg           >= 0.2 && < 0.3
+                    , configurator-pg           >= 0.2.11 && < 0.3
                    , containers                >= 0.5.7 && < 0.7
                    , cookie                    >= 0.4.2 && < 0.6
                    , directory                 >= 1.2.6 && < 1.4
@@ -112,10 +117,9 @@ library
                    , hasql-notifications       >= 0.2.2.2 && < 0.2.3
                    , hasql-pool                >= 1.0.1 && < 1.1
                    , hasql-transaction         >= 1.0.1 && < 1.2
-                    , heredoc                   >= 0.2 && < 0.3
+                    , http-client               >= 0.7.19 && < 0.8
                    , http-types                >= 0.12.2 && < 0.13
                    , insert-ordered-containers >= 0.2.2 && < 0.3
-                    , iproute                   >= 1.7.0 && < 1.8
                    , jose-jwt                  >= 0.9.6 && < 0.11
                    , lens                      >= 4.14 && < 5.4
                    , lens-aeson                >= 1.0.1 && < 1.3
@@ -132,11 +136,10 @@ library
                    , regex-tdfa                >= 1.2.2 && < 1.4
                    , retry                     >= 0.7.4 && < 0.10
                    , scientific                >= 0.3.4 && < 0.4
-                    , streaming-commons         >= 0.1.1 && < 0.3
+                    , streaming-commons         >= 0.2.3.1 && < 0.3
                    , swagger2                  >= 2.4 && < 2.9
                    , text                      >= 1.2.2 && < 2.2
                    , time                      >= 1.6 && < 1.13
-                    , timeit                    >= 2.0 && < 2.1
                    , unordered-containers      >= 0.2.8 && < 0.3
                    , unix-compat               >= 0.5.4 && < 0.8
                    , vault                     >= 0.3.1.5 && < 0.4
@@ -150,6 +153,10 @@ library
                    -- https://github.com/kazu-yamamoto/logger/commit/3a71ca70afdbb93d4ecf0083eeba1fbbbcab3fc3
                    , wai-logger                >= 2.4.0
                    , warp                      >= 3.3.19 && < 3.5
+                    , stm                       >= 2.5 && < 3
+                    , stm-hamt                  >= 1.2 && < 2
+                    , focus                     >= 1.0 && < 2
+                    , some                      >= 1.0.4.1 && < 2
                      -- -fno-spec-constr may help keep compile time memory use in check,
                      --   see https://gitlab.haskell.org/ghc/ghc/issues/16017#note_219304
                      -- -optP-Wno-nonportable-include-path
@@ -261,6 +268,7 @@ test-suite spec
                    , hasql-transaction >= 1.0.1 && < 1.2
                    , heredoc           >= 0.2 && < 0.3
                    , hspec             >= 2.3 && < 2.12
+                    , hspec-expectations >= 0.8.4 && < 0.9
                    , hspec-wai         >= 0.10 && < 0.12
                    , hspec-wai-json    >= 0.10 && < 0.12
                    , http-types        >= 0.12.3 && < 0.13
@@ -270,6 +278,7 @@ test-suite spec
                    , monad-control     >= 1.0.1 && < 1.1
                    , postgrest
                    , process           >= 1.4.2 && < 1.7
+                    , prometheus-client >= 1.1.1 && < 1.2.0
                    , protolude         >= 0.3.1 && < 0.4
                    , regex-tdfa        >= 1.2.2 && < 1.4
                    , scientific        >= 0.3.4 && < 0.4
@@ -284,6 +293,39 @@ test-suite spec
                      -- https://github.com/PostgREST/postgrest/issues/387
                      -with-rtsopts=-K33K

+test-suite observability
+  type:               exitcode-stdio-1.0
+  default-language:   Haskell2010
+  default-extensions: OverloadedStrings
+                      QuasiQuotes
+                      NoImplicitPrelude
+  hs-source-dirs:     test/observability
+  main-is:            Main.hs
+  other-modules:      ObsHelper
+                      Observation.JwtCache
+                      Observation.MetricsSpec
+  build-depends:      base              >= 4.9 && < 4.20
+                    , base64-bytestring >= 1 && < 1.3
+                    , bytestring        >= 0.10.8 && < 0.13
+                    , hasql-pool        >= 1.0.1 && < 1.1
+                    , hasql-transaction >= 1.0.1 && < 1.2
+                    , hspec             >= 2.3 && < 2.12
+                    , hspec-expectations >= 0.8.4 && < 0.9
+                    , hspec-wai         >= 0.10 && < 0.12
+                    , hspec-wai-json    >= 0.10 && < 0.12
+                    , http-types        >= 0.12.3 && < 0.13
+                    , jose-jwt          >= 0.9.6 && < 0.11
+                    , postgrest
+                    , prometheus-client >= 1.1.1 && < 1.2.0
+                    , protolude         >= 0.3.1 && < 0.4
+                    , text              >= 1.2.2 && < 2.2
+                    , wai               >= 3.2.1 && < 3.3
+  ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
+                      -fno-spec-constr -optP-Wno-nonportable-include-path
+                      -fwrite-ide-info
+                      -- https://github.com/PostgREST/postgrest/issues/387
+                      -with-rtsopts=-K33K
+
 test-suite doctests
  type:               exitcode-stdio-1.0
  default-language:   Haskell2010
@@ -21,7 +21,9 @@ let
      postgrest.cabalTools
      postgrest.devTools
      postgrest.docs
+      postgrest.gitTools
      postgrest.loadtest
+      postgrest.nixpkgsTools
      postgrest.release
      postgrest.style
      postgrest.tests
@@ -1,5 +1,3 @@
-{-# LANGUAGE NamedFieldPuns #-}
-
 module PostgREST.Admin
  ( runAdmin
  ) where
@@ -9,37 +7,35 @@ import qualified Network.HTTP.Types.Status as HTTP
 import qualified Network.Wai               as Wai
 import qualified Network.Wai.Handler.Warp  as Warp

-import Control.Monad.Extra (whenJust)
-
-import Network.Socket
+import Control.Monad.Extra       (whenJust)
+import Network.Socket            hiding (addrFamily)
 import Network.Socket.ByteString

 import PostgREST.AppState    (AppState)
-import PostgREST.Config      (AppConfig (..))
 import PostgREST.MediaType   (MediaType (..), toContentType)
 import PostgREST.Metrics     (metricsToText)
-import PostgREST.Network     (resolveHost)
+import PostgREST.Network     (resolveSocketToAddress)
 import PostgREST.Observation (Observation (..))

 import qualified PostgREST.AppState as AppState

-import Protolude
+import qualified Network.Socket as NS
+import           Protolude

-runAdmin :: AppState -> Warp.Settings -> IO ()
-runAdmin appState settings = do
-  AppConfig{configAdminServerPort} <- AppState.getConfig appState
-  whenJust (AppState.getSocketAdmin appState) $ \adminSocket -> do
-    host <- resolveHost adminSocket
-    observer $ AdminStartObs host configAdminServerPort
+runAdmin :: AppState -> Maybe NS.Socket -> NS.Socket -> Warp.Settings -> IO ()
+runAdmin appState maybeAdminSocket socketREST settings = do
+  whenJust maybeAdminSocket $ \adminSocket -> do
+    address <- resolveSocketToAddress adminSocket
+    observer $ AdminStartObs address
    void . forkIO $ Warp.runSettingsSocket settings adminSocket adminApp
  where
-    adminApp = admin appState
+    adminApp = admin appState socketREST
    observer = AppState.getObserver appState

 -- | PostgREST admin application
-admin :: AppState.AppState -> Wai.Application
-admin appState req respond  = do
-  isMainAppReachable  <- isRight <$> reachMainApp (AppState.getSocketREST appState)
+admin :: AppState.AppState -> NS.Socket -> Wai.Application
+admin appState socketREST req respond  = do
+  isMainAppReachable  <- isRight <$> reachMainApp socketREST
  isLoaded <- AppState.isLoaded appState
  isPending <- AppState.isPending appState

@@ -6,44 +6,30 @@ Description : PostgREST functions to translate HTTP request to a domain type cal
 {-# LANGUAGE NamedFieldPuns #-}
 module PostgREST.ApiRequest
  ( ApiRequest(..)
-  , InvokeMethod(..)
-  , Mutation(..)
-  , MediaType(..)
-  , Action(..)
-  , DbAction(..)
-  , Payload(..)
  , userApiRequest
  , userPreferences
  ) where

-import qualified Data.Aeson            as JSON
-import qualified Data.Aeson.Key        as K
-import qualified Data.Aeson.KeyMap     as KM
-import qualified Data.ByteString.Char8 as BS
-import qualified Data.ByteString.Lazy  as LBS
-import qualified Data.CaseInsensitive  as CI
-import qualified Data.Csv              as CSV
-import qualified Data.HashMap.Strict   as HM
-import qualified Data.List.NonEmpty    as NonEmptyList
-import qualified Data.Map.Strict       as M
-import qualified Data.Set              as S
-import qualified Data.Text.Encoding    as T
-import qualified Data.Vector           as V
+import qualified Data.CaseInsensitive as CI
+import qualified Data.HashMap.Strict  as HM
+import qualified Data.List.NonEmpty   as NonEmptyList
+import qualified Data.Set             as S
+import qualified Data.Text.Encoding   as T

-import Data.Either.Combinators (mapBoth)
-
-import Control.Arrow             ((***))
-import Data.Aeson.Types          (emptyArray, emptyObject)
 import Data.List                 (lookup)
 import Data.Ranged.Ranges        (emptyRange, rangeIntersection,
                                  rangeIsEmpty)
 import Network.HTTP.Types.Header (RequestHeaders, hCookie)
-import Network.HTTP.Types.URI    (parseSimpleQuery)
 import Network.Wai               (Request (..))
 import Network.Wai.Parse         (parseHttpAccept)
 import Web.Cookie                (parseCookies)

+import PostgREST.ApiRequest.Payload      (getPayload)
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
+import PostgREST.ApiRequest.Types        (Action (..), DbAction (..),
+                                          InvokeMethod (..),
+                                          Mutation (..), Payload (..),
+                                          RequestBody, Resource (..))
 import PostgREST.Config                  (AppConfig (..),
                                          OpenAPIMode (..))
 import PostgREST.Config.Database         (TimezoneNames)
@@ -64,44 +50,6 @@ import qualified PostgREST.MediaType              as MediaType

 import Protolude

-
-type RequestBody = LBS.ByteString
-
-data Payload
-  = ProcessedJSON -- ^ Cached attributes of a JSON payload
-      { payRaw  :: LBS.ByteString
-      -- ^ This is the raw ByteString that comes from the request body.  We
-      -- cache this instead of an Aeson Value because it was detected that for
-      -- large payloads the encoding had high memory usage, see
-      -- https://github.com/PostgREST/postgrest/pull/1005 for more details
-      , payKeys :: S.Set Text
-      -- ^ Keys of the object or if it's an array these keys are guaranteed to
-      -- be the same across all its objects
-      }
-  | ProcessedUrlEncoded { payArray  :: [(Text, Text)], payKeys :: S.Set Text }
-  | RawJSON { payRaw  :: LBS.ByteString }
-  | RawPay  { payRaw  :: LBS.ByteString }
-
-data InvokeMethod = Inv | InvRead Bool  deriving Eq
-data Mutation = MutationCreate | MutationDelete | MutationSingleUpsert | MutationUpdate deriving Eq
-
-data Resource
-  = ResourceRelation Text
-  | ResourceRoutine Text
-  | ResourceSchema
-
-data DbAction
-  = ActRelationRead {dbActQi :: QualifiedIdentifier, actHeadersOnly :: Bool}
-  | ActRelationMut  {dbActQi :: QualifiedIdentifier, actMutation :: Mutation}
-  | ActRoutine      {dbActQi :: QualifiedIdentifier, actInvMethod :: InvokeMethod}
-  | ActSchemaRead   Schema Bool
-
-data Action
-  = ActDb           DbAction
-  | ActRelationInfo QualifiedIdentifier
-  | ActRoutineInfo  QualifiedIdentifier InvokeMethod
-  | ActSchemaInfo
-
 {-|
  Describes what the user wants to do. This data type is a
  translation of the raw elements of an HTTP request into domain
@@ -207,7 +155,7 @@ getAction resource schema method =
 getSchema :: AppConfig -> RequestHeaders -> ByteString -> Either ApiRequestError (Schema, Bool)
 getSchema AppConfig{configDbSchemas} hdrs method = do
  case profile of
-    Just p | p `notElem` configDbSchemas -> Left $ UnacceptableSchema $ toList configDbSchemas
+    Just p | p `notElem` configDbSchemas -> Left $ UnacceptableSchema p $ toList configDbSchemas
           | otherwise                   -> Right (p, True)
    Nothing -> Right (defaultSchema, length configDbSchemas /= 1) -- if we have many schemas, assume the default schema was negotiated
  where
@@ -240,103 +188,3 @@ getRanges method QueryParams{qsRanges} hdrs
    -- The only emptyRange allowed is the limit zero range
    isInvalidRange = topLevelRange == emptyRange && not (hasLimitZero limitRange)
    topLevelRange = fromMaybe allRange $ HM.lookup "limit" ranges -- if no limit is specified, get all the request rows
-
-getPayload :: RequestBody -> MediaType -> QueryParams.QueryParams -> Action -> Either ApiRequestError (Maybe Payload, S.Set FieldName)
-getPayload reqBody contentMediaType QueryParams{qsColumns} action = do
-  checkedPayload <- if shouldParsePayload then payload else Right Nothing
-  let cols = case (checkedPayload, columns) of
-        (Just ProcessedJSON{payKeys}, _)       -> payKeys
-        (Just ProcessedUrlEncoded{payKeys}, _) -> payKeys
-        (Just RawJSON{}, Just cls)             -> cls
-        _                                      -> S.empty
-  return (checkedPayload, cols)
-  where
-    payload :: Either ApiRequestError (Maybe Payload)
-    payload = mapBoth InvalidBody Just $ case (contentMediaType, isProc) of
-      (MTApplicationJSON, _) ->
-        if isJust columns
-          then Right $ RawJSON reqBody
-          else note "All object keys must match" . payloadAttributes reqBody
-                 =<< if LBS.null reqBody && isProc
-                       then Right emptyObject
-                       else first BS.pack $
-                          -- Drop parsing error message in favor of generic one (https://github.com/PostgREST/postgrest/issues/2344)
-                          maybe (Left "Empty or invalid json") Right $ JSON.decode reqBody
-      (MTTextCSV, _) -> do
-        json <- csvToJson <$> first BS.pack (CSV.decodeByName reqBody)
-        note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json
-      (MTUrlEncoded, True) ->
-        Right $ ProcessedUrlEncoded params (S.fromList $ fst <$> params)
-      (MTUrlEncoded, False) ->
-        let paramsMap = HM.fromList $ (identity *** JSON.String) <$> params in
-        Right $ ProcessedJSON (JSON.encode paramsMap) $ S.fromList (HM.keys paramsMap)
-      (MTTextPlain, True) -> Right $ RawPay reqBody
-      (MTTextXML, True) -> Right $ RawPay reqBody
-      (MTOctetStream, True) -> Right $ RawPay reqBody
-      (ct, _) -> Left $ "Content-Type not acceptable: " <> MediaType.toMime ct
-
-    shouldParsePayload = case action of
-      ActDb (ActRelationMut _ MutationDelete) -> False
-      ActDb (ActRelationMut _ _)              -> True
-      ActDb (ActRoutine _  Inv)               -> True
-      _                                       -> False
-
-    columns = case action of
-      ActDb (ActRelationMut _ MutationCreate) -> qsColumns
-      ActDb (ActRelationMut _ MutationUpdate) -> qsColumns
-      ActDb (ActRoutine     _ Inv)            -> qsColumns
-      _                                       -> Nothing
-
-    isProc = case action of
-      ActDb (ActRoutine _ _) -> True
-      _                      -> False
-    params = (T.decodeUtf8 *** T.decodeUtf8) <$> parseSimpleQuery (LBS.toStrict reqBody)
-
-type CsvData = V.Vector (M.Map Text LBS.ByteString)
-
-{-|
-  Converts CSV like
-  a,b
-  1,hi
-  2,bye
-
-  into a JSON array like
-  [ {"a": "1", "b": "hi"}, {"a": 2, "b": "bye"} ]
-
-  The reason for its odd signature is so that it can compose
-  directly with CSV.decodeByName
-}
-csvToJson :: (CSV.Header, CsvData) -> JSON.Value
-csvToJson (_, vals) =
-  JSON.Array $ V.map rowToJsonObj vals
- where
-  rowToJsonObj = JSON.Object . KM.fromMapText .
-    M.map (\str ->
-        if str == "NULL"
-          then JSON.Null
-          else JSON.String . T.decodeUtf8 $ LBS.toStrict str
-      )
-
-payloadAttributes :: RequestBody -> JSON.Value -> Maybe Payload
-payloadAttributes raw json =
-  -- Test that Array contains only Objects having the same keys
-  case json of
-    JSON.Array arr ->
-      case arr V.!? 0 of
-        Just (JSON.Object o) ->
-          let canonicalKeys = S.fromList $ K.toText <$> KM.keys o
-              areKeysUniform = all (\case
-                JSON.Object x -> S.fromList (K.toText <$> KM.keys x) == canonicalKeys
-                _ -> False) arr in
-          if areKeysUniform
-            then Just $ ProcessedJSON raw canonicalKeys
-            else Nothing
-        Just _ -> Nothing
-        Nothing -> Just emptyPJArray
-
-    JSON.Object o -> Just $ ProcessedJSON raw (S.fromList $ K.toText <$> KM.keys o)
-
-    -- truncate everything else to an empty array.
-    _ -> Just emptyPJArray
-  where
-    emptyPJArray = ProcessedJSON (JSON.encode emptyArray) S.empty
@@ -0,0 +1,138 @@
+-- |
+-- Module      : PostgREST.ApiRequest.Payload
+-- Description : Parser for PostgREST Request Body
+--
+-- This module is in charge of parsing the request body (payload)
+--
+{-# LANGUAGE LambdaCase     #-}
+{-# LANGUAGE NamedFieldPuns #-}
+module PostgREST.ApiRequest.Payload
+  ( getPayload
+  ) where
+
+import qualified Data.Aeson            as JSON
+import qualified Data.Aeson.Key        as K
+import qualified Data.Aeson.KeyMap     as KM
+import qualified Data.ByteString.Char8 as BS
+import qualified Data.ByteString.Lazy  as LBS
+import qualified Data.Csv              as CSV
+import qualified Data.HashMap.Strict   as HM
+import qualified Data.Map.Strict       as M
+import qualified Data.Set              as S
+import qualified Data.Text.Encoding    as T
+import qualified Data.Vector           as V
+
+import Control.Arrow           ((***))
+import Data.Aeson.Types        (emptyArray, emptyObject)
+import Data.Either.Combinators (mapBoth)
+import Network.HTTP.Types.URI  (parseSimpleQuery)
+
+import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
+import PostgREST.ApiRequest.Types
+import PostgREST.Error                   (ApiRequestError (..))
+import PostgREST.MediaType               (MediaType (..))
+import PostgREST.SchemaCache.Identifiers (FieldName)
+
+import qualified PostgREST.MediaType as MediaType
+
+import Protolude
+
+getPayload :: RequestBody -> MediaType -> QueryParams -> Action -> Either ApiRequestError (Maybe Payload, S.Set FieldName)
+getPayload reqBody contentMediaType QueryParams{qsColumns} action = do
+  checkedPayload <- if shouldParsePayload then payload else Right Nothing
+  let cols = case (checkedPayload, columns) of
+        (Just ProcessedJSON{payKeys}, _)       -> payKeys
+        (Just ProcessedUrlEncoded{payKeys}, _) -> payKeys
+        (Just RawJSON{}, Just cls)             -> cls
+        _                                      -> S.empty
+  return (checkedPayload, cols)
+  where
+    payload :: Either ApiRequestError (Maybe Payload)
+    payload = mapBoth InvalidBody Just $ case (contentMediaType, isProc) of
+      (MTApplicationJSON, _) ->
+        if isJust columns
+          then Right $ RawJSON reqBody
+          else note "All object keys must match" . payloadAttributes reqBody
+                 =<< if LBS.null reqBody && isProc
+                       then Right emptyObject
+                       else first BS.pack $
+                          -- Drop parsing error message in favor of generic one (https://github.com/PostgREST/postgrest/issues/2344)
+                          maybe (Left "Empty or invalid json") Right $ JSON.decode reqBody
+      (MTTextCSV, _) -> do
+        json <- csvToJson <$> first BS.pack (CSV.decodeByName reqBody)
+        note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json
+      (MTUrlEncoded, True) ->
+        Right $ ProcessedUrlEncoded params (S.fromList $ fst <$> params)
+      (MTUrlEncoded, False) ->
+        let paramsMap = HM.fromList $ (identity *** JSON.String) <$> params in
+        Right $ ProcessedJSON (JSON.encode paramsMap) $ S.fromList (HM.keys paramsMap)
+      (MTTextPlain, True) -> Right $ RawPay reqBody
+      (MTTextXML, True) -> Right $ RawPay reqBody
+      (MTOctetStream, True) -> Right $ RawPay reqBody
+      (ct, _) -> Left $ "Content-Type not acceptable: " <> MediaType.toMime ct
+
+    shouldParsePayload = case action of
+      ActDb (ActRelationMut _ MutationDelete) -> False
+      ActDb (ActRelationMut _ _)              -> True
+      ActDb (ActRoutine _  Inv)               -> True
+      _                                       -> False
+
+    columns = case action of
+      ActDb (ActRelationMut _ MutationCreate) -> qsColumns
+      ActDb (ActRelationMut _ MutationUpdate) -> qsColumns
+      ActDb (ActRoutine     _ Inv)            -> qsColumns
+      _                                       -> Nothing
+
+    isProc = case action of
+      ActDb (ActRoutine _ _) -> True
+      _                      -> False
+    params = (T.decodeUtf8 *** T.decodeUtf8) <$> parseSimpleQuery (LBS.toStrict reqBody)
+
+type CsvData = V.Vector (M.Map Text LBS.ByteString)
+
+{-|
+  Converts CSV like
+  a,b
+  1,hi
+  2,bye
+
+  into a JSON array like
+  [ {"a": "1", "b": "hi"}, {"a": 2, "b": "bye"} ]
+
+  The reason for its odd signature is so that it can compose
+  directly with CSV.decodeByName
+-}
+csvToJson :: (CSV.Header, CsvData) -> JSON.Value
+csvToJson (_, vals) =
+  JSON.Array $ V.map rowToJsonObj vals
+ where
+  rowToJsonObj = JSON.Object . KM.fromMapText .
+    M.map (\str ->
+        if str == "NULL"
+          then JSON.Null
+          else JSON.String . T.decodeUtf8 $ LBS.toStrict str
+      )
+
+payloadAttributes :: RequestBody -> JSON.Value -> Maybe Payload
+payloadAttributes raw json =
+  -- Test that Array contains only Objects having the same keys
+  case json of
+    JSON.Array arr ->
+      case arr V.!? 0 of
+        Just (JSON.Object o) ->
+          let canonicalKeys = S.fromList $ K.toText <$> KM.keys o
+              areKeysUniform = all (\case
+                JSON.Object x -> S.fromList (K.toText <$> KM.keys x) == canonicalKeys
+                _ -> False) arr in
+          if areKeysUniform
+            then Just $ ProcessedJSON raw canonicalKeys
+            else Nothing
+        Just _ -> Nothing
+        Nothing -> Just emptyPJArray
+
+    JSON.Object o -> Just $ ProcessedJSON raw (S.fromList $ K.toText <$> KM.keys o)
+
+    -- truncate everything else to an empty array.
+    _ -> Just emptyPJArray
+  where
+    emptyPJArray = ProcessedJSON (JSON.encode emptyArray) S.empty
@@ -19,6 +19,7 @@ module PostgREST.ApiRequest.Preferences
  , PreferMaxAffected(..)
  , fromHeaders
  , shouldCount
+  , shouldExplainCount
  , prefAppliedHeader
  ) where

@@ -155,7 +156,7 @@ fromHeaders allowTxDbOverride acceptedTzNames headers =
    listStripPrefix prefix prefList = listToMaybe $ mapMaybe (BS.stripPrefix prefix) prefList

    timezonePref = listStripPrefix "timezone=" prefs
-    isTimezonePrefAccepted = (S.member <$> (decodeUtf8 <$> timezonePref) <*> pure acceptedTzNames) == Just True
+    isTimezonePrefAccepted = ((S.member . decodeUtf8 <$> timezonePref) <*> pure acceptedTzNames) == Just True

    maxAffectedPref = listStripPrefix "max-affected=" prefs >>= readMaybe . BS.unpack

@@ -238,6 +239,10 @@ shouldCount :: Maybe PreferCount -> Bool
 shouldCount prefCount =
  prefCount == Just ExactCount || prefCount == Just EstimatedCount

+shouldExplainCount :: Maybe PreferCount -> Bool
+shouldExplainCount prefCount =
+  prefCount == Just PlannedCount || prefCount == Just EstimatedCount
+
 -- | Whether to commit or roll back transactions.
 data PreferTransaction
  = Commit   -- ^ Commit transaction - the default.
@@ -30,12 +30,69 @@ module PostgREST.ApiRequest.Types
  , QuantOperator(..)
  , FtsOperator(..)
  , SelectItem(..)
+  , Payload (..)
+  , InvokeMethod (..)
+  , Mutation (..)
+  , Resource (..)
+  , DbAction (..)
+  , Action (..)
+  , RequestBody
  ) where

-import PostgREST.SchemaCache.Identifiers (FieldName)
+import qualified Data.ByteString.Lazy as LBS
+import qualified Data.Set             as S
+
+import PostgREST.SchemaCache.Identifiers (FieldName,
+                                          QualifiedIdentifier (..),
+                                          Schema)

 import Protolude

+data InvokeMethod = Inv | InvRead Bool
+  deriving Eq
+
+data Mutation
+  = MutationCreate
+  | MutationDelete
+  | MutationSingleUpsert
+  | MutationUpdate
+  deriving Eq
+
+data Resource
+  = ResourceRelation Text
+  | ResourceRoutine Text
+  | ResourceSchema
+
+data DbAction
+  = ActRelationRead {dbActQi :: QualifiedIdentifier, actHeadersOnly :: Bool}
+  | ActRelationMut  {dbActQi :: QualifiedIdentifier, actMutation :: Mutation}
+  | ActRoutine      {dbActQi :: QualifiedIdentifier, actInvMethod :: InvokeMethod}
+  | ActSchemaRead   Schema Bool
+
+data Action
+  = ActDb           DbAction
+  | ActRelationInfo QualifiedIdentifier
+  | ActRoutineInfo  QualifiedIdentifier InvokeMethod
+  | ActSchemaInfo
+
+type RequestBody = LBS.ByteString
+
+data Payload
+  = ProcessedJSON -- ^ Cached attributes of a JSON payload
+      { payRaw  :: LBS.ByteString
+      -- ^ This is the raw ByteString that comes from the request body.  We
+      -- cache this instead of an Aeson Value because it was detected that for
+      -- large payloads the encoding had high memory usage, see
+      -- https://github.com/PostgREST/postgrest/pull/1005 for more details
+      , payKeys :: S.Set Text
+      -- ^ Keys of the object or if it's an array these keys are guaranteed to
+      -- be the same across all its objects
+      }
+  | ProcessedUrlEncoded { payArray  :: [(Text, Text)], payKeys :: S.Set Text }
+  | RawJSON { payRaw  :: LBS.ByteString }
+  | RawPay  { payRaw  :: LBS.ByteString }
+
+
 -- | The value in `/tbl?select=alias:field.aggregateFunction()::cast`
 data SelectItem
  = SelectField
@@ -9,18 +9,24 @@ Some of its functionality includes:
 - Producing HTTP Headers according to RFCs.
 - Content Negotiation
 -}
-{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE RecordWildCards     #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE ViewPatterns        #-}
 module PostgREST.App
  ( postgrest
  , run
  ) where


+import GHC.IO.Exception (IOErrorType (..))
+import System.IO.Error  (ioeGetErrorType)
+
 import Control.Monad.Except     (liftEither)
 import Data.Either.Combinators  (mapLeft, whenLeft)
 import Data.Maybe               (fromJust)
 import Data.String              (IsString (..))
-import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
+import Network.Wai.Handler.Warp (defaultSettings, setHost,
+                                 setOnException, setPort,
                                 setServerName)

 import qualified Data.Text.Encoding       as T
@@ -35,6 +41,7 @@ import qualified PostgREST.Cors       as Cors
 import qualified PostgREST.Error      as Error
 import qualified PostgREST.Listener   as Listener
 import qualified PostgREST.Logger     as Logger
+import qualified PostgREST.MainTx     as MainTx
 import qualified PostgREST.Plan       as Plan
 import qualified PostgREST.Query      as Query
 import qualified PostgREST.Response   as Response
@@ -43,49 +50,65 @@ import qualified PostgREST.Unix       as Unix (installSignalHandlers)
 import PostgREST.ApiRequest           (ApiRequest (..))
 import PostgREST.AppState             (AppState)
 import PostgREST.Auth.Types           (AuthResult (..))
-import PostgREST.Config               (AppConfig (..), LogLevel (..),
-                                       LogQuery (..))
-import PostgREST.Config.PgVersion     (PgVersion (..))
+import PostgREST.Config               (AppConfig (..), LogLevel (..))
 import PostgREST.Error                (Error)
-import PostgREST.Network              (resolveHost)
+import PostgREST.Network              (resolveSocketToAddress)
 import PostgREST.Observation          (Observation (..))
 import PostgREST.Response.Performance (ServerTiming (..),
                                       serverTimingHeader)
 import PostgREST.SchemaCache          (SchemaCache (..))
+import PostgREST.TimeIt               (timeItT)
 import PostgREST.Version              (docsVersion, prettyVersion)

-import qualified Data.ByteString.Char8 as BS
-import qualified Data.List             as L
-import qualified Network.HTTP.Types    as HTTP
-import qualified Network.Socket        as NS
-import           Protolude             hiding (Handler)
-import           System.TimeIt         (timeItT)
+import qualified Data.ByteString.Char8  as BS
+import qualified Data.List              as L
+import           Data.Streaming.Network (bindPortTCP,
+                                         bindRandomPortTCP)
+import qualified Data.Text              as T
+import qualified Network.HTTP.Types     as HTTP
+import qualified Network.Socket         as NS
+import           PostgREST.Unix         (createAndBindDomainSocket)
+import           Protolude              hiding (Handler)

 type Handler = ExceptT Error

 run :: AppState -> IO ()
 run appState = do
-  let observer = AppState.getObserver appState
  conf@AppConfig{..} <- AppState.getConfig appState

  AppState.schemaCacheLoader appState -- Loads the initial SchemaCache
-  Unix.installSignalHandlers (AppState.getMainThreadId appState) (AppState.schemaCacheLoader appState) (AppState.readInDbConfig False appState)
+  (mainSocket, adminSocket) <- initSockets conf
+
+  Unix.installSignalHandlers observer (AppState.getMainThreadId appState) (AppState.schemaCacheLoader appState) (AppState.readInDbConfig False appState)

  Listener.runListener appState

-  Admin.runAdmin appState (serverSettings conf)
+  Admin.runAdmin appState adminSocket mainSocket (serverSettings conf)

  let app = postgrest configLogLevel appState (AppState.schemaCacheLoader appState)

-  case configServerUnixSocket of
-    Just path -> do
-      observer $ AppServerUnixObs path
-    Nothing   -> do
-      port <- NS.socketPort $ AppState.getSocketREST appState
-      host <- resolveHost $ AppState.getSocketREST appState
-      observer $ AppServerPortObs (fromJust host) port
+  do
+    address <- resolveSocketToAddress mainSocket
+    observer $ AppServerAddressObs address

-  Warp.runSettingsSocket (serverSettings conf) (AppState.getSocketREST appState) app
+  Warp.runSettingsSocket (serverSettings conf & setOnException onWarpException) mainSocket app
+  where
+    observer = AppState.getObserver appState
+
+    onWarpException :: Maybe Wai.Request -> SomeException -> IO ()
+    onWarpException _ ex =
+      when (shouldDisplayException ex) $
+        observer $ WarpErrorObs $ show ex
+
+    -- Similar to wai defaultShouldDisplayException in
+    -- https://github.com/yesodweb/wai//blob/8c3882c60f6abe043889fc20c7efd3fa9747fa4a/warp/Network/Wai/Handler/Warp/Settings.hs#L251-L258
+    -- but without omitting AsyncException since it's important to log for ThreadKilled, StackOverflow and other cases.
+    -- We want to reuse this to avoid flooding the logs for some transient failure cases.
+    shouldDisplayException :: SomeException -> Bool
+    shouldDisplayException se
+        | Just (_ :: Warp.InvalidRequest) <- fromException se = False
+        | Just (ioeGetErrorType -> et) <- fromException se, et == ResourceVanished || et == InvalidArgument = False
+        | otherwise = True

 serverSettings :: AppConfig -> Warp.Settings
 serverSettings AppConfig{..} =
@@ -108,18 +131,21 @@ postgrest logLevel appState connWorker =
      Right authResult -> do
        appConf <- AppState.getConfig appState -- the config must be read again because it can reload
        maybeSchemaCache <- AppState.getSchemaCache appState
-        pgVer <- AppState.getPgVersion appState

        let
          eitherResponse :: IO (Either Error Wai.Response)
          eitherResponse =
-            runExceptT $ postgrestResponse appState appConf maybeSchemaCache pgVer authResult req
+            runExceptT $ postgrestResponse appState appConf maybeSchemaCache authResult req

        response <- either Error.errorResponseFor identity <$> eitherResponse
-        -- Launch the connWorker when the connection is down.  The postgrest
+        -- Launch the connWorker when the connection is down. The postgrest
        -- function can respond successfully (with a stale schema cache) before
-        -- the connWorker is done.
-        when (isServiceUnavailable response) connWorker
+        -- the connWorker is done. However, when there's an empty schema cache
+        -- postgrest responds with the error `PGRST002`; this means that the schema
+        -- cache is still loading, so we don't launch the connWorker here because
+        -- it would duplicate the loading process, e.g. https://github.com/PostgREST/postgrest/issues/3704
+        -- TODO: this process may be unnecessary when the Listener is enabled. Revisit once https://github.com/PostgREST/postgrest/issues/1766 is done
+        when (isServiceUnavailable response && isJust maybeSchemaCache) connWorker
        resp <- do
          delay <- AppState.getNextDelay appState
          return $ addRetryHint delay response
@@ -129,16 +155,18 @@ postgrestResponse
  :: AppState.AppState
  -> AppConfig
  -> Maybe SchemaCache
-  -> PgVersion
  -> AuthResult
  -> Wai.Request
  -> Handler IO Wai.Response
-postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@AuthResult{..} req = do
+postgrestResponse appState conf@AppConfig{..} maybeSchemaCache authResult@AuthResult{..} req = do
+  let observer = AppState.getObserver appState
+
  sCache <-
    case maybeSchemaCache of
      Just sCache ->
        return sCache
-      Nothing ->
+      Nothing -> do
+        lift $ observer SchemaCacheEmptyObs
        throwError Error.NoSchemaCacheError

  body <- lift $ Wai.strictRequestBody req
@@ -150,24 +178,32 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@
  (parseTime, apiReq@ApiRequest{..}) <- withTiming $ liftEither . mapLeft Error.ApiRequestError $ ApiRequest.userApiRequest conf prefs req body
  (planTime, plan)                   <- withTiming $ liftEither $ Plan.actionPlan iAction conf apiReq sCache

-  let query = Query.query conf authResult apiReq plan sCache pgVer
-      logSQL = lift . AppState.getObserver appState . DBQuery (Query.getSQLQuery query)
+  let mainQ = Query.mainQuery plan conf apiReq authResult configDbPreRequest
+      tx = MainTx.mainTx mainQ conf authResult apiReq plan sCache
+      obsQuery s = when configLogQuery $ observer $ QueryObs mainQ s

-  (queryTime, queryResult) <- withTiming $ do
-    case query of
-      Query.NoDbQuery r -> pure r
-      Query.DbQuery{..} -> do
+  (txTime, txResult) <- withTiming $ do
+    case tx of
+      MainTx.NoDbTx r -> pure r
+      MainTx.DbTx{..} -> do
        dbRes <- lift $ AppState.usePool appState (dqTransaction dqIsoLevel dqTxMode $ runExceptT dqDbHandler)
-        let eitherResp = mapLeft Error.PgErr . mapLeft (Error.PgError (Just authRole /= configDbAnonRole)) $ dbRes
-        when (configLogQuery /= LogQueryDisabled) $ whenLeft eitherResp $ logSQL . Error.status
-        liftEither eitherResp >>= liftEither
+        let eitherResp = join $ mapLeft (Error.PgErr . Error.PgError (Just authRole /= configDbAnonRole)) dbRes
+
+        -- TODO: we use obsQuery twice, one here and one below because in case of an error with the usePool above, the request will finish here and return an error message.
+        -- This is because of a combination of ExceptT + our Error module which has Wai.responseLBS.
+        -- This needs refactoring so only the below obsQuery is used.
+        lift $ whenLeft eitherResp $ obsQuery . Error.status
+        liftEither eitherResp

  (respTime, resp) <- withTiming $ do
-    let response = Response.actionResponse queryResult apiReq (T.decodeUtf8 prettyVersion, docsVersion) conf sCache iSchema iNegotiatedByProfile
-    when (configLogQuery /= LogQueryDisabled) $ logSQL $ either Error.status Response.pgrstStatus response
+    let response = Response.actionResponse txResult apiReq (T.decodeUtf8 prettyVersion, docsVersion) conf sCache iSchema iNegotiatedByProfile
+        status' = either Error.status Response.pgrstStatus response
+
+    -- TODO: see above obsQuery, only this obsQuery should remain after refactoring (because the QueryObs depends on the status)
+    lift $ obsQuery status'
    liftEither response

-  return $ toWaiResponse (ServerTiming jwtTime parseTime planTime queryTime respTime) resp
+  return $ toWaiResponse (ServerTiming jwtTime parseTime planTime txTime respTime) resp

  where
    toWaiResponse :: ServerTiming -> Response.PgrstResponse -> Wai.Response
@@ -199,3 +235,40 @@ addRetryHint delay response = do

 isServiceUnavailable :: Wai.Response -> Bool
 isServiceUnavailable response = Wai.responseStatus response == HTTP.status503
+
+type AppSockets = (NS.Socket, Maybe NS.Socket)
+
+initSockets :: AppConfig -> IO AppSockets
+initSockets AppConfig{..} = do
+  let
+    cfg'usp = configServerUnixSocket
+    cfg'uspm = configServerUnixSocketMode
+    cfg'host = configServerHost
+    cfg'port = configServerPort
+    cfg'adminHost = configAdminServerHost
+    cfg'adminPort = configAdminServerPort
+
+  sock <- case cfg'usp of
+    -- I'm not using `streaming-commons`' bindPath function here because it's not defined for Windows,
+    -- but we need to have runtime error if we try to use it in Windows, not compile time error
+    Just path -> createAndBindDomainSocket path cfg'uspm
+    Nothing -> do
+      (_, sock) <-
+        if cfg'port /= 0
+          then do
+            sock <- bindPortTCP cfg'port (fromString $ T.unpack cfg'host)
+            pure (cfg'port, sock)
+          else do
+            -- explicitly bind to a random port, returning bound port number
+            (num, sock) <- bindRandomPortTCP (fromString $ T.unpack cfg'host)
+            pure (num, sock)
+      pure sock
+
+  adminSock <- case cfg'adminPort of
+    Just adminPort -> do
+      adminSock <- bindPortTCP adminPort (fromString $ T.unpack cfg'adminHost)
+      pure $ Just adminSock
+    Nothing -> pure Nothing
+
+  pure (sock, adminSock)
+
@@ -13,11 +13,9 @@ module PostgREST.AppState
  , getNextListenerDelay
  , getTime
  , getJwtCacheState
-  , getSocketREST
-  , getSocketAdmin
  , init
-  , initSockets
  , initWithPool
+  , putConfig -- For tests TODO refactoring
  , putNextListenerDelay
  , putSchemaCache
  , putPgVersion
@@ -32,20 +30,18 @@ module PostgREST.AppState

 import qualified Data.ByteString.Char8      as BS
 import           Data.Either.Combinators    (whenLeft)
-import qualified Data.Text                  as T (unpack)
 import qualified Hasql.Pool                 as SQL
 import qualified Hasql.Pool.Config          as SQL
 import qualified Hasql.Session              as SQL
 import qualified Hasql.Transaction.Sessions as SQL
 import qualified Network.HTTP.Types.Status  as HTTP
-import qualified Network.Socket             as NS
 import qualified PostgREST.Auth.JwtCache    as JwtCache
 import qualified PostgREST.Error            as Error
 import qualified PostgREST.Logger           as Logger
 import qualified PostgREST.Metrics          as Metrics
 import           PostgREST.Observation
+import           PostgREST.TimeIt           (timeItT)
 import           PostgREST.Version          (prettyVersion)
-import           System.TimeIt              (timeItT)

 import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
                           updateAction)
@@ -57,7 +53,7 @@ import Data.IORef         (IORef, atomicWriteIORef, newIORef,
                           readIORef)
 import Data.Time.Clock    (UTCTime, getCurrentTime)

-import PostgREST.Auth.JwtCache           (JwtCacheState)
+import PostgREST.Auth.JwtCache           (JwtCacheState, update)
 import PostgREST.Config                  (AppConfig (..),
                                          addFallbackAppName,
                                          readAppConfig)
@@ -69,11 +65,8 @@ import PostgREST.Config.PgVersion        (PgVersion (..),
 import PostgREST.SchemaCache             (SchemaCache (..),
                                          querySchemaCache,
                                          showSummary)
-import PostgREST.SchemaCache.Identifiers (dumpQi)
-import PostgREST.Unix                    (createAndBindDomainSocket)
+import PostgREST.SchemaCache.Identifiers (quoteQi)

-import Data.Streaming.Network (bindPortTCP, bindRandomPortTCP)
-import Data.String            (IsString (..))
 import Protolude

 data AppState = AppState
@@ -99,10 +92,6 @@ data AppState = AppState
  , stateNextDelay         :: IORef Int
  -- | Keeps track of the next delay for the listener
  , stateNextListenerDelay :: IORef Int
-  -- | Network socket for REST API
-  , stateSocketREST        :: NS.Socket
-  -- | Network socket for the admin UI
-  , stateSocketAdmin       :: Maybe NS.Socket
  -- | Observation handler
  , stateObserver          :: ObservationHandler
  -- | JWT Cache
@@ -117,8 +106,6 @@ data SchemaCacheStatus
  | SCPending
  deriving Eq

-type AppSockets = (NS.Socket, Maybe NS.Socket)
-
 init :: AppConfig -> IO AppState
 init conf@AppConfig{configLogLevel, configDbPoolSize} = do
  loggerState  <- Logger.init
@@ -127,14 +114,11 @@ init conf@AppConfig{configLogLevel, configDbPoolSize} = do

  observer $ AppStartObs prettyVersion

-  jwtCacheState <- JwtCache.init
  pool <- initPool conf observer
-  (sock, adminSock) <- initSockets conf
-  state' <- initWithPool (sock, adminSock) pool conf jwtCacheState loggerState metricsState observer
-  pure state' { stateSocketREST = sock, stateSocketAdmin = adminSock}
+  initWithPool pool conf loggerState metricsState observer --{ stateSocketREST = sock, stateSocketAdmin = adminSock}

-initWithPool :: AppSockets -> SQL.Pool -> AppConfig -> JwtCache.JwtCacheState -> Logger.LoggerState -> Metrics.MetricsState -> ObservationHandler -> IO AppState
-initWithPool (sock, adminSock) pool conf jwtCacheState loggerState metricsState observer = do
+initWithPool :: SQL.Pool -> AppConfig -> Logger.LoggerState -> Metrics.MetricsState -> ObservationHandler -> IO AppState
+initWithPool pool conf loggerState metricsState observer = do

  appState <- AppState pool
    <$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step
@@ -147,10 +131,8 @@ initWithPool (sock, adminSock) pool conf jwtCacheState loggerState metricsState
    <*> myThreadId
    <*> newIORef 0
    <*> newIORef 1
-    <*> pure sock
-    <*> pure adminSock
    <*> pure observer
-    <*> pure jwtCacheState
+    <*> JwtCache.init conf observer
    <*> pure loggerState
    <*> pure metricsState

@@ -167,40 +149,6 @@ initWithPool (sock, adminSock) pool conf jwtCacheState loggerState metricsState
 destroy :: AppState -> IO ()
 destroy = destroyPool

-initSockets :: AppConfig -> IO AppSockets
-initSockets AppConfig{..} = do
-  let
-    cfg'usp = configServerUnixSocket
-    cfg'uspm = configServerUnixSocketMode
-    cfg'host = configServerHost
-    cfg'port = configServerPort
-    cfg'adminHost = configAdminServerHost
-    cfg'adminPort = configAdminServerPort
-
-  sock <- case cfg'usp of
-    -- I'm not using `streaming-commons`' bindPath function here because it's not defined for Windows,
-    -- but we need to have runtime error if we try to use it in Windows, not compile time error
-    Just path -> createAndBindDomainSocket path cfg'uspm
-    Nothing -> do
-      (_, sock) <-
-        if cfg'port /= 0
-          then do
-            sock <- bindPortTCP cfg'port (fromString $ T.unpack cfg'host)
-            pure (cfg'port, sock)
-          else do
-            -- explicitly bind to a random port, returning bound port number
-            (num, sock) <- bindRandomPortTCP (fromString $ T.unpack cfg'host)
-            pure (num, sock)
-      pure sock
-
-  adminSock <- case cfg'adminPort of
-    Just adminPort -> do
-      adminSock <- bindPortTCP adminPort (fromString $ T.unpack cfg'adminHost)
-      pure $ Just adminSock
-    Nothing -> pure Nothing
-
-  pure (sock, adminSock)
-
 initPool :: AppConfig -> ObservationHandler -> IO SQL.Pool
 initPool AppConfig{..} observer = do
  SQL.acquire $ SQL.settings
@@ -314,12 +262,6 @@ getTime = stateGetTime
 getJwtCacheState :: AppState -> JwtCacheState
 getJwtCacheState = stateJwtCache

-getSocketREST :: AppState -> NS.Socket
-getSocketREST = stateSocketREST
-
-getSocketAdmin :: AppState -> Maybe NS.Socket
-getSocketAdmin = stateSocketAdmin
-
 getMainThreadId :: AppState -> ThreadId
 getMainThreadId = stateMainThreadId

@@ -383,14 +325,16 @@ retryingSchemaCacheLoad appState@AppState{stateObserver=observer, stateMainThrea
            observer ExitDBNoRecoveryObs
            killThread mainThreadId
          return Nothing
-        Right actualPgVersion -> do
-          when (actualPgVersion < minimumPgVersion) $ do
+        Right actualPgVersion ->
+          if actualPgVersion < minimumPgVersion then do
            observer $ ExitUnsupportedPgVersion actualPgVersion minimumPgVersion
            killThread mainThreadId
-          observer $ DBConnectedObs $ pgvFullName actualPgVersion
-          observer $ PoolInit configDbPoolSize
-          putPgVersion appState actualPgVersion
-          return $ Just actualPgVersion
+            return Nothing
+          else do
+            observer $ DBConnectedObs $ pgvFullName actualPgVersion
+            observer $ PoolInit configDbPoolSize
+            putPgVersion appState actualPgVersion
+            return $ Just actualPgVersion

    qInDbConfig :: IO ()
    qInDbConfig = do
@@ -442,7 +386,7 @@ readInDbConfig startingUp appState@AppState{stateObserver=observer} = do
  pgVer <- getPgVersion appState
  dbSettings <-
    if configDbConfig conf then do
-      qDbSettings <- usePool appState (queryDbSettings (dumpQi <$> configDbPreConfig conf) (configDbPreparedStatements conf))
+      qDbSettings <- usePool appState (queryDbSettings (quoteQi <$> configDbPreConfig conf) (configDbPreparedStatements conf))
      case qDbSettings of
        Left e -> do
          observer $ ConfigReadErrorObs e
@@ -471,10 +415,7 @@ readInDbConfig startingUp appState@AppState{stateObserver=observer} = do
      -- After the config has reloaded, jwt-secret might have changed, so
      -- if it has changed, it is important to invalidate the jwt cache
      -- entries, because they were cached using the old secret
-      if configJwtSecret conf == configJwtSecret newConf then
-        pass
-      else
-        JwtCache.emptyCache (getJwtCacheState appState) -- atomic O(1) operation
+      update (getJwtCacheState appState) newConf

      if startingUp then
        pass
@@ -1,3 +1,4 @@
+{-# LANGUAGE RecordWildCards #-}
 {-|
 Module      : PostgREST.Auth
 Description : PostgREST authentication functions.
@@ -10,8 +11,6 @@ Authentication should always be implemented in an external service.
 In the test suite there is an example of simple login function that can be used for a
 very simple authentication system inside the PostgreSQL database.
 -}
-{-# LANGUAGE LambdaCase      #-}
-{-# LANGUAGE RecordWildCards #-}
 module PostgREST.Auth
  ( getResult
  , getJwtDur
@@ -19,177 +18,44 @@ module PostgREST.Auth
  , middleware
  ) where

-import qualified Data.Aeson                      as JSON
-import qualified Data.Aeson.Key                  as K
-import qualified Data.Aeson.KeyMap               as KM
-import qualified Data.Aeson.Types                as JSON
 import qualified Data.ByteString                 as BS
-import qualified Data.ByteString.Internal        as BS
-import qualified Data.ByteString.Lazy.Char8      as LBS
-import qualified Data.Scientific                 as Sci
-import qualified Data.Text                       as T
 import qualified Data.Vault.Lazy                 as Vault
-import qualified Data.Vector                     as V
-import qualified Jose.Jwk                        as JWT
-import qualified Jose.Jwt                        as JWT
 import qualified Network.HTTP.Types.Header       as HTTP
 import qualified Network.Wai                     as Wai
 import qualified Network.Wai.Middleware.HttpAuth as Wai

-import Control.Monad.Except    (liftEither)
-import Data.Either.Combinators (mapLeft)
-import Data.List               (lookup)
-import Data.Time.Clock         (UTCTime, nominalDiffTimeToSeconds)
-import Data.Time.Clock.POSIX   (utcTimeToPOSIXSeconds)
-import System.IO.Unsafe        (unsafePerformIO)
-import System.TimeIt           (timeItT)
+import Data.List        (lookup)
+import PostgREST.TimeIt (timeItT)
+import System.IO.Unsafe (unsafePerformIO)

 import PostgREST.AppState      (AppState, getConfig, getJwtCacheState,
                                getTime)
+import PostgREST.Auth.Jwt      (parseClaims)
 import PostgREST.Auth.JwtCache (lookupJwtCache)
 import PostgREST.Auth.Types    (AuthResult (..))
-import PostgREST.Config        (AppConfig (..), FilterExp (..),
-                                JSPath, JSPathExp (..))
-import PostgREST.Error         (Error (..), JwtError (..))
+import PostgREST.Config        (AppConfig (..))
+import PostgREST.Error         (Error (..))

 import Protolude

-- | Receives the JWT secret and audience (from config) and a JWT and returns a
-- JSON object of JWT claims.
-parseToken :: AppConfig -> Maybe ByteString -> UTCTime -> ExceptT Error IO JSON.Value
-parseToken _ Nothing _ = return JSON.emptyObject
-parseToken _ (Just "") _ = throwE . JwtErr $ JwtDecodeError "Empty JWT is sent in Authorization header"
-parseToken AppConfig{..} (Just tkn) time = do
-  secret <- liftEither . maybeToRight (JwtErr JwtSecretMissing) $ configJWKS
-  tknWith3Parts <- liftEither $ hasThreeParts tkn
-  eitherContent <- liftIO $ JWT.decode (JWT.keys secret) Nothing tknWith3Parts
-  content <- liftEither . mapLeft (JwtErr . jwtDecodeError) $ eitherContent
-  liftEither $ mapLeft JwtErr $ verifyClaims content
-  where
-      hasThreeParts :: ByteString -> Either Error ByteString
-      hasThreeParts token = case length $ BS.split (BS.c2w '.') token of
-        3 -> Right token
-        n -> Left $ JwtErr $ JwtDecodeError ("Expected 3 parts in JWT; got " <> show n)
-      jwtDecodeError :: JWT.JwtError -> JwtError
-      -- The only errors we can get from JWT.decode function are:
-      --   BadAlgorithm
-      --   KeyError
-      --   BadCrypto
-      jwtDecodeError (JWT.KeyError _)     = JwtDecodeError "No suitable key or wrong key type"
-      jwtDecodeError (JWT.BadAlgorithm _) = JwtDecodeError "Wrong or unsupported encoding algorithm"
-      jwtDecodeError JWT.BadCrypto        = JwtDecodeError "JWT cryptographic operation failed"
-      -- Control never reaches here, the decode function only returns the above three
-      jwtDecodeError _                    = JwtDecodeError "JWT couldn't be decoded"
-
-      verifyClaims :: JWT.JwtContent -> Either JwtError JSON.Value
-      verifyClaims (JWT.Jws (_, claims)) = case JSON.decodeStrict claims of
-        Just jclaims@(JSON.Object mclaims) ->
-          verifyClaim mclaims "exp" isValidExpClaim "JWT expired" >>
-          verifyClaim mclaims "nbf" isValidNbfClaim "JWT not yet valid" >>
-          verifyClaim mclaims "iat" isValidIatClaim "JWT issued at future" >>
-          verifyClaim mclaims "aud" isValidAudClaim "JWT not in audience" >>
-          return jclaims
-        _ -> Left $ JwtClaimsError "Parsing claims failed"
-      -- TODO: We could enable JWE support here (encrypted tokens)
-      verifyClaims _ = Left $ JwtDecodeError "Unsupported token type"
-
-      verifyClaim mclaims claim func err = do
-        isValid <- maybe (Right True) func (KM.lookup claim mclaims)
-        unless isValid $ Left $ JwtClaimsError err
-
-      allowedSkewSeconds = 30 :: Int64
-      now = floor . nominalDiffTimeToSeconds $ utcTimeToPOSIXSeconds time
-      sciToInt = fromMaybe 0 . Sci.toBoundedInteger
-      allStrings = all (\case (JSON.String _) -> True; _ -> False)
-
-      isValidExpClaim :: JSON.Value -> Either JwtError Bool
-      isValidExpClaim (JSON.Number secs) = Right $ now <= (sciToInt secs + allowedSkewSeconds)
-      isValidExpClaim _ = Left $ JwtClaimsError "The JWT 'exp' claim must be a number"
-
-      isValidNbfClaim :: JSON.Value -> Either JwtError Bool
-      isValidNbfClaim (JSON.Number secs) = Right $ now >= (sciToInt secs - allowedSkewSeconds)
-      isValidNbfClaim _ = Left $ JwtClaimsError "The JWT 'nbf' claim must be a number"
-
-      isValidIatClaim :: JSON.Value -> Either JwtError Bool
-      isValidIatClaim (JSON.Number secs) = Right $ now >= (sciToInt secs - allowedSkewSeconds)
-      isValidIatClaim _ = Left $ JwtClaimsError "The JWT 'iat' claim must be a number"
-
-      isValidAudClaim :: JSON.Value -> Either JwtError Bool
-      isValidAudClaim JSON.Null = Right True -- {"aud": null} is valid for all audiences
-      isValidAudClaim (JSON.String str) = Right $ maybe (const True) (==) configJwtAudience str
-      isValidAudClaim (JSON.Array arr)
-        | null arr = Right True              -- {"aud": []} is valid for all audiences
-        | allStrings arr = Right $ maybe True (\a -> JSON.String a `elem` arr) configJwtAudience
-      isValidAudClaim _ = Left $ JwtClaimsError "The JWT 'aud' claim must be a string or an array of strings"
-
-parseClaims :: Monad m =>
-  AppConfig -> JSON.Value -> ExceptT Error m AuthResult
-parseClaims AppConfig{..} jclaims@(JSON.Object mclaims) = do
-  -- role defaults to anon if not specified in jwt
-  role <- liftEither . maybeToRight (JwtErr JwtTokenRequired) $
-    unquoted <$> walkJSPath (Just jclaims) configJwtRoleClaimKey <|> configDbAnonRole
-  return AuthResult
-           { authClaims = mclaims & KM.insert "role" (JSON.toJSON $ decodeUtf8 role)
-           , authRole = role
-           }
-  where
-    walkJSPath :: Maybe JSON.Value -> JSPath -> Maybe JSON.Value
-    walkJSPath x                      []                = x
-    walkJSPath (Just (JSON.Object o)) (JSPKey key:rest) = walkJSPath (KM.lookup (K.fromText key) o) rest
-    walkJSPath (Just (JSON.Array ar)) (JSPIdx idx:rest) = walkJSPath (ar V.!? idx) rest
-    walkJSPath (Just (JSON.Array ar)) [JSPFilter (EqualsCond txt)] = findFirstMatch (==) txt ar
-    walkJSPath (Just (JSON.Array ar)) [JSPFilter (NotEqualsCond txt)] = findFirstMatch (/=) txt ar
-    walkJSPath (Just (JSON.Array ar)) [JSPFilter (StartsWithCond txt)] = findFirstMatch T.isPrefixOf txt ar
-    walkJSPath (Just (JSON.Array ar)) [JSPFilter (EndsWithCond txt)] = findFirstMatch T.isSuffixOf txt ar
-    walkJSPath (Just (JSON.Array ar)) [JSPFilter (ContainsCond txt)] = findFirstMatch T.isInfixOf txt ar
-    walkJSPath _                      _                 = Nothing
-
-    findFirstMatch matchWith pattern = foldr checkMatch Nothing
-      where
-        checkMatch (JSON.String txt) acc
-            | pattern `matchWith` txt = Just $ JSON.String txt
-            | otherwise = acc
-        checkMatch _ acc = acc
-
-    unquoted :: JSON.Value -> BS.ByteString
-    unquoted (JSON.String t) = encodeUtf8 t
-    unquoted v               = LBS.toStrict $ JSON.encode v
-- impossible case - just added to please -Wincomplete-patterns
-parseClaims _ _ = return AuthResult { authClaims = KM.empty, authRole = mempty }
-
-- | Validate authorization header.
+-- | Validate authorization header
 --   Parse and store JWT claims for future use in the request.
 middleware :: AppState -> Wai.Middleware
 middleware appState app req respond = do
-  conf <- getConfig appState
+  conf@AppConfig{..} <- getConfig appState
  time <- getTime appState

  let token  = Wai.extractBearerAuth =<< lookup HTTP.hAuthorization (Wai.requestHeaders req)
-      parseJwt = runExceptT $ parseToken conf token time >>= parseClaims conf
+      parseJwt = runExceptT $ lookupJwtCache jwtCacheState token >>= parseClaims conf time
      jwtCacheState = getJwtCacheState appState

-- If ServerTimingEnabled -> calculate JWT validation time
-- If JwtCacheMaxLifetime -> cache JWT validation result
-  req' <- case (configServerTimingEnabled conf, configJwtCacheMaxLifetime conf) of
-    (True, 0)            -> do
-          (dur, authResult) <- timeItT parseJwt
-          return $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult & Vault.insert jwtDurKey dur }
-
-    (True, maxLifetime)  -> do
-          (dur, authResult) <- timeItT $ case token of
-            Just tkn -> lookupJwtCache jwtCacheState tkn maxLifetime parseJwt time
-            Nothing  -> parseJwt
-          return $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult & Vault.insert jwtDurKey dur }
-
-    (False, 0)           -> do
-          authResult <- parseJwt
-          return $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult }
-
-    (False, maxLifetime) -> do
-          authResult <- case token of
-            Just tkn -> lookupJwtCache jwtCacheState tkn maxLifetime parseJwt time
-            Nothing -> parseJwt
-          return $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult }
+  -- If ServerTimingEnabled -> calculate JWT validation time
+  req' <- if configServerTimingEnabled then do
+      (dur, authResult) <- timeItT parseJwt
+      pure $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult & Vault.insert jwtDurKey dur }
+    else do
+      authResult <- parseJwt
+      pure $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult }

  app req' respond

@@ -0,0 +1,151 @@
+{-|
+Module      : PostgREST.Auth.Jwt
+Description : PostgREST JWT support functions.
+
+This module provides functions to deal with JWT parsing and validation (http://jwt.io).
+-}
+{-# LANGUAGE DeriveGeneric         #-}
+{-# LANGUAGE FlexibleContexts      #-}
+{-# LANGUAGE ImpredicativeTypes    #-}
+{-# LANGUAGE LambdaCase            #-}
+{-# LANGUAGE NamedFieldPuns        #-}
+{-# LANGUAGE QuantifiedConstraints #-}
+
+module PostgREST.Auth.Jwt
+  ( parseAndDecodeClaims
+  , parseClaims) where
+
+import qualified Data.Aeson                 as JSON
+import qualified Data.Aeson.Key             as K
+import qualified Data.Aeson.KeyMap          as KM
+import qualified Data.ByteString            as BS
+import qualified Data.ByteString.Internal   as BS
+import qualified Data.ByteString.Lazy.Char8 as LBS
+import qualified Data.Scientific            as Sci
+import qualified Data.Text                  as T
+import qualified Data.Vector                as V
+import qualified Jose.Jwk                   as JWT
+import qualified Jose.Jwt                   as JWT
+
+import Control.Monad.Except    (liftEither)
+import Data.Either.Combinators (mapLeft)
+import Data.Text               ()
+import Data.Time.Clock         (UTCTime, nominalDiffTimeToSeconds)
+import Data.Time.Clock.POSIX   (utcTimeToPOSIXSeconds)
+
+import PostgREST.Auth.Types (AuthResult (..))
+import PostgREST.Config     (AppConfig (..), FilterExp (..), JSPath,
+                             JSPathExp (..), audMatchesCfg)
+import PostgREST.Error      (Error (..),
+                             JwtClaimsError (AudClaimNotStringOrArray, ExpClaimNotNumber, IatClaimNotNumber, JWTExpired, JWTIssuedAtFuture, JWTNotInAudience, JWTNotYetValid, NbfClaimNotNumber, ParsingClaimsFailed),
+                             JwtDecodeError (..), JwtError (..))
+
+import Data.Aeson       ((.:?))
+import Data.Aeson.Types (parseMaybe)
+import Jose.Jwk         (JwkSet)
+import Protolude        hiding (first)
+
+parseAndDecodeClaims :: (MonadError Error m, MonadIO m) => JwkSet -> ByteString -> m JSON.Object
+parseAndDecodeClaims jwkSet token = parseToken jwkSet token >>= decodeClaims
+
+decodeClaims :: MonadError Error m => JWT.JwtContent -> m JSON.Object
+decodeClaims (JWT.Jws (_, claims)) = maybe (throwError (JwtErr $ JwtClaimsErr ParsingClaimsFailed)) pure (JSON.decodeStrict claims)
+decodeClaims _ = throwError $ JwtErr $ JwtDecodeErr UnsupportedTokenType
+
+validateClaims :: MonadError Error m => UTCTime -> (Text -> Bool) -> JSON.Object -> m ()
+validateClaims time audMatches claims = liftEither $ maybeToLeft () (fmap JwtErr . getAlt $ JwtClaimsErr <$> checkForErrors time audMatches claims)
+
+data ValidAud = VAString Text | VAArray [Text] deriving Generic
+instance JSON.FromJSON ValidAud where
+  parseJSON = JSON.genericParseJSON JSON.defaultOptions { JSON.sumEncoding = JSON.UntaggedValue }
+
+checkForErrors :: (Applicative m, Monoid (m JwtClaimsError)) => UTCTime -> (Text -> Bool) -> JSON.Object -> m JwtClaimsError
+checkForErrors time audMatches = mconcat
+  [
+    claim "exp" ExpClaimNotNumber $ inThePast JWTExpired
+  , claim "nbf" NbfClaimNotNumber $ inTheFuture JWTNotYetValid
+  , claim "iat" IatClaimNotNumber $ inTheFuture JWTIssuedAtFuture
+  , claim "aud" AudClaimNotStringOrArray $ checkValue (not . validAud) JWTNotInAudience
+  ]
+  where
+      allowedSkewSeconds = 30 :: Int64
+      sciToInt = fromMaybe 0 . Sci.toBoundedInteger
+      toSec = floor . nominalDiffTimeToSeconds . utcTimeToPOSIXSeconds
+      now = toSec time
+
+      inTheFuture = checkTime ((now + allowedSkewSeconds) <)
+      inThePast = checkTime ((now - allowedSkewSeconds) >)
+
+      checkTime cond = checkValue (cond. sciToInt)
+
+      validAud = \case
+        (VAString aud) -> audMatches aud
+        (VAArray auds) -> null auds || any audMatches auds
+
+      checkValue invalid msg val =
+        if invalid val then
+          pure msg
+        else
+          mempty
+
+      claim key parseError checkParsed = maybe (pure parseError) (maybe mempty checkParsed) . parseMaybe (.:? key)
+
+-- | Receives the JWT secret and audience (from config) and a JWT and returns a
+-- JSON object of JWT claims.
+parseToken :: (MonadError Error m, MonadIO m) => JwkSet -> ByteString -> m JWT.JwtContent
+parseToken _ "" = throwError $ JwtErr $ JwtDecodeErr EmptyAuthHeader
+parseToken secret tkn = do
+  -- secret <- liftEither . maybeToRight (JwtErr JwtSecretMissing) $ configJWKS
+  tknWith3Parts <- hasThreeParts tkn
+  eitherContent <- liftIO $ JWT.decode (JWT.keys secret) Nothing tknWith3Parts
+  liftEither . mapLeft (JwtErr . jwtDecodeError) $ eitherContent
+  --liftEither $ mapLeft JwtErr $ verifyClaims content
+  where
+      --hasThreeParts :: ByteString -> Either Error ByteString
+      hasThreeParts token = case length $ BS.split (BS.c2w '.') token of
+        3 -> pure token
+        n -> throwError $ JwtErr $ JwtDecodeErr $ UnexpectedParts n
+
+      jwtDecodeError :: JWT.JwtError -> JwtError
+      -- The only errors we can get from JWT.decode function are:
+      --   BadAlgorithm
+      --   KeyError
+      --   BadCrypto
+      jwtDecodeError (JWT.KeyError m)     = JwtDecodeErr $ KeyError m
+      jwtDecodeError (JWT.BadAlgorithm m) = JwtDecodeErr $ BadAlgorithm m
+      jwtDecodeError JWT.BadCrypto        = JwtDecodeErr BadCrypto
+      -- Control never reaches here, the decode function only returns the above three
+      jwtDecodeError _                    = JwtDecodeErr UnreachableDecodeError
+
+parseClaims :: (MonadError Error m, MonadIO m) => AppConfig -> UTCTime -> JSON.Object -> m AuthResult
+parseClaims cfg@AppConfig{configJwtRoleClaimKey, configDbAnonRole} time mclaims = do
+  validateClaims time (audMatchesCfg cfg) mclaims
+  -- role defaults to anon if not specified in jwt
+  role <- liftEither . maybeToRight (JwtErr JwtTokenRequired) $
+    unquoted <$> walkJSPath (Just $ JSON.Object mclaims) configJwtRoleClaimKey <|> configDbAnonRole
+  pure AuthResult
+           { authClaims = mclaims & KM.insert "role" (JSON.toJSON $ decodeUtf8 role)
+           , authRole = role
+           }
+  where
+    walkJSPath :: Maybe JSON.Value -> JSPath -> Maybe JSON.Value
+    walkJSPath x                      []                = x
+    walkJSPath (Just (JSON.Object o)) (JSPKey key:rest) = walkJSPath (KM.lookup (K.fromText key) o) rest
+    walkJSPath (Just (JSON.Array ar)) (JSPIdx idx:rest) = walkJSPath (ar V.!? idx) rest
+    walkJSPath (Just (JSON.Array ar)) [JSPFilter (EqualsCond txt)] = findFirstMatch (==) txt ar
+    walkJSPath (Just (JSON.Array ar)) [JSPFilter (NotEqualsCond txt)] = findFirstMatch (/=) txt ar
+    walkJSPath (Just (JSON.Array ar)) [JSPFilter (StartsWithCond txt)] = findFirstMatch T.isPrefixOf txt ar
+    walkJSPath (Just (JSON.Array ar)) [JSPFilter (EndsWithCond txt)] = findFirstMatch T.isSuffixOf txt ar
+    walkJSPath (Just (JSON.Array ar)) [JSPFilter (ContainsCond txt)] = findFirstMatch T.isInfixOf txt ar
+    walkJSPath _                      _                 = Nothing
+
+    findFirstMatch matchWith pattern = foldr checkMatch Nothing
+      where
+        checkMatch (JSON.String txt) acc
+            | pattern `matchWith` txt = Just $ JSON.String txt
+            | otherwise = acc
+        checkMatch _ acc = acc
+
+    unquoted :: JSON.Value -> BS.ByteString
+    unquoted (JSON.String t) = encodeUtf8 t
+    unquoted v               = LBS.toStrict $ JSON.encode v
@@ -1,99 +1,114 @@
 {-|
 Module      : PostgREST.Auth.JwtCache
-Description : PostgREST Jwt Authentication Result Cache.
+Description : PostgREST JWT validation results Cache.

-This module provides functions to deal with the JWT cache
+This module provides functions to deal with the JWT cache.
 -}
-{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE ExistentialQuantification #-}
+{-# LANGUAGE FlexibleInstances         #-}
+{-# LANGUAGE LambdaCase                #-}
+{-# LANGUAGE MultiParamTypeClasses     #-}
+{-# LANGUAGE NamedFieldPuns            #-}
+{-# LANGUAGE StrictData                #-}
+
 module PostgREST.Auth.JwtCache
  ( init
+  , update
  , JwtCacheState
  , lookupJwtCache
-  , emptyCache
  ) where

 import qualified Data.Aeson        as JSON
 import qualified Data.Aeson.KeyMap as KM
-import qualified Data.Cache        as C
-import qualified Data.Scientific   as Sci

-import Control.Debounce
+import PostgREST.Error (Error (..), JwtError (JwtSecretMissing))

-import Data.Time.Clock       (UTCTime, nominalDiffTimeToSeconds)
-import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds)
-import System.Clock          (TimeSpec (..))
+import           Control.Concurrent.STM      (newTVarIO, readTVar,
+                                              writeTVar)
+import           Control.Concurrent.STM.TVar (TVar)
+import           Control.Monad.Error.Class   (liftEither)
+import           Data.ByteString             hiding (all, init)
+import           Data.IORef                  (IORef, newIORef,
+                                              readIORef, writeIORef)
+import           Jose.Jwk                    (JwkSet)
+import           PostgREST.Auth.Jwt          (parseAndDecodeClaims)
+import           PostgREST.Cache.Sieve       (alwaysValid)
+import qualified PostgREST.Cache.Sieve       as SC
+import           PostgREST.Config            (AppConfig (..))
+import           PostgREST.Observation       (Observation (JwtCacheEviction, JwtCacheLookup),
+                                              ObservationHandler)
+import           Protolude

-import PostgREST.Auth.Types (AuthResult (..))
-import PostgREST.Error      (Error (..))
+data JwtCacheState = JwtCacheState ObservationHandler (IORef JwtCache)

-import Protolude
+class CacheVariant m v where
+  cached :: SC.Cache m ByteString v -> ByteString -> ExceptT Error IO JSON.Object

-- | JWT Cache and IO action that triggers purging old entries from the cache
-data JwtCacheState = JwtCacheState
-  { jwtCache   :: C.Cache ByteString AuthResult
-  , purgeCache :: IO ()
-  }
+{-|
+Jwt caching can have three different configurations:
+* missing JWT Key (no caching and throw error when JWT token present in the request)
+* JWT cache turned off
+* JWT cache turned on
+
+All three options are represented by JwtCache data type.
+
+Handling of reconfiguration is centralized in this module.
+-}
+data JwtCache =
+  JwtNoJwks |
+  JwtNoCache JwkSet |
+  forall m v. CacheVariant m v => JwtCache JwkSet (TVar Int) (SC.Cache m ByteString v)
+
+instance CacheVariant IO (Either Error JSON.Object) where
+  cached c = lift . SC.cached c >=> liftEither
+
+instance CacheVariant (ExceptT Error IO) JSON.Object where
+  cached = SC.cached
+
+decode :: JwtCache -> ByteString -> ExceptT Error IO JSON.Object
+decode JwtNoJwks        = const $ throwError (JwtErr JwtSecretMissing)
+decode (JwtNoCache key) = parseAndDecodeClaims key
+decode (JwtCache _ _ c) = cached c
+
+-- | Reconfigure JWT caching and update JwtCacheState accordingly
+update :: JwtCacheState -> AppConfig -> IO ()
+update (JwtCacheState observationHandler jwtCacheState) config@AppConfig{configJWKS, configJwtCacheMaxEntries} =
+  let reinitialize =
+        newJwtCache config observationHandler
+          >>= writeIORef jwtCacheState
+  in
+  readIORef jwtCacheState >>= \case
+    (JwtCache decodingKey maxSize _) ->
+      if configJWKS /= Just decodingKey || configJwtCacheMaxEntries <= 0 then
+        -- reinitialize if key changed or cache disabled
+        reinitialize
+      else
+        -- max size changed - set it and let the cache shrink itself if necessary
+        atomically $ writeTVar maxSize configJwtCacheMaxEntries
+
+    _ -> reinitialize
+
+init :: AppConfig -> ObservationHandler -> IO JwtCacheState
+init config = fmap (<$>) JwtCacheState <*> (newJwtCache config >=> newIORef)

 -- | Initialize JwtCacheState
-init :: IO JwtCacheState
-init = do
-  cache <- C.newCache Nothing -- no default expiration
-  -- purgeExpired has O(n^2) complexity
-  -- so we wrap it in debounce to make sure it:
-  -- 1) is executed asynchronously
-  -- 2) only a single purge operation is running at a time
-  debounce <- mkDebounce defaultDebounceSettings
-    -- debounceFreq is set to default 1 second
-    { debounceAction = C.purgeExpired cache
-    , debounceEdge = leadingEdge
-    }
-  pure $ JwtCacheState cache debounce
+newJwtCache :: AppConfig -> ObservationHandler -> IO JwtCache
+newJwtCache AppConfig{configJWKS, configJwtCacheMaxEntries} observationHandler = do
+  maybe (pure JwtNoJwks) initCache configJWKS
+  where
+    initCache key = if configJwtCacheMaxEntries <= 0 then pure (JwtNoCache key) else createCache key configJwtCacheMaxEntries

-- | Used to retrieve and insert JWT to JWT Cache
-lookupJwtCache :: JwtCacheState -> ByteString -> Int -> IO (Either Error AuthResult) -> UTCTime -> IO (Either Error AuthResult)
-lookupJwtCache JwtCacheState{jwtCache, purgeCache} token maxLifetime parseJwt utc = do
-  checkCache <- C.lookup jwtCache token
-  authResult <- maybe parseJwt (pure . Right) checkCache
+    createCache key maxSize = do
+          maxSizeTVar <- newTVarIO maxSize
+          JwtCache key maxSizeTVar <$>
+            notCachingErrors (readTVar maxSizeTVar) key

-  case (authResult,checkCache) of
-    -- From comment:
-    -- https://github.com/PostgREST/postgrest/pull/3801#discussion_r1857987914
-    --
-    -- We purge expired cache entries on a cache miss
-    -- The reasoning is that:
-    --
-    -- 1. We expect it to be rare (otherwise there is no point of the cache)
-    -- 2. It makes sure the cache is not growing (as inserting new entries
-    --    does garbage collection)
-    -- 3. Since this is time expiration based cache there is no real risk of
-    --    starvation - sooner or later we are going to have a cache miss.
+    notCachingErrors :: STM Int -> JwkSet -> IO (SC.Cache (ExceptT Error IO) ByteString JSON.Object)
+    notCachingErrors maxSize key = SC.cacheIO (SC.CacheConfig maxSize
+            (parseAndDecodeClaims key)
+            (lift . observationHandler . JwtCacheLookup) -- lookup metrics
+            (const . const $ lift $ observationHandler JwtCacheEviction) -- evictions metrics
+            alwaysValid) -- no invalidation for now

-    (Right res, Nothing) -> do -- cache miss
-
-      let timeSpec = getTimeSpec res maxLifetime utc
-
-      -- insert new cache entry
-      C.insert' jwtCache (Just timeSpec) token res
-
-      -- Execute IO action to purge the cache
-      -- It is assumed this action returns immidiately
-      -- so that request processing is not blocked.
-      purgeCache
-
-    _                    -> pure ()
-
-  return authResult
-
-- Used to extract JWT exp claim and add to JWT Cache
-getTimeSpec :: AuthResult -> Int -> UTCTime -> TimeSpec
-getTimeSpec res maxLifetime utc = do
-  let expireJSON = KM.lookup "exp" (authClaims res)
-      utcToSecs = floor . nominalDiffTimeToSeconds . utcTimeToPOSIXSeconds
-      sciToInt = fromMaybe 0 . Sci.toBoundedInteger
-  case expireJSON of
-    Just (JSON.Number seconds) -> TimeSpec (sciToInt seconds - utcToSecs utc) 0
-    _                          -> TimeSpec (fromIntegral maxLifetime :: Int64) 0
-
-- | Empty the cache (done when the config is reloaded)
-emptyCache :: JwtCacheState -> IO ()
-emptyCache JwtCacheState{jwtCache} = C.purge jwtCache
+lookupJwtCache :: JwtCacheState -> Maybe ByteString -> ExceptT Error IO JSON.Object
+lookupJwtCache (JwtCacheState _ cacheState) k = liftIO (readIORef cacheState) >>= flip (maybe (pure KM.empty)) k . decode
@@ -1,5 +1,4 @@
 {-# LANGUAGE NamedFieldPuns  #-}
-{-# LANGUAGE QuasiQuotes     #-}
 {-# LANGUAGE RecordWildCards #-}
 module PostgREST.CLI
  ( main
@@ -14,8 +13,6 @@ import qualified Data.ByteString.Lazy       as LBS
 import qualified Hasql.Transaction.Sessions as SQL
 import qualified Options.Applicative        as O

-import Text.Heredoc (str)
-
 import PostgREST.AppState    (AppState)
 import PostgREST.Config      (AppConfig (..))
 import PostgREST.Observation (Observation (..))
@@ -24,6 +21,7 @@ import PostgREST.Version     (prettyVersion)

 import qualified PostgREST.App      as App
 import qualified PostgREST.AppState as AppState
+import qualified PostgREST.Client   as Client
 import qualified PostgREST.Config   as Config

 import Protolude
@@ -31,16 +29,26 @@ import Protolude

 main :: CLI -> IO ()
 main CLI{cliCommand, cliPath} = do
-  conf@AppConfig{..} <-
+  conf <-
    either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty mempty
+  case cliCommand of
+    Client adminCmd -> runClientCommand conf adminCmd
+    Run runCmd      -> runAppCommand conf runCmd

+-- | Run command using http-client to communicate with an already running postgrest
+runClientCommand :: AppConfig -> ClientCommand -> IO ()
+runClientCommand conf CmdReady = Client.ready conf
+
+-- | Run postgrest with command
+runAppCommand :: AppConfig -> RunCommand -> IO ()
+runAppCommand conf@AppConfig{..} runCmd = do
  -- Per https://github.com/PostgREST/postgrest/issues/268, we want to
  -- explicitly close the connections to PostgreSQL on shutdown.
  -- 'AppState.destroy' takes care of that.
  bracket
    (AppState.init conf)
    AppState.destroy
-    (\appState -> case cliCommand of
+    (\appState -> case runCmd of
      CmdDumpConfig -> do
        when configDbConfig $ AppState.readInDbConfig True appState
        putStr . Config.toText =<< AppState.getConfig appState
@@ -71,6 +79,13 @@ data CLI = CLI
  }

 data Command
+  = Client ClientCommand
+  | Run RunCommand
+
+data ClientCommand
+  = CmdReady
+
+data RunCommand
  = CmdRun
  | CmdDumpConfig
  | CmdDumpSchema
@@ -97,7 +112,7 @@ readCLIShowHelp =
        <> O.help "Show the version information"

    exampleParser =
-      O.infoOption exampleConfigFile $
+      O.infoOption Config.exampleConfigFile $
        O.long "example"
        <> O.short 'e'
        <> O.help "Show an example configuration file"
@@ -105,7 +120,7 @@ readCLIShowHelp =
    cliParser :: O.Parser CLI
    cliParser =
      CLI
-        <$> (dumpConfigFlag <|> dumpSchemaFlag)
+        <$> (dumpConfigFlag <|> dumpSchemaFlag <|> readyFlag)
        <*> O.optional configFileOption

    configFileOption =
@@ -114,125 +129,16 @@ readCLIShowHelp =
        <> O.help "Path to configuration file"

    dumpConfigFlag =
-      O.flag CmdRun CmdDumpConfig $
+      O.flag (Run CmdRun) (Run CmdDumpConfig) $
        O.long "dump-config"
        <> O.help "Dump loaded configuration and exit"

    dumpSchemaFlag =
-      O.flag CmdRun CmdDumpSchema $
+      O.flag (Run CmdRun) (Run CmdDumpSchema) $
        O.long "dump-schema"
        <> O.help "Dump loaded schema as JSON and exit (for debugging, output structure is unstable)"

-exampleConfigFile :: [Char]
-exampleConfigFile =
-  [str|## Admin server used for checks. It's disabled by default unless a port is specified.
-      |# admin-server-port = 3001
-      |
-      |## The database role to use when no client authentication is provided
-      |# db-anon-role = "anon"
-      |
-      |## Notification channel for reloading the schema cache
-      |db-channel = "pgrst"
-      |
-      |## Enable or disable the notification channel
-      |db-channel-enabled = true
-      |
-      |## Enable in-database configuration
-      |db-config = true
-      |
-      |## Function for in-database configuration
-      |## db-pre-config = "postgrest.pre_config"
-      |
-      |## Extra schemas to add to the search_path of every request
-      |db-extra-search-path = "public"
-      |
-      |## Limit rows in response
-      |# db-max-rows = 1000
-      |
-      |## Allow getting the EXPLAIN plan through the `Accept: application/vnd.pgrst.plan` header
-      |# db-plan-enabled = false
-      |
-      |## Number of open connections in the pool
-      |db-pool = 10
-      |
-      |## Time in seconds to wait to acquire a slot from the connection pool
-      |# db-pool-acquisition-timeout = 10
-      |
-      |## Time in seconds after which to recycle pool connections
-      |# db-pool-max-lifetime = 1800
-      |
-      |## Time in seconds after which to recycle unused pool connections
-      |# db-pool-max-idletime = 30
-      |
-      |## Allow automatic database connection retrying
-      |# db-pool-automatic-recovery = true
-      |
-      |## Stored proc to exec immediately after auth
-      |# db-pre-request = "stored_proc_name"
-      |
-      |## Enable or disable prepared statements. disabling is only necessary when behind a connection pooler.
-      |## When disabled, statements will be parametrized but won't be prepared.
-      |db-prepared-statements = true
-      |
-      |## The name of which database schema to expose to REST clients
-      |db-schemas = "public"
-      |
-      |## How to terminate database transactions
-      |## Possible values are:
-      |## commit (default)
-      |##   Transaction is always committed, this can not be overriden
-      |## commit-allow-override
-      |##   Transaction is committed, but can be overriden with Prefer tx=rollback header
-      |## rollback
-      |##   Transaction is always rolled back, this can not be overriden
-      |## rollback-allow-override
-      |##   Transaction is rolled back, but can be overriden with Prefer tx=commit header
-      |db-tx-end = "commit"
-      |
-      |## The standard connection URI format, documented at
-      |## https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
-      |db-uri = "postgresql://"
-      |
-      |# jwt-aud = "your_audience_claim"
-      |
-      |## Jspath to the role claim key
-      |jwt-role-claim-key = ".role"
-      |
-      |## Choose a secret, JSON Web Key (or set) to enable JWT auth
-      |## (use "@filename" to load from separate file)
-      |# jwt-secret = "secret_with_at_least_32_characters"
-      |jwt-secret-is-base64 = false
-      |
-      |## Enables and set JWT Cache max lifetime, disables caching with 0
-      |# jwt-cache-max-lifetime = 0
-      |
-      |## Logging level, the admitted values are: crit, error, warn, info and debug.
-      |log-level = "error"
-      |
-      |## Log the requested SQL query at the current log-level.
-      |log-query = "disabled"
-      |
-      |## Determine if the OpenAPI output should follow or ignore role privileges or be disabled entirely.
-      |## Admitted values: follow-privileges, ignore-privileges, disabled
-      |openapi-mode = "follow-privileges"
-      |
-      |## Base url for the OpenAPI output
-      |openapi-server-proxy-uri = ""
-      |
-      |## Configurable CORS origins
-      |# server-cors-allowed-origins = ""
-      |
-      |server-host = "!4"
-      |server-port = 3000
-      |
-      |## Allow getting the request-response timing information through the `Server-Timing` header
-      |server-timing-enabled = false
-      |
-      |## Unix socket location
-      |## if specified it takes precedence over server-port
-      |# server-unix-socket = "/tmp/pgrst.sock"
-      |
-      |## Unix socket file mode
-      |## When none is provided, 660 is applied by default
-      |# server-unix-socket-mode = "660"
-      |]
+    readyFlag =
+      O.flag (Run CmdRun) (Client CmdReady) $
+        O.long "ready"
+        <> O.help "Checks the health of PostgREST by doing a request on the admin server /ready endpoint"
@@ -0,0 +1,218 @@
+{-|
+Module      : PostgREST.Cache.Sieve
+Description : PostgREST cache implementation based on Sieve algorithm.
+
+This module provides implementation of a mutable cache on Sieve algorithm.
+-}
+{-# LANGUAGE DataKinds       #-}
+{-# LANGUAGE GADTs           #-}
+{-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE NamedFieldPuns  #-}
+{-# LANGUAGE PolyKinds       #-}
+{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE RecursiveDo     #-}
+{-# LANGUAGE StrictData      #-}
+{-# LANGUAGE TupleSections   #-}
+
+module PostgREST.Cache.Sieve (
+      Cache
+    , CacheConfig (..)
+    , Discard (..)
+    , alwaysValid
+    , cache
+    , cacheIO
+    , cached
+)
+where
+
+import           Control.Concurrent.STM
+import           Control.Monad.Extra    (whileM)
+import           Data.Some
+import qualified Focus                  as F
+import           Protolude              hiding (elem, head)
+import qualified StmHamt.SizedHamt      as SH
+
+data ListNode k v (b :: Bool) = ListNode {
+        nextPtr        :: NodePtr k v,
+        prevNextPtrPtr :: NodePtrPtr k v,
+        elem           :: NodeElem k v b
+    }
+
+data NodeElem :: Type -> Type -> Bool -> Type where
+    Head :: {
+            entries          :: SH.SizedHamt (HamtEntry k v),
+            finger           :: NodePtrPtr k v
+        } -> NodeElem k v False
+    Entry :: Hashable k => {
+            visited :: TVar Bool,
+            ekey :: k,
+            entryValue :: v
+        } -> NodeElem k v True
+
+type HamtEntry k v = ListNode k v True
+type AnyNode k v = Some (ListNode k v)
+type NodePtr k v = TVar (AnyNode k v)
+type NodePtrPtr k v = TVar (NodePtr k v)
+
+data Discard m v = Refresh (m ()) | Invalid (m v)
+
+data Cache m k v = (MonadIO m, Hashable k) => Cache (ListNode k v False) (CacheConfig m k v)
+
+data CacheConfig m k v = CacheConfig {
+        maxSize          :: STM Int,
+        load             :: k -> m v,
+        requestListener  :: Bool -> m (),
+        evictionListener :: k -> v -> m (),
+        validator        :: m (k -> v -> Maybe (Discard m v))
+}
+
+alwaysValid :: Applicative m => m (k -> v -> Maybe (Discard m v))
+alwaysValid = pure (const . const Nothing)
+
+cacheIO :: (MonadIO m, Hashable k) => CacheConfig m k v -> IO (Cache m k v)
+cacheIO = atomically . cache
+
+cache :: (MonadIO m, Hashable k) => CacheConfig m k v -> STM (Cache m k v)
+cache cacheConfig = mdo
+    tail <- newTVar (Some head)
+    entries <- SH.new
+    finger <- newTVar tail
+    head <- ListNode tail <$> newTVar tail <*> pure Head {..}
+    pure $ Cache head cacheConfig
+
+cached :: Cache m k v -> k -> m v
+cached (Cache head@ListNode{prevNextPtrPtr=neck, elem=Head{..}} CacheConfig{..}) k = do
+    checkValid <- validator
+    tryMaybe
+        -- Fast path: lookup value, update stats and return the value if found and valid
+        ((liftIO . atomically) (lookup checkValid) >>= notify (requestListener . isJust) >>= validate)
+        -- Slow path: load/calculate value and insert it (if still not found)
+        (do
+            value <- load k
+            whileM (not <$> tryInsert value)
+            pure value)
+    where
+        tryMaybe f notFound = f >>= maybe notFound pure
+
+        notify = ((<$) <*>)
+
+        validate = fmap join . traverse (\case
+            -- valid value
+            (Right v) -> pure $ Just v
+            -- refresh value
+            (Left (Refresh act)) -> act $> Nothing
+            -- discard value and return alt result
+            (Left (Invalid res)) -> Just <$> res)
+
+        lookup checkValid = SH.focus focus (ekey . elem) k entries
+            where
+                focus = F.Focus
+                    -- not found
+                    (pure (Nothing, F.Leave))
+                    -- found
+                    -- check entry validity
+                    (\e@ListNode{elem=Entry{visited, entryValue}} ->
+                        maybe
+                            -- entry valid
+                            (mark visited True $> (Just $ Right entryValue, F.Leave))
+                            -- entry invalid
+                            -- remove it
+                            ((removeEntry e $>) . (, F.Remove) . Just . Left)
+                            (checkValid k entryValue)
+                    )
+
+        mark t b = whenM ((/= b) <$> readTVar t) (writeTVar t b)
+
+        -- perform a single entry eviction and possibly insertion atomically
+        -- returning False if could not insert
+        -- (either because entry currently pointed by the finger was visited
+        --  or because after this entry eviction the cache is still full)
+        -- so that other threads don't have to wait when visiting entries.
+        -- First check if entry is still not in the cache - this time inside transaction.
+        --
+        -- Execute evictionListener if an entry was evicted
+        tryInsert value = do
+            (result, evicted) <- liftIO . atomically $ do
+                -- Use SH.focus to performa a single lookup instead of 2
+                -- we cannot modify Hamt from inside focus
+                -- so if there is any entry to remove
+                -- we need to delete it after
+                (res, evictedKey) <- SH.focus focus (ekey . elem) k entries
+                case evictedKey of
+                    (Just Entry{ekey=entryKey, entryValue}) -> do
+                        SH.focus F.delete (ekey . elem) entryKey entries
+                        pure (res, evictionListener entryKey entryValue)
+                    Nothing -> pure (res, pure ())
+
+            evicted $> result
+            where
+                focus = F.Focus (do
+                    (hasSpace, evictedKey) <- evictionStep
+                    if hasSpace then do
+                        entry <- newLinkedEntry value
+                        -- done, maybe evicted, insert entry
+                        pure ((True, evictedKey), F.Set entry)
+                    else
+                        -- not done, maybe evicted, don't modify entries
+                        pure ((False, evictedKey), F.Leave))
+                    -- Entry found case
+                    (\ListNode{elem=Entry{visited}} -> do
+                        -- mark as visited
+                        mark visited True
+                        -- done, no evictions, don't modify entries
+                        pure ((True, Nothing), F.Leave))
+
+        -- if the cache is full precoesses a single node
+        -- removing it if it is marked as unvisited
+        -- or clearing visited mark
+        -- returns True if there is space in the cache
+        -- puts evictionListener in state if an entry was evicted
+        evictionStep = do
+            currDiff <- liftA2 (-) (SH.size entries) (max 1 <$> maxSize)
+            if currDiff >= 0 then do
+                -- no space in the cache
+                -- need to evict an entry
+                (nextFinger, evictedKey) <- readTVar finger >>= evict
+                writeTVar finger nextFinger
+                -- return if enough space and evicted key if any
+                pure (isJust evictedKey && currDiff == 0, evictedKey)
+            else
+                -- there is space in the cache
+                pure (True, Nothing)
+
+        evict :: TVar (Some (ListNode k v)) -> STM (NodePtr k v, Maybe (NodeElem k v True))
+        evict = readTVar >=> \case
+            (Some e@ListNode{nextPtr, prevNextPtrPtr, elem=elem@Entry{visited}}) -> do
+                ifM (readTVar visited)
+
+                    (writeTVar visited False $> (nextPtr, Nothing))
+
+                    (unlinkEntry e *> fmap (, Just elem) (readTVar prevNextPtrPtr))
+            -- skip head
+            (Some ListNode{nextPtr, elem=Head{}}) -> evict nextPtr
+
+        unlinkEntry :: HamtEntry k v -> STM ()
+        unlinkEntry (ListNode{nextPtr, prevNextPtrPtr=currPrev}) = do
+            nextEntry <- readTVar nextPtr
+            withSome nextEntry $ \e -> do
+                prevNextPtr <- readTVar currPrev
+                writeTVar (prevNextPtrPtr e) prevNextPtr
+                writeTVar prevNextPtr nextEntry
+
+        newLinkedEntry v = do
+            oldNeckNextPtr <- readTVar neck
+            newNeckNextPtr <- newTVar (Some head)
+            newNeck <- ListNode newNeckNextPtr <$>
+                newTVar oldNeckNextPtr <*>
+                (Entry <$> newTVar False <*> pure k <*> pure v)
+            -- update pointers
+            writeTVar oldNeckNextPtr (Some newNeck)
+            writeTVar neck newNeckNextPtr
+            -- return HAMT entry
+            pure newNeck
+
+        removeEntry = fmap (*>) unlinkEntry <*> adjustFinger
+
+        adjustFinger ListNode{nextPtr, prevNextPtrPtr} =
+            whenM ((nextPtr ==) <$> readTVar finger) $
+                readTVar prevNextPtrPtr >>= writeTVar finger
@@ -0,0 +1,100 @@
+{-|
+Module      : PostgREST.Client
+Description : PostgREST HTTP client
+-}
+{-# LANGUAGE NamedFieldPuns #-}
+module PostgREST.Client
+  ( ready
+  ) where
+
+import qualified Data.Text                 as T
+import qualified Network.HTTP.Client       as HC
+import qualified Network.HTTP.Types.Status as HTTP
+
+import Network.HTTP.Client (HttpException (..))
+import System.IO           (hFlush)
+
+import PostgREST.Config  (AppConfig (..))
+import PostgREST.Network (isSpecialHostName)
+
+import Protolude
+
+data PgrstClientError
+  = NoAdminServer
+  | NoSpecialHostNamesAllowed Text
+  | PostgRESTNotReady         Text
+  | HTTPConnectionRefused     Text
+  | HTTPExceptionInvalidURL   Text
+
+-- | This is invoked by the CLI "--ready" flag.
+--   The http-client sends and a request to /ready endpoint
+--   and exits with success or failure.
+ready :: AppConfig -> IO ()
+ready AppConfig{configAdminServerHost, configAdminServerPort} = do
+
+  client <- HC.newManager HC.defaultManagerSettings
+  readyURL <- getURL
+  req <- HC.parseRequest (T.unpack readyURL) `catch` handleHttpException
+  resp <- HC.httpLbs req client `catch` handleHttpException
+
+  let status = HC.responseStatus resp
+
+  if status >= HTTP.status200 && status < HTTP.status300
+    then printAndExitWithSuccess $ "OK: " <> readyURL
+    else printAndExitWithFailure $ clientErrorMsg (PostgRESTNotReady readyURL)
+        where
+          getURL :: IO Text
+          getURL =
+            -- Here, we have three cases:
+            -- 1. If the admin port config is not defined, we exit
+            --      with "no admin server error"
+            -- 2. Otherwise, if admin server is running, then we check if
+            --      postgrest server-host is configured with special hostname like "*4",
+            --      if it is, we fail with "no special hostname allowed with "--ready".
+            --      The reason for this is that we can't know the actual address.
+            -- 3. Finally, if we know the "actual" hostname and the port, then we
+            --      construct the URL and return it.
+            case configAdminServerPort of
+              Nothing   -> printAndExitWithFailure $ clientErrorMsg NoAdminServer
+              Just port ->
+                if isSpecialHostName configAdminServerHost
+                  then printAndExitWithFailure $ clientErrorMsg (NoSpecialHostNamesAllowed configAdminServerHost)
+                  else return $ makeReadyUrl port
+
+          -- NOTE: http-client automatically resolves hostnames
+          makeReadyUrl :: Int -> Text
+          makeReadyUrl p = "http://" <> wrapIfIpv6 configAdminServerHost <> ":" <> (T.pack . show) p <> "/ready"
+            where
+              -- IPv6 needs to wrapped in [], it has ':' as separator
+              wrapIfIpv6 :: Text -> Text
+              wrapIfIpv6 s
+                | T.any (== ':') s = "[" <> s <> "]"
+                | otherwise = s
+
+-- | Handle HTTP exception for "http-client" requests
+handleHttpException :: HttpException -> IO a
+handleHttpException (HttpExceptionRequest req _) = do
+  let url = show (HC.getUri req)
+  printAndExitWithFailure $ clientErrorMsg (HTTPConnectionRefused $ T.pack url)
+handleHttpException (InvalidUrlException url _) = do
+  printAndExitWithFailure $ clientErrorMsg (HTTPExceptionInvalidURL $ T.pack url)
+
+-- | Print the message on stdout and exit with success
+printAndExitWithSuccess :: Text -> IO a
+printAndExitWithSuccess msg = putStrLn (T.unpack msg) >> hFlush stdout >> exitSuccess
+
+-- | Print the message on stderr and exit with failure
+printAndExitWithFailure :: Text -> IO a
+printAndExitWithFailure msg = hPutStrLn stderr (T.unpack msg) >> hFlush stderr >> exitWith (ExitFailure 1)
+
+-- | Pgrst client error to error message
+clientErrorMsg :: PgrstClientError -> Text
+clientErrorMsg err = "ERROR: " <>
+  case err of
+    NoAdminServer -> "Admin server is not running. Please check admin-server-port config."
+    NoSpecialHostNamesAllowed host ->
+      "The `--ready` flag cannot be used when server-host is configured as \"" <> host <> "\". "
+      <> "Please update your server-host config to \"localhost\"."
+    PostgRESTNotReady url -> url
+    HTTPConnectionRefused url -> "connection refused to " <> url
+    HTTPExceptionInvalidURL url -> "invalid url - " <> url
@@ -17,7 +17,6 @@ module PostgREST.Config
  , JSPathExp(..)
  , FilterExp(..)
  , LogLevel(..)
-  , LogQuery(..)
  , OpenAPIMode(..)
  , Proxy(..)
  , toText
@@ -28,6 +27,8 @@ module PostgREST.Config
  , parseSecret
  , addFallbackAppName
  , addTargetSessionAttrs
+  , exampleConfigFile
+  , audMatchesCfg
  ) where

 import qualified Data.Aeson             as JSON
@@ -36,6 +37,7 @@ import qualified Data.ByteString.Base64 as B64
 import qualified Data.CaseInsensitive   as CI
 import qualified Data.Configurator      as C
 import qualified Data.Map.Strict        as M
+import qualified Data.String            as S
 import qualified Data.Text              as T
 import qualified Data.Text.Encoding     as T
 import qualified Jose.Jwa               as JWT
@@ -66,6 +68,8 @@ import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier, dumpQi,

 import Protolude hiding (Proxy, toList)

+audMatchesCfg :: AppConfig -> Text -> Bool
+audMatchesCfg =  maybe (const True) (==) . configJwtAudience

 data AppConfig = AppConfig
  { configAppSettings              :: [(Text, Text)]
@@ -97,9 +101,9 @@ data AppConfig = AppConfig
  , configJwtRoleClaimKey          :: JSPath
  , configJwtSecret                :: Maybe BS.ByteString
  , configJwtSecretIsBase64        :: Bool
-  , configJwtCacheMaxLifetime      :: Int
+  , configJwtCacheMaxEntries       :: Int
  , configLogLevel                 :: LogLevel
-  , configLogQuery                 :: LogQuery
+  , configLogQuery                 :: Bool
  , configOpenApiMode              :: OpenAPIMode
  , configOpenApiSecurityActive    :: Bool
  , configOpenApiServerProxyUri    :: Maybe Text
@@ -114,7 +118,9 @@ data AppConfig = AppConfig
  , configAdminServerPort          :: Maybe Int
  , configRoleSettings             :: RoleSettings
  , configRoleIsoLvl               :: RoleIsolationLvl
-  , configInternalSCSleep          :: Maybe Int32
+  , configInternalSCQuerySleep     :: Maybe Int32
+  , configInternalSCLoadSleep      :: Maybe Int32
+  , configInternalSCRelLoadSleep   :: Maybe Int32
  }

 data LogLevel = LogCrit | LogError | LogWarn | LogInfo | LogDebug
@@ -128,14 +134,6 @@ dumpLogLevel = \case
  LogInfo  -> "info"
  LogDebug -> "debug"

-data LogQuery = LogQueryMain | LogQueryDisabled
-  deriving (Eq)
-
-dumpLogQuery :: LogQuery -> Text
-dumpLogQuery = \case
-  LogQueryMain     -> "main-query"
-  LogQueryDisabled -> "disabled"
-
 data OpenAPIMode = OAFollowPriv | OAIgnorePriv | OADisabled
  deriving Eq

@@ -177,9 +175,9 @@ toText conf =
      ,("jwt-role-claim-key",        q . T.intercalate mempty . fmap dumpJSPath . configJwtRoleClaimKey)
      ,("jwt-secret",                q . T.decodeUtf8 . showJwtSecret)
      ,("jwt-secret-is-base64",          T.toLower . show . configJwtSecretIsBase64)
-      ,("jwt-cache-max-lifetime",                   show . configJwtCacheMaxLifetime)
+      ,("jwt-cache-max-entries",         show . configJwtCacheMaxEntries)
      ,("log-level",                 q . dumpLogLevel . configLogLevel)
-      ,("log-query",                 q . dumpLogQuery . configLogQuery)
+      ,("log-query",                     T.toLower . show . configLogQuery)
      ,("openapi-mode",              q . dumpOpenApiMode . configOpenApiMode)
      ,("openapi-security-active",       T.toLower . show . configOpenApiSecurityActive)
      ,("openapi-server-proxy-uri",  q . fromMaybe mempty . configOpenApiServerProxyUri)
@@ -287,9 +285,9 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
    <*> (fromMaybe False <$> optWithAlias
          (optBool "jwt-secret-is-base64")
          (optBool "secret-is-base64"))
-    <*> (fromMaybe 0 <$> optInt "jwt-cache-max-lifetime")
+    <*> (fromMaybe 1000 <$> optInt "jwt-cache-max-entries")
    <*> parseLogLevel "log-level"
-    <*> parseLogQuery "log-query"
+    <*> (fromMaybe False <$> optBool "log-query")
    <*> parseOpenAPIMode "openapi-mode"
    <*> (fromMaybe False <$> optBool "openapi-security-active")
    <*> parseOpenAPIServerProxyURI "openapi-server-proxy-uri"
@@ -305,7 +303,9 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
    <*> parseAdminServerPort "admin-server-port"
    <*> pure roleSettings
    <*> pure roleIsolationLvl
-    <*> optInt "internal-schema-cache-sleep"
+    <*> optInt "internal-schema-cache-query-sleep"
+    <*> optInt "internal-schema-cache-load-sleep"
+    <*> optInt "internal-schema-cache-relationship-load-sleep"
  where
    parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
    parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
@@ -365,14 +365,6 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
        Just "debug" -> pure LogDebug
        Just _       -> fail "Invalid logging level. Check your configuration."

-    parseLogQuery :: C.Key -> C.Parser C.Config LogQuery
-    parseLogQuery k =
-      optString k >>= \case
-        Nothing           -> pure  LogQueryDisabled
-        Just "disabled"   -> pure  LogQueryDisabled
-        Just "main-query" -> pure  LogQueryMain
-        Just _            -> fail "Invalid SQL logging value. Check your configuration."
-
    parseTxEnd :: C.Key -> ((Bool, Bool) -> Bool) -> C.Parser C.Config Bool
    parseTxEnd k f =
      optString k >>= \case
@@ -628,3 +620,118 @@ addConnStringOption dbUri key val = dbUri <>
    uriFmt = key <> "=" <> toS (escapeURIString isUnescapedInURIComponent $ toS val)
    keyValFmt = key <> "=" <> "'" <> T.replace "'" "\\'" val <> "'"
    lookAtOptions x =  T.breakOn "?" . snd $ T.breakOnEnd "@" x -- start from after `@` to not mess passwords that include `?`, see https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING-URIS
+
+-- | Example config file displayed on postgrest "--example" flag
+exampleConfigFile :: [Char]
+exampleConfigFile = S.unlines
+  [ "## Admin server used for checks. It's disabled by default unless a port is specified."
+  , "# admin-server-port = 3001"
+  , ""
+  , "## The database role to use when no client authentication is provided"
+  , "# db-anon-role = \"anon\""
+  , ""
+  , "## Notification channel for reloading the schema cache"
+  , "db-channel = \"pgrst\""
+  , ""
+  , "## Enable or disable the notification channel"
+  , "db-channel-enabled = true"
+  , ""
+  , "## Enable in-database configuration"
+  , "db-config = true"
+  , ""
+  , "## Function for in-database configuration"
+  , "## db-pre-config = \"postgrest.pre_config\""
+  , ""
+  , "## Extra schemas to add to the search_path of every request"
+  , "db-extra-search-path = \"public\""
+  , ""
+  , "## Limit rows in response"
+  , "# db-max-rows = 1000"
+  , ""
+  , "## Allow getting the EXPLAIN plan through the `Accept: application/vnd.pgrst.plan` header"
+  , "# db-plan-enabled = false"
+  , ""
+  , "## Number of open connections in the pool"
+  , "db-pool = 10"
+  , ""
+  , "## Time in seconds to wait to acquire a slot from the connection pool"
+  , "# db-pool-acquisition-timeout = 10"
+  , ""
+  , "## Time in seconds after which to recycle pool connections"
+  , "# db-pool-max-lifetime = 1800"
+  ,  ""
+  ,  "## Time in seconds after which to recycle unused pool connections"
+  ,  "# db-pool-max-idletime = 30"
+  ,  ""
+  , "## Allow automatic database connection retrying"
+  , "# db-pool-automatic-recovery = true"
+  , ""
+  , "## Stored proc to exec immediately after auth"
+  , "# db-pre-request = \"stored_proc_name\""
+  , ""
+  , "## Enable or disable prepared statements. disabling is only necessary when behind a connection pooler."
+  , "## When disabled, statements will be parametrized but won't be prepared."
+  , "db-prepared-statements = true"
+  , ""
+  , "## The name of which database schema to expose to REST clients"
+  , "db-schemas = \"public\""
+  , ""
+  , "## How to terminate database transactions"
+  , "## Possible values are:"
+  , "## commit (default)"
+  , "##   Transaction is always committed, this can not be overriden"
+  , "## commit-allow-override"
+  , "##   Transaction is committed, but can be overriden with Prefer tx=rollback header"
+  , "## rollback"
+  , "##   Transaction is always rolled back, this can not be overriden"
+  , "## rollback-allow-override"
+  , "##   Transaction is rolled back, but can be overriden with Prefer tx=commit header"
+  , "db-tx-end = \"commit\""
+  , ""
+  , "## The standard connection URI format, documented at"
+  , "## https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING"
+  , "db-uri = \"postgresql://\""
+  , ""
+  , "# jwt-aud = \"your_audience_claim\""
+  , ""
+  , "## Jspath to the role claim key"
+  ,  "jwt-role-claim-key = \".role\""
+  ,  ""
+  ,  "## Choose a secret, JSON Web Key (or set) to enable JWT auth"
+  ,  "## (use \"@filename\" to load from separate file)"
+  ,  "# jwt-secret = \"secret_with_at_least_32_characters\""
+  ,  "jwt-secret-is-base64 = false"
+  , ""
+  , "## Enables JWT Cache and sets its max size, disables caching with 0"
+  , "# jwt-cache-max-entries = 0"
+  , ""
+  , "## Logging level, the admitted values are: crit, error, warn, info and debug."
+  , "log-level = \"error\""
+  , ""
+  , "## Log the SQL query at the current log-level."
+  , "log-query = false"
+  , ""
+  , "## Determine if the OpenAPI output should follow or ignore role privileges or be disabled entirely."
+  , "## Admitted values: follow-privileges, ignore-privileges, disabled"
+  , "openapi-mode = \"follow-privileges\""
+  , ""
+  , "## Base url for the OpenAPI output"
+  , "openapi-server-proxy-uri = \"\""
+  , ""
+  , "## Configurable CORS origins"
+  , "# server-cors-allowed-origins = \"\""
+  , ""
+  , "server-host = \"!4\""
+  , "server-port = 3000"
+  , ""
+  , "## Allow getting the request-response timing information through the `Server-Timing` header"
+  , "server-timing-enabled = false"
+  , ""
+  , "## Unix socket location"
+  , "## if specified it takes precedence over server-port"
+  , "# server-unix-socket = \"/tmp/pgrst.sock\""
+  , ""
+  , "## Unix socket file mode"
+  , "## When none is provided, 660 is applied by default"
+  , "# server-unix-socket-mode = \"660\""
+  ]
@@ -3,7 +3,6 @@
 module PostgREST.Config.PgVersion
  ( PgVersion(..)
  , minimumPgVersion
-  , pgVersion130
  , pgVersion140
  , pgVersion150
  , pgVersion170
@@ -26,10 +25,7 @@ instance Ord PgVersion where

 -- | Tells the minimum PostgreSQL version required by this version of PostgREST
 minimumPgVersion :: PgVersion
-minimumPgVersion = pgVersion121
-
-pgVersion121 :: PgVersion
-pgVersion121 = PgVersion 120001 "12.1" "12.1"
+minimumPgVersion = pgVersion130

 pgVersion130 :: PgVersion
 pgVersion130 = PgVersion 130000 "13.0" "13.0"
@@ -3,6 +3,7 @@ Module      : PostgREST.Error
 Description : PostgREST error HTTP responses
 -}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
+{-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}

 module PostgREST.Error
@@ -14,6 +15,8 @@ module PostgREST.Error
  , PgError(..)
  , Error(..)
  , JwtError (..)
+  , JwtDecodeError(..)
+  , JwtClaimsError(..)
  , errorPayload
  , status
  ) where
@@ -39,6 +42,7 @@ import Network.HTTP.Types.Header (Header)
 import           PostgREST.MediaType (MediaType (..))
 import qualified PostgREST.MediaType as MediaType

+import PostgREST.SchemaCache              (SchemaCache (SchemaCache, dbTablesFuzzyIndex))
 import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..),
                                           Schema)
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
@@ -47,10 +51,8 @@ import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                           RelationshipsMap)
 import PostgREST.SchemaCache.Routine      (Routine (..),
                                           RoutineParam (..))
-import PostgREST.SchemaCache.Table        (Table (..))
 import Protolude

-
 class (ErrorBody a, JSON.ToJSON a) => PgrstError a where
  status   :: a -> HTTP.Status
  headers  :: a -> [Header]
@@ -86,7 +88,7 @@ data ApiRequestError
  | QueryParamError QPError
  | RelatedOrderNotToOne Text Text
  | UnacceptableFilter Text
-  | UnacceptableSchema [Text]
+  | UnacceptableSchema Text [Text]
  | UnsupportedMethod ByteString
  | GucHeadersError
  | GucStatusError
@@ -194,7 +196,7 @@ instance ErrorBody ApiRequestError where
  message (InvalidBody errorMessage)   = T.decodeUtf8 errorMessage
  message (InvalidRange _)             = "Requested range not satisfiable"
  message InvalidFilters               = "Filters must include all and only primary key columns with 'eq' operators"
-  message (UnacceptableSchema schemas) = "The schema must be one of the following: " <> T.intercalate ", " schemas
+  message (UnacceptableSchema sch _)   = "Invalid schema: " <> sch
  message (MediaTypeError cts)         = "None of these media types are available: " <> T.intercalate ", " (map T.decodeUtf8 cts)
  message (NotEmbedded resource)       = "'" <> resource <> "' is not an embedded resource in this request"
  message GucHeadersError              = "response.headers guc must be a JSON array composed of objects with a single key and a string value"
@@ -234,6 +236,7 @@ instance ErrorBody ApiRequestError where
  -- HINT: Maybe JSON.Value
  hint (NotEmbedded resource) = Just $ JSON.String $ "Verify that '" <> resource <> "' is included in the 'select' query parameter."
  hint (PGRSTParseError raiseErr) = Just $ JSON.String $ pgrstParseErrorHint raiseErr
+  hint (UnacceptableSchema _ schemas) = Just $ JSON.String $ "Only the following schemas are exposed: "  <> T.intercalate ", " schemas

  hint _ = Nothing

@@ -247,7 +250,7 @@ data SchemaCacheError
  | NoRelBetween Text Text (Maybe Text) Text RelationshipsMap
  | NoRpc Text Text [Text] MediaType Bool [QualifiedIdentifier] [Routine]
  | ColumnNotFound Text Text
-  | TableNotFound Text Text [Table]
+  | TableNotFound Text Text SchemaCache
  deriving Show

 instance PgrstError SchemaCacheError where
@@ -310,7 +313,7 @@ instance ErrorBody SchemaCacheError where
      where
        onlySingleParams = isInvPost && contentType `elem` [MTTextPlain, MTTextXML, MTOctetStream]
  hint (AmbiguousRpc _)      = Just "Try renaming the parameters or the function itself in the database so function overloading can be resolved"
-  hint (TableNotFound schemaName relName tbls) = JSON.String <$> tableNotFoundHint schemaName relName tbls
+  hint (TableNotFound schemaName relName schemaCache) = JSON.String <$> tableNotFoundHint schemaName relName schemaCache

  hint _ = Nothing

@@ -383,7 +386,7 @@ noRelBetweenHint parent child schema allRels = ("Perhaps you meant '" <>) <$>
 -- Just "Perhaps you meant to call the function api.test"
 --
 -- >>> noRpcHint "api" "other" [] procs []
-- Just "Perhaps you meant to call the function api.another"
+-- Nothing
 --
 -- >>> noRpcHint "api" "noclosealternative" [] procs []
 -- Nothing
@@ -420,18 +423,30 @@ noRpcHint schema procName params allProcs overloadedProcs =
    -- E.g. ["val", "param", "name"] into "(name, param, val)"
    listToText       = ("(" <>) . (<> ")") . T.intercalate ", " . sort
    possibleProcs
-      | null overloadedProcs = Fuzzy.getOne fuzzySetOfProcs procName
-      | otherwise            = (procName <>) <$> Fuzzy.getOne fuzzySetOfParams (listToText params)
+      | null overloadedProcs = getFuzzyHint HintProcedure fuzzySetOfProcs procName
+      | otherwise            = (procName <>) <$> getFuzzyHint HintParams fuzzySetOfParams (listToText params)

 -- |
 -- Do a fuzzy search in all tables in the same schema and return closest result
-tableNotFoundHint :: Text -> Text -> [Table] -> Maybe Text
-tableNotFoundHint schema tblName tblList
+tableNotFoundHint :: Text -> Text -> SchemaCache -> Maybe Text
+tableNotFoundHint schema tblName SchemaCache{dbTablesFuzzyIndex}
  = fmap (\tbl -> "Perhaps you meant the table '" <> schema <> "." <> tbl <> "'") perhapsTable
    where
-      perhapsTable = Fuzzy.getOne fuzzyTableSet tblName
-      fuzzyTableSet = Fuzzy.fromList [ tableName tbl | tbl <- tblList, tableSchema tbl == schema]
+      perhapsTable = (\fuzzySet -> getFuzzyHint HintTable fuzzySet tblName) =<< HM.lookup schema dbTablesFuzzyIndex

+data HintType
+  = HintTable
+  | HintProcedure
+  | HintParams
+
+-- | Get hint using Fuzzy Search with at least 0.75 similarity score
+getFuzzyHint :: HintType -> Fuzzy.FuzzySet -> Text -> Maybe Text
+getFuzzyHint hintType =
+  let minScore = 0.75 :: Double -- used for table and procedure name hints
+  in case hintType of
+    HintTable     -> Fuzzy.getOneWithMinScore minScore
+    HintProcedure -> Fuzzy.getOneWithMinScore minScore
+    HintParams    -> Fuzzy.getOne -- For params, we stick to `getOne` which defaults to 0.33 min score, not a security risk to reveal params

 compressedRel :: Relationship -> JSON.Value
 -- An ambiguousness error cannot happen for computed relationships TODO refactor so this mempty is not needed
@@ -647,10 +662,32 @@ data Error
  deriving Show

 data JwtError
-  = JwtDecodeError Text
+  = JwtDecodeErr JwtDecodeError
  | JwtSecretMissing
  | JwtTokenRequired
-  | JwtClaimsError Text
+  | JwtClaimsErr JwtClaimsError
+  deriving Show
+
+data JwtDecodeError
+  = EmptyAuthHeader
+  | UnexpectedParts Int
+  | KeyError Text
+  | BadAlgorithm Text
+  | BadCrypto
+  | UnsupportedTokenType
+  | UnreachableDecodeError
+  deriving Show
+
+data JwtClaimsError
+  = JWTExpired
+  | JWTNotYetValid
+  | JWTIssuedAtFuture
+  | JWTNotInAudience
+  | ParsingClaimsFailed
+  | ExpClaimNotNumber
+  | NbfClaimNotNumber
+  | IatClaimNotNumber
+  | AudClaimNotStringOrArray
  deriving Show

 instance PgrstError Error where
@@ -696,14 +733,14 @@ instance ErrorBody Error where
  hint (PgErr err)           = hint err

 instance PgrstError JwtError where
-  status JwtDecodeError{} = HTTP.unauthorized401
+  status JwtDecodeErr{}   = HTTP.unauthorized401
  status JwtSecretMissing = HTTP.status500
  status JwtTokenRequired = HTTP.unauthorized401
-  status JwtClaimsError{} = HTTP.unauthorized401
+  status JwtClaimsErr{}   = HTTP.unauthorized401

-  headers (JwtDecodeError m) = [invalidTokenHeader m]
+  headers e@(JwtDecodeErr _) = [invalidTokenHeader $ message e]
  headers JwtTokenRequired   = [requiredTokenHeader]
-  headers (JwtClaimsError m) = [invalidTokenHeader m]
+  headers e@(JwtClaimsErr _) = [invalidTokenHeader $ message e]
  headers _                  = mempty

 instance JSON.ToJSON JwtError where
@@ -711,16 +748,36 @@ instance JSON.ToJSON JwtError where
    (code err) (message err) (details err) (hint err)

 instance ErrorBody JwtError where
-  code JwtSecretMissing   = "PGRST300"
-  code (JwtDecodeError _) = "PGRST301"
-  code JwtTokenRequired   = "PGRST302"
-  code (JwtClaimsError _) = "PGRST303"
+  code JwtSecretMissing = "PGRST300"
+  code (JwtDecodeErr _) = "PGRST301"
+  code JwtTokenRequired = "PGRST302"
+  code (JwtClaimsErr _) = "PGRST303"

-  message JwtSecretMissing     = "Server lacks JWT secret"
-  message (JwtDecodeError msg) = msg
-  message JwtTokenRequired     = "Anonymous access is disabled"
-  message (JwtClaimsError msg) = msg
+  message JwtSecretMissing = "Server lacks JWT secret"
+  message (JwtDecodeErr e) = case e of
+    EmptyAuthHeader        -> "Empty JWT is sent in Authorization header"
+    UnexpectedParts n      -> "Expected 3 parts in JWT; got " <> show n
+    KeyError _             -> "No suitable key or wrong key type"
+    BadAlgorithm _         -> "Wrong or unsupported encoding algorithm"
+    BadCrypto              -> "JWT cryptographic operation failed"
+    UnsupportedTokenType   -> "Unsupported token type"
+    UnreachableDecodeError -> "JWT couldn't be decoded"
+  message JwtTokenRequired = "Anonymous access is disabled"
+  message (JwtClaimsErr e) = case e of
+    JWTExpired               -> "JWT expired"
+    JWTNotYetValid           -> "JWT not yet valid"
+    JWTIssuedAtFuture        -> "JWT issued at future"
+    JWTNotInAudience         -> "JWT not in audience"
+    ParsingClaimsFailed      -> "Parsing claims failed"
+    ExpClaimNotNumber        -> "The JWT 'exp' claim must be a number"
+    NbfClaimNotNumber        -> "The JWT 'nbf' claim must be a number"
+    IatClaimNotNumber        -> "The JWT 'iat' claim must be a number"
+    AudClaimNotStringOrArray -> "The JWT 'aud' claim must be a string or an array of strings"

+  details (JwtDecodeErr jde) = case jde of
+    KeyError dets     -> Just $ JSON.String dets
+    BadAlgorithm dets -> Just $ JSON.String dets
+    _                 -> Nothing
  details _ = Nothing

  hint _    = Nothing
@@ -1,3 +1,4 @@
+{-# LANGUAGE LambdaCase      #-}
 {-# LANGUAGE MultiWayIf      #-}
 {-# LANGUAGE RecordWildCards #-}

@@ -9,12 +10,14 @@ import qualified Hasql.Connection      as SQL
 import qualified Hasql.Notifications   as SQL
 import           PostgREST.AppState    (AppState, getConfig)
 import           PostgREST.Config      (AppConfig (..))
-import           PostgREST.Observation (Observation (..))
+import           PostgREST.Observation (Observation (..),
+                                        isDbListenerBug)
 import           PostgREST.Version     (prettyVersion)

 import qualified PostgREST.AppState as AppState
 import qualified PostgREST.Config   as Config

+import Data.Either.Combinators (whenRight)
 import Protolude

 -- | Starts the Listener in a thread
@@ -22,17 +25,20 @@ runListener :: AppState -> IO ()
 runListener appState = do
  AppConfig{..} <- getConfig appState
  when configDbChannelEnabled $
-    void . forkIO $ retryingListen appState
+    void . forkIO . void $ retryingListen appState

 -- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
-retryingListen :: AppState -> IO ()
+-- | This function never returns (but can throw) and return type enforces that.
+retryingListen :: AppState -> IO Void
 retryingListen appState = do
  AppConfig{..} <- AppState.getConfig appState
  let
    dbChannel = toS configDbChannel
-    handleFinally err = do
+    onError err = do
      AppState.putIsListenerOn appState False
      observer $ DBListenFail dbChannel (Right err)
+      when (isDbListenerBug err) $
+        observer DBListenBugHint
      unless configDbPoolAutomaticRecovery $
        killThread mainThreadId

@@ -42,29 +48,39 @@ retryingListen appState = do
      threadDelay (delay * oneSecondInMicro)
      unless (delay == maxDelay) $
        AppState.putNextListenerDelay appState (delay * 2)
+      -- loop running the listener
      retryingListen appState

-  -- forkFinally allows to detect if the thread dies
-  void . flip forkFinally handleFinally $ do
-    dbOrError <- SQL.acquire $ toUtf8 (Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri)
-    case dbOrError of
-      Right db -> do
-        SQL.listen db $ SQL.toPgIdentifier dbChannel
-        AppState.putIsListenerOn appState True
+  -- Execute the listener with with error handling
+  handle onError $ do
+    -- Make sure we don't leak connections on errors
+    bracket
+      -- acquire connection
+      (SQL.acquire $ toUtf8 (Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri))
+      -- release connection
+      (`whenRight` releaseConnection) $
+      -- use connection
+      \case
+        Right db -> do
+          SQL.listen db $ SQL.toPgIdentifier dbChannel
+          AppState.putIsListenerOn appState True

-        delay <- AppState.getNextListenerDelay appState
-        when (delay > 1) $ do -- if we did a retry
-          -- assume we lost notifications, refresh the schema cache
-          AppState.schemaCacheLoader appState
-          -- reset the delay
-          AppState.putNextListenerDelay appState 1
+          delay <- AppState.getNextListenerDelay appState
+          when (delay > 1) $ do -- if we did a retry
+            -- assume we lost notifications, refresh the schema cache
+            AppState.schemaCacheLoader appState
+            -- reset the delay
+            AppState.putNextListenerDelay appState 1

-        observer $ DBListenStart dbChannel
-        SQL.waitForNotifications handleNotification db
+          observer $ DBListenStart dbChannel

-      Left err -> do
-        observer $ DBListenFail dbChannel (Left err)
-        exitFailure
+          -- wait for notifications
+          -- this will never return, in case of an error it will throw and be caught by onError
+          forever $ SQL.waitForNotifications handleNotification db
+
+        Left err -> do
+          observer $ DBListenFail dbChannel (Left err)
+          exitFailure
  where
    observer = AppState.getObserver appState
    mainThreadId = AppState.getMainThreadId appState
@@ -79,3 +95,5 @@ retryingListen appState = do

    cacheReloader =
      AppState.schemaCacheLoader appState
+
+    releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release
@@ -1,3 +1,4 @@
+{-# LANGUAGE RecordWildCards #-}
 {-|
 Module      : PostgREST.Logger
 Description : Logging based on the Observation.hs module. Access logs get sent to stdout and server diagnostic get sent to stderr.
@@ -10,10 +11,16 @@ module PostgREST.Logger
  , LoggerState
  ) where

-import           Control.AutoUpdate    (defaultUpdateSettings,
-                                        mkAutoUpdate, updateAction)
+import           Control.AutoUpdate                (defaultUpdateSettings,
+                                                    mkAutoUpdate,
+                                                    updateAction)
 import           Control.Debounce
-import qualified Data.ByteString.Char8 as BS
+import qualified Data.ByteString.Char8             as BS
+import qualified Data.Text.Encoding                as T
+import qualified Hasql.Decoders                    as HD
+import qualified Hasql.DynamicStatements.Snippet   as SQL hiding (sql)
+import qualified Hasql.DynamicStatements.Statement as SQL
+import qualified Hasql.Statement                   as SQL

 import Data.Time (ZonedTime, defaultTimeLocale, formatTime,
                  getZonedTime)
@@ -26,6 +33,7 @@ import System.IO.Unsafe          (unsafePerformIO)

 import PostgREST.Config      (LogLevel (..))
 import PostgREST.Observation
+import PostgREST.Query       (MainQuery (..))

 import Protolude

@@ -87,19 +95,27 @@ observationLogger loggerState logLevel obs = case obs of
  o@(QueryErrorCodeHighObs _) -> do
    when (logLevel >= LogError) $ do
      logWithZTime loggerState $ observationMessage o
+  o@SchemaCacheEmptyObs ->
+    when (logLevel >= LogError) $ do
+    logWithZTime loggerState $ observationMessage o
  o@(HasqlPoolObs _) -> do
    when (logLevel >= LogDebug) $ do
      logWithZTime loggerState $ observationMessage o
-  o@(DBQuery sql status) -> do
-    -- Does not log SQL when it's empty (for OPTIONS requests or for the default OpenAPI output)
-    when (sql /= mempty && shouldLogResponse logLevel status) $ do
-      logWithZTime loggerState $ observationMessage o
+  QueryObs gq status -> do
+    when (shouldLogResponse logLevel status) $
+      logMainQ loggerState gq
  o@PoolRequest ->
    when (logLevel >= LogDebug) $ do
      logWithZTime loggerState $ observationMessage o
  o@PoolRequestFullfilled ->
    when (logLevel >= LogDebug) $ do
      logWithZTime loggerState $ observationMessage o
+  o@JwtCacheEviction ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessage o
+  o@(JwtCacheLookup _) ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessage o
  o ->
    logWithZTime loggerState $ observationMessage o

@@ -107,3 +123,20 @@ logWithZTime :: LoggerState -> Text -> IO ()
 logWithZTime loggerState txt = do
  zTime <- stateGetZTime loggerState
  hPutStrLn stderr $ toS (formatTime defaultTimeLocale "%d/%b/%Y:%T %z: " zTime) <> txt
+
+logMainQ :: LoggerState -> MainQuery -> IO ()
+logMainQ loggerState MainQuery{mqOpenAPI=(x, y, z),..} =
+  let snipts  = renderSnippet <$> [mqTxVars, fromMaybe mempty mqPreReq, mqMain, x, y, z, fromMaybe mempty mqExplain]
+      -- Does not log SQL when it's empty (happens on OPTIONS requests and when the openapi queries are not generated)
+      logQ q = when (q /= mempty) $ logWithZTime loggerState $ showOnSingleLine '\n' $ T.decodeUtf8 q in
+  mapM_ logQ snipts
+
+-- TODO: maybe patch upstream hasql-dynamic-statements so we have a less hackish way to convert
+-- the SQL.Snippet or maybe don't use hasql-dynamic-statements and resort to plain strings for the queries and use regular hasql
+renderSnippet :: SQL.Snippet -> ByteString
+renderSnippet snippet =
+  let SQL.Statement sql _ _ _ = SQL.dynamicallyParameterized snippet decoder prepared
+      decoder = HD.noResult -- unused
+      prepared = False  -- unused
+  in
+    sql
@@ -0,0 +1,279 @@
+{-# LANGUAGE NamedFieldPuns  #-}
+{-# LANGUAGE RecordWildCards #-}
+{-|
+Module      : PostgREST.MainTx
+Description : PostgREST transaction executor
+
+This module parametrizes, prepares, executes SQL queries and decodes their results.
+-}
+module PostgREST.MainTx
+  ( MainTx (..)
+  , DbResult (..)
+  , ResultSet (..)
+  , mainTx
+  ) where
+
+import           Control.Lens                      ((^?))
+import           Control.Monad.Extra               (whenJust)
+import qualified Data.Aeson.Lens                   as L
+import qualified Data.ByteString                   as BS hiding
+                                                         (break)
+import qualified Data.ByteString.Char8             as BS
+import qualified Data.HashMap.Strict               as HM
+import qualified Data.Set                          as S
+import qualified Hasql.Decoders                    as HD
+import qualified Hasql.DynamicStatements.Statement as SQL
+import qualified Hasql.Session                     as SQL (Session)
+import qualified Hasql.Transaction                 as SQL
+import qualified Hasql.Transaction.Sessions        as SQL
+
+import qualified PostgREST.Error       as Error
+import qualified PostgREST.SchemaCache as SchemaCache
+
+
+import PostgREST.ApiRequest              (ApiRequest (..))
+import PostgREST.ApiRequest.Preferences  (PreferCount (..),
+                                          PreferHandling (..),
+                                          PreferMaxAffected (..),
+                                          PreferTransaction (..),
+                                          Preferences (..))
+import PostgREST.ApiRequest.Types        (Mutation (..))
+import PostgREST.Auth.Types              (AuthResult (..))
+import PostgREST.Config                  (AppConfig (..),
+                                          OpenAPIMode (..))
+import PostgREST.Error                   (Error)
+import PostgREST.MediaType               (MediaType (..))
+import PostgREST.Plan                    (ActionPlan (..),
+                                          CrudPlan (..),
+                                          DbActionPlan (..),
+                                          InfoPlan (..),
+                                          InspectPlan (..))
+import PostgREST.Query                   (MainQuery (..))
+import PostgREST.SchemaCache             (SchemaCache (..))
+import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..))
+import PostgREST.SchemaCache.Routine     (Routine (..), RoutineMap)
+import PostgREST.SchemaCache.Table       (TablesMap)
+
+import Protolude hiding (Handler)
+
+type DbHandler = ExceptT Error SQL.Transaction
+
+data MainTx
+  = DbTx {
+      dqIsoLevel    :: SQL.IsolationLevel
+    , dqTxMode      :: SQL.Mode
+    , dqDbHandler   :: DbHandler DbResult
+    , dqTransaction :: SQL.IsolationLevel -> SQL.Mode -> SQL.Transaction (Either Error DbResult) -> SQL.Session (Either Error DbResult)
+    }
+  | NoDbTx DbResult
+
+data DbResult
+  = DbCrudResult  CrudPlan ResultSet
+  | DbPlanResult  MediaType BS.ByteString
+  | MaybeDbResult InspectPlan  (Maybe (TablesMap, RoutineMap, Maybe Text))
+  | NoDbResult    InfoPlan
+
+-- | Standard result set format used for the mqMain query
+data ResultSet
+  = RSStandard
+  { rsTableTotal :: Maybe Int64
+  -- ^ count of all the table rows
+  , rsQueryTotal :: Int64
+  -- ^ count of the query rows
+  , rsLocation   :: [(BS.ByteString, BS.ByteString)]
+  -- ^ The Location header(only used for inserts) is represented as a list of strings containing
+  -- variable bindings like @"k1=eq.42"@, or the empty list if there is no location header.
+  , rsBody       :: BS.ByteString
+  -- ^ the aggregated body of the query
+  , rsGucHeaders :: Maybe BS.ByteString
+  -- ^ the HTTP headers to be added to the response
+  , rsGucStatus  :: Maybe Text
+  -- ^ the HTTP status to be added to the response
+  , rsInserted   :: Maybe Int64
+  -- ^ the number of rows inserted (Only used for upserts)
+  }
+
+mainTx :: MainQuery -> AppConfig -> AuthResult -> ApiRequest -> ActionPlan -> SchemaCache -> MainTx
+mainTx _ _ _ _ (NoDb x) _ = NoDbTx $ NoDbResult x
+mainTx genQ@MainQuery{..} conf@AppConfig{..} AuthResult{..} apiReq (Db plan) sCache =
+  DbTx isoLvl txMode dbHandler transaction
+  where
+    transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction
+    isoLvl = planIsoLvl conf authRole plan
+    txMode = planTxMode plan
+    dbHandler = do
+      lift $ SQL.statement mempty $ SQL.dynamicallyParameterized mqTxVars
+          HD.noResult configDbPreparedStatements
+      lift $ whenJust mqPreReq $ \q ->
+        SQL.statement mempty $ SQL.dynamicallyParameterized q
+          HD.noResult configDbPreparedStatements
+      actionResult genQ plan conf apiReq sCache
+
+planTxMode :: DbActionPlan -> SQL.Mode
+planTxMode (DbCrud _ x) = pTxMode x
+planTxMode (MayUseDb x) = ipTxmode x
+
+planIsoLvl :: AppConfig -> ByteString -> DbActionPlan -> SQL.IsolationLevel
+planIsoLvl AppConfig{configRoleIsoLvl} role actPlan = case actPlan of
+  DbCrud _ CallReadPlan{crProc} -> fromMaybe roleIsoLvl $ pdIsoLvl crProc
+  _                           -> roleIsoLvl
+  where
+    roleIsoLvl = HM.findWithDefault SQL.ReadCommitted role configRoleIsoLvl
+
+actionResult :: MainQuery -> DbActionPlan -> AppConfig -> ApiRequest -> SchemaCache -> ExceptT Error SQL.Transaction DbResult
+actionResult MainQuery{..} (DbCrud True plan) conf@AppConfig{..} apiReq _ = do
+  explRes <- lift $ SQL.statement mempty $ SQL.dynamicallyParameterized mqMain planRow configDbPreparedStatements
+  optionalRollback conf apiReq
+  pure $ DbPlanResult (pMedia plan) explRes
+
+actionResult MainQuery{..} (DbCrud _ plan@WrappedReadPlan{..}) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} _ = do
+  resultSet@RSStandard{rsTableTotal=tableTotal} <- lift $ SQL.statement mempty $ dynStmt (HD.singleRow $ standardRow True)
+  failNotSingular pMedia resultSet
+  optionalRollback conf apiReq
+  explainTotal <- lift . fmap join $ traverse (\snip ->
+                    SQL.statement mempty $ SQL.dynamicallyParameterized snip decodeExplain configDbPreparedStatements)
+                    mqExplain
+
+  pure $ DbCrudResult plan
+    resultSet{rsTableTotal=case preferCount of
+      Just PlannedCount   -> explainTotal
+      Just EstimatedCount -> if tableTotal > (fromIntegral <$> configDbMaxRows)
+        then max <$> tableTotal <*> explainTotal
+        else tableTotal
+      _                   -> tableTotal}
+  where
+    dynStmt decod = SQL.dynamicallyParameterized mqMain decod configDbPreparedStatements
+
+    decodeExplain :: HD.Result (Maybe Int64)
+    decodeExplain =
+      let row = HD.singleRow $ column HD.bytea in
+      (^? L.nth 0 . L.key "Plan" .  L.key "Plan Rows" . L._Integral) <$> row
+
+actionResult MainQuery{..} (DbCrud _ plan@MutateReadPlan{..}) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} _ = do
+  resultSet <- lift $ SQL.statement mempty $ dynStmt decodeRow
+  failMutation resultSet
+  optionalRollback conf apiReq
+  pure $ DbCrudResult plan resultSet
+  where
+    dynStmt decod = SQL.dynamicallyParameterized mqMain decod configDbPreparedStatements
+    failMutation resultSet = case mrMutation of
+      MutationCreate -> do
+        failNotSingular pMedia resultSet
+      MutationUpdate -> do
+        failNotSingular pMedia resultSet
+        failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet
+      MutationSingleUpsert -> do
+        failPut resultSet
+      MutationDelete -> do
+        failNotSingular pMedia resultSet
+        failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet
+    decodeRow = fromMaybe (RSStandard Nothing 0 mempty mempty Nothing Nothing Nothing) <$> HD.rowMaybe (standardRow False)
+
+actionResult MainQuery{..} (DbCrud _ plan@CallReadPlan{..}) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} _ = do
+  resultSet <- lift $ SQL.statement mempty $ dynStmt decodeRow
+  optionalRollback conf apiReq
+  failNotSingular pMedia resultSet
+  failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet
+  pure $ DbCrudResult plan resultSet
+  where
+    dynStmt decod = SQL.dynamicallyParameterized mqMain decod configDbPreparedStatements
+    decodeRow = fromMaybe (RSStandard (Just 0) 0 mempty mempty Nothing Nothing Nothing) <$> HD.rowMaybe (standardRow True)
+
+actionResult MainQuery{mqOpenAPI=(tblsQ, funcsQ, schQ)} (MayUseDb plan@InspectPlan{ipSchema=tSchema}) AppConfig{..} _ sCache =
+  mainActionQuery
+  where
+    mainActionQuery = lift $
+      case configOpenApiMode of
+        OAFollowPriv -> do
+          tableAccess <- SQL.statement mempty $ SQL.dynamicallyParameterized tblsQ  decodeAccessibleIdentifiers configDbPreparedStatements
+          accFuncs <-  SQL.statement mempty $ SQL.dynamicallyParameterized   funcsQ   SchemaCache.decodeFuncs configDbPreparedStatements
+          schDesc <- SQL.statement mempty $ SQL.dynamicallyParameterized     schQ decodeSchemaDesc configDbPreparedStatements
+          let tbls = HM.filterWithKey (\qi _ -> S.member qi tableAccess) $ SchemaCache.dbTables sCache
+
+          pure $ MaybeDbResult plan (Just (tbls, accFuncs, schDesc))
+        OAIgnorePriv -> do
+          schDesc <- SQL.statement mempty (SQL.dynamicallyParameterized schQ decodeSchemaDesc configDbPreparedStatements)
+
+          let tbls = HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) (SchemaCache.dbTables sCache)
+              routs = HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) (SchemaCache.dbRoutines sCache)
+
+          pure $ MaybeDbResult plan (Just (tbls, routs, schDesc))
+        OADisabled ->
+          pure $ MaybeDbResult plan Nothing
+
+    decodeSchemaDesc :: HD.Result (Maybe Text)
+    decodeSchemaDesc = join <$> HD.rowMaybe (nullableColumn HD.text)
+
+    decodeAccessibleIdentifiers :: HD.Result (S.Set QualifiedIdentifier)
+    decodeAccessibleIdentifiers =
+      let
+        row = QualifiedIdentifier
+          <$> column HD.text
+          <*> column HD.text
+      in
+      S.fromList <$> HD.rowList row
+
+-- Makes sure the querystring pk matches the payload pk
+-- e.g. PUT /items?id=eq.1 { "id" : 1, .. } is accepted,
+-- PUT /items?id=eq.14 { "id" : 2, .. } is rejected.
+-- If this condition is not satisfied then nothing is inserted,
+-- check the WHERE for INSERT in QueryBuilder.hs to see how it's done
+failPut :: ResultSet -> DbHandler ()
+failPut RSStandard{rsQueryTotal=queryTotal} =
+  when (queryTotal /= 1) $ do
+    lift SQL.condemn
+    throwError $ Error.ApiRequestError Error.PutMatchingPkError
+
+-- |
+-- Fail a response if a single JSON object was requested and not exactly one
+-- was found.
+failNotSingular :: MediaType -> ResultSet -> DbHandler ()
+failNotSingular mediaType RSStandard{rsQueryTotal=queryTotal} =
+  when (elem mediaType [MTVndSingularJSON True, MTVndSingularJSON False] && queryTotal /= 1) $ do
+    lift SQL.condemn
+    throwError $ Error.ApiRequestError . Error.SingularityError $ toInteger queryTotal
+
+failExceedsMaxAffectedPref :: (Maybe PreferMaxAffected, Maybe PreferHandling) -> ResultSet -> DbHandler ()
+failExceedsMaxAffectedPref (Nothing,_) _ = pure ()
+failExceedsMaxAffectedPref (Just (PreferMaxAffected n), handling) RSStandard{rsQueryTotal=queryTotal} = when ((queryTotal > n) && (handling == Just Strict)) $ do
+  lift SQL.condemn
+  throwError $ Error.ApiRequestError . Error.MaxAffectedViolationError $ toInteger queryTotal
+
+-- | Set a transaction to roll back if requested
+optionalRollback :: AppConfig -> ApiRequest -> DbHandler ()
+optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} = do
+  lift $ when (shouldRollback || (configDbTxRollbackAll && not shouldCommit)) $ do
+    SQL.sql "SET CONSTRAINTS ALL IMMEDIATE"
+    SQL.condemn
+  where
+    shouldCommit =
+      preferTransaction == Just Commit
+    shouldRollback =
+      preferTransaction == Just Rollback
+
+-- | We use rowList because when doing EXPLAIN (FORMAT TEXT), the result comes as many rows. FORMAT JSON comes as one.
+planRow :: HD.Result BS.ByteString
+planRow = BS.unlines <$> HD.rowList (column HD.bytea)
+
+column :: HD.Value a -> HD.Row a
+column = HD.column . HD.nonNullable
+
+nullableColumn :: HD.Value a -> HD.Row (Maybe a)
+nullableColumn = HD.column . HD.nullable
+
+arrayColumn :: HD.Value a -> HD.Row [a]
+arrayColumn = column . HD.listArray . HD.nonNullable
+
+standardRow :: Bool -> HD.Row ResultSet
+standardRow noLocation =
+  RSStandard <$> nullableColumn HD.int8 <*> column HD.int8
+             <*> (if noLocation then pure mempty else fmap splitKeyValue <$> arrayColumn HD.bytea)
+             <*> (fromMaybe mempty <$> nullableColumn HD.bytea)
+             <*> nullableColumn HD.bytea
+             <*> nullableColumn HD.text
+             <*> nullableColumn HD.int8
+  where
+    splitKeyValue :: ByteString -> (ByteString, ByteString)
+    splitKeyValue kv =
+      let (k, v) = BS.break (== '=') kv in
+      (k, BS.tail v)
@@ -1,3 +1,4 @@
+{-# LANGUAGE RecordWildCards #-}
 {-|
 Module      : PostgREST.Logger
 Description : Metrics based on the Observation module. See Observation.hs.
@@ -19,22 +20,36 @@ import PostgREST.Observation
 import Protolude

 data MetricsState =
-  MetricsState Counter Gauge Gauge Gauge (Vector Label1 Counter) Gauge
+  MetricsState {
+    poolTimeouts         :: Counter,
+    poolAvailable        :: Gauge,
+    poolWaiting          :: Gauge,
+    poolMaxSize          :: Gauge,
+    schemaCacheLoads     :: Vector Label1 Counter,
+    schemaCacheQueryTime :: Gauge,
+    jwtCacheRequests     :: Counter,
+    jwtCacheHits         :: Counter,
+    jwtCacheEvictions    :: Counter
+  }

 init :: Int -> IO MetricsState
 init configDbPoolSize = do
-  poolTimeouts <- register $ counter (Info "pgrst_db_pool_timeouts_total" "The total number of pool connection timeouts")
-  poolAvailable <- register $ gauge (Info "pgrst_db_pool_available" "Available connections in the pool")
-  poolWaiting <- register $ gauge (Info "pgrst_db_pool_waiting" "Requests waiting to acquire a pool connection")
-  poolMaxSize <- register $ gauge (Info "pgrst_db_pool_max" "Max pool connections")
-  schemaCacheLoads <- register $ vector "status" $ counter (Info "pgrst_schema_cache_loads_total" "The total number of times the schema cache was loaded")
-  schemaCacheQueryTime <- register $ gauge (Info "pgrst_schema_cache_query_time_seconds" "The query time in seconds of the last schema cache load")
-  setGauge poolMaxSize (fromIntegral configDbPoolSize)
-  pure $ MetricsState poolTimeouts poolAvailable poolWaiting poolMaxSize schemaCacheLoads schemaCacheQueryTime
+  metricState <- MetricsState <$>
+    register (counter (Info "pgrst_db_pool_timeouts_total" "The total number of pool connection timeouts")) <*>
+    register (gauge (Info "pgrst_db_pool_available" "Available connections in the pool")) <*>
+    register (gauge (Info "pgrst_db_pool_waiting" "Requests waiting to acquire a pool connection")) <*>
+    register (gauge (Info "pgrst_db_pool_max" "Max pool connections")) <*>
+    register (vector "status" $ counter (Info "pgrst_schema_cache_loads_total" "The total number of times the schema cache was loaded")) <*>
+    register (gauge (Info "pgrst_schema_cache_query_time_seconds" "The query time in seconds of the last schema cache load")) <*>
+    register (counter (Info "pgrst_jwt_cache_requests_total" "The total number of JWT cache lookups")) <*>
+    register (counter (Info "pgrst_jwt_cache_hits_total" "The total number of JWT cache hits")) <*>
+    register (counter (Info "pgrst_jwt_cache_evictions_total" "The total number of JWT cache evictions"))
+  setGauge (poolMaxSize metricState) (fromIntegral configDbPoolSize)
+  pure metricState

 -- Only some observations are used as metrics
 observationMetrics :: MetricsState -> ObservationHandler
-observationMetrics (MetricsState poolTimeouts poolAvailable poolWaiting _ schemaCacheLoads schemaCacheQueryTime) obs = case obs of
+observationMetrics MetricsState{..} obs = case obs of
  (PoolAcqTimeoutObs _) -> do
    incCounter poolTimeouts
  (HasqlPoolObs (SQL.ConnectionObservation _ status)) -> case status of
@@ -54,6 +69,9 @@ observationMetrics (MetricsState poolTimeouts poolAvailable poolWaiting _ schema
    setGauge schemaCacheQueryTime resTime
  SchemaCacheErrorObs{} -> do
    withLabel schemaCacheLoads "FAIL" incCounter
+  JwtCacheLookup True -> incCounter jwtCacheRequests *> incCounter jwtCacheHits
+  JwtCacheLookup False -> incCounter jwtCacheRequests
+  JwtCacheEviction -> incCounter jwtCacheEvictions
  _ ->
    pure ()

@@ -1,17 +1,61 @@
 module PostgREST.Network
-  ( resolveHost
+  ( resolveSocketToAddress
+  , escapeHostName
+  , isSpecialHostName
  ) where

-import           Data.IP        (fromHostAddress, fromHostAddress6)
 import           Data.String    (IsString (..))
 import qualified Network.Socket as NS

 import Protolude

-resolveHost :: NS.Socket -> IO (Maybe Text)
-resolveHost sock = do
+-- | Resolves the socket to an address depending on the socket type. The Show
+--   instance of the socket types automatically resolves it to the correct
+--   address. Example resolution:
+-- -----------------------------------------------------
+-- | IPv4         | IPv6             | Unix            |
+-- -----------------------------------------------------
+-- | 127.0.0.1:80 | [2001:db8::1]:80 | /tmp/pgrst.sock |
+-- -----------------------------------------------------
+resolveSocketToAddress :: NS.Socket -> IO Text
+resolveSocketToAddress sock = do
  sn <- NS.getSocketName sock
-  case sn of
-    NS.SockAddrInet _ hostAddr ->  pure $ Just $ fromString $ show $ fromHostAddress hostAddr
-    NS.SockAddrInet6 _ _ hostAddr6 _ -> pure $ Just $ fromString $ show $ fromHostAddress6 hostAddr6
-    _ -> pure Nothing
+  return $ showSocketAddr sn
+
+-- |
+-- >>> let addr_ipv4 = NS.SockAddrInet 80 (NS.tupleToHostAddress (127,0,0,1))
+-- >>> let addr_ipv6 = NS.SockAddrInet6 80 0 (0,0,0,1) 0
+-- >>> let addr_unix = NS.SockAddrUnix "/tmp/pgrst.sock"
+--
+-- >>> showSocketAddr addr_ipv4
+-- "127.0.0.1:80"
+
+-- >>> showSocketAddr addr_ipv6
+-- "[::1]:80"
+--
+-- >>> showSocketAddr addr_unix
+-- "/tmp/pgrst.sock"
+showSocketAddr :: NS.SockAddr -> Text
+showSocketAddr = fromString . show
+
+-- | When printing special addresses like !4 or *6, we use the following mapping.
+--   These special addresses come from:
+--     https://hackage.haskell.org/package/streaming-commons-0.2.3.0/docs/\
+--     Data-Streaming-Network.html#t:HostPreference
+-- TODO: "!6" should not be printed as "0.0.0.0" address.
+escapeHostName :: Text -> Text
+escapeHostName "*"  = "0.0.0.0"
+escapeHostName "*4" = "0.0.0.0"
+escapeHostName "!4" = "0.0.0.0"
+escapeHostName "*6" = "0.0.0.0"
+escapeHostName "!6" = "0.0.0.0"
+escapeHostName h    = h
+
+-- | Check if a hostname is special
+isSpecialHostName :: Text -> Bool
+isSpecialHostName "*"  = True
+isSpecialHostName "*4" = True
+isSpecialHostName "!4" = True
+isSpecialHostName "*6" = True
+isSpecialHostName "!6" = True
+isSpecialHostName _    = False
@@ -1,4 +1,5 @@
-{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE LambdaCase    #-}
 {-|
 Module      : PostgREST.Observation
 Description : This module holds an Observation type which is the core of Observability for PostgREST.
@@ -11,6 +12,8 @@ module PostgREST.Observation
  , ObsFatalError(..)
  , observationMessage
  , ObservationHandler
+  , showOnSingleLine
+  , isDbListenerBug
  ) where

 import qualified Data.ByteString.Lazy       as LBS
@@ -21,34 +24,35 @@ import qualified Hasql.Connection           as SQL
 import qualified Hasql.Pool                 as SQL
 import qualified Hasql.Pool.Observation     as SQL
 import           Network.HTTP.Types.Status  (Status)
-import qualified Network.Socket             as NS
 import           Numeric                    (showFFloat)
 import           PostgREST.Config.PgVersion
 import qualified PostgREST.Error            as Error
+import           PostgREST.Query            (MainQuery)

-import Protolude         hiding (toList)
-import Protolude.Partial (fromJust)
+import Protolude hiding (toList)

 data Observation
-  = AdminStartObs (Maybe Text) (Maybe Int)
+  = AdminStartObs Text
  | AppStartObs ByteString
-  | AppServerPortObs Text NS.PortNumber
-  | AppServerUnixObs FilePath
+  | AppServerAddressObs Text
  | ExitUnsupportedPgVersion PgVersion PgVersion
  | ExitDBNoRecoveryObs
  | ExitDBFatalError ObsFatalError SQL.UsageError
  | DBConnectedObs Text
+  | SchemaCacheEmptyObs
  | SchemaCacheErrorObs (NonEmpty Text) [Text] SQL.UsageError
  | SchemaCacheQueriedObs Double
  | SchemaCacheSummaryObs Text
  | SchemaCacheLoadedObs Double
  | ConnectionRetryObs Int
  | DBListenStart Text
-  | DBListenFail Text (Either SQL.ConnectionError (Either SomeException ()))
+  | DBListenFail Text (Either SQL.ConnectionError SomeException)
  | DBListenRetry Int
+  | DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147
  | DBListenerGotSCacheMsg ByteString
  | DBListenerGotConfigMsg ByteString
-  | DBQuery ByteString Status
+  | DBListenerConnectionCleanupFail SomeException
+  | QueryObs MainQuery Status
  | ConfigReadErrorObs SQL.UsageError
  | ConfigInvalidObs Text
  | ConfigSucceededObs
@@ -60,6 +64,11 @@ data Observation
  | HasqlPoolObs SQL.Observation
  | PoolRequest
  | PoolRequestFullfilled
+  | JwtCacheLookup Bool
+  | JwtCacheEviction
+  | TerminationUnixSignalObs Text
+  | WarpErrorObs Text
+  deriving (Generic)

 data ObsFatalError = ServerAuthError | ServerPgrstBug | ServerError42P05 | ServerError08P01

@@ -67,14 +76,12 @@ type ObservationHandler = Observation -> IO ()

 observationMessage :: Observation -> Text
 observationMessage = \case
-  AdminStartObs host port ->
-    "Admin server listening on " <> fromJust host <> ":" <> show (fromIntegral (fromJust port) :: Integer)
+  AdminStartObs address ->
+    "Admin server listening on " <> address
  AppStartObs ver ->
    "Starting PostgREST " <> T.decodeUtf8 ver <> "..."
-  AppServerPortObs host port ->
-    "API server listening on " <> host <> ":" <> show port
-  AppServerUnixObs sock ->
-    "API server listening on unix socket " <> show sock
+  AppServerAddressObs address ->
+    "API server listening on " <> address
  DBConnectedObs ver ->
    "Successfully connected to " <> ver
  ExitUnsupportedPgVersion pgVer minPgVer ->
@@ -89,6 +96,8 @@ observationMessage = \case
    "If you are using connection poolers in transaction mode, try setting db-prepared-statements to false. " <> jsonMessage usageErr
  ExitDBFatalError ServerError08P01 usageErr ->
    "Connection poolers in statement mode are not supported." <> jsonMessage usageErr
+  SchemaCacheEmptyObs ->
+    T.decodeUtf8 . LBS.toStrict . Error.errorPayload $ Error.NoSchemaCacheError
  SchemaCacheErrorObs dbSchemas extraPaths usageErr ->
    "Failed to load the schema cache using "
      <> "db-schemas=" <> T.intercalate "," (toList dbSchemas)
@@ -108,19 +117,20 @@ observationMessage = \case
  DBListenStart channel -> do
    "Listening for database notifications on the " <> show channel <> " channel"
  DBListenFail channel listenErr ->
-    "Failed listening for database notifications on the " <> show channel <> " channel. " <> (
-      case listenErr of
-        Left err  -> show err
-        Right err -> showListenerError err
-    )
+    "Failed listening for database notifications on the " <> show channel <> " channel. " <>
+      either showListenerConnError showListenerException listenErr
  DBListenRetry delay ->
    "Retrying listening for database notifications in " <> (show delay::Text) <> " seconds..."
+  DBListenBugHint ->
+    "HINT:  This is likely a bug in the notification queue, try executing the following to solve it: select pg_notification_queue_usage();"
  DBListenerGotSCacheMsg channel ->
    "Received a schema cache reload message on the " <> show channel <> " channel"
  DBListenerGotConfigMsg channel ->
    "Received a config reload message on the " <> show channel <> " channel"
-  DBQuery sql _ ->
-    T.decodeUtf8 sql
+  DBListenerConnectionCleanupFail ex ->
+    "Failed during listener connection cleanup: " <> showOnSingleLine '\t' (show ex)
+  QueryObs{} ->
+    mempty -- TODO pending refactor: The logic for printing the query cannot be done here. Join the observationMessage function into observationLogger to avoid this mempty.
  ConfigReadErrorObs usageErr ->
    "Failed to query database settings for the config parameters." <> jsonMessage usageErr
  QueryRoleSettingsErrorObs usageErr ->
@@ -151,14 +161,30 @@ observationMessage = \case
    "Trying to borrow a connection from pool"
  PoolRequestFullfilled ->
    "Borrowed a connection from the pool"
+  JwtCacheLookup _ ->
+    "Looked up a JWT in JWT cache"
+  JwtCacheEviction ->
+    "Evicted entry from JWT cache"
+  TerminationUnixSignalObs signal ->
+    "Received termination unix signal " <> signal
+  WarpErrorObs txt ->
+    "Warp server error: " <> txt
  where
    showMillis :: Double -> Text
-    showMillis x = toS $ showFFloat (Just 1) (x * 1000) ""
+    showMillis x = toS $ showFFloat (Just 1) x ""

    jsonMessage err = T.decodeUtf8 . LBS.toStrict . Error.errorPayload $ Error.PgError False err

-    showListenerError :: Either SomeException () -> Text
-    showListenerError (Right _) = "Failed getting notifications" -- should not happen as the listener will never finish (hasql-notifications uses `forever` internally) with a Right result
-    showListenerError (Left e)  =
-      let showOnSingleLine txt = T.intercalate " " $ T.filter (/= '\t') <$> T.lines txt in -- the errors from hasql-notifications come intercalated with "\t\n"
-      showOnSingleLine $ show e
+
+    showListenerConnError :: SQL.ConnectionError -> Text
+    showListenerConnError = maybe "Connection error" (showOnSingleLine '\t' . T.decodeUtf8)
+
+    showListenerException :: SomeException -> Text
+    showListenerException = showOnSingleLine '\t' . show
+
+
+showOnSingleLine :: Char -> Text -> Text
+showOnSingleLine split txt = T.intercalate " " $ T.filter (/= split) <$> T.lines txt -- the errors from hasql-notifications come intercalated with "\t\n"
+
+isDbListenerBug :: SomeException -> Bool
+isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e
@@ -4,7 +4,8 @@ Description : PostgREST Request Planner

 This module is in charge of building an intermediate
 representation between the HTTP request and the
-final resulting SQL query.
+final response, which may or not result in SQL execution
+(computing OpenAPI or OPTIONS requests don't require database interaction)

 A query tree is built in case of resource embedding. By inferring the
 relationship between tables, join conditions are added for every embedded
@@ -22,7 +23,6 @@ module PostgREST.Plan
  , InspectPlan(..)
  , InfoPlan(..)
  , CrudPlan(..)
-  , CallReadPlan(..)
  ) where

 import qualified Data.HashMap.Strict           as HM
@@ -37,12 +37,7 @@ import Data.List               (delete, lookup)
 import Data.Maybe              (fromJust)
 import Data.Tree               (Tree (..))

-import PostgREST.ApiRequest                  (Action (..),
-                                              ApiRequest (..),
-                                              DbAction (..),
-                                              InvokeMethod (..),
-                                              Mutation (..),
-                                              Payload (..))
+import PostgREST.ApiRequest                  (ApiRequest (..))
 import PostgREST.Config                      (AppConfig (..))
 import PostgREST.Error                       (ApiRequestError (..),
                                              Error (..),
@@ -70,7 +65,6 @@ import PostgREST.SchemaCache.Routine         (MediaHandler (..),
                                              Routine (..),
                                              RoutineMap,
                                              RoutineParam (..),
-                                              funcReturnsCompositeAlias,
                                              funcReturnsScalar,
                                              funcReturnsSetOfScalar,
                                              funcReturnsSingle)
@@ -96,12 +90,13 @@ import Protolude hiding (from)
 -- Setup for doctests
 -- >>> import Data.Ranged.Ranges (fullRange)

+-- Plan for reading or writing to the db
 data CrudPlan
  = WrappedReadPlan
  { wrReadPlan :: ReadPlanTree
  , pTxMode    :: SQL.Mode
  , wrHandler  :: MediaHandler
-  , wrMedia    :: MediaType
+  , pMedia     :: MediaType
  , wrHdrsOnly :: Bool
  , crudQi     :: QualifiedIdentifier
  }
@@ -110,22 +105,22 @@ data CrudPlan
  , mrMutatePlan :: MutatePlan
  , pTxMode      :: SQL.Mode
  , mrHandler    :: MediaHandler
-  , mrMedia      :: MediaType
+  , pMedia       :: MediaType
  , mrMutation   :: Mutation
  , crudQi       :: QualifiedIdentifier
  }
-
-data CallReadPlan = CallReadPlan {
+  | CallReadPlan {
    crReadPlan :: ReadPlanTree
  , crCallPlan :: CallPlan
-  , crTxMode   :: SQL.Mode
+  , pTxMode    :: SQL.Mode
  , crProc     :: Routine
  , crHandler  :: MediaHandler
-  , crMedia    :: MediaType
+  , pMedia     :: MediaType
  , crInvMthd  :: InvokeMethod
  , crQi       :: QualifiedIdentifier
  }

+-- Plan for reading db object metadadta
 data InspectPlan = InspectPlan {
    ipMedia    :: MediaType
  , ipTxmode   :: SQL.Mode
@@ -133,31 +128,51 @@ data InspectPlan = InspectPlan {
  , ipSchema   :: Schema
  }

-data DbActionPlan = DbCrud CrudPlan | DbCall CallReadPlan | MaybeDb InspectPlan
-data InfoPlan     = RelInfoPlan QualifiedIdentifier | RoutineInfoPlan CallReadPlan | SchemaInfoPlan
-data ActionPlan   = Db DbActionPlan | NoDb InfoPlan
+-- A Plan may use the the database or not
+data ActionPlan
+  = Db DbActionPlan
+  | NoDb InfoPlan
+
+type IsDbExplain = Bool
+
+-- A db plan can consist on read/write, rpc call or reading metadata (which may use the db or just use cached objects)
+data DbActionPlan
+  = DbCrud   IsDbExplain CrudPlan
+  | MayUseDb InspectPlan
+
+-- Plans that don't use the database
+data InfoPlan
+  = RelInfoPlan QualifiedIdentifier -- info about relation
+  | RoutineInfoPlan Routine -- info about function
+  | SchemaInfoPlan -- info about schema cache

 actionPlan :: Action -> AppConfig -> ApiRequest -> SchemaCache -> Either Error ActionPlan
 actionPlan act conf apiReq sCache = case act of
-    ActDb dbAct              -> Db <$> dbActionPlan dbAct conf apiReq sCache
-    ActRelationInfo ident    -> pure . NoDb $ RelInfoPlan ident
-    ActRoutineInfo ident inv -> NoDb . RoutineInfoPlan <$> callReadPlan ident conf sCache apiReq inv
-    ActSchemaInfo            -> pure $ NoDb SchemaInfoPlan
+  ActDb dbAct              -> Db <$> dbActionPlan dbAct conf apiReq sCache
+  ActRelationInfo ident    -> pure . NoDb $ RelInfoPlan ident
+  ActRoutineInfo ident inv ->
+    let crPln = callReadPlan ident conf sCache apiReq inv in
+    NoDb . RoutineInfoPlan . crProc <$> crPln
+  ActSchemaInfo            -> pure $ NoDb SchemaInfoPlan

 dbActionPlan :: DbAction -> AppConfig -> ApiRequest -> SchemaCache -> Either Error DbActionPlan
 dbActionPlan dbAct conf apiReq sCache = case dbAct of
  ActRelationRead identifier headersOnly ->
-    DbCrud <$> wrappedReadPlan identifier conf sCache apiReq headersOnly
+    toDbActPlan <$> wrappedReadPlan identifier conf sCache apiReq headersOnly
  ActRelationMut identifier mut ->
-    DbCrud <$> mutateReadPlan mut apiReq identifier conf sCache
+    toDbActPlan <$> mutateReadPlan mut apiReq identifier conf sCache
  ActRoutine identifier invMethod ->
-    DbCall <$> callReadPlan identifier conf sCache apiReq invMethod
+    toDbActPlan <$> callReadPlan identifier conf sCache apiReq invMethod
  ActSchemaRead tSchema headersOnly ->
-    MaybeDb <$> inspectPlan apiReq headersOnly tSchema
+    MayUseDb <$> inspectPlan apiReq headersOnly tSchema
+  where
+    toDbActPlan pl = case pMedia pl of
+      MTVndPlan{} -> DbCrud True pl
+      _           -> DbCrud False pl

 wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Bool -> Either Error CrudPlan
 wrappedReadPlan  identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} headersOnly = do
-  qi <- findTable identifier (dbTables sCache)
+  qi <- findTable identifier sCache
  rPlan <- readPlan qi conf sCache apiRequest
  (handler, mediaType)  <- mapLeft ApiRequestError $ negotiateContent conf apiRequest qi iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan)
  if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
@@ -165,14 +180,14 @@ wrappedReadPlan  identifier conf sCache apiRequest@ApiRequest{iPreferences=Prefe

 mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error CrudPlan
 mutateReadPlan  mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf sCache = do
-  qi <- findTable identifier (dbTables sCache)
+  qi <- findTable identifier sCache
  rPlan <- readPlan qi conf sCache apiRequest
  mPlan <- mutatePlan mutation qi apiRequest sCache rPlan
  if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
  (handler, mediaType)  <- mapLeft ApiRequestError $ negotiateContent conf apiRequest qi iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan)
  return $ MutateReadPlan rPlan mPlan SQL.Write handler mediaType mutation qi

-callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CallReadPlan
+callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CrudPlan
 callReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{preferHandling, invalidPrefs, preferMaxAffected},..} invMethod = do
  let paramKeys = case invMethod of
        InvRead _ -> S.fromList $ fst <$> qsParams'
@@ -241,14 +256,16 @@ findProc qi argumentsKeys allProcs contentMediaType isInvPost =
      | hasSingleUnnamedParam proc = (ts,proc:fs)
      | otherwise                  = (ts,fs)
    -- If the function is called with post and has a single unnamed parameter
-    -- it can be called depending on content type and the parameter type
-    hasSingleUnnamedParam Function{pdParams=[RoutineParam{ppType}]} = isInvPost && case (contentMediaType, ppType) of
-      (MTApplicationJSON, "json")  -> True
-      (MTApplicationJSON, "jsonb") -> True
-      (MTTextPlain, "text")        -> True
-      (MTTextXML, "xml")           -> True
-      (MTOctetStream, "bytea")     -> True
-      _                            -> False
+    -- it can be called depending on content type and the parameter type.
+    -- The parameter must have no declared name (ppName == mempty).
+    hasSingleUnnamedParam Function{pdParams=[RoutineParam{ppName, ppType}]} =
+      isInvPost && ppName == mempty && case (contentMediaType, ppType) of
+        (MTApplicationJSON, "json")  -> True
+        (MTApplicationJSON, "jsonb") -> True
+        (MTTextPlain, "text")        -> True
+        (MTTextXML, "xml")           -> True
+        (MTOctetStream, "bytea")     -> True
+        _                            -> False
    hasSingleUnnamedParam _ = False
    matchesParams proc =
      let
@@ -795,10 +812,10 @@ validateAggFunctions aggFunctionsAllowed (Node rp@ReadPlan {select} forest)
  | otherwise = Node rp <$> traverse (validateAggFunctions aggFunctionsAllowed) forest

 -- | Lookup table in the schema cache before creating read plan
-findTable :: QualifiedIdentifier -> TablesMap -> Either Error QualifiedIdentifier
-findTable qi@QualifiedIdentifier{..} tableMap =
-  case HM.lookup qi tableMap of
-    Nothing -> Left $ SchemaCacheErr $ TableNotFound qiSchema qiName (HM.elems tableMap)
+findTable :: QualifiedIdentifier -> SchemaCache -> Either Error QualifiedIdentifier
+findTable qi@QualifiedIdentifier{..} sc@SchemaCache{dbTables} =
+  case HM.lookup qi dbTables of
+    Nothing -> Left $ SchemaCacheErr $ TableNotFound qiSchema qiName sc
    Just _ -> Right qi

 addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
@@ -948,10 +965,17 @@ addRanges ApiRequest{..} rReq =

 addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
 addLogicTrees ctx ApiRequest{..} rReq =
-  foldr addLogicTreeToNode (Right rReq) qsLogic
+  foldr addLogicTreeToNode (Right rReq) logic
  where
    QueryParams.QueryParams{..} = iQueryParams

+    logic =
+      case iAction of
+        ActDb (ActRelationRead _  _) -> qsLogic
+        ActDb (ActRoutine _ _)       -> qsLogic
+        -- For mutations, take the non-root logic filters. These will only affect the embeddings and not the top level of the returned representation.
+        _                            -> filter (not . null . fst) qsLogic
+
    addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either Error ReadPlanTree -> Either Error ReadPlanTree
    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)

@@ -1024,7 +1048,6 @@ callPlan proc ApiRequest{} paramKeys args readReq = FunctionCall {
 , funCArgs = args
 , funCScalar = funcReturnsScalar proc
 , funCSetOfScalar = funcReturnsSetOfScalar proc
-, funCRetCompositeAlias = funcReturnsCompositeAlias proc
 , funCFilterFields = getFilterFieldNames readReq
 , funCReturning = inferColsEmbedNeeds readReq []
 }
@@ -1121,7 +1144,7 @@ negotiateContent conf ApiRequest{iAction=act, iPreferences=Preferences{preferRep
      m@(MTVndPlan (MTVndSingularJSON strip) _ _) -> mtPlanToNothing $ Just (BuiltinAggSingleJson strip, m)
      m@(MTVndPlan MTVndArrayJSONStrip _ _)       -> mtPlanToNothing $ Just (BuiltinAggArrayJsonStrip, m)
      -- TODO the plan should have its own MediaHandler instead of relying on MediaType
-      m@(MTVndPlan mType _ _)                     -> mtPlanToNothing $ (,) <$> (fst <$> lookupHandler mType) <*> pure m
+      m@(MTVndPlan mType _ _)                     -> mtPlanToNothing $ ((,) . fst <$> lookupHandler mType) <*> pure m
      -- all the other media types can be overridden
      x                                           -> lookupHandler x
    mtPlanToNothing x = if configDbPlanEnabled conf then x else Nothing -- don't find anything if the plan media type is not allowed
@@ -19,14 +19,13 @@ import           PostgREST.SchemaCache.Routine     (Routine (..),
 import Protolude

 data CallPlan = FunctionCall
-  { funCQi                :: QualifiedIdentifier
-  , funCParams            :: CallParams
-  , funCArgs              :: CallArgs
-  , funCScalar            :: Bool
-  , funCSetOfScalar       :: Bool
-  , funCRetCompositeAlias :: Bool
-  , funCFilterFields      :: Set FieldName
-  , funCReturning         :: Set FieldName
+  { funCQi           :: QualifiedIdentifier
+  , funCParams       :: CallParams
+  , funCArgs         :: CallArgs
+  , funCScalar       :: Bool
+  , funCSetOfScalar  :: Bool
+  , funCFilterFields :: Set FieldName
+  , funCReturning    :: Set FieldName
  }

 data CallParams
@@ -1,304 +1,58 @@
-{-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
-- TODO: This module shouldn't depend on SchemaCache
+{-|
+Module      : PostgREST.Query
+Description : PostgREST query building
+
+TODO: This module shouldn't depend on SchemaCache: once OpenAPI is removed, this can be done
+-}
 module PostgREST.Query
-  ( Query (..)
-  , QueryResult (..)
-  , query
-  , getSQLQuery
+  ( mainQuery
+  , MainQuery (..)
  ) where

-import qualified Data.Aeson                        as JSON
-import qualified Data.Aeson.KeyMap                 as KM
-import qualified Data.ByteString                   as BS
-import qualified Data.ByteString.Lazy.Char8        as LBS
-import qualified Data.HashMap.Strict               as HM
-import qualified Data.Set                          as S
-import qualified Hasql.Decoders                    as HD
-import qualified Hasql.DynamicStatements.Snippet   as SQL (Snippet)
-import qualified Hasql.DynamicStatements.Statement as SQL
-import qualified Hasql.Session                     as SQL (Session)
-import qualified Hasql.Transaction                 as SQL
-import qualified Hasql.Transaction.Sessions        as SQL
+import qualified Hasql.DynamicStatements.Snippet as SQL hiding (sql)

-import qualified PostgREST.Error              as Error
+import qualified PostgREST.Query.PreQuery     as PreQuery
 import qualified PostgREST.Query.QueryBuilder as QueryBuilder
+import qualified PostgREST.Query.SqlFragment  as SqlFragment
 import qualified PostgREST.Query.Statements   as Statements
-import qualified PostgREST.SchemaCache        as SchemaCache


-import PostgREST.ApiRequest              (ApiRequest (..),
-                                          Mutation (..))
-import PostgREST.ApiRequest.Preferences  (PreferCount (..),
-                                          PreferHandling (..),
-                                          PreferMaxAffected (..),
-                                          PreferTimezone (..),
-                                          PreferTransaction (..),
-                                          Preferences (..),
-                                          shouldCount)
+import PostgREST.ApiRequest              (ApiRequest (..))
+import PostgREST.ApiRequest.Preferences  (Preferences (..),
+                                          shouldExplainCount)
 import PostgREST.Auth.Types              (AuthResult (..))
-import PostgREST.Config                  (AppConfig (..),
-                                          OpenAPIMode (..))
-import PostgREST.Config.PgVersion        (PgVersion (..))
-import PostgREST.Error                   (Error)
-import PostgREST.MediaType               (MediaType (..))
+import PostgREST.Config                  (AppConfig (..))
 import PostgREST.Plan                    (ActionPlan (..),
-                                          CallReadPlan (..),
                                          CrudPlan (..),
                                          DbActionPlan (..),
-                                          InfoPlan (..),
                                          InspectPlan (..))
-import PostgREST.Plan.MutatePlan         (MutatePlan (..))
-import PostgREST.Query.SqlFragment       (escapeIdentList, fromQi,
-                                          intercalateSnippet,
-                                          setConfigWithConstantName,
-                                          setConfigWithConstantNameJSON,
-                                          setConfigWithDynamicName)
-import PostgREST.Query.Statements        (ResultSet (..))
-import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..))
-import PostgREST.SchemaCache.Routine     (Routine (..), RoutineMap)
-import PostgREST.SchemaCache.Table       (TablesMap)

 import Protolude hiding (Handler)

-type DbHandler = ExceptT Error SQL.Transaction
+-- The Queries that run on every request
+data MainQuery = MainQuery
+  { mqTxVars  :: SQL.Snippet       -- ^ the transaction variables that always run on each query
+  , mqPreReq  :: Maybe SQL.Snippet -- ^ the pre-request function that runs if enabled
+  -- TODO only one of the following queries actually runs on each request, once OpenAPI is removed from core it will be easier to refactor this
+  , mqMain    :: SQL.Snippet
+  , mqOpenAPI :: (SQL.Snippet, SQL.Snippet, SQL.Snippet)
+  , mqExplain :: Maybe SQL.Snippet     -- ^ the explain query that gets generated for the "Prefer: count=estimated" case
+  }

-data Query
-  = DbQuery {
-      dqIsoLevel    :: SQL.IsolationLevel
-    , dqTxMode      :: SQL.Mode
-    , dqDbHandler   :: DbHandler QueryResult
-    , dqTransaction :: SQL.IsolationLevel -> SQL.Mode -> SQL.Transaction (Either Error QueryResult) -> SQL.Session (Either Error QueryResult)
-    , dqSQL         :: ByteString
-    }
-  | NoDbQuery QueryResult
-
-data QueryResult
-  = DbCrudResult  CrudPlan ResultSet
-  | DbCallResult  CallReadPlan  ResultSet
-  | MaybeDbResult InspectPlan  (Maybe (TablesMap, RoutineMap, Maybe Text))
-  | NoDbResult    InfoPlan
-
-query :: AppConfig -> AuthResult -> ApiRequest -> ActionPlan -> SchemaCache -> PgVersion -> Query
-query _ _ _ (NoDb x) _ _ = NoDbQuery $ NoDbResult x
-query config AuthResult{..} apiReq (Db plan) sCache pgVer =
-  DbQuery isoLvl txMode dbHandler transaction mainSQLQuery
-  where
-    transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction
-    prepared = configDbPreparedStatements config
-    isoLvl = planIsoLvl config authRole plan
-    txMode = planTxMode plan
-    (mainActionQuery, mainSQLQuery) = actionQuery plan config apiReq pgVer sCache
-    dbHandler = do
-      setPgLocals plan config authClaims authRole apiReq
-      runPreReq config
-      mainActionQuery
-
-planTxMode :: DbActionPlan -> SQL.Mode
-planTxMode (DbCrud x)  = pTxMode x
-planTxMode (DbCall x)  = crTxMode x
-planTxMode (MaybeDb x) = ipTxmode x
-
-planIsoLvl :: AppConfig -> ByteString -> DbActionPlan -> SQL.IsolationLevel
-planIsoLvl AppConfig{configRoleIsoLvl} role actPlan = case actPlan of
-  DbCall CallReadPlan{crProc} -> fromMaybe roleIsoLvl $ pdIsoLvl crProc
-  _                           -> roleIsoLvl
-  where
-    roleIsoLvl = HM.findWithDefault SQL.ReadCommitted role configRoleIsoLvl
-
-- TODO: Generate the Hasql Statement in a diferent module after the OpenAPI functionality is removed
-actionQuery :: DbActionPlan -> AppConfig -> ApiRequest -> PgVersion -> SchemaCache -> (DbHandler QueryResult, ByteString)
-actionQuery (DbCrud plan@WrappedReadPlan{..}) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} _ _ =
-  (mainActionQuery, mainSQLQuery)
-  where
-    countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan
-    (result, mainSQLQuery) = Statements.prepareRead
-      (QueryBuilder.readPlanToQuery wrReadPlan)
-      (if preferCount == Just EstimatedCount then
-         -- LIMIT maxRows + 1 so we can determine below that maxRows was surpassed
-         QueryBuilder.limitedQuery countQuery ((+ 1) <$> configDbMaxRows)
-       else
-         countQuery
-      )
-      (shouldCount preferCount)
-      wrMedia
-      wrHandler
-      configDbPreparedStatements
-    mainActionQuery = do
-      resultSet <- lift $ SQL.statement mempty result
-      failNotSingular wrMedia resultSet
-      optionalRollback conf apiReq
-      DbCrudResult plan <$> resultSetWTotal conf apiReq resultSet countQuery
-
-actionQuery (DbCrud plan@MutateReadPlan{..}) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} _ _ =
-  (mainActionQuery, mainSQLQuery)
-  where
-    (isPut, isInsert, pkCols) = case mrMutatePlan of {Insert{where_,insPkCols} -> ((not . null) where_, True, insPkCols); _ -> (False,False, mempty);}
-    (result, mainSQLQuery) = Statements.prepareWrite
-      (QueryBuilder.readPlanToQuery mrReadPlan)
-      (QueryBuilder.mutatePlanToQuery mrMutatePlan)
-      isInsert
-      isPut
-      mrMedia
-      mrHandler
-      preferRepresentation
-      preferResolution
-      pkCols
-      configDbPreparedStatements
-    failMutation resultSet = case mrMutation of
-      MutationCreate -> do
-        failNotSingular mrMedia resultSet
-      MutationUpdate -> do
-        failNotSingular mrMedia resultSet
-        failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet
-      MutationSingleUpsert -> do
-        failPut resultSet
-      MutationDelete -> do
-        failNotSingular mrMedia resultSet
-        failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet
-    mainActionQuery = do
-      resultSet <- lift $ SQL.statement mempty result
-      failMutation resultSet
-      optionalRollback conf apiReq
-      pure $ DbCrudResult plan resultSet
-
-actionQuery (DbCall plan@CallReadPlan{..}) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} pgVer _ =
-  (mainActionQuery, mainSQLQuery)
-  where
-    (result, mainSQLQuery) = Statements.prepareCall
-      crProc
-      (QueryBuilder.callPlanToQuery crCallPlan pgVer)
-      (QueryBuilder.readPlanToQuery crReadPlan)
-      (QueryBuilder.readPlanToCountQuery crReadPlan)
-      (shouldCount preferCount)
-      crMedia
-      crHandler
-      configDbPreparedStatements
-    mainActionQuery = do
-      resultSet <- lift $ SQL.statement mempty result
-      optionalRollback conf apiReq
-      failNotSingular crMedia resultSet
-      failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet
-      pure $ DbCallResult plan resultSet
-
-actionQuery (MaybeDb plan@InspectPlan{ipSchema=tSchema}) AppConfig{..} _ _ sCache =
-  (mainActionQuery, mempty)
-  where
-    mainActionQuery = lift $
-      case configOpenApiMode of
-        OAFollowPriv -> do
-          tableAccess <- SQL.statement [tSchema] (SchemaCache.accessibleTables configDbPreparedStatements)
-          MaybeDbResult plan . Just <$> ((,,)
-                (HM.filterWithKey (\qi _ -> S.member qi tableAccess) $ SchemaCache.dbTables sCache)
-            <$> SQL.statement ([tSchema], configDbHoistedTxSettings) (SchemaCache.accessibleFuncs configDbPreparedStatements)
-            <*> SQL.statement tSchema (SchemaCache.schemaDescription configDbPreparedStatements))
-        OAIgnorePriv ->
-          MaybeDbResult plan . Just <$> ((,,)
-                (HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) $ SchemaCache.dbTables sCache)
-                (HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) $ SchemaCache.dbRoutines sCache)
-            <$> SQL.statement tSchema (SchemaCache.schemaDescription configDbPreparedStatements))
-        OADisabled ->
-          pure $ MaybeDbResult plan Nothing
-
-- Makes sure the querystring pk matches the payload pk
-- e.g. PUT /items?id=eq.1 { "id" : 1, .. } is accepted,
-- PUT /items?id=eq.14 { "id" : 2, .. } is rejected.
-- If this condition is not satisfied then nothing is inserted,
-- check the WHERE for INSERT in QueryBuilder.hs to see how it's done
-failPut :: ResultSet -> DbHandler ()
-failPut RSPlan{} = pure ()
-failPut RSStandard{rsQueryTotal=queryTotal} =
-  when (queryTotal /= 1) $ do
-    lift SQL.condemn
-    throwError $ Error.ApiRequestError Error.PutMatchingPkError
-
-resultSetWTotal :: AppConfig -> ApiRequest -> ResultSet -> SQL.Snippet -> DbHandler ResultSet
-resultSetWTotal _ _ rs@RSPlan{} _ = return rs
-resultSetWTotal AppConfig{..} ApiRequest{iPreferences=Preferences{..}} rs@RSStandard{rsTableTotal=tableTotal} countQuery =
-  case preferCount of
-    Just PlannedCount -> do
-      total <- explain
-      return rs{rsTableTotal=total}
-    Just EstimatedCount ->
-      if tableTotal > (fromIntegral <$> configDbMaxRows) then do
-        total <- max tableTotal <$> explain
-        return rs{rsTableTotal=total}
-      else
-        return rs
-    Just ExactCount ->
-      return rs
-    Nothing ->
-      return rs
-  where
-    explain =
-      lift . SQL.statement mempty . Statements.preparePlanRows countQuery $
-        configDbPreparedStatements
-
-- |
-- Fail a response if a single JSON object was requested and not exactly one
-- was found.
-failNotSingular :: MediaType -> ResultSet -> DbHandler ()
-failNotSingular _ RSPlan{} = pure ()
-failNotSingular mediaType RSStandard{rsQueryTotal=queryTotal} =
-  when (elem mediaType [MTVndSingularJSON True, MTVndSingularJSON False] && queryTotal /= 1) $ do
-    lift SQL.condemn
-    throwError $ Error.ApiRequestError . Error.SingularityError $ toInteger queryTotal
-
-failExceedsMaxAffectedPref :: (Maybe PreferMaxAffected, Maybe PreferHandling) -> ResultSet -> DbHandler ()
-failExceedsMaxAffectedPref (Nothing,_) _ = pure ()
-failExceedsMaxAffectedPref _ RSPlan{} = pure ()
-failExceedsMaxAffectedPref (Just (PreferMaxAffected n), handling) RSStandard{rsQueryTotal=queryTotal} = when ((queryTotal > n) && (handling == Just Strict)) $ do
-  lift SQL.condemn
-  throwError $ Error.ApiRequestError . Error.MaxAffectedViolationError $ toInteger queryTotal
-
-- | Set a transaction to roll back if requested
-optionalRollback :: AppConfig -> ApiRequest -> DbHandler ()
-optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} = do
-  lift $ when (shouldRollback || (configDbTxRollbackAll && not shouldCommit)) $ do
-    SQL.sql "SET CONSTRAINTS ALL IMMEDIATE"
-    SQL.condemn
-  where
-    shouldCommit =
-      preferTransaction == Just Commit
-    shouldRollback =
-      preferTransaction == Just Rollback
-
-- | Set transaction scoped settings
-setPgLocals :: DbActionPlan -> AppConfig -> KM.KeyMap JSON.Value -> BS.ByteString -> ApiRequest -> DbHandler ()
-setPgLocals dbActPlan AppConfig{..} claims role ApiRequest{..} = lift $
-  SQL.statement mempty $ SQL.dynamicallyParameterized
-    -- To ensure `GRANT SET ON PARAMETER <superuser_setting> TO authenticator` works, the role settings must be set before the impersonated role.
-    -- Otherwise the GRANT SET would have to be applied to the impersonated role. See https://github.com/PostgREST/postgrest/issues/3045
-    ("select " <> intercalateSnippet ", " (searchPathSql : roleSettingsSql ++ roleSql ++ claimsSql ++ [methodSql, pathSql] ++ headersSql ++ cookiesSql ++ timezoneSql ++ funcSettingsSql ++ appSettingsSql))
-    HD.noResult configDbPreparedStatements
-  where
-    methodSql = setConfigWithConstantName ("request.method", iMethod)
-    pathSql = setConfigWithConstantName ("request.path", iPath)
-    headersSql = setConfigWithConstantNameJSON "request.headers" iHeaders
-    cookiesSql = setConfigWithConstantNameJSON "request.cookies" iCookies
-    claimsSql = [setConfigWithConstantName ("request.jwt.claims", LBS.toStrict $ JSON.encode claims)]
-    roleSql = [setConfigWithConstantName ("role", role)]
-    roleSettingsSql = setConfigWithDynamicName <$> HM.toList (fromMaybe mempty $ HM.lookup role configRoleSettings)
-    appSettingsSql = setConfigWithDynamicName <$> (join bimap toUtf8 <$> configAppSettings)
-    timezoneSql = maybe mempty (\(PreferTimezone tz) -> [setConfigWithConstantName ("timezone", tz)]) $ preferTimezone iPreferences
-    funcSettingsSql = setConfigWithDynamicName <$> (join bimap toUtf8 <$> funcSettings)
-    searchPathSql =
-      let schemas = escapeIdentList (iSchema : configDbExtraSearchPath) in
-      setConfigWithConstantName ("search_path", schemas)
-    funcSettings = case dbActPlan of
-      DbCall CallReadPlan{crProc} -> pdFuncSettings crProc
-      _                           -> mempty
-
-- | Runs the pre-request function.
-runPreReq :: AppConfig -> DbHandler ()
-runPreReq conf = lift $ traverse_ (SQL.statement mempty . stmt) (configDbPreRequest conf)
-  where
-    stmt req = SQL.dynamicallyParameterized
-      ("select " <> fromQi req <> "()")
-      HD.noResult
-      (configDbPreparedStatements conf)
-
-getSQLQuery :: Query -> ByteString
-getSQLQuery DbQuery{dqSQL} = dqSQL
-getSQLQuery _              = mempty
+mainQuery :: ActionPlan -> AppConfig -> ApiRequest -> AuthResult -> Maybe QualifiedIdentifier -> MainQuery
+mainQuery (NoDb _) _ _ _ _ = MainQuery mempty Nothing mempty (mempty, mempty, mempty) mempty
+mainQuery (Db plan) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} authRes preReq =
+  let genQ = MainQuery (PreQuery.txVarQuery plan conf authRes apiReq) (PreQuery.preReqQuery <$> preReq) in
+  case plan of
+    DbCrud _ WrappedReadPlan{..} ->
+      let countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan in
+      genQ (Statements.mainRead wrReadPlan countQuery preferCount configDbMaxRows pMedia wrHandler) (mempty, mempty, mempty)
+      (if shouldExplainCount preferCount then Just (Statements.postExplain countQuery) else Nothing)
+    DbCrud _ MutateReadPlan{..} ->
+      genQ (Statements.mainWrite mrReadPlan mrMutatePlan pMedia mrHandler preferRepresentation preferResolution) (mempty, mempty, mempty) mempty
+    DbCrud _ CallReadPlan{..} ->
+      genQ (Statements.mainCall crProc crCallPlan crReadPlan preferCount pMedia crHandler) (mempty, mempty, mempty) mempty
+    MayUseDb InspectPlan{ipSchema=tSchema} ->
+      genQ mempty (SqlFragment.accessibleTables tSchema, SqlFragment.accessibleFuncs tSchema, SqlFragment.schemaDescription tSchema) mempty
@@ -0,0 +1,64 @@
+{-# LANGUAGE NamedFieldPuns  #-}
+{-# LANGUAGE RecordWildCards #-}
+{-|
+Module      : PostgREST.Query.PreQuery
+Description : Builds queries that run prior to the main query
+-}
+module PostgREST.Query.PreQuery
+  ( txVarQuery
+  , preReqQuery
+  ) where
+
+import qualified Data.Aeson                      as JSON
+import qualified Data.ByteString.Lazy.Char8      as LBS
+import qualified Data.HashMap.Strict             as HM
+import qualified Hasql.DynamicStatements.Snippet as SQL hiding (sql)
+
+
+
+import PostgREST.ApiRequest              (ApiRequest (..))
+import PostgREST.ApiRequest.Preferences  (PreferTimezone (..),
+                                          Preferences (..))
+import PostgREST.Auth.Types              (AuthResult (..))
+import PostgREST.Config                  (AppConfig (..))
+import PostgREST.Plan                    (CrudPlan (..),
+                                          DbActionPlan (..))
+import PostgREST.Query.SqlFragment       (escapeIdentList, fromQi,
+                                          intercalateSnippet,
+                                          setConfigWithConstantName,
+                                          setConfigWithConstantNameJSON,
+                                          setConfigWithDynamicName)
+import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..))
+import PostgREST.SchemaCache.Routine     (Routine (..))
+
+import Protolude hiding (Handler)
+
+-- sets transaction variables
+txVarQuery :: DbActionPlan -> AppConfig -> AuthResult -> ApiRequest -> SQL.Snippet
+txVarQuery dbActPlan AppConfig{..} AuthResult{..} ApiRequest{..} =
+    -- To ensure `GRANT SET ON PARAMETER <superuser_setting> TO authenticator` works, the role settings must be set before the impersonated role.
+    -- Otherwise the GRANT SET would have to be applied to the impersonated role. See https://github.com/PostgREST/postgrest/issues/3045
+    "select " <> intercalateSnippet ", " (
+      searchPathSql : roleSettingsSql ++ roleSql ++ claimsSql ++ [methodSql, pathSql] ++ headersSql ++ cookiesSql ++ timezoneSql ++ funcSettingsSql ++ appSettingsSql
+    )
+  where
+    methodSql = setConfigWithConstantName ("request.method", iMethod)
+    pathSql = setConfigWithConstantName ("request.path", iPath)
+    headersSql = setConfigWithConstantNameJSON "request.headers" iHeaders
+    cookiesSql = setConfigWithConstantNameJSON "request.cookies" iCookies
+    claimsSql = [setConfigWithConstantName ("request.jwt.claims", LBS.toStrict $ JSON.encode authClaims)]
+    roleSql = [setConfigWithConstantName ("role", authRole)]
+    roleSettingsSql = setConfigWithDynamicName <$> HM.toList (fromMaybe mempty $ HM.lookup authRole configRoleSettings)
+    appSettingsSql = setConfigWithDynamicName . join bimap toUtf8 <$> configAppSettings
+    timezoneSql = maybe mempty (\(PreferTimezone tz) -> [setConfigWithConstantName ("timezone", tz)]) $ preferTimezone iPreferences
+    funcSettingsSql = setConfigWithDynamicName . join bimap toUtf8 <$> funcSettings
+    searchPathSql =
+      let schemas = escapeIdentList (iSchema : configDbExtraSearchPath) in
+      setConfigWithConstantName ("search_path", schemas)
+    funcSettings = case dbActPlan of
+      DbCrud _ CallReadPlan{crProc} -> pdFuncSettings crProc
+      _                             -> mempty
+
+-- runs the pre-request function
+preReqQuery :: QualifiedIdentifier -> SQL.Snippet
+preReqQuery preRequest = "select " <> fromQi preRequest <> "()"
@@ -28,7 +28,6 @@ import Data.Maybe (fromJust)
 import Data.Tree  (Tree (..))

 import PostgREST.ApiRequest.Preferences   (PreferResolution (..))
-import PostgREST.Config.PgVersion         (PgVersion, pgVersion130)
 import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                           Junction (..),
@@ -169,8 +168,8 @@ mutatePlanToQuery (Delete mainQi logicForest returnings) =
  where
    whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)

-callPlanToQuery :: CallPlan -> PgVersion -> SQL.Snippet
-callPlanToQuery (FunctionCall qi params arguments returnsScalar returnsSetOfScalar returnsCompositeAlias filterFields returnings) pgVer =
+callPlanToQuery :: CallPlan -> SQL.Snippet
+callPlanToQuery (FunctionCall qi params arguments returnsScalar returnsSetOfScalar filterFields returnings) =
  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call.pgrst_scalar" else returnedColumns) <> " " <>
  fromCall
  where
@@ -186,9 +185,8 @@ callPlanToQuery (FunctionCall qi params arguments returnsScalar returnsSetOfScal
                         "LATERAL " <> callIt (fmtParams prms)

    callIt :: SQL.Snippet -> SQL.Snippet
-    callIt argument | pgVer < pgVersion130 && returnsCompositeAlias = "(SELECT (" <> fromQi qi <> "(" <> argument <> ")).*) pgrst_call"
-                    | returnsScalar || returnsSetOfScalar           = "(SELECT " <> fromQi qi <> "(" <> argument <> ") pgrst_scalar) pgrst_call"
-                    | otherwise                                     = fromQi qi <> "(" <> argument <> ") pgrst_call"
+    callIt argument | returnsScalar || returnsSetOfScalar = "(SELECT " <> fromQi qi <> "(" <> argument <> ") pgrst_scalar) pgrst_call"
+                    | otherwise                           = fromQi qi <> "(" <> argument <> ") pgrst_call"

    fmtParams :: [RoutineParam] -> SQL.Snippet
    fmtParams prms = intercalateSnippet ", "
@@ -6,13 +6,22 @@ Module      : PostgREST.Query.SqlFragment
 Description : Helper functions for PostgREST.QueryBuilder.
 -}
 module PostgREST.Query.SqlFragment
-  ( noLocationF
-  , handlerF
+  ( accessibleFuncs
+  , accessibleTables
+  , addConfigPgrstInserted
  , countF
-  , groupF
+  , currentSettingF
+  , escapeIdent
+  , escapeIdentList
+  , explainF
+  , fromJsonBodyF
  , fromQi
+  , groupF
+  , handlerF
+  , intercalateSnippet
  , limitOffsetF
  , locationF
+  , noLocationF
  , orderF
  , pgFmtColumn
  , pgFmtFilter
@@ -21,25 +30,19 @@ module PostgREST.Query.SqlFragment
  , pgFmtLogicTree
  , pgFmtOrderTerm
  , pgFmtSelectItem
-  , pgFmtSpreadSelectItem
  , pgFmtSpreadJoinSelectItem
-  , fromJsonBodyF
+  , pgFmtSpreadSelectItem
  , responseHeadersF
  , responseStatusF
-  , addConfigPgrstInserted
-  , currentSettingF
  , returningF
+  , schemaDescription
+  , setConfigWithConstantName
+  , setConfigWithConstantNameJSON
+  , setConfigWithDynamicName
  , singleParameter
  , sourceCTE
  , sourceCTEName
  , unknownEncoder
-  , intercalateSnippet
-  , explainF
-  , setConfigWithConstantName
-  , setConfigWithDynamicName
-  , setConfigWithConstantNameJSON
-  , escapeIdent
-  , escapeIdentList
  ) where

 import qualified Data.Aeson                      as JSON
@@ -87,7 +90,8 @@ import PostgREST.RangeQuery              (NonnegRange, allRange,
                                          rangeLimit, rangeOffset)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                          QualifiedIdentifier (..),
-                                          RelIdentifier (..))
+                                          RelIdentifier (..),
+                                          escapeIdent, trimNullChars)
 import PostgREST.SchemaCache.Routine     (MediaHandler (..),
                                          Routine (..),
                                          funcReturnsScalar,
@@ -160,9 +164,6 @@ pgBuildArrayLiteral vals =
 pgFmtIdent :: Text -> SQL.Snippet
 pgFmtIdent x = SQL.sql . encodeUtf8 $ escapeIdent x

-escapeIdent :: Text -> Text
-escapeIdent x = "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
-
 -- Only use it if the input comes from the database itself, like on `jsonb_build_object('column_from_a_table', val)..`
 pgFmtLit :: Text -> Text
 pgFmtLit x =
@@ -173,9 +174,6 @@ pgFmtLit x =
    then "E" <> slashed
    else slashed

-trimNullChars :: Text -> Text
-trimNullChars = T.takeWhile (/= '\x0')
-
 -- |
 -- Format a list of identifiers and separate them by commas.
 --
@@ -314,41 +312,45 @@ pgFmtFullSelName aggAlias fieldName = case fieldName of
 -- TODO: At this stage there shouldn't be a Maybe since ApiRequest should ensure that an INSERT/UPDATE has a body
 fromJsonBodyF :: Maybe LBS.ByteString -> [CoercibleField] -> Bool -> Bool -> Bool -> SQL.Snippet
 fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
-  (if includeSelect then "SELECT " <> namedCols <> " " else mempty) <>
-  "FROM (SELECT " <> jsonPlaceHolder <> " AS json_data) pgrst_payload, " <>
-  (if includeDefaults
-    then if isJsonObject
-      then "LATERAL (SELECT " <> defsJsonb <> " || pgrst_payload.json_data AS val) pgrst_json_defs, "
-      else "LATERAL (SELECT jsonb_agg(" <> defsJsonb <> " || elem) AS val from jsonb_array_elements(pgrst_payload.json_data) elem) pgrst_json_defs, "
-    else mempty) <>
-  "LATERAL (SELECT " <> parsedCols <> " FROM " <>
-    (if null fields -- when json keys are empty, e.g. when payload is `{}` or `[{}, {}]`
-      then SQL.sql $
-        if isJsonObject
-          then "(values(1)) _ "                                  -- only 1 row for an empty json object '{}'
-          else jsonArrayElementsF <> "(" <> finalBodyF <> ") _ " -- extract rows of a json array of empty objects `[{}, {}]`
-      else jsonToRecordsetF <> "(" <> SQL.sql finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
-    ) <>
-  ") pgrst_body "
+  selectClause <> fromClause <> defaultsClause <> lateralClause <> " pgrst_body "
  where
+    selectClause = if includeSelect then "SELECT " <> namedCols <> " " else mempty
+    fromClause = "FROM (SELECT " <> jsonPlaceHolder <> " AS json_data) pgrst_payload, "
+    defaultsClause
+      | includeDefaults && isJsonObject     = "LATERAL (SELECT " <> defsJsonb <> " || pgrst_payload.json_data AS val) pgrst_json_defs, "
+      | includeDefaults && not isJsonObject = "LATERAL (SELECT jsonb_agg(" <> defsJsonb <> " || elem) AS val from jsonb_array_elements(pgrst_payload.json_data) elem) pgrst_json_defs, "
+      | otherwise = mempty
+    lateralClause = "LATERAL (SELECT " <> parsedCols <> " FROM " <> lateralFieldsSource <> ")"
+
    namedCols = intercalateSnippet ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . cfName <$> fields
    parsedCols = intercalateSnippet ", " $ pgFmtCoerceNamed <$> fields
    typedCols = intercalateSnippet ", " $ pgFmtIdent . cfName <> const " " <> SQL.sql . encodeUtf8 . cfIRType <$> fields
+
+    lateralFieldsSource = if null fields then emptyFieldsSource else nonEmptyFieldsSource
+      where
+        limitClause = if includeLimitOne then "LIMIT 1" else mempty
+        nonEmptyFieldsSource = jsonToRecordsetF <> "(" <> finalBodyF <> ") AS _(" <> typedCols <> ") " <> limitClause
+        -- when json keys are empty, e.g. when payload is `{}` or `[{}, {}]`
+        emptyFieldsSource = if isJsonObject
+                              then "(values(1)) _ " -- only 1 row for an empty json object '{}'
+                              else jsonArrayElementsF <> "(" <> finalBodyF <> ") _ " -- extract rows of a json array of empty objects `[{}, {}]`
+
    defsJsonb = SQL.sql $ "jsonb_build_object(" <> BS.intercalate "," fieldsWDefaults <> ")"
-    fieldsWDefaults = mapMaybe (\case
-        CoercibleField{cfName=nam, cfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
-        CoercibleField{cfDefault=Nothing} -> Nothing
-      ) fields
+    fieldsWDefaults = mapMaybe extractFieldDefault fields
+      where
+        extractFieldDefault CoercibleField{cfName=nam, cfDefault=Just def} = Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
+        extractFieldDefault CoercibleField{cfDefault=Nothing}              = Nothing
+
    (finalBodyF, jsonArrayElementsF, jsonToRecordsetF) =
      if includeDefaults
        then ("pgrst_json_defs.val", "jsonb_array_elements", if isJsonObject then "jsonb_to_record" else "jsonb_to_recordset")
        else ("pgrst_payload.json_data", "json_array_elements", if isJsonObject then "json_to_record" else "json_to_recordset")
+
    jsonPlaceHolder = SQL.encoderAndParam (HE.nullable $ if includeDefaults then HE.jsonbLazyBytes else HE.jsonLazyBytes) body
    isJsonObject = -- light validation as pg's json_to_record(set) already validates that the body is valid JSON. We just need to know whether the body looks like an object or not.
-      let
-        insignificantWhitespace = [32,9,10,13] --" \t\n\r" [32,9,10,13] https://datatracker.ietf.org/doc/html/rfc8259#section-2
-      in
      LBS.take 1 (LBS.dropWhile (`elem` insignificantWhitespace) (fromMaybe mempty body)) == "{"
+        where
+          insignificantWhitespace = [32,9,10,13] --" \t\n\r" [32,9,10,13] https://datatracker.ietf.org/doc/html/rfc8259#section-2

 pgFmtOrderTerm :: QualifiedIdentifier -> CoercibleOrderTerm -> SQL.Snippet
 pgFmtOrderTerm qi ot =
@@ -586,3 +588,117 @@ handlerF rout = \case
  BuiltinOvAggCsv            -> asCsvF
  CustomFunc funcQi target   -> customFuncF rout funcQi target
  NoAgg                      -> "''::text"
+
+schemaDescription :: Text -> SQL.Snippet
+schemaDescription schema =
+  "SELECT pg_catalog.obj_description(" <> encoded <> "::regnamespace, 'pg_namespace')"
+  where
+    encoded = SQL.encoderAndParam (HE.nonNullable HE.unknown) $ encodeUtf8 schema
+
+accessibleTables :: Text -> SQL.Snippet
+accessibleTables schema = SQL.sql (encodeUtf8 [trimming|
+  SELECT
+    n.nspname AS table_schema,
+    c.relname AS table_name
+  FROM pg_class c
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  WHERE c.relkind IN ('v','r','m','f','p')
+  AND c.relnamespace = |]) <> encodedSchema <> "::regnamespace " <> SQL.sql (encodeUtf8 [trimming|
+  AND (
+    pg_has_role(c.relowner, 'USAGE')
+    or has_table_privilege(c.oid, 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER')
+    or has_any_column_privilege(c.oid, 'SELECT, INSERT, UPDATE, REFERENCES')
+  )
+  AND not c.relispartition
+  ORDER BY table_schema, table_name|])
+  where
+    encodedSchema = SQL.encoderAndParam (HE.nonNullable HE.text) schema
+
+accessibleFuncs :: Text -> SQL.Snippet
+accessibleFuncs schema = baseFuncSqlQuery <> "AND p.pronamespace = " <> encodedSchema <> "::regnamespace"
+  where
+    encodedSchema = SQL.encoderAndParam (HE.nonNullable HE.text) schema
+
+baseFuncSqlQuery :: SQL.Snippet
+baseFuncSqlQuery = SQL.sql $ encodeUtf8 [trimming|
+  WITH
+  base_types AS (
+    WITH RECURSIVE
+    recurse AS (
+      SELECT
+        oid,
+        typbasetype,
+        typnamespace AS base_namespace,
+        COALESCE(NULLIF(typbasetype, 0), oid) AS base_type
+      FROM pg_type
+      UNION
+      SELECT
+        t.oid,
+        b.typbasetype,
+        b.typnamespace AS base_namespace,
+        COALESCE(NULLIF(b.typbasetype, 0), b.oid) AS base_type
+      FROM recurse t
+      JOIN pg_type b ON t.typbasetype = b.oid
+    )
+    SELECT
+      oid,
+      base_namespace,
+      base_type
+    FROM recurse
+    WHERE typbasetype = 0
+  ),
+  arguments AS (
+    SELECT
+      oid,
+      array_agg((
+        COALESCE(name, ''), -- name
+        type::regtype::text, -- type
+        CASE type
+          WHEN 'bit'::regtype THEN 'bit varying'
+          WHEN 'bit[]'::regtype THEN 'bit varying[]'
+          WHEN 'character'::regtype THEN 'character varying'
+          WHEN 'character[]'::regtype THEN 'character varying[]'
+          ELSE type::regtype::text
+        END, -- convert types that ignore the length and accept any value till maximum size
+        idx <= (pronargs - pronargdefaults), -- is_required
+        COALESCE(mode = 'v', FALSE) -- is_variadic
+      ) ORDER BY idx) AS args,
+      CASE COUNT(*) - COUNT(name) -- number of unnamed arguments
+        WHEN 0 THEN true
+        WHEN 1 THEN (array_agg(type))[1] IN ('bytea'::regtype, 'json'::regtype, 'jsonb'::regtype, 'text'::regtype, 'xml'::regtype)
+        ELSE false
+      END AS callable
+    FROM pg_proc,
+         unnest(proargnames, proargtypes, proargmodes)
+           WITH ORDINALITY AS _ (name, type, mode, idx)
+    WHERE type IS NOT NULL -- only input arguments
+    GROUP BY oid
+  )
+  SELECT
+    pn.nspname AS proc_schema,
+    p.proname AS proc_name,
+    d.description AS proc_description,
+    COALESCE(a.args, '{}') AS args,
+    tn.nspname AS schema,
+    COALESCE(comp.relname, t.typname) AS name,
+    p.proretset AS rettype_is_setof,
+    (t.typtype = 'c'
+     -- if any TABLE, INOUT or OUT arguments present, treat as composite
+     or COALESCE(proargmodes::text[] && '{t,b,o}', false)
+    ) AS rettype_is_composite,
+    bt.oid <> bt.base_type as rettype_is_composite_alias,
+    p.provolatile,
+    p.provariadic > 0 as hasvariadic,
+    'ignored' AS transaction_isolation_level,
+    '{}'::text[] as kvs
+  FROM pg_proc p
+  LEFT JOIN arguments a ON a.oid = p.oid
+  JOIN pg_namespace pn ON pn.oid = p.pronamespace
+  JOIN base_types bt ON bt.oid = p.prorettype
+  JOIN pg_type t ON t.oid = bt.base_type
+  JOIN pg_namespace tn ON tn.oid = t.typnamespace
+  LEFT JOIN pg_class comp ON comp.oid = t.typrelid
+  LEFT JOIN pg_description as d ON d.objoid = p.oid AND d.classoid = 'pg_proc'::regclass
+  WHERE t.oid <> 'trigger'::regtype AND COALESCE(a.callable, true)
+  AND has_function_privilege(p.oid, 'execute')
+  AND prokind = 'f' |]
@@ -1,66 +1,34 @@
+{-# LANGUAGE NamedFieldPuns #-}
 {-|
 Module      : PostgREST.Query.Statements
-Description : PostgREST single SQL statements.
-
-This module constructs single SQL statements that can be parametrized and prepared.
-
- It consumes the SqlQuery types generated by the QueryBuilder module.
- It generates the body format and some headers of the final HTTP response.
+Description : PostgREST main queries
 -}
 module PostgREST.Query.Statements
-  ( prepareWrite
-  , prepareRead
-  , prepareCall
-  , preparePlanRows
-  , ResultSet (..)
+  ( mainWrite
+  , mainRead
+  , mainCall
+  , postExplain
  ) where

-import qualified Data.Aeson.Lens                   as L
-import qualified Data.ByteString.Char8             as BS
-import qualified Hasql.Decoders                    as HD
-import qualified Hasql.DynamicStatements.Snippet   as SQL
-import qualified Hasql.DynamicStatements.Statement as SQL
-import qualified Hasql.Statement                   as SQL
-
-import Control.Lens ((^?))
+import qualified Hasql.DynamicStatements.Snippet as SQL

 import PostgREST.ApiRequest.Preferences
 import PostgREST.MediaType              (MTVndPlanFormat (..),
                                         MediaType (..))
+import PostgREST.Plan.CallPlan
+import PostgREST.Plan.MutatePlan        as MTPlan
+import PostgREST.Plan.ReadPlan
+import PostgREST.Query.QueryBuilder
 import PostgREST.Query.SqlFragment
 import PostgREST.SchemaCache.Routine    (MediaHandler (..), Routine,
                                         funcReturnsSingle)

 import Protolude

-- | Standard result set format used for all queries
-data ResultSet
-  = RSStandard
-  { rsTableTotal :: Maybe Int64
-  -- ^ count of all the table rows
-  , rsQueryTotal :: Int64
-  -- ^ count of the query rows
-  , rsLocation   :: [(BS.ByteString, BS.ByteString)]
-  -- ^ The Location header(only used for inserts) is represented as a list of strings containing
-  -- variable bindings like @"k1=eq.42"@, or the empty list if there is no location header.
-  , rsBody       :: BS.ByteString
-  -- ^ the aggregated body of the query
-  , rsGucHeaders :: Maybe BS.ByteString
-  -- ^ the HTTP headers to be added to the response
-  , rsGucStatus  :: Maybe Text
-  -- ^ the HTTP status to be added to the response
-  , rsInserted   :: Maybe Int64
-  -- ^ the number of rows inserted (Only used for upserts)
-  }
-  | RSPlan BS.ByteString -- ^ the plan of the query
-
-
-prepareWrite :: SQL.Snippet -> SQL.Snippet -> Bool -> Bool -> MediaType -> MediaHandler ->
-                Maybe PreferRepresentation -> Maybe PreferResolution -> [Text] -> Bool -> (SQL.Statement () ResultSet, ByteString)
-prepareWrite selectQuery mutateQuery isInsert isPut mt handler rep resolution pKeys prepared =
- (result, sql)
+mainWrite :: ReadPlanTree -> MutatePlan -> MediaType -> MediaHandler ->
+             Maybe PreferRepresentation -> Maybe PreferResolution -> SQL.Snippet
+mainWrite rPlan mtplan mt handler rep resolution = mtSnippet mt snippet
 where
-  result@(SQL.Statement sql _ _ _) = SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt prepared
  checkUpsert snip = if isInsert && (isPut || resolution == Just MergeDuplicates) then snip else "''"
  pgrstInsertedF = checkUpsert "nullif(current_setting('pgrst.inserted', true),'')::int"
  snippet =
@@ -79,7 +47,7 @@ prepareWrite selectQuery mutateQuery isInsert isPut mt handler rep resolution pK
    if isInsert && rep == Just HeadersOnly
      then
        "CASE WHEN pg_catalog.count(_postgrest_t) = 1 " <>
-          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ") " <>
+          "THEN coalesce(" <> locationF pkCols <> ", " <> noLocationF <> ") " <>
          "ELSE " <> noLocationF <> " " <>
        "END"
      else noLocationF
@@ -89,16 +57,16 @@ prepareWrite selectQuery mutateQuery isInsert isPut mt handler rep resolution pK
    | handler == NoAgg = "SELECT * FROM " <> sourceCTE
    | otherwise        = selectQuery

-  decodeIt :: HD.Result ResultSet
-  decodeIt = case mt of
-    MTVndPlan{} -> planRow
-    _           -> fromMaybe (RSStandard Nothing 0 mempty mempty Nothing Nothing Nothing) <$> HD.rowMaybe (standardRow False)
+  selectQuery = readPlanToQuery rPlan
+  mutateQuery = mutatePlanToQuery mtplan
+  (isPut, isInsert, pkCols) = case mtplan of
+    MTPlan.Insert{MTPlan.where_,insPkCols} -> ((not . null) where_, True, insPkCols)
+    _ -> (False,False, mempty);

-prepareRead :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> MediaHandler -> Bool -> (SQL.Statement () ResultSet, ByteString)
-prepareRead selectQuery countQuery countTotal mt handler prepared =
- (result, sql)
+mainRead :: ReadPlanTree -> SQL.Snippet -> Maybe PreferCount -> Maybe Integer ->
+            MediaType -> MediaHandler -> SQL.Snippet
+mainRead rPlan countQuery pCount maxRows mt handler = mtSnippet mt snippet
 where
-  result@(SQL.Statement sql _ _ _) = SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt prepared
  snippet =
    "WITH " <> sourceCTE <> " AS ( " <> selectQuery <> " ) " <>
    countCTEF <> " " <>
@@ -111,20 +79,19 @@ prepareRead selectQuery countQuery countTotal mt handler prepared =
      "''" <> " AS response_inserted " <>
    "FROM ( SELECT * FROM " <> sourceCTE <> " ) _postgrest_t"

-  (countCTEF, countResultF) = countF countQuery countTotal
+  (countCTEF, countResultF) = countF countQ $ shouldCount pCount
+  selectQuery = readPlanToQuery rPlan
+  countQ =
+    if pCount == Just EstimatedCount then
+      -- LIMIT maxRows + 1 so we can determine below that maxRows was surpassed
+      limitedQuery countQuery ((+ 1) <$> maxRows)
+    else
+      countQuery

-  decodeIt :: HD.Result ResultSet
-  decodeIt = case mt of
-    MTVndPlan{} -> planRow
-    _           -> HD.singleRow $ standardRow True
-
-prepareCall :: Routine -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
-               MediaType -> MediaHandler -> Bool ->
-               (SQL.Statement () ResultSet, ByteString)
-prepareCall rout callProcQuery selectQuery countQuery countTotal mt handler prepared =
-  (result, sql)
+mainCall :: Routine -> CallPlan -> ReadPlanTree -> Maybe PreferCount ->
+            MediaType -> MediaHandler -> SQL.Snippet
+mainCall rout cPlan rPlan pCount mt handler = mtSnippet mt snippet
  where
-    result@(SQL.Statement sql _ _ _) = SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt prepared
    snippet =
      "WITH " <> sourceCTE <> " AS (" <> callProcQuery <> ") " <>
      countCTEF <>
@@ -139,51 +106,16 @@ prepareCall rout callProcQuery selectQuery countQuery countTotal mt handler prep
        "''" <> " AS response_inserted " <>
      "FROM (" <> selectQuery <> ") _postgrest_t"

-    (countCTEF, countResultF) = countF countQuery countTotal
+    (countCTEF, countResultF) = countF countQuery $ shouldCount pCount
+    selectQuery = readPlanToQuery rPlan
+    callProcQuery = callPlanToQuery cPlan
+    countQuery = readPlanToCountQuery rPlan

-    decodeIt :: HD.Result ResultSet
-    decodeIt = case mt of
-      MTVndPlan{} -> planRow
-      _           -> fromMaybe (RSStandard (Just 0) 0 mempty mempty Nothing Nothing Nothing) <$> HD.rowMaybe (standardRow True)
-
-preparePlanRows :: SQL.Snippet -> Bool -> SQL.Statement () (Maybe Int64)
-preparePlanRows countQuery =
-  SQL.dynamicallyParameterized snippet decodeIt
-  where
-    snippet = explainF PlanJSON mempty countQuery
-    decodeIt :: HD.Result (Maybe Int64)
-    decodeIt =
-      let row = HD.singleRow $ column HD.bytea in
-      (^? L.nth 0 . L.key "Plan" .  L.key "Plan Rows" . L._Integral) <$> row
-
-standardRow :: Bool -> HD.Row ResultSet
-standardRow noLocation =
-  RSStandard <$> nullableColumn HD.int8 <*> column HD.int8
-             <*> (if noLocation then pure mempty else fmap splitKeyValue <$> arrayColumn HD.bytea)
-             <*> (fromMaybe mempty <$> nullableColumn HD.bytea)
-             <*> nullableColumn HD.bytea
-             <*> nullableColumn HD.text
-             <*> nullableColumn HD.int8
-  where
-    splitKeyValue :: ByteString -> (ByteString, ByteString)
-    splitKeyValue kv =
-      let (k, v) = BS.break (== '=') kv in
-      (k, BS.tail v)
+-- This occurs after the main query runs, that's why it's prefixed with "post"
+postExplain :: SQL.Snippet -> SQL.Snippet
+postExplain = explainF PlanJSON mempty

 mtSnippet :: MediaType -> SQL.Snippet -> SQL.Snippet
 mtSnippet mediaType snippet = case mediaType of
  MTVndPlan _ fmt opts -> explainF fmt opts snippet
  _                    -> snippet
-
-- | We use rowList because when doing EXPLAIN (FORMAT TEXT), the result comes as many rows. FORMAT JSON comes as one.
-planRow :: HD.Result ResultSet
-planRow = RSPlan . BS.unlines <$> HD.rowList (column HD.bytea)
-
-column :: HD.Value a -> HD.Row a
-column = HD.column . HD.nonNullable
-
-nullableColumn :: HD.Value a -> HD.Row (Maybe a)
-nullableColumn = HD.column . HD.nullable
-
-arrayColumn :: HD.Value a -> HD.Row [a]
-arrayColumn = column . HD.listArray . HD.nonNullable
@@ -24,24 +24,23 @@ import qualified PostgREST.MediaType        as MediaType
 import qualified PostgREST.RangeQuery       as RangeQuery
 import qualified PostgREST.Response.OpenAPI as OpenAPI

-import PostgREST.ApiRequest              (ApiRequest (..),
-                                          InvokeMethod (..),
-                                          Mutation (..))
+import PostgREST.ApiRequest              (ApiRequest (..))
 import PostgREST.ApiRequest.Preferences  (PreferRepresentation (..),
                                          PreferResolution (..),
                                          Preferences (..),
                                          prefAppliedHeader,
                                          shouldCount)
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
+import PostgREST.ApiRequest.Types        (InvokeMethod (..),
+                                          Mutation (..))
 import PostgREST.Config                  (AppConfig (..))
+import PostgREST.MainTx                  (DbResult (..),
+                                          ResultSet (..))
 import PostgREST.MediaType               (MediaType (..))
-import PostgREST.Plan                    (CallReadPlan (..),
-                                          CrudPlan (..),
+import PostgREST.Plan                    (CrudPlan (..),
                                          InfoPlan (..),
                                          InspectPlan (..))
 import PostgREST.Plan.MutatePlan         (MutatePlan (..))
-import PostgREST.Query                   (QueryResult (..))
-import PostgREST.Query.Statements        (ResultSet (..))
 import PostgREST.Response.GucHeader      (GucHeader, unwrapGucHeader)
 import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
@@ -61,185 +60,163 @@ data PgrstResponse = PgrstResponse {
 , pgrstBody    :: LBS.ByteString
 }

-actionResponse :: QueryResult -> ApiRequest -> (Text, Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Either Error.Error PgrstResponse
+actionResponse :: DbResult -> ApiRequest -> (Text, Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Either Error.Error PgrstResponse

-actionResponse (DbCrudResult WrappedReadPlan{wrMedia, wrHdrsOnly=headersOnly, crudQi=identifier} resultSet) ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} _ _ _ _ _ =
-  case resultSet of
-    RSStandard{..} -> do
-      let
-        (status, contentRange) = RangeQuery.rangeStatusHeader iTopLevelRange rsQueryTotal rsTableTotal
-        cLHeader = if headersOnly then mempty else [contentLengthHeaderStrict rsBody]
-        prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing Nothing preferCount preferTransaction Nothing preferHandling preferTimezone Nothing []
-        headers =
-          [ contentRange
-          , ( "Content-Location"
-            , "/"
-                <> toUtf8 (qiName identifier)
-                <> if BS.null (qsCanonical iQueryParams) then mempty else "?" <> qsCanonical iQueryParams
-            )
-          ]
-          ++ cLHeader
-          ++ contentTypeHeaders wrMedia ctxApiRequest
-          ++ prefHeader
+actionResponse (DbCrudResult plan@WrappedReadPlan{pMedia, wrHdrsOnly=headersOnly, crudQi=identifier} RSStandard{..}) ctxApiRequest@ApiRequest{..} _ _ _ _ _ = do
+  let
+    (status, contentRange) = RangeQuery.rangeStatusHeader iTopLevelRange rsQueryTotal rsTableTotal
+    cLHeader = if headersOnly then mempty else [ contentLengthHeader bod ]
+    prefHeader = maybeToList . prefAppliedHeader $ responsePreferences plan ctxApiRequest

-      (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers
+    headers =
+      [ contentRange
+      , ( "Content-Location"
+        , "/"
+            <> toUtf8 (qiName identifier)
+            <> if BS.null (qsCanonical iQueryParams) then mempty else "?" <> qsCanonical iQueryParams
+        )
+      ]
+      ++ cLHeader
+      ++ contentTypeHeaders pMedia ctxApiRequest
+      ++ prefHeader
+    bod | status == HTTP.status416 = Error.errorPayload $ Error.ApiRequestError $ Error.InvalidRange $
+                                     Error.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
+        | headersOnly              = mempty
+        | otherwise                = LBS.fromStrict rsBody

-      let bod | status == HTTP.status416 = Error.errorPayload $ Error.ApiRequestError $ Error.InvalidRange $
-                                           Error.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
-              | headersOnly              = mempty
-              | otherwise                = LBS.fromStrict rsBody
+  (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers

-      Right $ PgrstResponse ovStatus ovHeaders bod
+  Right $ PgrstResponse ovStatus ovHeaders bod

-    RSPlan plan ->
-      Right $ PgrstResponse HTTP.status200 (contentLengthHeaderStrict plan : contentTypeHeaders wrMedia ctxApiRequest) $ LBS.fromStrict plan
+actionResponse (DbCrudResult plan@MutateReadPlan{mrMutation=MutationCreate, pMedia, crudQi=QualifiedIdentifier{..}} RSStandard{..}) ctxApiRequest@ApiRequest{..} _ _ _ _ _ = do
+  let
+    prefHeader = prefAppliedHeader $ responsePreferences plan ctxApiRequest

-actionResponse (DbCrudResult MutateReadPlan{mrMutation=MutationCreate, mrMutatePlan, mrMedia, crudQi=QualifiedIdentifier{..}} resultSet) ctxApiRequest@ApiRequest{iPreferences=Preferences{..}, ..} _ _ _ _ _ = case resultSet of
-  RSStandard{..} -> do
-    let
-      pkCols = case mrMutatePlan of { Insert{insPkCols} -> insPkCols; _ -> mempty;}
-      prefHeader = prefAppliedHeader $
-        Preferences (if null pkCols && isNothing (qsOnConflict iQueryParams) then Nothing else preferResolution)
-        preferRepresentation preferCount preferTransaction preferMissing preferHandling preferTimezone Nothing []
-      headers =
-        catMaybes
-          [ if null rsLocation then
-              Nothing
-            else
-              Just
-                ( HTTP.hLocation
-                , "/"
-                    <> toUtf8 qiName
-                    <> HTTP.renderSimpleQuery True rsLocation
-                )
-          , Just . RangeQuery.contentRangeH 1 0 $
-              if shouldCount preferCount then Just rsQueryTotal else Nothing
-          , Just $ contentLengthHeaderStrict rsBody
-          , prefHeader ]
-
-    let isInsertIfGTZero i =
-          if i <= 0 && preferResolution == Just MergeDuplicates then
-            HTTP.status200
+    headers =
+      catMaybes
+        [ if null rsLocation then
+            Nothing
          else
-            HTTP.status201
-        status = maybe HTTP.status200 isInsertIfGTZero rsInserted
-        (headers', bod) = case preferRepresentation of
-          Just Full -> (headers ++ contentTypeHeaders mrMedia ctxApiRequest, LBS.fromStrict rsBody)
-          Just None -> (headers, mempty)
-          Just HeadersOnly -> (headers, mempty)
-          Nothing -> (headers, mempty)
+            Just
+              ( HTTP.hLocation
+              , "/"
+                  <> toUtf8 qiName
+                  <> HTTP.renderSimpleQuery True rsLocation
+              )
+        , Just . RangeQuery.contentRangeH 1 0 $
+            if shouldCount (preferCount iPreferences) then Just rsQueryTotal else Nothing
+        , prefHeader ]

-    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers'
+    isInsertIfGTZero i =
+        if i <= 0 && preferResolution iPreferences == Just MergeDuplicates then
+          HTTP.status200
+        else
+          HTTP.status201
+    status = maybe HTTP.status200 isInsertIfGTZero rsInserted
+    (headers', bod) = case preferRepresentation iPreferences of
+      Just Full -> (headers ++ contentTypeHeaders pMedia ctxApiRequest, LBS.fromStrict rsBody)
+      Just None -> (headers, mempty)
+      Just HeadersOnly -> (headers, mempty)
+      Nothing -> (headers, mempty)

-    Right $ PgrstResponse ovStatus ovHeaders bod
-  RSPlan plan ->
-    Right $ PgrstResponse HTTP.status200 (contentLengthHeaderStrict plan : contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan
+  (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status $ contentLengthHeader bod:headers'

-actionResponse (DbCrudResult MutateReadPlan{mrMutation=MutationUpdate, mrMedia} resultSet) ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} _ _ _ _ _ = case resultSet of
-  RSStandard{..} -> do
-    let
-      contentRangeHeader =
-        Just . RangeQuery.contentRangeH 0 (rsQueryTotal - 1) $
-          if shouldCount preferCount then Just rsQueryTotal else Nothing
-      prefHeader = prefAppliedHeader $ Preferences Nothing preferRepresentation preferCount preferTransaction preferMissing preferHandling preferTimezone preferMaxAffected []
-      headers = catMaybes [contentRangeHeader, prefHeader]
+  Right $ PgrstResponse ovStatus ovHeaders bod

-    let (status, headers', body) =
-          case preferRepresentation of
-            Just Full -> (HTTP.status200, headers ++ [contentLengthHeaderStrict rsBody] ++ contentTypeHeaders mrMedia ctxApiRequest, LBS.fromStrict rsBody)
+actionResponse (DbCrudResult plan@MutateReadPlan{mrMutation=MutationUpdate, pMedia} RSStandard{..}) ctxApiRequest@ApiRequest{..} _ _ _ _ _ = do
+  let
+    contentRangeHeader =
+      Just . RangeQuery.contentRangeH 0 (rsQueryTotal - 1) $
+        if shouldCount (preferCount iPreferences) then Just rsQueryTotal else Nothing
+
+    prefHeader = prefAppliedHeader $ responsePreferences plan ctxApiRequest
+
+    headers = catMaybes [contentRangeHeader, prefHeader]
+    lbsBody = LBS.fromStrict rsBody
+
+  let (status, headers', body) =
+        case preferRepresentation iPreferences of
+          Just Full -> (HTTP.status200, headers ++ [contentLengthHeader lbsBody] ++ contentTypeHeaders pMedia ctxApiRequest, lbsBody)
+          Just None -> (HTTP.status204, headers, mempty)
+          _         -> (HTTP.status204, headers, mempty)
+
+  (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers'
+
+  Right $ PgrstResponse ovStatus ovHeaders body
+
+actionResponse (DbCrudResult plan@MutateReadPlan{mrMutation=MutationSingleUpsert, pMedia} RSStandard{..}) ctxApiRequest@ApiRequest{..} _ _ _ _ _ = do
+  let
+    prefHeader = maybeToList . prefAppliedHeader $ responsePreferences plan ctxApiRequest
+    lbsBody = LBS.fromStrict rsBody
+    cLHeader = [contentLengthHeader lbsBody]
+    cTHeader = contentTypeHeaders pMedia ctxApiRequest
+
+  let isInsertIfGTZero i = if i > 0 then HTTP.status201 else HTTP.status200
+      upsertStatus       = isInsertIfGTZero $ fromJust rsInserted
+      (status, headers, body) =
+        case preferRepresentation iPreferences of
+          Just Full -> (upsertStatus, cLHeader ++ cTHeader ++ prefHeader, lbsBody)
+          Just None -> (HTTP.status204,  prefHeader, mempty)
+          _ -> (HTTP.status204, prefHeader, mempty)
+  (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers
+
+  Right $ PgrstResponse ovStatus ovHeaders body
+
+actionResponse (DbCrudResult plan@MutateReadPlan{mrMutation=MutationDelete, pMedia} RSStandard{..}) ctxApiRequest@ApiRequest{..} _ _ _ _ _ = do
+  let
+    contentRangeHeader = RangeQuery.contentRangeH 1 0 $ if shouldCount (preferCount iPreferences) then Just rsQueryTotal else Nothing
+    prefHeader = maybeToList . prefAppliedHeader $ responsePreferences plan ctxApiRequest
+    headers = contentRangeHeader : prefHeader
+    lbsBody = LBS.fromStrict rsBody
+    (status, headers', body) =
+        case preferRepresentation iPreferences of
+            Just Full -> (HTTP.status200, headers ++ [contentLengthHeader lbsBody] ++ contentTypeHeaders pMedia ctxApiRequest, lbsBody)
            Just None -> (HTTP.status204, headers, mempty)
-            _         -> (HTTP.status204, headers, mempty)
+            _ -> (HTTP.status204, headers, mempty)

-    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers'
+  (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers'

-    Right $ PgrstResponse ovStatus ovHeaders body
+  Right $ PgrstResponse ovStatus ovHeaders body

-  RSPlan plan ->
-    Right $ PgrstResponse HTTP.status200 (contentLengthHeaderStrict plan : contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan
+actionResponse (DbCrudResult plan@CallReadPlan{pMedia, crInvMthd=invMethod, crProc=proc} RSStandard {..}) ctxApiRequest@ApiRequest{..} _ _ _ _ _ = do
+  let
+    (status, contentRange) =
+      RangeQuery.rangeStatusHeader iTopLevelRange rsQueryTotal rsTableTotal
+    rsOrErrBody = if status == HTTP.status416
+      then Error.errorPayload $ Error.ApiRequestError $ Error.InvalidRange
+        $ Error.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
+      else LBS.fromStrict rsBody
+    isHeadMethod = invMethod == InvRead True
+    prefHeader = maybeToList . prefAppliedHeader $ responsePreferences plan ctxApiRequest
+    cLHeader = if isHeadMethod then mempty else [contentLengthHeader rsOrErrBody]
+    headers = contentRange : prefHeader
+    (status', headers', body) =
+        if Routine.funcReturnsVoid proc then
+            (HTTP.status204, headers, mempty)
+          else
+            (status,
+              headers ++ cLHeader ++ contentTypeHeaders pMedia ctxApiRequest,
+              if isHeadMethod then mempty else rsOrErrBody)

-actionResponse (DbCrudResult MutateReadPlan{mrMutation=MutationSingleUpsert, mrMedia} resultSet) ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} _ _ _ _ _ = case resultSet of
-  RSStandard {..} -> do
-    let
-      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing preferRepresentation preferCount preferTransaction Nothing preferHandling preferTimezone Nothing []
-      cLHeader = [contentLengthHeaderStrict rsBody]
-      cTHeader = contentTypeHeaders mrMedia ctxApiRequest
+  (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status' headers'

-    let isInsertIfGTZero i = if i > 0 then HTTP.status201 else HTTP.status200
-        upsertStatus       = isInsertIfGTZero $ fromJust rsInserted
-        (status, headers, body) =
-          case preferRepresentation of
-            Just Full -> (upsertStatus, cLHeader ++ cTHeader ++ prefHeader, LBS.fromStrict rsBody)
-            Just None -> (HTTP.status204,  prefHeader, mempty)
-            _ -> (HTTP.status204, prefHeader, mempty)
-    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers
+  Right $ PgrstResponse ovStatus ovHeaders body

-    Right $ PgrstResponse ovStatus ovHeaders body
-
-  RSPlan plan ->
-    Right $ PgrstResponse HTTP.status200 (contentLengthHeaderStrict plan : contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan
-
-actionResponse (DbCrudResult MutateReadPlan{mrMutation=MutationDelete, mrMedia} resultSet) ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} _ _ _ _ _ = case resultSet of
-  RSStandard {..} -> do
-    let
-      contentRangeHeader =
-        RangeQuery.contentRangeH 1 0 $
-          if shouldCount preferCount then Just rsQueryTotal else Nothing
-      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing preferRepresentation preferCount preferTransaction Nothing preferHandling preferTimezone preferMaxAffected []
-      headers = contentRangeHeader : prefHeader
-
-    let (status, headers', body) =
-          case preferRepresentation of
-              Just Full -> (HTTP.status200, headers ++ [contentLengthHeaderStrict rsBody] ++ contentTypeHeaders mrMedia ctxApiRequest, LBS.fromStrict rsBody)
-              Just None -> (HTTP.status204, headers, mempty)
-              _ -> (HTTP.status204, headers, mempty)
-
-    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers'
-
-    Right $ PgrstResponse ovStatus ovHeaders body
-
-  RSPlan plan ->
-    Right $ PgrstResponse HTTP.status200 (contentLengthHeaderStrict plan : contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan
-
-actionResponse (DbCallResult CallReadPlan{crMedia, crInvMthd=invMethod, crProc=proc} resultSet) ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} _ _ _ _ _ = case resultSet of
-  RSStandard {..} -> do
-    let
-      (status, contentRange) =
-        RangeQuery.rangeStatusHeader iTopLevelRange rsQueryTotal rsTableTotal
-      rsOrErrBody = if status == HTTP.status416
-        then Error.errorPayload $ Error.ApiRequestError $ Error.InvalidRange
-          $ Error.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
-        else LBS.fromStrict rsBody
-      isHeadMethod = invMethod == InvRead True
-      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing Nothing preferCount preferTransaction Nothing preferHandling preferTimezone preferMaxAffected []
-      cLHeader = if isHeadMethod then mempty else [contentLengthHeaderLazy rsOrErrBody]
-      headers = contentRange : prefHeader
-
-    let (status', headers', body) =
-          if Routine.funcReturnsVoid proc then
-              (HTTP.status204, headers, mempty)
-            else
-              (status,
-                headers ++ cLHeader ++ contentTypeHeaders crMedia ctxApiRequest,
-                if isHeadMethod then mempty else rsOrErrBody)
-
-    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status' headers'
-
-    Right $ PgrstResponse ovStatus ovHeaders body
-
-  RSPlan plan ->
-    Right $ PgrstResponse HTTP.status200 (contentLengthHeaderStrict plan : contentTypeHeaders crMedia ctxApiRequest) $ LBS.fromStrict plan
+actionResponse (DbPlanResult media plan) ctxApiRequest _ _ _ _ _ =
+  let body = LBS.fromStrict plan in
+  Right $ PgrstResponse HTTP.status200 (contentLengthHeader body : contentTypeHeaders media ctxApiRequest) body

 actionResponse (MaybeDbResult InspectPlan{ipHdrsOnly=headersOnly} body) _ versions conf sCache schema negotiatedByProfile =
  let
    rsBody = maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache x y z) body
-    cLHeader = if headersOnly then mempty else [contentLengthHeaderLazy rsBody]
+    cLHeader = if headersOnly then mempty else [contentLengthHeader rsBody]
  in
  Right $ PgrstResponse HTTP.status200 (MediaType.toContentType MTOpenAPI : cLHeader ++ maybeToList (profileHeader schema negotiatedByProfile)) rsBody

-actionResponse (NoDbResult (RelInfoPlan qi@QualifiedIdentifier{..})) _ _ _ SchemaCache{dbTables} _ _ =
+actionResponse (NoDbResult (RelInfoPlan qi@QualifiedIdentifier{..})) _ _ _ sc@SchemaCache{dbTables} _ _ =
  case HM.lookup qi dbTables of
    Just tbl -> respondInfo $ allowH tbl
-    Nothing  -> Left $ Error.SchemaCacheErr $ Error.TableNotFound qiSchema qiName (HM.elems dbTables)
+    Nothing  -> Left $ Error.SchemaCacheErr $ Error.TableNotFound qiSchema qiName sc
  where
    allowH table =
      let hasPK = not . null $ tablePKCols table in
@@ -250,7 +227,7 @@ actionResponse (NoDbResult (RelInfoPlan qi@QualifiedIdentifier{..})) _ _ _ Schem
          ["PATCH" | tableUpdatable table] ++
          ["DELETE" | tableDeletable table]

-actionResponse (NoDbResult (RoutineInfoPlan CallReadPlan{crProc=proc})) _ _ _ _ _ _
+actionResponse (NoDbResult (RoutineInfoPlan proc)) _ _ _ _ _ _
  | pdVolatility proc == Volatile = respondInfo "OPTIONS,POST"
  | otherwise                     = respondInfo "OPTIONS,GET,HEAD,POST"

@@ -259,7 +236,7 @@ actionResponse (NoDbResult SchemaInfoPlan) _ _ _ _ _ _ = respondInfo "OPTIONS,GE
 respondInfo :: ByteString -> Either Error.Error PgrstResponse
 respondInfo allowHeader =
  let allOrigins = ("Access-Control-Allow-Origin", "*") in
-  Right $ PgrstResponse HTTP.status200 [contentLengthHeaderStrict mempty, allOrigins, (HTTP.hAllow, allowHeader)] mempty
+  Right $ PgrstResponse HTTP.status200 [contentLengthHeader mempty, allOrigins, (HTTP.hAllow, allowHeader)] mempty

 -- Status and headers can be overridden as per https://postgrest.org/en/stable/references/transactions.html#response-headers
 overrideStatusHeaders :: Maybe Text -> Maybe BS.ByteString -> HTTP.Status -> [HTTP.Header]-> Either Error.Error (HTTP.Status, [HTTP.Header])
@@ -276,14 +253,8 @@ decodeGucStatus :: Maybe Text -> Either Error.Error (Maybe HTTP.Status)
 decodeGucStatus =
  maybe (Right Nothing) $ first (const . Error.ApiRequestError $ Error.GucStatusError) . fmap (Just . toEnum . fst) . decimal

-contentLengthHeader :: Show b => (a -> b) -> a -> HTTP.Header
-contentLengthHeader lenFn body = ("Content-Length", show (lenFn body))
-
-contentLengthHeaderStrict :: BS.ByteString -> HTTP.Header
-contentLengthHeaderStrict = contentLengthHeader BS.length
-
-contentLengthHeaderLazy :: LBS.ByteString -> HTTP.Header
-contentLengthHeaderLazy = contentLengthHeader LBS.length
+contentLengthHeader :: LBS.ByteString -> HTTP.Header
+contentLengthHeader body = ("Content-Length", show (LBS.length body))

 contentTypeHeaders :: MediaType -> ApiRequest -> [HTTP.Header]
 contentTypeHeaders mediaType ApiRequest{..} =
@@ -301,3 +272,31 @@ addHeadersIfNotIncluded :: [HTTP.Header] -> [HTTP.Header] -> [HTTP.Header]
 addHeadersIfNotIncluded newHeaders initialHeaders =
  filter (\(nk, _) -> isNothing $ find (\(ik, _) -> ik == nk) initialHeaders) newHeaders ++
  initialHeaders
+
+-- | Get Preferences for Preference-Applied header per plan
+responsePreferences :: CrudPlan -> ApiRequest -> Preferences
+responsePreferences plan ApiRequest{iPreferences=Preferences{..}, iQueryParams=QueryParams{..}} =
+  let
+    -- Only returned on Inserts
+    preferResolution' = case plan of
+      MutateReadPlan{mrMutation=MutationCreate, mrMutatePlan} ->
+        let pkCols = case mrMutatePlan of { Insert{insPkCols} -> insPkCols ; _ -> mempty; }
+        in (if null pkCols && isNothing qsOnConflict then Nothing else preferResolution)
+      _ -> Nothing
+
+    preferRepresentation' = case plan of
+      MutateReadPlan{} -> preferRepresentation
+      _                -> Nothing
+
+    preferMissing' = case plan of
+      MutateReadPlan{mrMutation=MutationCreate} -> preferMissing
+      MutateReadPlan{mrMutation=MutationUpdate} -> preferMissing
+      _                                         -> Nothing
+
+    preferMaxAffected' = case plan of
+      MutateReadPlan{mrMutation=MutationUpdate} -> preferMaxAffected
+      MutateReadPlan{mrMutation=MutationDelete} -> preferMaxAffected
+      CallReadPlan{}                            -> preferMaxAffected
+      _                                         -> Nothing
+
+    in Preferences preferResolution' preferRepresentation' preferCount preferTransaction preferMissing' preferHandling preferTimezone preferMaxAffected' []
--- a/Show More
+++ b/Show More