bump version to 12.0.0

feat: custom SQL handler for the "*/*" media type
refactor: use a data type instead of Map for Server Timing
2023-12-01 19:18:07 -05:00 · 2023-11-28 23:14:38 -05:00 · 2023-11-28 18:26:56 -05:00 · 2023-11-28 12:48:57 -05:00 · 2023-11-28 12:45:39 -05:00 · 2023-11-28 11:52:52 -05:00
115 changed files with 7123 additions and 2717 deletions
@@ -3,4 +3,16 @@ When submitting a new feature or fix:

 - Add a new entry to the CHANGELOG - https://github.com/PostgREST/postgrest/blob/main/CHANGELOG.md#unreleased
 - If relevant, update the docs     - https://github.com/PostgREST/postgrest-docs
+- Use a prefix for the PR title or commits, e.g. "fix: description of the fix".
+  + `fix`, bug fixes
+  + `feat`, new features added
+  + `perf`, performance improvements
+  + `nix`, related to the Nix development environment
+  + `ci`, related to the Continuous Integration modules
+  + `test`, related to the testing modules
+  + `refactor`, refactoring code
+  + `deprecate`, deprecating a feature
+  + `chore`, maintenance (changelog, build process, etc.)
+  + Other prefixes may be used if necessary
+- If there's a breaking change, add `BREAKING CHANGE` and an explanation to your commit message
 -->
@@ -7,13 +7,23 @@ inputs:
    description: Token to pass to cachix
  tools:
    description: Tools to install with nix-env -iA <tools>
+  cache-id:
+    description: Cache id to use for cache-nix-action
+    default: "default"

 runs:
  using: composite
  steps:
-    - uses: cachix/install-nix-action@v18
+    - uses: nixbuild/nix-quick-install-action@v26
      with:
-        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
+        nix_version: '2.13.6'
+    - name: Restore and cache Nix store
+      uses: nix-community/cache-nix-action@v4.0.3
+      with:
+        key: cache-nix-${{ runner.os }}-id-${{ inputs.cache-id }}-${{ hashFiles('nix/**/*.nix') }}
+        restore-keys: |
+          cache-nix-${{ runner.os }}-common-
+        restore-key-hit: true
    - uses: cachix/cachix-action@v12
      with:
        name: postgrest
@@ -1,6 +1,11 @@
 version: 2
 updates:
- package-ecosystem: github-actions
-  directory: /
-  schedule:
-    interval: weekly
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+
+  - package-ecosystem: github-actions
+    directory: /.github/actions/setup-nix
+    schedule:
+      interval: weekly
@@ -4,13 +4,15 @@

 [ -z "$1" ] && { echo "Missing 1st argument: PostgREST github commit SHA"; exit 1; }
 [ -z "$2" ] && { echo "Missing 2nd argument: Build environment directory name"; exit 1; }
+[ -z "$3" ] && { echo "Missing 3rd argument: GHC version"; exit 1; }

 PGRST_GITHUB_COMMIT="$1"
 SCRIPT_DIR="$2"

 DOCKER_BUILD_DIR="$SCRIPT_DIR/docker-env"
-CABAL_VERSION="3.6.0.0"
-GHC_VERSION="9.2.4"
+# latest is a shortcut documented on https://www.haskell.org/ghcup/guide/#tags-and-shortcuts
+CABAL_VERSION="latest"
+GHC_VERSION="$3"

 install_packages() {
  sudo apt-get update -y
@@ -13,4 +13,6 @@ EXPOSE 3000

 USER 1000

-CMD postgrest
+# Use the array form to avoid running the command using bash, which does not handle `SIGTERM` properly. 
+# See https://docs.docker.com/compose/faq/#why-do-my-services-take-10-seconds-to-recreate-or-stop 
+CMD ["postgrest"]
@@ -28,7 +28,7 @@ jobs:
    name: Seed ${{ matrix.os }}
    runs-on: ${{ matrix.runs-on }}
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
@@ -17,11 +17,12 @@ jobs:
    name: Lint & check code style
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: style
+          cache-id: style
      - name: Run linter (check locally with `nix-shell --run postgrest-lint`)
        run: postgrest-lint
      - name: Run style check (auto-format with `nix-shell --run postgrest-style`)
@@ -37,16 +38,17 @@ jobs:
        # https://github.com/actions/runner/issues/241#issuecomment-842566950
        shell: script -qec "bash --noprofile --norc -eo pipefail {0}"
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: tests
+          cache-id: test-pg

      - name: Run coverage (IO tests and Spec tests against PostgreSQL 15)
        run: postgrest-coverage
      - name: Upload coverage to codecov
-        uses: codecov/codecov-action@v3.1.3
+        uses: codecov/codecov-action@v3.1.4
        with:
          files: ./coverage/codecov.json

@@ -63,7 +65,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        pgVersion: [9.6, 10, 11, 12, 13, 14, 15]
+        pgVersion: [9.6, 10, 11, 12, 13, 14, 15, 16]
    name: Test PG ${{ matrix.pgVersion }} (Nix)
    runs-on: ubuntu-latest
    defaults:
@@ -72,11 +74,14 @@ jobs:
        # https://github.com/actions/runner/issues/241#issuecomment-842566950
        shell: script -qec "bash --noprofile --norc -eo pipefail {0}"
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: tests withTools
+          # It seems like they are installing the same set of derivations, so we can assign them the same cache id. 
+          # This would decrease the amount of caches dowloaded on merge cache step and will prevent disk space issues.
+          cache-id: test-pg 

      - name: Run spec tests
        if: always()
@@ -91,11 +96,12 @@ jobs:
    name: Test memory (Nix)
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: memory
+          cache-id: test-memory
      - name: Run memory tests
        run: postgrest-test-memory

@@ -104,11 +110,12 @@ jobs:
    name: Build Linux static (Nix)
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: tests
+          cache-id: static-nix

      - name: Build static executable
        run: nix-build -A postgrestStatic
@@ -130,12 +137,38 @@ jobs:
          path: postgrest-docker.tar.gz
          if-no-files-found: error

+  # TODO: Enable this again in a PR by PostgREST admins, because regular users don't have permission to delete cache entries, which this job does.
+  #
+  # merge-nix-caches-linux:
+  #   name: "Merge Nix caches (Linux)"
+  #   needs: [Test-Nix, Test-Pg-Nix, Test-Memory-Nix, Build-Static-Nix, Lint-Style]
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     max-parallel: 1
+  #     matrix:
+  #       cache-id: ['static-nix', 'test-pg', 'style', 'test-memory']
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: nixbuild/nix-quick-install-action@v26
+  #       with:
+  #         nix_version: '2.13.6'
+  #     - name: Restore and cache Nix store
+  #       uses: nix-community/cache-nix-action@v4
+  #       with:
+  #         key: cache-nix-${{ runner.os }}-common-${{ hashFiles('nix/**/*.nix') }}
+  #         extra-restore-keys: |
+  #           cache-nix-${{ runner.os }}-cid-
+  #         purge: true
+  #         purge-keys: |
+  #           cache-nix-${{ runner.os }}-cid-
+  #           cache-nix-${{ runner.os }}-common-
+  #         purge-created-max-age: 0

  Build-Macos-Nix:
    name: Build MacOS (Nix)
    runs-on: macos-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix

@@ -175,12 +208,12 @@ jobs:
    name: Build ${{ matrix.name }} (Stack)
    runs-on: ${{ matrix.runs-on }}
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Stack working files cache
        uses: actions/cache@v3
        with:
          path: ${{ matrix.cache }}
-          key: ${{ runner.os }}-${{ hashFiles('stack.yaml.lock') }}
+          key: cache-stack-${{ runner.os }}-${{ hashFiles('stack.yaml.lock') }}
      - name: Install dependencies
        if: ${{ matrix.deps }}
        run: ${{ matrix.deps }}
@@ -199,7 +232,7 @@ jobs:
    name: Get FreeBSD build from CirrusCI
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Get FreeBSD executable from CirrusCI
        env:
          # GITHUB_SHA does weird things for pull request, so we roll our own:
@@ -216,12 +249,12 @@ jobs:
  Build-Cabal:
    strategy:
      matrix:
-        ghc: ['9.2.4']
+        ghc: ['9.0.2', '9.2.4']
      fail-fast: false
    name: Build Linux (Cabal, GHC ${{ matrix.ghc }})
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Workaround runner image issue
        # https://github.com/actions/runner-images/issues/7061
        run: sudo chown -R "$USER" /usr/local/.ghcup
@@ -229,16 +262,20 @@ jobs:
        run: |
          ghcup install ghc ${{ matrix.ghc }}
          ghcup set ghc ${{ matrix.ghc }}
-      - name: Copy cabal.project
+      - name: Copy cabal.project & fix caching
        run: |
+          mkdir ~/.cabal
          cp cabal.project.non-nix cabal.project
      - name: Cache
        uses: actions/cache@v3
        with:
-          path: ~/.cabal
-          key: ${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('**/*.cabal') }}-${{ hashFiles('**/cabal.project') }}
+          path: |
+            ~/.cabal/packages
+            ~/.cabal/store
+            dist-newstyle
+          key: cache-cabal-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('**/*.cabal', '**/cabal.project') }}
          restore-keys: |
-            ${{ runner.os }}-${{ matrix.ghc }}-
+            cache-cabal-${{ runner.os }}-${{ matrix.ghc }}-
      - name: Install dependencies
        run: |
          cabal update
@@ -247,15 +284,20 @@ jobs:
        run: cabal build --enable-tests --enable-benchmarks all

  Build-Cabal-Arm:
-    name: Build aarch64 (Cabal)
+    strategy:
+      matrix:
+        ghc: ['9.2.4']
+      fail-fast: false
+    name: Build aarch64 (Cabal, GHC ${{ matrix.ghc }})
    if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/rel-') }}
    runs-on: ubuntu-latest
    outputs:
      remotepath: ${{ steps.Remote-Dir.outputs.remotepath }}
    env:
      GITHUB_COMMIT: ${{ github.sha }}
+      GHC_VERSION: ${{ matrix.ghc }}
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - id: Remote-Dir
        name: Unique directory name for the remote build
        run: echo "remotepath=postgrest-build-$(uuidgen)" >> "$GITHUB_OUTPUT"
@@ -280,8 +322,8 @@ jobs:
          fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
          command_timeout: 120m
          script_stop: true
-          envs: GITHUB_COMMIT,REMOTE_DIR
-          script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR"
+          envs: GITHUB_COMMIT,REMOTE_DIR,GHC_VERSION
+          script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR" "GHC_VERSION"
      - name: Download binaries from remote server
        uses: nicklasfrahm/scp-action@main
        with:
@@ -319,7 +361,7 @@ jobs:
      version: ${{ steps.Identify-Version.outputs.version }}
      isprerelease: ${{ steps.Identify-Version.outputs.isprerelease }}
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - id: Identify-Version
        name: Identify the version to be released
        run: |
@@ -372,7 +414,7 @@ jobs:
    env:
      VERSION: ${{ needs.Prepare-Release.outputs.version }}
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Download all artifacts
        uses: actions/download-artifact@v3
        with:
@@ -438,7 +480,7 @@ jobs:
      VERSION: ${{ needs.Prepare-Release.outputs.version }}
      ISPRERELEASE: ${{ needs.Prepare-Release.outputs.isprerelease }}
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
@@ -491,7 +533,7 @@ jobs:
      VERSION: ${{ needs.Prepare-Release.outputs.version }}
      ISPRERELEASE: ${{ needs.Prepare-Release.outputs.isprerelease }}
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Publish images for ARM builds on Docker Hub
        uses: appleboy/ssh-action@master
        env:
@@ -518,7 +560,7 @@ jobs:
    env:
      REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Remove uploaded files from server
        uses: appleboy/ssh-action@master
        with:
@@ -11,20 +11,26 @@ on:
      - main

 jobs:
-  Loadtest-Nix:
-    name: Loadtest (Nix)
+  Loadtest-PR-Nix:
+    name: Loadtest PR (Nix)
+    if: ${{ github.event_name == 'pull_request' }}
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: loadtest
+          cache-id: test-pg
+      - uses: actions-ecosystem/action-get-latest-tag@v1
+        id: get-latest-tag
+        with:
+          semver_only: true          
      - name: Run loadtest
        run: |
-          postgrest-loadtest-against main
+          postgrest-loadtest-against main ${{ steps.get-latest-tag.outputs.tag }}
          postgrest-loadtest-report > loadtest/loadtest.md
      - name: Upload report
        uses: actions/upload-artifact@v3
@@ -32,3 +38,32 @@ jobs:
          name: loadtest.md
          path: loadtest/loadtest.md
          if-no-files-found: error
+
+  Loadtest-Merge-Nix:
+    name: Loadtest Merge (Nix)
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions-ecosystem/action-get-latest-tag@v1
+        id: get-latest-tag
+        with:
+          semver_only: true          
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+        with:
+          tools: loadtest
+          cache-id: test-pg
+      - name: Run loadtest
+        run: |
+          postgrest-loadtest-against ${{ steps.get-latest-tag.outputs.tag }}
+          postgrest-loadtest-report > loadtest/loadtest.md
+      - name: Upload report
+        uses: actions/upload-artifact@v3
+        with:
+          name: loadtest.md
+          path: loadtest/loadtest.md
+          if-no-files-found: error
+        
@@ -22,7 +22,7 @@ jobs:
          name: loadtest.md
          path: artifacts
      - name: Upload to GitHub Checks
-        uses: LouisBrunner/checks-action@v1.6.0
+        uses: LouisBrunner/checks-action@v1.6.2
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ github.event.workflow_run.head_sha }}
@@ -55,12 +55,6 @@ It builds the OpenAPI response using the schema cache.

 This module provides functions to deal with JWT authorization.

-### Workers.hs
-
-This spawns threads which are used to execute concurrent jobs.
-
-Jobs include connection recovery, a listener for the PostgreSQL LISTEN command, and an admin server.
-
 ### SchemaCache.hs

 This queries the PostgreSQL system catalogs and caches the metadata into a SchemaCache type,
@@ -68,3 +62,7 @@ This queries the PostgreSQL system catalogs and caches the metadata into a Schem
 ### AppState.hs

 The state of the App which is kept across requests.
+
+This spawns threads which are used to execute concurrent jobs.
+
+Jobs include connection recover and a listener for the PostgreSQL LISTEN command.
@@ -4,40 +4,30 @@ PostgREST ongoing development is only possible thanks to our Sponsors and Backer

 ## Sponsors

-<table>
+<table align="center">
  <tbody>
    <tr>
      <td align="center" valign="middle">
        <a href="https://www.cybertec-postgresql.com/en/?utm_source=postgrest.org&utm_medium=referral&utm_campaign=postgrest" target="_blank">
-          <img width="222px" src="static/cybertec-new.png">
+          <img width="296px" src="static/cybertec-new.png">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://www.2ndquadrant.com/en/?utm_campaign=External%20Websites&utm_source=PostgREST&utm_medium=Logo" target="_blank">
-          <img width="296px" src="static/2ndquadrant.png">
-        </a>
-      </td>
-      <td align="center" valign="middle">
-        <a href="https://tryretool.com/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/retool.png">
+        <a href="https://gnuhost.eu/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/gnuhost.png">
        </a>
      </td>
    </tr>
    <tr></tr>
    <tr>
-      <td align="center" valign="middle">
-        <a href="https://gnuhost.eu/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/gnuhost.png">
-        </a>
-      </td>
      <td align="center" valign="middle">
        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
-          <img width="296px" src="static/supabase.png">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://oblivious.ai/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/oblivious.jpg">
+        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/neon.jpg">
        </a>
      </td>
    </tr>
@@ -75,7 +65,22 @@ PostgREST ongoing development is only possible thanks to our Sponsors and Backer
    <tr>
      <td align="center" valign="middle">
        <a href="https://www.timescale.com?utm_campaign=postgrest&utm_source=sponsor&utm_medium=referral&utm_content=github" target="_blank">
-          <img width="222px" src="static/timescaledb.png">
+          <img width="222px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/timescaledb.png">
+        </a>
+      </td>
+      <td align="center" valign="middle">
+        <a href="https://tryretool.com/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img max-width="222px" height="88" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/retool.png">
+        </a>
+      </td>
+      <td align="center" valign="middle">
+        <a href="https://www.2ndquadrant.com/en/?utm_campaign=External%20Websites&utm_source=PostgREST&utm_medium=Logo" target="_blank">
+          <img width="222px" src="static/2ndquadrant.png">
+        </a>
+      </td>
+      <td align="center" valign="middle">
+        <a href="https://oblivious.ai/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="222px" src="static/oblivious.jpg">
        </a>
      </td>
    </tr>
@@ -5,6 +5,131 @@ This project adheres to [Semantic Versioning](http://semver.org/).

 ## Unreleased

+## [12.0.0] - 2023-12-01
+
+### Added
+
+ - #1614, Add `db-pool-automatic-recovery` configuration to disable connection retrying - @taimoorzaeem
+ - #2492, Allow full response control when raising exceptions - @taimoorzaeem, @laurenceisla
+ - #2771, #2983, #3062, #3055 Add `Server-Timing` response header - @taimoorzaeem, @develop7, @laurenceisla
+ - #2698, Add config `jwt-cache-max-lifetime` and implement JWT caching - @taimoorzaeem
+ - #2943, Add `handling=strict/lenient` for Prefer header - @taimoorzaeem
+ - #2441, Add config `server-cors-allowed-origins` to specify CORS origins - @taimoorzaeem
+ - #2825, SQL handlers for custom media types - @steve-chavez
+   + Solves #1548, #2699, #2763, #2170, #1462, #1102, #1374, #2901
+ - #2799, Add timezone in Prefer header - @taimoorzaeem
+ - #3001, Add `statement_timeout` set on functions - @taimoorzaeem
+ - #3045, Apply superuser settings on impersonated roles if they have PostgreSQL 15 `GRANT SET ON PARAMETER` privilege - @steve-chavez
+ - #915, Add support for aggregate functions - @timabdulla
+    + The aggregate functions SUM(), MAX(), MIN(), AVG(), and COUNT() are now supported.
+    + It's disabled by default, you can enable it with `db-aggregates-enabled`.
+ - #3057, Log all internal database errors to stderr - @laurenceisla
+
+### Fixed
+
+ - #3015, Fix unnecessary count() on RPC returning single - @steve-chavez
+ - #1070, Fix HTTP status responses for upserts - @taimoorzaeem
+   + `PUT` returns `201` instead of `200` when rows are inserted
+   + `POST` with `Prefer: resolution=merge-duplicates` returns `200` instead of `201` when no rows are inserted
+ - #3019, Transaction-Scoped Settings are now shown clearly in the Postgres logs - @laurenceisla
+   + Shows `set_config('pgrst.setting_name', $1)` instead of `setconfig($1, $2)`
+   + Does not apply to role settings and `app.settings.*`
+ - #2420, Fix bogus message when listening on port 0 - @develop7
+ - #3067, Fix Acquision Timeout errors logging to stderr when `log-level=crit` - @laurenceisla
+
+### Changed
+
+ - Removed [raw-media-types config](https://postgrest.org/en/v11.1/references/configuration.html#raw-media-types) - @steve-chavez
+ - Removed `application/octet-stream`, `text/plain`, `text/xml` [builtin support for scalar results](https://postgrest.org/en/v11.1/references/api/resource_representation.html#scalar-function-response-format) - @steve-chavez
+ - Removed default `application/openapi+json` media type for [db-root-spec](https://postgrest.org/en/v11.1/references/configuration.html#db-root-spec) - @steve-chavez
+ - Removed [db-use-legacy-gucs](https://postgrest.org/en/v11.2/references/configuration.html#db-use-legacy-gucs) - @laurenceisla
+
+## [11.2.2] - 2023-10-25
+
+### Fixed
+
+ - #2824, Fix regression by reverting fix that returned 206 when first position = length in a `Range` header - @laurenceisla, @strengthless
+
+## [11.2.1] - 2023-10-03
+
+### Fixed
+
+ - #2899, Fix `application/vnd.pgrst.array` not accepted as a valid mediatype - @taimoorzaeem
+ - #2524, Fix schema cache and configuration reloading with `NOTIFY` not working on Windows - @diogob, @laurenceisla
+ - #2915, Fix duplicate headers in response - @taimoorzaeem
+ - #2824, Fix range request with first position same as length return status 206 - @taimoorzaeem
+ - #2939, Fix wrong `Preference-Applied` with `Prefer: tx=commit` when transaction is rollbacked - @steve-chavez
+ - #2939, Fix `count=exact` not being included in `Preference-Applied` - @steve-chavez
+ - #2800, Fix not including to-one embed resources that had a `NULL` value in any of the selected fields when doing null filtering on them - @laurenceisla
+ - #2846, Fix error when requesting `Prefer: count=<type>` and doing null filtering on embedded resources - @laurenceisla
+ - #2959, Fix setting `default_transaction_isolation` unnecessarily - @steve-chavez
+ - #2929, Fix arrow filtering on RPC returning dynamic TABLE with composite type - @steve-chavez
+ - #2963, Fix RPCs not embedding correctly when using overloaded functions for computed relationships - @laurenceisla
+ - #2970, Fix regression that rejects URI connection strings with certain unescaped characters in the password - @laurenceisla, @steve-chavez
+
+## [11.2.0] - 2023-08-10
+
+### Added
+
+ - #2523, Data representations - @aljungberg
+   + Allows for flexible API output formatting and input parsing on a per-column type basis using regular SQL functions configured in the database
+   + Enables greater flexibility in the form and shape of your APIs, both for output and input, making PostgREST a more versatile general-purpose API server
+   + Examples include base64 encode/decode your binary data (like a `bytea` column containing an image), choose whether to present a timestamp column as seconds since the Unix epoch or as an ISO 8601 string, or represent fixed precision decimals as strings, not doubles, to preserve precision
+   + ...and accept the same in `POST/PUT/PATCH` by configuring the reverse transformation(s)
+   + Other use-cases include custom representation of enums, arrays, nested objects, CSS hex colour strings, gzip compressed fields, metric to imperial conversions, and much more
+   + Works when using the `select` parameter to select only a subset of columns, embedding through complex joins, renaming fields, with views and computed columns
+   + Works when filtering on a formatted column without extra indexes by parsing to the canonical representation
+   + Works for data `RETURNING` operations, such as requesting the full body in a POST/PUT/PATCH with `Prefer: return=representation`
+   + Works for batch updates and inserts
+   + Completely optional, define the functions in the database and they will be used automatically everywhere
+   + Data representations preserve the ability to write to the original column and require no extra storage or complex triggers (compared to using `GENERATED ALWAYS` columns)
+   + Note: data representations require Postgres 10 (Postgres 11 if using `IN` predicates); data representations are not implemented for RPC
+ - #2647, Allow to verify the PostgREST version in SQL: `select distinct application_name from pg_stat_activity`. - @laurenceisla
+ - #2856, Add the `--version` CLI option that prints the version information - @laurenceisla
+ - #1655, Improve `details` field of the singular error response - @taimoorzaeem
+ - #740, Add `Preference-Applied` in response for `Prefer: return=representation/headers-only/minimal` - @taimoorzaeem
+ - #1601, Add optional `nulls=stripped` parameter for mediatypes `application/vnd.pgrst.array+json` and `application/vnd.pgrst.object+json` - @taimoorzaeem
+
+### Fixed
+
+ - #2821, Fix OPTIONS not accepting all available media types - @steve-chavez
+ - #2834, Fix compilation on Ubuntu by being compatible with GHC 9.0.2 - @steve-chavez
+ - #2840, Fix `Prefer: missing=default` with DOMAIN default values - @steve-chavez
+ - #2849, Fix HEAD unnecessarily executing aggregates - @steve-chavez
+ - #2594, Fix unused index on jsonb/jsonb arrow filter and order (``/bets?data->>contractId=eq.1`` and ``/bets?order=data->>contractId``) - @steve-chavez
+ - #2861, Fix character and bit columns with fixed length not inserting/updating properly - @laurenceisla
+   + Fixes the error "value too long for type character(1)" when the char length of the column was bigger than one.
+ - #2862, Fix null filtering on embedded resource when using a column name equal to the relation name - @steve-chavez
+ - #1586, Fix function parameters of type character and bit not ignoring length - @laurenceisla
+   + Fixes the error "value too long for type character(1)" when the char length of the parameter was bigger than one.
+ - #2881, Fix error when a function returns `RECORD` or `SET OF RECORD` - @laurenceisla
+ - #2896, Fix applying superuser settings for impersonated role - @steve-chavez
+
+### Deprecated
+
+ - #2863, Deprecate resource embedding target disambiguation - @steve-chavez
+   + The `/table?select=*,other!fk(*)` must be used to disambiguate
+   + The server aids in choosing the `!fk` by sending a `hint` on the error whenever an ambiguous request happens.
+
+## [11.1.0] - 2023-06-07
+
+### Added
+
+ - #2786, Limit idle postgresql connection lifetime - @robx
+   + New option `db-pool-max-idletime` (default 30s).
+   + This is equivalent to the old option `db-pool-timeout` of PostgREST 10.0.0.
+   + A config alias for `db-pool-timeout` is included.
+ - #2703, Add pre-config function - @steve-chavez
+    + New config option `db-pre-config`(empty by default)
+    + Allows using the in-database configuration without SUPERUSER
+ - #2781, When `db-channel-enabled` is false, start automatic connection recovery on a new request when pool connections are closed with `pg_terminate_backend` - @steve-chavez
+    + Mitigates the lack of LISTEN/NOTIFY for schema cache reloading on read replicas.
+
+### Fixed
+
+ - #2791, Fix dropping schema cache reload notifications  - @steve-chavez
+ - #2801, Stop retrying connection when "no password supplied" - @steve-chavez
+
 ## [11.0.1] - 2023-04-27

 ### Fixed
@@ -15,40 +15,30 @@ API than you are likely to write from scratch.

 ## Sponsors

-<table>
+<table align="center">
  <tbody>
    <tr>
      <td align="center" valign="middle">
        <a href="https://www.cybertec-postgresql.com/en/?utm_source=postgrest.org&utm_medium=referral&utm_campaign=postgrest" target="_blank">
-          <img width="222px" src="static/cybertec-new.png">
+          <img width="296px" src="static/cybertec-new.png">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://www.2ndquadrant.com/en/?utm_campaign=External%20Websites&utm_source=PostgREST&utm_medium=Logo" target="_blank">
-          <img width="296px" src="static/2ndquadrant.png">
-        </a>
-      </td>
-      <td align="center" valign="middle">
-        <a href="https://tryretool.com/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/retool.png">
+        <a href="https://gnuhost.eu/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/gnuhost.png">
        </a>
      </td>
    </tr>
    <tr></tr>
    <tr>
-      <td align="center" valign="middle">
-        <a href="https://gnuhost.eu/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/gnuhost.png">
-        </a>
-      </td>
      <td align="center" valign="middle">
        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
-          <img width="296px" src="static/supabase.png">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://oblivious.ai/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="static/oblivious.jpg">
+        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/neon.jpg">
        </a>
      </td>
    </tr>
@@ -0,0 +1 @@
+index-state: hackage.haskell.org 2023-10-13T13:54:33Z
@@ -50,6 +50,19 @@ let

  postgresqlVersions =
    [
+      {
+        name = "postgresql-16";
+        postgresql = pkgs.postgresql_16.withPackages (p: [
+          p.postgis
+          (p.pg_safeupdate.overrideAttrs (old: {
+            installPhase = ''
+              mkdir -p $out/bin
+              cp safeupdate.dylib safeupdate.so || true
+              install -D safeupdate.so -t $out/lib
+            '';
+          }))
+        ]);
+      }
      { name = "postgresql-15"; postgresql = pkgs.postgresql_15.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
      { name = "postgresql-14"; postgresql = pkgs.postgresql_14.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
      { name = "postgresql-13"; postgresql = pkgs.postgresql_13.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
@@ -1,37 +1,16 @@
-{-# LANGUAGE CPP #-}
-
 module Main (main) where

 import System.IO (BufferMode (..), hSetBuffering)

-import qualified PostgREST.App as App
 import qualified PostgREST.CLI as CLI

 import Protolude

-#ifndef mingw32_HOST_OS
-import qualified PostgREST.Unix as Unix
-#endif
-
 main :: IO ()
 main = do
  setBuffering
  opts <- CLI.readCLIShowHelp
-  CLI.main installSignalHandlers runAppInSocket opts
-
-installSignalHandlers :: App.SignalHandlerInstaller
-#ifndef mingw32_HOST_OS
-installSignalHandlers = Unix.installSignalHandlers
-#else
-installSignalHandlers _ = pass
-#endif
-
-runAppInSocket :: Maybe App.SocketRunner
-#ifndef mingw32_HOST_OS
-runAppInSocket = Just Unix.runAppWithSocket
-#else
-runAppInSocket = Nothing
-#endif
+  CLI.main opts

 setBuffering :: IO ()
 setBuffering = do
@@ -5,24 +5,14 @@ for developing, testing and building PostgREST.

 ## Getting started with Nix

-You'll need to [get Nix](https://nixos.org/download.html). The installer will
-create your Nix store in the `/nix/` directory, where all build artifacts and
-their dependencies will be stored. It will also link the Nix executables like
-`nix-env`, `nix-build` and `nix-shell` into your PATH. Nix will manage all
-other PostgREST dependencies from here on out. To clean up older build
-artifacts from the `/nix/store`, you can run `nix-collect-garbage`.
-
-If you are on a system that does not support nix, for example Windows, you can
-run the nix development environment in a docker container. Inside the `nix/`
-directory run `docker-compose run --rm nix` to start the docker container. This
-will set up the binary cache and launch `nix-shell` automatically.
+You'll need to [get Nix](https://nixos.org/download.html). Follow the recommended installation for your operating system from the official download website.

 ## Building PostgREST

 To build PostgREST from your local checkout of the repository, run:

 ```bash
-nix-build --attr postgrestPackage
+$ nix-build --attr postgrestPackage

 ```

@@ -32,6 +22,15 @@ build the `postgrestPackage` attribute from the Nix expression it finds in our
 `default.nix` (see below for details). Nix will take care of getting the right
 GHC version and all the build dependencies.

+You can also build a statically linked binary with:
+
+```bash
+$ nix-build --attr postgrestStatic
+
+$ ldd result/bin/postgrest
+$       not a dynamic executable
+```
+
 ## Binary cache

 We recommend that you use the PostgREST binary cache on
@@ -39,10 +38,10 @@ We recommend that you use the PostgREST binary cache on

 ```bash
 # Install cachix:
-nix-env -iA cachix -f https://cachix.org/api/v1/install
+$ nix-env -iA cachix -f https://cachix.org/api/v1/install

 # Set cachix up to use the PostgREST binary cache:
-cachix use postgrest
+$ cachix use postgrest

 ```

@@ -56,7 +55,7 @@ following command will put you into a new shell that has GHC and Cabal on the
 PATH:

 ```bash
-nix-shell
+$ nix-shell

 ```

@@ -92,7 +91,7 @@ Some additional modules like `memory`, `docker` and `release`
 have large dependencies that would need to be built before the shell becomes
 available, which could take an especially long time if the cachix binary cache
 is not used. You can activate those by passing a flag to `nix-shell` with
-`nix-shell --arg <module> true`. This will make the respective utilites available:
+`nix-shell --arg <module> true`. This will make the respective utilities available:

 ```bash
 $ nix-shell --arg memory true
@@ -114,7 +113,7 @@ postgrest-test-memory
 Note that `postgrest-test-memory` is now also available.

 To run one-off commands, you can also use `nix-shell --run <command>`, which
-will lauch the Nix shell, run that one command and exit. Note that the tab
+will launch the Nix shell, run that one command and exit. Note that the tab
 completion will not work with `nix-shell --run`, as Nix has yet to evaluate
 our Nix expressions to see which utilities are available.

@@ -146,10 +145,10 @@ Note: Once inside nix-shell, the utilities work from any directory inside
 the PostgREST repo. Paths are resolved relative to the repo root:

 ```bash
-$ cd src
+[nix-shell]$ cd src
 # Even though the current directory is ./src, the config path must still start
 # from the repo root:
-$ postgrest-run test/io/configs/simple.conf
+[nix-shell]$ postgrest-run test/io/configs/simple.conf
 ```

 ## Testing
@@ -177,21 +176,21 @@ run with `postgrest-test-io`. The test runner under the hood is

 ```bash
 # Filter the tests to run by name, including all that contain 'config':
-postgrest-test-io -k config
+[nix-shell]$ postgrest-test-io -k config

 # Run tests in parallel using xdist, specifying the number of processes:
-postgrest-test-io -n auto
-postgrest-test-io -n 8
+[nix-shell]$ postgrest-test-io -n auto
+[nix-shell]$ postgrest-test-io -n 8
 ```

 The memory tests check that we don't surpass a memory threshold for big request bodies.

 ```bash
 # Build the dependencies needed for the memory test
-nix-shell --arg memory true
+$ nix-shell --arg memory true

 # Run the memory test
-postgrest-test-memory
+[nix-shell]$ postgrest-test-memory
 ```

 The loadtests ensure that performance doesn't drop on a change. Underlyingly they use
@@ -199,38 +198,37 @@ The loadtests ensure that performance doesn't drop on a change. Underlyingly the

 ```bash
 # Run the loadtests on the latest commit(HEAD)
-postgrest-loadtest
+[nix-shell]$ postgrest-loadtest

 # You can loadtest comparing to a different branch
-postgrest-loadtest-against master
+[nix-shell]$ postgrest-loadtest-against master

 # You can simulate latency client/postgrest and postgrest/database
-PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest
+[nix-shell]$ PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest

 # You can build postgrest directly with cabal for faster iteration
-PGRST_BUILD_CABAL=1 postgrest-loadtest
+[nix-shell]$ PGRST_BUILD_CABAL=1 postgrest-loadtest

 # Produce a markdown report to be used on CI
-postgrest-loadtest-report
+[nix-shell]$ postgrest-loadtest-report
 ```

 doctests for some of our modules are also available:

 ```bash
-postgrest-test-doctest
+[nix-shell]$ postgrest-test-doctest
 ```

 ## Code coverage

-Code coverage is available under the `postgrest-coverage` command. This will produce a `./coverage` directory that can be visualized with a simple http server.
+Code coverage is available under the `postgrest-coverage` command. This will produce a `./coverage` directory that can be visualized on a browser.

 ```bash
 # Will run all the tests and produce a coverage dir
-postgrest-coverage
+[nix-shell]$ postgrest-coverage
+...

-# Visualize the output
-cd coverage
-python -mSimpleHTTPServer 8080
+postgrest-coverage: To see the results, visit file://$(pwd)/coverage/check/hpc_index.html
 ```

 ## Linting and styling code
@@ -248,11 +246,11 @@ $ nix-shell --run postgrest-style
 ```

 There is also `postgrest-style-check` that exits with a non-zero exit code if
-the check resulted in any uncommited changes. It's mostly useful for CI.
+the check resulted in any uncommitted changes. It's mostly useful for CI.

 ## General development tools

-Tools like `postgrest-build`, `postgrest-run` etc. are simple wrappers around
+Tools like `postgrest-build`, `postgrest-run`, `postgrest-repl` etc. are simple wrappers around
 `cabal` and should do what you expect. `postgrest-check` runs most checks that will
 also run in CI, with the exception of the IO and Memory checks that need to be run
 separately.
@@ -266,6 +264,23 @@ run against the latest PostgreSQL version by default.
 file is changed. For example, `postgrest-watch postgrest-with-all postgrest-test-spec`
 will re-run the full spec test suite against all PostgreSQL versions on every change.

+## REPL
+
+You can use `postgrest-repl` to manually inspect the PostgREST modules.
+
+```bash
+$ postgrest-repl
+
+ghci> import PostgREST.<tab>
+PostgREST.Admin                        PostgREST.Config.Database              PostgREST.Plan.MutatePlan              PostgREST.Response.OpenAPI
+PostgREST.ApiRequest                   PostgREST.Config.JSPath                PostgREST.Plan.ReadPlan                PostgREST.SchemaCache
+...
+
+ghci> import PostgREST.MediaType
+ghci> decodeMediaType "application/json"
+MTApplicationJSON
+```
+
 ## Tour

 The following is not required for working on PostgREST with Nix, but it will
@@ -294,7 +309,7 @@ version.
 ### `shell.nix`

 [`shell.nix`](../shell.nix) defines an environment in which PostgREST can be
-built and developed. It extends the build enviroment from our `postgrest`
+built and developed. It extends the build environment from our `postgrest`
 attribute with useful utilities that will be put on the PATH in `nix-shell`.

 ### `nix/overlays`
@@ -74,7 +74,7 @@ required to avoid build timeouts in CI.

 You'll need to set the `CACHIX_SIGNING_KEY` before proceeding, e.g. by creating
 a file containing `export CACHIX_SIGNING_KEY=...` and sourcing that file, which
-avoids having the secret in you shell history.
+avoids having the secret in your shell history.

 To push all new artifacts to Cachix, run:

@@ -43,8 +43,8 @@ let
        (prev.callHackageDirect
          {
            pkg = "hasql-notifications";
-            ver = "0.2.0.4";
-            sha256 = "sha256-fm1xiDyvDkb5WLOJ73/s8wrWEW23XFS7luAv2brfr8I=";
+            ver = "0.2.0.6";
+            sha256 = "sha256-7PyFlB2B70njudOjaX6tk1m77ol9vnF5fI0LF86kVAI=";
          }
          { });

@@ -52,8 +52,8 @@ let
        (prev.callHackageDirect
          {
            pkg = "hasql-pool";
-            ver = "0.9";
-            sha256 = "sha256-5UshbbaBVY8eJ/9VagNVVxonRwMcd7UmGqDc35pJNFY=";
+            ver = "0.10";
+            sha256 = "sha256-kHzoqtNV9BFWnn1h560JRqMooQRwxokVKgDRBexamNI=";
          }
          { });
    } // extraOverrides final prev;
@@ -4,16 +4,16 @@ self: super:
 {
  ## Example for including a postgresql version from a specific nixpks commit:
  ##
-  #  postgresql_14 =
-  #    let
-  #      rev = "76b1e16c6659ccef7187ca69b287525fea133244";
-  #      tarballHash = "1vsahpcx80k2bgslspb0sa6j4bmhdx77sw6la455drqcrqhdqj6a";
-  #
-  #      pinnedPkgs =
-  #        builtins.fetchTarball {
-  #          url = "https://github.com/nixos/nixpkgs/archive/${rev}.tar.gz";
-  #          sha256 = tarballHash;
-  #        };
-  #    in
-  #    (import pinnedPkgs { }).pkgs.postgresql_14;
+  postgresql_16 =
+    let
+      rev = "5148520bfab61f99fd25fb9ff7bfbb50dad3c9db";
+      tarballHash = "1dfjmz65h8z4lk845724vypzmf3dbgsdndjpj8ydlhx6c7rpcq3p";
+
+      pinnedPkgs =
+        builtins.fetchTarball {
+          url = "https://github.com/nixos/nixpkgs/archive/${rev}.tar.gz";
+          sha256 = tarballHash;
+        };
+    in
+    (import pinnedPkgs { }).pkgs.postgresql_16;
 }
@@ -41,6 +41,8 @@ let
        args =
          [
            "ARG_USE_ENV([PGRST_DB_ANON_ROLE], [postgrest_test_anonymous], [PostgREST anonymous role])"
+            "ARG_USE_ENV([PGRST_DB_POOL], [1], [PostgREST pool size])"
+            "ARG_USE_ENV([PGRST_DB_POOL_ACQUISITION_TIMEOUT], [1], [PostgREST pool size])"
            "ARG_LEFTOVERS([PostgREST arguments])"
          ];
        inRootDir = true;
@@ -48,11 +50,25 @@ let
      }
      ''
        export PGRST_DB_ANON_ROLE
+        export PGRST_DB_POOL
+        export PGRST_DB_POOL_ACQUISITION_TIMEOUT

        exec ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
          postgrest "''${_arg_leftovers[@]}"
      '';

+  repl =
+    checkedShellScript
+      {
+        name = "postgrest-repl";
+        docs = "Interact with PostgREST modules using the cabal repl";
+        args = [ "ARG_LEFTOVERS([cabal v2-repl arguments])" ];
+        inRootDir = true;
+        withEnv = postgrest.env;
+      }
+      ''
+        exec ${cabal-install}/bin/cabal v2-repl "''${_arg_leftovers[@]}"
+      '';
 in
 buildToolbox
 {
@@ -61,5 +77,6 @@ buildToolbox
    build
    clean
    run
+    repl
  ];
 }
@@ -76,13 +76,12 @@ let
        inherit name;
        docs =
          ''
-            Run the vegeta loadtest twice:
-            - once on the <target> branch
+            Run the vegeta loadtest against every target branch and HEAD:
+            - once on the every <target-#> branch
            - once in the current worktree
          '';
        args = [
-          "ARG_POSITIONAL_SINGLE([target], [Commit-ish reference to compare with])"
-          "ARG_LEFTOVERS([additional vegeta arguments])"
+          "ARG_POSITIONAL_INF([target], [Commit-ish reference to compare with], 1)"
        ];
        positionalCompletion =
          ''
@@ -93,9 +92,11 @@ let
        inRootDir = true;
      }
      ''
+        for tgt in "''${_arg_target[@]}"; do
+
        cat << EOF

-        Running loadtest on "$_arg_target"...
+        Running loadtest on "$tgt"...

        EOF

@@ -104,21 +105,23 @@ let
        # Save the results in the current working tree, too,
        # otherwise they'd be lost in the temporary working tree
        # created by withTools.withGit.
-        ${withTools.withGit} "$_arg_target" ${loadtest} --output "$PWD/loadtest/$_arg_target.bin" --testdir "$PWD/test/load" "''${_arg_leftovers[@]}"
+        ${withTools.withGit} "$tgt" ${loadtest} --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"

        cat << EOF

-        Done running on "$_arg_target".
+        Done running on "$tgt".

        EOF

+        done
+
        cat << EOF

        Running loadtest on HEAD...

        EOF

-        ${loadtest} --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load" "''${_arg_leftovers[@]}"
+        ${loadtest} --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load"

        cat << EOF

@@ -56,8 +56,8 @@ let
        inRootDir = true;
      }
      ''
-        trap "echo You need to be on the main branch to proceed. Exiting ..." ERR
-        [ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]
+        trap "echo You need to be on the main branch or a release branch to proceed. Exiting ..." ERR
+        [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^main$|^rel- ]]
        trap "" ERR

        trap "echo You have uncommitted changes in postgrest.cabal. Exiting ..." ERR
@@ -111,7 +111,7 @@ let
        remote="$(git remote -v | grep PostgREST/postgrest | grep push | cut -f1)"
        trap "" ERR

-        push="git push --atomic $remote main v$new_version"
+        push="git push --atomic $remote $(git rev-parse --abbrev-ref HEAD) v$new_version"

        echo "To push both the branch and the new tag, the following will be run:"
        echo
@@ -12,40 +12,30 @@ write from scratch.

 ## Sponsors

-<table>
+<table align="center">
  <tbody>
    <tr>
      <td align="center" valign="middle">
        <a href="https://www.cybertec-postgresql.com/en/?utm_source=postgrest.org&utm_medium=referral&utm_campaign=postgrest" target="_blank">
-          <img width="222px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/cybertec-new.png">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/cybertec-new.png">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://www.2ndquadrant.com/en/?utm_campaign=External%20Websites&utm_source=PostgREST&utm_medium=Logo" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/2ndquadrant.png">
-        </a>
-      </td>
-      <td align="center" valign="middle">
-        <a href="https://tryretool.com/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/retool.png">
+        <a href="https://gnuhost.eu/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/gnuhost.png">
        </a>
      </td>
    </tr>
    <tr></tr>
    <tr>
-      <td align="center" valign="middle">
-        <a href="https://gnuhost.eu/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/gnuhost.png">
-        </a>
-      </td>
      <td align="center" valign="middle">
        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
        </a>
      </td>
      <td align="center" valign="middle">
-        <a href="https://oblivious.ai/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/oblivious.jpg">
+        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/neon.jpg">
        </a>
      </td>
    </tr>
@@ -58,13 +48,13 @@ To learn how to use this container, see the [PostgREST Docker
 documentation](https://postgrest.org/en/stable/install.html#docker).

 You can configure the PostgREST image by setting
-[enviroment variables](https://postgrest.org/en/stable/configuration.html).
+[environment variables](https://postgrest.org/en/stable/configuration.html).

 # How this image is built

 The image is built from scratch using
 [Nix](https://nixos.org/nixpkgs/manual/#sec-pkgs-dockerTools) instead of a
-`Dockerfile`, which yields a higly secure and optimized image. This is also why
+`Dockerfile`, which yields a highly secure and optimized image. This is also why
 no commands are listed in the image history. See the [PostgREST
 respository](https://github.com/PostgREST/postgrest/tree/main/nix/tools/docker) for
 details on the build process and how to inspect the image.
@@ -164,7 +164,7 @@ let
            ${ghc}/bin/hpc markup --highlight-covered --destdir=coverage/overlay "$tmpdir"/overlay.tix || true
            ${ghc}/bin/hpc markup --highlight-covered --destdir=coverage/check "$tmpdir"/check.tix || true
            echo "ERROR: Something is covered by both the tests and the overlay:"
-            echo "file://$(pwd)/coverage/check/hpc_index.html"
+            echo "postgrest-coverage: To see the results, visit file://$(pwd)/coverage/check/hpc_index.html"
            exit 1
          else
            # copy the result .tix file to the coverage/ dir to make it available to postgrest-coverage-draft-overlay, too
@@ -174,7 +174,7 @@ let

            # create html and stdout reports
            ${ghc}/bin/hpc markup --destdir=coverage coverage/postgrest.tix
-            echo "file://$(pwd)/coverage/hpc_index.html"
+            echo "postgrest-coverage: To see the results, visit file://$(pwd)/coverage/hpc_index.html"
            ${ghc}/bin/hpc report coverage/postgrest.tix "''${_arg_leftovers[@]}"
          fi
        ''
@@ -32,6 +32,7 @@ let
            "ARG_USE_ENV([PGDATABASE], [postgres], [PG database name])"
            "ARG_USE_ENV([PGRST_DB_SCHEMAS], [test], [Schema to expose])"
            "ARG_USE_ENV([PGTZ], [utc], [Timezone to use])"
+            "ARG_USE_ENV([PGOPTIONS], [-c search_path=public,test], [PG options to use])"
          ];
        positionalCompletion = "_command";
        inRootDir = true;
@@ -61,6 +62,7 @@ let
        export PGDATABASE
        export PGRST_DB_SCHEMAS
        export PGTZ
+        export PGOPTIONS

        HBA_FILE="$tmpdir/pg_hba.conf"
        echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
@@ -70,7 +72,7 @@ let
        # We try to make the database cluster as independent as possible from the host
        # by specifying the timezone, locale and encoding.
        # initdb -U creates a superuser(man initdb)
-        PGTZ=UTC initdb --no-locale --encoding=UTF8 --nosync -U "${superuserRole}" --auth=trust \
+        TZ=$PGTZ initdb --no-locale --encoding=UTF8 --nosync -U "${superuserRole}" --auth=trust \
          >> "$setuplog"

        log "Starting the database cluster..."
@@ -94,7 +96,7 @@ let

        log "Done. Running command..."

-        echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U $PGUSER"
+        echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U ${superuserRole}"
        echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log"

        ("$_arg_command" "''${_arg_leftovers[@]}")
@@ -1,5 +1,5 @@
 name:               postgrest
-version:            11.0.1
+version:            12.0.0
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                    for tables, views, and functions, supporting all HTTP methods that security
@@ -34,7 +34,8 @@ library
  default-extensions: OverloadedStrings
                      NoImplicitPrelude
  hs-source-dirs:     src
-  exposed-modules:    PostgREST.App
+  exposed-modules:    PostgREST.Admin
+                      PostgREST.App
                      PostgREST.AppState
                      PostgREST.Auth
                      PostgREST.CLI
@@ -48,6 +49,7 @@ library
                      PostgREST.SchemaCache.Identifiers
                      PostgREST.SchemaCache.Routine
                      PostgREST.SchemaCache.Relationship
+                      PostgREST.SchemaCache.Representations
                      PostgREST.SchemaCache.Table
                      PostgREST.Error
                      PostgREST.Logger
@@ -62,6 +64,7 @@ library
                      PostgREST.Plan.ReadPlan
                      PostgREST.Plan.Types
                      PostgREST.RangeQuery
+                      PostgREST.Unix
                      PostgREST.ApiRequest
                      PostgREST.ApiRequest.Preferences
                      PostgREST.ApiRequest.QueryParams
@@ -69,8 +72,8 @@ library
                      PostgREST.Response
                      PostgREST.Response.OpenAPI
                      PostgREST.Response.GucHeader
+                      PostgREST.Response.Performance
                      PostgREST.Version
-                      PostgREST.Workers
  other-modules:      Paths_postgrest
  build-depends:      base                      >= 4.9 && < 4.17
                    , HTTP                      >= 4000.3.7 && < 4000.5
@@ -79,19 +82,23 @@ library
                    , auto-update               >= 0.1.4 && < 0.2
                    , base64-bytestring         >= 1 && < 1.3
                    , bytestring                >= 0.10.8 && < 0.12
+                    , cache                     >= 0.1.3 && < 0.2.0
                    , case-insensitive          >= 1.2 && < 1.3
                    , cassava                   >= 0.4.5 && < 0.6
+                    , clock                     >= 0.8.3 && < 0.9.0
                    , configurator-pg           >= 0.2 && < 0.3
                    , containers                >= 0.5.7 && < 0.7
                    , contravariant-extras      >= 0.3.3 && < 0.4
                    , cookie                    >= 0.4.2 && < 0.5
+                    , directory                 >= 1.2.6 && < 1.4
                    , either                    >= 4.4.1 && < 5.1
+                    , extra                     >= 1.7.0 && < 2.0
                    , fuzzyset                  >= 0.2.3
                    , gitrev                    >= 1.2 && < 1.4
                    , hasql                     >= 1.6.1.1 && < 1.7
                    , hasql-dynamic-statements  >= 0.3.1 && < 0.4
-                    , hasql-notifications       >= 0.1 && < 0.3
-                    , hasql-pool                >= 0.9 && < 0.10
+                    , hasql-notifications       >= 0.2.0.6 && < 0.3
+                    , hasql-pool                >= 0.10 && < 0.11
                    , hasql-transaction         >= 1.0.1 && < 1.1
                    , heredoc                   >= 0.2 && < 0.3
                    , http-types                >= 0.12.2 && < 0.13
@@ -109,10 +116,13 @@ library
                    , regex-tdfa                >= 1.2.2 && < 1.4
                    , retry                     >= 0.7.4 && < 0.10
                    , scientific                >= 0.3.4 && < 0.4
+                    , streaming-commons         >= 0.1.1 && < 0.3
                    , swagger2                  >= 2.4 && < 2.9
                    , text                      >= 1.2.2 && < 1.3
                    , time                      >= 1.6 && < 1.12
+                    , timeit                    >= 2.0 && < 2.1
                    , unordered-containers      >= 0.2.8 && < 0.3
+                    , unix-compat               >= 0.5.4 && < 0.6
                    , vault                     >= 0.3.1.5 && < 0.4
                    , vector                    >= 0.11 && < 0.14
                    , wai                       >= 3.2.1 && < 3.3
@@ -142,9 +152,6 @@ library
  if !os(windows)
    build-depends:
      unix
-      , directory >= 1.2.6 && < 1.4
-    exposed-modules:
-      PostgREST.Unix

 executable postgrest
  default-language:   Haskell2010
@@ -186,7 +193,6 @@ test-suite spec
                      Feature.ConcurrentSpec
                      Feature.CorsSpec
                      Feature.ExtraSearchPathSpec
-                      Feature.LegacyGucsSpec
                      Feature.NoSuperuserSpec
                      Feature.ObservabilitySpec
                      Feature.OpenApi.DisabledOpenApiSpec
@@ -196,25 +202,29 @@ test-suite spec
                      Feature.OpenApi.RootSpec
                      Feature.OpenApi.SecurityOpenApiSpec
                      Feature.OptionsSpec
+                      Feature.Query.AggregateFunctionsSpec
                      Feature.Query.AndOrParamsSpec
                      Feature.Query.ComputedRelsSpec
+                      Feature.Query.CustomMediaSpec
                      Feature.Query.DeleteSpec
                      Feature.Query.EmbedDisambiguationSpec
                      Feature.Query.EmbedInnerJoinSpec
-                      Feature.Query.PlanSpec
-                      Feature.Query.HtmlRawOutputSpec
+                      Feature.Query.ErrorSpec
                      Feature.Query.InsertSpec
                      Feature.Query.JsonOperatorSpec
                      Feature.Query.MultipleSchemaSpec
-                      Feature.Query.ErrorSpec
+                      Feature.Query.NullsStripSpec
                      Feature.Query.PgSafeUpdateSpec
+                      Feature.Query.PlanSpec
                      Feature.Query.PostGISSpec
+                      Feature.Query.PreferencesSpec
                      Feature.Query.QueryLimitedSpec
                      Feature.Query.QuerySpec
                      Feature.Query.RangeSpec
                      Feature.Query.RawOutputTypesSpec
                      Feature.Query.RelatedQueriesSpec
                      Feature.Query.RpcSpec
+                      Feature.Query.ServerTimingSpec
                      Feature.Query.SingularSpec
                      Feature.Query.SpreadQueriesSpec
                      Feature.Query.UnicodeSpec
@@ -232,7 +242,7 @@ test-suite spec
                    , bytestring        >= 0.10.8 && < 0.12
                    , case-insensitive  >= 1.2 && < 1.3
                    , containers        >= 0.5.7 && < 0.7
-                    , hasql-pool        >= 0.9 && < 0.10
+                    , hasql-pool        >= 0.10 && < 0.11
                    , hasql-transaction >= 1.0.1 && < 1.1
                    , heredoc           >= 0.2 && < 0.3
                    , hspec             >= 2.3 && < 2.10
@@ -0,0 +1,64 @@
+{-# LANGUAGE NamedFieldPuns #-}
+
+module PostgREST.Admin
+  ( runAdmin
+  ) where
+
+import qualified Hasql.Session             as SQL
+import qualified Network.HTTP.Types.Status as HTTP
+import qualified Network.Wai               as Wai
+import qualified Network.Wai.Handler.Warp  as Warp
+
+import Control.Monad.Extra (whenJust)
+
+import Network.Socket
+import Network.Socket.ByteString
+
+import PostgREST.AppState (AppState)
+import PostgREST.Config   (AppConfig (..))
+
+import qualified PostgREST.AppState as AppState
+
+import Protolude
+import Protolude.Partial (fromJust)
+
+runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
+runAdmin conf@AppConfig{configAdminServerPort} appState settings =
+  whenJust (AppState.getSocketAdmin appState) $ \adminSocket -> do
+    AppState.logWithZTime appState $ "Admin server listening on port " <> show (fromIntegral (fromJust configAdminServerPort) :: Integer)
+    void . forkIO $ Warp.runSettingsSocket settings adminSocket adminApp
+  where
+    adminApp = admin appState conf
+
+-- | PostgREST admin application
+admin :: AppState.AppState -> AppConfig -> Wai.Application
+admin appState appConfig req respond  = do
+  isMainAppReachable  <- isRight <$> reachMainApp (AppState.getSocketREST appState)
+  isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
+  isConnectionUp      <-
+    if configDbChannelEnabled appConfig
+      then AppState.getIsListenerOn appState
+      else isRight <$> AppState.usePool appState appConfig (SQL.sql "SELECT 1")
+
+  case Wai.pathInfo req of
+    ["ready"] ->
+      respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
+    ["live"] ->
+      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
+    _ ->
+      respond $ Wai.responseLBS HTTP.status404 [] mempty
+
+-- Try to connect to the main app socket
+-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
+reachMainApp :: Socket -> IO (Either IOException ())
+reachMainApp appSock = do
+  sockAddr <- getSocketName appSock
+  sock <- socket (addrFamily sockAddr) Stream defaultProtocol
+  try $ do
+    connect sock sockAddr
+    withSocketsDo $ bracket (pure sock) close sendEmpty
+  where
+    sendEmpty sock = void $ send sock mempty
+    addrFamily (SockAddrInet _ _) = AF_INET
+    addrFamily (SockAddrInet6 {}) = AF_INET6
+    addrFamily (SockAddrUnix _)   = AF_UNIX
@@ -26,7 +26,6 @@ import qualified Data.ByteString.Lazy  as LBS
 import qualified Data.CaseInsensitive  as CI
 import qualified Data.Csv              as CSV
 import qualified Data.HashMap.Strict   as HM
-import qualified Data.List             as L
 import qualified Data.List.NonEmpty    as NonEmptyList
 import qualified Data.Map.Strict       as M
 import qualified Data.Set              as S
@@ -37,7 +36,7 @@ import Data.Either.Combinators (mapBoth)

 import Control.Arrow             ((***))
 import Data.Aeson.Types          (emptyArray, emptyObject)
-import Data.List                 (lookup, union)
+import Data.List                 (lookup)
 import Data.Ranged.Ranges        (emptyRange, rangeIntersection,
                                  rangeIsEmpty)
 import Network.HTTP.Types.Header (RequestHeaders, hCookie)
@@ -51,13 +50,12 @@ import PostgREST.ApiRequest.Types        (ApiRequestError (..),
                                          RangeError (..))
 import PostgREST.Config                  (AppConfig (..),
                                          OpenAPIMode (..))
-import PostgREST.MediaType               (MTPlanAttrs (..),
-                                          MTPlanFormat (..),
-                                          MediaType (..))
+import PostgREST.MediaType               (MediaType (..))
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                          convertToLimitZeroRange,
                                          hasLimitZero,
                                          rangeRequested)
+import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                          QualifiedIdentifier (..),
                                          Schema)
@@ -132,17 +130,16 @@ data ApiRequest = ApiRequest {
  , iMethod              :: ByteString                       -- ^ Raw request method
  , iSchema              :: Schema                           -- ^ The request schema. Can vary depending on profile headers.
  , iNegotiatedByProfile :: Bool                             -- ^ If schema was was chosen according to the profile spec https://www.w3.org/TR/dx-prof-conneg/
-  , iAcceptMediaType     :: MediaType                        -- ^ The media type in the Accept header
+  , iAcceptMediaType     :: [MediaType]                      -- ^ The resolved media types in the Accept, considering quality(q) factors
  , iContentMediaType    :: MediaType                        -- ^ The media type in the Content-Type header
  }

 -- | Examines HTTP request and translates it into user intent.
-userApiRequest :: AppConfig -> Request -> RequestBody -> Either ApiRequestError ApiRequest
-userApiRequest conf req reqBody = do
+userApiRequest :: AppConfig -> Request -> RequestBody -> SchemaCache -> Either ApiRequestError ApiRequest
+userApiRequest conf req reqBody sCache = do
  pInfo@PathInfo{..} <- getPathInfo conf $ pathInfo req
  act <- getAction pInfo method
  qPrms <- first QueryParamError $ QueryParams.parse (pathIsProc && act `elem` [ActionInvoke InvGet, ActionInvoke InvHead]) $ rawQueryString req
-  (acceptMediaType, contentMediaType) <- getMediaTypes conf hdrs act pInfo
  (schema, negotiatedByProfile) <- getSchema conf hdrs method
  (topLevelRange, ranges) <- getRanges method qPrms hdrs
  (payload, columns) <- getPayload reqBody contentMediaType qPrms act pInfo
@@ -154,7 +151,7 @@ userApiRequest conf req reqBody = do
  , iRange = ranges
  , iTopLevelRange = topLevelRange
  , iPayload = payload
-  , iPreferences = Preferences.fromHeaders hdrs
+  , iPreferences = Preferences.fromHeaders (configDbTxAllowOverride conf) (dbTimezones sCache) hdrs
  , iQueryParams = qPrms
  , iColumns = columns
  , iHeaders = iHdrs
@@ -163,7 +160,7 @@ userApiRequest conf req reqBody = do
  , iMethod = method
  , iSchema = schema
  , iNegotiatedByProfile = negotiatedByProfile
-  , iAcceptMediaType = acceptMediaType
+  , iAcceptMediaType = maybe [MTAny] (map MediaType.decodeMediaType . parseHttpAccept) $ lookupHeader "accept"
  , iContentMediaType = contentMediaType
  }
  where
@@ -172,6 +169,7 @@ userApiRequest conf req reqBody = do
    lookupHeader    = flip lookup hdrs
    iHdrs = [ (CI.foldedCase k, v) | (k,v) <- hdrs, k /= hCookie]
    iCkies = maybe [] parseCookies $ lookupHeader "Cookie"
+    contentMediaType = maybe MTApplicationJSON MediaType.decodeMediaType $ lookupHeader "content-type"

 getPathInfo :: AppConfig -> [Text] -> Either ApiRequestError PathInfo
 getPathInfo AppConfig{configOpenApiMode, configDbRootSpec} path =
@@ -205,15 +203,6 @@ getAction PathInfo{pathIsProc, pathIsDefSpec} method =
      "OPTIONS"                  -> Right ActionInfo
      _                          -> Left $ UnsupportedMethod method

-getMediaTypes :: AppConfig -> RequestHeaders -> Action -> PathInfo -> Either ApiRequestError (MediaType, MediaType)
-getMediaTypes conf hdrs action path = do
-   acceptMediaType <- findAcceptMediaType conf action path accepts
-   pure (acceptMediaType, contentMediaType)
-  where
-    accepts = maybe [MTAny] (map MediaType.decodeMediaType . parseHttpAccept) $ lookupHeader "accept"
-    contentMediaType = maybe MTApplicationJSON MediaType.decodeMediaType $ lookupHeader "content-type"
-    lookupHeader    = flip lookup hdrs
-
 getSchema :: AppConfig -> RequestHeaders -> ByteString -> Either ApiRequestError (Schema, Bool)
 getSchema AppConfig{configDbSchemas} hdrs method = do
  case profile of
@@ -299,19 +288,6 @@ getPayload reqBody contentMediaType QueryParams{qsColumns} action PathInfo{pathI
      ActionInvoke InvPost        -> qsColumns
      _                           -> Nothing

-{-|
-  Find the best match from a list of media types accepted by the
-  client in order of decreasing preference and a list of types
-  producible by the server.  If there is no match but the client
-  accepts */* then return the top server pick.
-}
-mutuallyAgreeable :: [MediaType] -> [MediaType] -> Maybe MediaType
-mutuallyAgreeable sProduces cAccepts =
-  let exact = listToMaybe $ L.intersect cAccepts sProduces in
-  if isNothing exact && MTAny `elem` cAccepts
-     then listToMaybe sProduces
-     else exact
-
 type CsvData = V.Vector (M.Map Text LBS.ByteString)

 {-|
@@ -360,29 +336,3 @@ payloadAttributes raw json =
    _ -> Just emptyPJArray
  where
    emptyPJArray = ProcessedJSON (JSON.encode emptyArray) S.empty
-
-findAcceptMediaType :: AppConfig -> Action -> PathInfo -> [MediaType] -> Either ApiRequestError MediaType
-findAcceptMediaType conf action path accepts =
-  case mutuallyAgreeable (requestMediaTypes conf action path) accepts of
-    Just ct ->
-      Right ct
-    Nothing ->
-      Left . MediaTypeError $ map MediaType.toMime accepts
-
-requestMediaTypes :: AppConfig -> Action -> PathInfo -> [MediaType]
-requestMediaTypes conf action path =
-  case action of
-    ActionRead _    -> defaultMediaTypes ++ rawMediaTypes
-    ActionInvoke _  -> invokeMediaTypes
-    ActionInspect _ -> [MTOpenAPI, MTApplicationJSON]
-    ActionInfo      -> [MTTextCSV]
-    _               -> defaultMediaTypes
-  where
-    invokeMediaTypes =
-      defaultMediaTypes
-        ++ rawMediaTypes
-        ++ [MTOpenAPI | pathIsRootSpec path]
-    defaultMediaTypes =
-      [MTApplicationJSON, MTSingularJSON, MTGeoJSON, MTTextCSV] ++
-      [MTPlan $ MTPlanAttrs Nothing PlanJSON mempty | configDbPlanEnabled conf]
-    rawMediaTypes = configRawMediaTypes conf `union` [MTOctetStream, MTTextPlain, MTTextXML]
@@ -6,25 +6,30 @@
 --
 -- [1] https://datatracker.ietf.org/doc/html/rfc7240
 --
+{-# LANGUAGE NamedFieldPuns #-}
 module PostgREST.ApiRequest.Preferences
  ( Preferences(..)
  , PreferCount(..)
+  , PreferHandling(..)
  , PreferMissing(..)
  , PreferParameters(..)
  , PreferRepresentation(..)
  , PreferResolution(..)
  , PreferTransaction(..)
+  , PreferTimezone(..)
  , fromHeaders
-  , ToAppliedHeader(..)
  , shouldCount
+  , prefAppliedHeader
  ) where

 import qualified Data.ByteString.Char8     as BS
 import qualified Data.Map                  as Map
+import qualified Data.Set                  as S
 import qualified Network.HTTP.Types.Header as HTTP

-import Protolude
+import PostgREST.Config.Database (TimezoneNames)

+import Protolude

 -- $setup
 -- Setup for doctests
@@ -35,92 +40,122 @@ import Protolude
 -- >>> deriving instance Show PreferCount
 -- >>> deriving instance Show PreferTransaction
 -- >>> deriving instance Show PreferMissing
+-- >>> deriving instance Show PreferHandling
+-- >>> deriving instance Show PreferTimezone
 -- >>> deriving instance Show Preferences

 -- | Preferences recognized by the application.
 data Preferences
  = Preferences
    { preferResolution     :: Maybe PreferResolution
-    , preferRepresentation :: PreferRepresentation
+    , preferRepresentation :: Maybe PreferRepresentation
    , preferParameters     :: Maybe PreferParameters
    , preferCount          :: Maybe PreferCount
    , preferTransaction    :: Maybe PreferTransaction
    , preferMissing        :: Maybe PreferMissing
+    , preferHandling       :: Maybe PreferHandling
+    , preferTimezone       :: Maybe PreferTimezone
+    , invalidPrefs         :: [ByteString]
    }

 -- |
 -- Parse HTTP headers based on RFC7240[1] to identify preferences.
 --
-- One header with comma-separated values can be used to set multiple preferences:
+-- >>> let sc = S.fromList ["America/Los_Angeles"]
 --
-- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates, count=exact")]
+-- One header with comma-separated values can be used to set multiple preferences:
+-- >>> pPrint $ fromHeaders True sc [("Prefer", "resolution=ignore-duplicates, count=exact, timezone=America/Los_Angeles")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
--     , preferRepresentation = None
+--     , preferRepresentation = Nothing
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
 --     , preferMissing = Nothing
+--     , preferHandling = Nothing
+--     , preferTimezone = Just
+--         ( PreferTimezone "America/Los_Angeles" )
+--     , invalidPrefs = []
 --     }
 --
 -- Multiple headers can also be used:
 --
-- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null")]
+-- >>> pPrint $ fromHeaders True sc [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null"), ("Prefer", "handling=lenient"), ("Prefer", "invalid")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
--     , preferRepresentation = None
+--     , preferRepresentation = Nothing
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
 --     , preferMissing = Just ApplyNulls
+--     , preferHandling = Just Lenient
+--     , preferTimezone = Nothing
+--     , invalidPrefs = [ "invalid" ]
 --     }
 --
 -- If a preference is set more than once, only the first is used:
 --
-- >>> preferTransaction $ fromHeaders [("Prefer", "tx=commit, tx=rollback")]
+-- >>> preferTransaction $ fromHeaders True sc [("Prefer", "tx=commit, tx=rollback")]
 -- Just Commit
 --
 -- This is also the case across multiple headers:
 --
 -- >>> :{
--   preferResolution . fromHeaders $
+--   preferResolution . fromHeaders True sc $
 --     [ ("Prefer", "resolution=ignore-duplicates")
 --     , ("Prefer", "resolution=merge-duplicates")
 --     ]
 -- :}
 -- Just IgnoreDuplicates
 --
-- Preferences not recognized by the application are ignored:
--
-- >>> preferResolution $ fromHeaders [("Prefer", "resolution=foo")]
-- Nothing
 --
 -- Preferences can be separated by arbitrary amounts of space, lower-case header is also recognized:
 --
-- >>> pPrint $ fromHeaders [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default")]
+-- >>> pPrint $ fromHeaders True sc [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default, handling=strict, anything")]
 -- Preferences
 --     { preferResolution = Nothing
--     , preferRepresentation = Full
+--     , preferRepresentation = Just Full
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Just Commit
 --     , preferMissing = Just ApplyDefaults
+--     , preferHandling = Just Strict
+--     , preferTimezone = Nothing
+--     , invalidPrefs = [ "anything" ]
 --     }
 --
-fromHeaders :: [HTTP.Header] -> Preferences
-fromHeaders headers =
+fromHeaders :: Bool -> TimezoneNames -> [HTTP.Header] -> Preferences
+fromHeaders allowTxDbOverride acceptedTzNames headers =
  Preferences
-    { preferResolution = parsePrefs [MergeDuplicates, IgnoreDuplicates]
-    , preferRepresentation = fromMaybe None $ parsePrefs [Full, None, HeadersOnly]
-    , preferParameters = parsePrefs [SingleObject]
-    , preferCount = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
-    , preferTransaction = parsePrefs [Commit, Rollback]
-    , preferMissing = parsePrefs [ApplyDefaults, ApplyNulls]
+    { preferResolution     = parsePrefs [MergeDuplicates, IgnoreDuplicates]
+    , preferRepresentation = parsePrefs [Full, None, HeadersOnly]
+    , preferParameters     = parsePrefs [SingleObject]
+    , preferCount          = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
+    , preferTransaction    = if allowTxDbOverride then parsePrefs [Commit, Rollback] else Nothing
+    , preferMissing        = parsePrefs [ApplyDefaults, ApplyNulls]
+    , preferHandling       = parsePrefs [Strict, Lenient]
+    , preferTimezone       = if isTimezonePrefAccepted then PreferTimezone <$> timezonePref else Nothing
+    , invalidPrefs         = filter checkPrefs prefs
    }
  where
+    mapToHeadVal :: ToHeaderValue a => [a] -> [ByteString]
+    mapToHeadVal = map toHeaderValue
+    acceptedPrefs = mapToHeadVal [MergeDuplicates, IgnoreDuplicates] ++
+                    mapToHeadVal [Full, None, HeadersOnly] ++
+                    mapToHeadVal [SingleObject] ++
+                    mapToHeadVal [ExactCount, PlannedCount, EstimatedCount] ++
+                    mapToHeadVal [Commit, Rollback] ++
+                    mapToHeadVal [ApplyDefaults, ApplyNulls] ++
+                    mapToHeadVal [Strict, Lenient]
+
    prefHeaders = filter ((==) HTTP.hPrefer . fst) headers
    prefs = fmap BS.strip . concatMap (BS.split ',' . snd) $ prefHeaders

+    timezonePref = listToMaybe $ mapMaybe (BS.stripPrefix "timezone=") prefs
+    isTimezonePrefAccepted = (S.member <$> timezonePref <*> pure acceptedTzNames) == Just True
+
+    checkPrefs p = p `notElem` acceptedPrefs && not isTimezonePrefAccepted
+
    parsePrefs :: ToHeaderValue a => [a] -> Maybe a
    parsePrefs vals =
      head $ mapMaybe (flip Map.lookup $ prefMap vals) prefs
@@ -128,6 +163,24 @@ fromHeaders headers =
    prefMap :: ToHeaderValue a => [a] -> Map.Map ByteString a
    prefMap = Map.fromList . fmap (\pref -> (toHeaderValue pref, pref))

+prefAppliedHeader :: Preferences -> Maybe HTTP.Header
+prefAppliedHeader Preferences {preferResolution, preferRepresentation, preferParameters, preferCount, preferTransaction, preferMissing, preferHandling, preferTimezone } =
+  if null prefsVals
+    then Nothing
+    else Just (HTTP.hPreferenceApplied, combined)
+  where
+    combined = BS.intercalate ", " prefsVals
+    prefsVals = catMaybes [
+        toHeaderValue <$> preferResolution
+      , toHeaderValue <$> preferMissing
+      , toHeaderValue <$> preferRepresentation
+      , toHeaderValue <$> preferParameters
+      , toHeaderValue <$> preferCount
+      , toHeaderValue <$> preferTransaction
+      , toHeaderValue <$> preferHandling
+      , toHeaderValue <$> preferTimezone
+      ]
+
 -- |
 -- Convert a preference into the value that we look for in the 'Prefer' headers.
 --
@@ -137,27 +190,16 @@ fromHeaders headers =
 class ToHeaderValue a where
  toHeaderValue :: a -> ByteString

-- |
-- Header to indicate that a preference has been applied.
--
-- >>> toAppliedHeader MergeDuplicates
-- ("Preference-Applied","resolution=merge-duplicates")
--
-class ToHeaderValue a => ToAppliedHeader a where
-  toAppliedHeader :: a -> HTTP.Header
-  toAppliedHeader x = (HTTP.hPreferenceApplied, toHeaderValue x)
-
 -- | How to handle duplicate values.
 data PreferResolution
  = MergeDuplicates
  | IgnoreDuplicates
+  deriving Eq

 instance ToHeaderValue PreferResolution where
  toHeaderValue MergeDuplicates  = "resolution=merge-duplicates"
  toHeaderValue IgnoreDuplicates = "resolution=ignore-duplicates"

-instance ToAppliedHeader PreferResolution
-
 -- |
 -- How to return the mutated data.
 --
@@ -207,8 +249,6 @@ instance ToHeaderValue PreferTransaction where
  toHeaderValue Commit   = "tx=commit"
  toHeaderValue Rollback = "tx=rollback"

-instance ToAppliedHeader PreferTransaction
-
 -- |
 -- How to handle the insertion/update when the keys specified in ?columns are not present
 -- in the json body.
@@ -221,4 +261,20 @@ instance ToHeaderValue PreferMissing where
  toHeaderValue ApplyDefaults = "missing=default"
  toHeaderValue ApplyNulls    = "missing=null"

-instance ToAppliedHeader PreferMissing
+-- |
+-- Handling of unrecognised preferences
+data PreferHandling
+  = Strict  -- ^ Throw error on unrecognised preferences
+  | Lenient -- ^ Ignore unrecognised preferences
+  deriving Eq
+
+instance ToHeaderValue PreferHandling where
+  toHeaderValue Strict  = "handling=strict"
+  toHeaderValue Lenient = "handling=lenient"
+
+-- |
+-- Change timezone
+newtype PreferTimezone = PreferTimezone ByteString
+
+instance ToHeaderValue PreferTimezone where
+  toHeaderValue (PreferTimezone tz) = "timezone=" <> tz
@@ -31,8 +31,8 @@ import Data.Tree                     (Tree (..))
 import Text.Parsec.Error             (errorMessages,
                                      showErrorMessages)
 import Text.ParserCombinators.Parsec (GenParser, ParseError, Parser,
-                                      anyChar, between, char, digit,
-                                      eof, errorPos, letter,
+                                      anyChar, between, char, choice,
+                                      digit, eof, errorPos, letter,
                                      lookAhead, many1, noneOf,
                                      notFollowedBy, oneOf,
                                      optionMaybe, sepBy, sepBy1,
@@ -43,7 +43,8 @@ import PostgREST.RangeQuery              (NonnegRange, allRange,
                                          rangeOffset, restrictRange)
 import PostgREST.SchemaCache.Identifiers (FieldName)

-import PostgREST.ApiRequest.Types (EmbedParam (..), EmbedPath, Field,
+import PostgREST.ApiRequest.Types (AggregateFunction (..),
+                                   EmbedParam (..), EmbedPath, Field,
                                   Filter (..), FtsOperator (..),
                                   Hint, JoinType (..),
                                   JsonOperand (..),
@@ -58,30 +59,7 @@ import PostgREST.ApiRequest.Types (EmbedParam (..), EmbedPath, Field,
                                   SimpleOperator (..), SingleVal,
                                   TrileanVal (..))

-import Protolude hiding (try)
-
-
-- $setup
-- Setup for doctests
-- >>> import Text.Pretty.Simple (pPrint)
-- >>> deriving instance Show QPError
-- >>> deriving instance Show TrileanVal
-- >>> deriving instance Show FtsOperator
-- >>> deriving instance Show QuantOperator
-- >>> deriving instance Show SimpleOperator
-- >>> deriving instance Show OpQuantifier
-- >>> deriving instance Show Operation
-- >>> deriving instance Show OpExpr
-- >>> deriving instance Show JsonOperand
-- >>> deriving instance Show JsonOperation
-- >>> deriving instance Show Filter
-- >>> deriving instance Show JoinType
-- >>> deriving instance Show SelectItem
-- >>> deriving instance Show OrderDirection
-- >>> deriving instance Show OrderNulls
-- >>> deriving instance Show OrderTerm
-- >>> deriving instance Show LogicOperator
-- >>> deriving instance Show LogicTree
+import Protolude hiding (Sum, try)

 data QueryParams =
  QueryParams
@@ -122,7 +100,7 @@ data QueryParams =
 -- 'select' is a reserved parameter that selects the fields to be returned:
 --
 -- >>> qsSelect <$> parse False "select=name,location"
-- Right [Node {rootLabel = SelectField {selField = ("name",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectField {selField = ("location",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]
+-- Right [Node {rootLabel = SelectField {selField = ("name",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectField {selField = ("location",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []}]
 --
 -- Filters are parameters whose value contains an operator, separated by a '.' from its value:
 --
@@ -305,16 +283,16 @@ pTreePath = do
 -- Parse select= into a Forest of SelectItems
 --
 -- >>> P.parse pFieldForest "" "id"
-- Right [Node {rootLabel = SelectField {selField = ("id",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]
+-- Right [Node {rootLabel = SelectField {selField = ("id",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []}]
 --
 -- >>> P.parse pFieldForest "" "client(id)"
-- Right [Node {rootLabel = SelectRelation {selRelation = "client", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("id",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]}]
+-- Right [Node {rootLabel = SelectRelation {selRelation = "client", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("id",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []}]}]
 --
 -- >>> P.parse pFieldForest "" "*,client(*,nested(*))"
-- Right [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectRelation {selRelation = "client", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectRelation {selRelation = "nested", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]}]}]
+-- Right [Node {rootLabel = SelectField {selField = ("*",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectRelation {selRelation = "client", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectRelation {selRelation = "nested", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []}]}]}]
 --
 -- >>> P.parse pFieldForest "" "*,...client(*),other(*)"
-- Right [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SpreadRelation {selRelation = "client", selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]},Node {rootLabel = SelectRelation {selRelation = "other", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]}]
+-- Right [Node {rootLabel = SelectField {selField = ("*",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SpreadRelation {selRelation = "client", selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []}]},Node {rootLabel = SelectRelation {selRelation = "other", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing}, subForest = []}]}]
 --
 -- >>> P.parse pFieldForest "" ""
 -- Right []
@@ -322,7 +300,7 @@ pTreePath = do
 -- >>> P.parse pFieldForest "" "id,clients(name[])"
 -- Left (line 1, column 16):
 -- unexpected '['
-- expecting letter, digit, "-", "->>", "->", "::", ")", "," or end of input
+-- expecting letter, digit, "-", "->>", "->", "::", ".", ")", "," or end of input
 --
 -- >>> P.parse pFieldForest "" "data->>-78xy"
 -- Left (line 1, column 11):
@@ -475,35 +453,37 @@ pRelationSelect :: Parser SelectItem
 pRelationSelect = lexeme $ do
    alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
    name <- pFieldName
+    guard (name /= "count")
    (hint, jType) <- pEmbedParams
    try (void $ lookAhead (string "("))
    return $ SelectRelation name alias hint jType

+
 -- |
 -- Parse regular fields in select
 --
 -- >>> P.parse pFieldSelect "" "name"
-- Right (SelectField {selField = ("name",[]), selCast = Nothing, selAlias = Nothing})
+-- Right (SelectField {selField = ("name",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing})
 --
 -- >>> P.parse pFieldSelect "" "name->jsonpath"
-- Right (SelectField {selField = ("name",[JArrow {jOp = JKey {jVal = "jsonpath"}}]), selCast = Nothing, selAlias = Nothing})
+-- Right (SelectField {selField = ("name",[JArrow {jOp = JKey {jVal = "jsonpath"}}]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing})
 --
 -- >>> P.parse pFieldSelect "" "name::cast"
-- Right (SelectField {selField = ("name",[]), selCast = Just "cast", selAlias = Nothing})
+-- Right (SelectField {selField = ("name",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Just "cast", selAlias = Nothing})
 --
 -- >>> P.parse pFieldSelect "" "alias:name"
-- Right (SelectField {selField = ("name",[]), selCast = Nothing, selAlias = Just "alias"})
+-- Right (SelectField {selField = ("name",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Just "alias"})
 --
 -- >>> P.parse pFieldSelect "" "alias:name->jsonpath::cast"
-- Right (SelectField {selField = ("name",[JArrow {jOp = JKey {jVal = "jsonpath"}}]), selCast = Just "cast", selAlias = Just "alias"})
+-- Right (SelectField {selField = ("name",[JArrow {jOp = JKey {jVal = "jsonpath"}}]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Just "cast", selAlias = Just "alias"})
 --
 -- >>> P.parse pFieldSelect "" "*"
-- Right (SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing})
+-- Right (SelectField {selField = ("*",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing})
 --
 -- >>> P.parse pFieldSelect "" "name!hint"
 -- Left (line 1, column 5):
 -- unexpected '!'
-- expecting letter, digit, "-", "->>", "->", "::", ")", "," or end of input
+-- expecting letter, digit, "-", "->>", "->", "::", ".", ")", "," or end of input
 --
 -- >>> P.parse pFieldSelect "" "*!hint"
 -- Left (line 1, column 2):
@@ -518,18 +498,36 @@ pFieldSelect :: Parser SelectItem
 pFieldSelect = lexeme $ try (do
    s <- pStar
    pEnd
-    return $ SelectField (s, []) Nothing Nothing)
-  <|> do
-    alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
-    fld <- pField
-    cast' <- optionMaybe (string "::" *> pIdentifier)
+    return $ SelectField (s, []) Nothing Nothing Nothing Nothing)
+  <|> try (do
+    alias    <- optionMaybe ( try(pFieldName <* aliasSeparator) )
+    _        <- string "count()"
+    aggCast' <- optionMaybe (string "::" *> pIdentifier)
    pEnd
-    return $ SelectField fld (toS <$> cast') alias
+    return $ SelectField ("*", []) (Just Count) (toS <$> aggCast') Nothing alias)
+  <|> do
+    alias    <- optionMaybe ( try(pFieldName <* aliasSeparator) )
+    fld      <- pField
+    cast'    <- optionMaybe (string "::" *> pIdentifier)
+    agg      <- optionMaybe (try (char '.' *> pAggregation <* string "()"))
+    aggCast' <- optionMaybe (string "::" *> pIdentifier)
+    pEnd
+    return $ SelectField fld agg (toS <$> aggCast') (toS <$> cast') alias
  where
    pEnd = try (void $ lookAhead (string ")")) <|>
           try (void $ lookAhead (string ",")) <|>
           try eof
    pStar = string "*" $> "*"
+    pAggregation = choice
+      [ string "sum"   $> Sum
+      , string "avg"   $> Avg
+      , string "count" $> Count
+      -- Using 'try' for "min" and "max" to allow backtracking.
+      -- This is necessary because both start with the same character 'm',
+      -- and without 'try', a partial match on "max" would prevent "min" from being tried.
+      , try (string "max") $> Max
+      , try (string "min") $> Min
+      ]


 -- |
@@ -1,6 +1,7 @@
 {-# LANGUAGE DuplicateRecordFields #-}
 module PostgREST.ApiRequest.Types
-  ( Alias
+  ( AggregateFunction(..)
+  , Alias
  , Cast
  , Depth
  , EmbedParam(..)
@@ -42,12 +43,14 @@ import PostgREST.SchemaCache.Routine      (Routine (..))

 import Protolude

-- | The value in `/tbl?select=alias:field::cast`
+-- | The value in `/tbl?select=alias:field.aggregateFunction()::cast`
 data SelectItem
  = SelectField
-    { selField :: Field
-    , selCast  :: Maybe Cast
-    , selAlias :: Maybe Alias
+    { selField             :: Field
+    , selAggregateFunction :: Maybe AggregateFunction
+    , selAggregateCast     :: Maybe Cast
+    , selCast              :: Maybe Cast
+    , selAlias             :: Maybe Alias
    }
 -- | The value in `/tbl?select=alias:another_tbl(*)`
  | SelectRelation
@@ -62,15 +65,16 @@ data SelectItem
    , selHint     :: Maybe Hint
    , selJoinType :: Maybe JoinType
    }
-  deriving (Eq)
+  deriving (Eq, Show)

 data ApiRequestError
-  = AmbiguousRelBetween Text Text [Relationship]
+  = AggregatesNotAllowed
+  | AmbiguousRelBetween Text Text [Relationship]
  | AmbiguousRpc [Routine]
-  | BinaryFieldError MediaType
  | MediaTypeError [ByteString]
  | InvalidBody ByteString
  | InvalidFilters
+  | InvalidPreferences [ByteString]
  | InvalidRange RangeError
  | InvalidRpcMethod ByteString
  | LimitNoOrderError
@@ -86,12 +90,21 @@ data ApiRequestError
  | UnacceptableSchema [Text]
  | UnsupportedMethod ByteString
  | ColumnNotFound Text Text
+  | GucHeadersError
+  | GucStatusError
+  | OffLimitsChangesError Int64 Integer
+  | PutMatchingPkError
+  | SingularityError Integer
+  | PGRSTParseError
+  deriving Show

 data QPError = QPError Text Text
+  deriving Show
 data RangeError
  = NegativeLimit
  | LowerGTUpper
  | OutOfBounds Text Text
+  deriving Show

 type NodeName = Text
 type Depth = Integer
@@ -108,23 +121,26 @@ data OrderTerm
    , otDirection :: Maybe OrderDirection
    , otNullOrder :: Maybe OrderNulls
    }
-  deriving Eq
+  deriving (Eq, Show)

 data OrderDirection
  = OrderAsc
  | OrderDesc
-  deriving (Eq)
+  deriving (Eq, Show)

 data OrderNulls
  = OrderNullsFirst
  | OrderNullsLast
-  deriving (Eq)
+  deriving (Eq, Show)

 type Field = (FieldName, JsonPath)
 type Cast = Text
 type Alias = Text
 type Hint = Text

+data AggregateFunction = Sum | Avg | Max | Min | Count
+  deriving (Show, Eq)
+
 data EmbedParam
  -- | Disambiguates an embedding operation when there's multiple relationships
  -- between two tables. Can be the name of a foreign key constraint, column
@@ -135,7 +151,7 @@ data EmbedParam
 data JoinType
  = JTInner
  | JTLeft
-  deriving Eq
+  deriving (Eq, Show)

 -- | Path of the embedded levels, e.g "clients.projects.name=eq.." gives Path
 -- ["clients", "projects"]
@@ -149,7 +165,7 @@ type JsonPath = [JsonOperation]
 data JsonOperation
  = JArrow { jOp :: JsonOperand }
  | J2Arrow { jOp :: JsonOperand }
-  deriving (Eq, Ord)
+  deriving (Eq, Show, Ord)

 -- | Represents the key(`->'key'`) or index(`->'1`::int`), the index is Text
 -- because we reuse our escaping functons and let pg do the casting with
@@ -157,7 +173,7 @@ data JsonOperation
 data JsonOperand
  = JKey { jVal :: Text }
  | JIdx { jVal :: Text }
-  deriving (Eq, Ord)
+  deriving (Eq, Show, Ord)

 -- | Boolean logic expression tree e.g. "and(name.eq.N,or(id.eq.1,id.eq.2))" is:
 --
@@ -169,28 +185,27 @@ data JsonOperand
 data LogicTree
  = Expr Bool LogicOperator [LogicTree]
  | Stmnt Filter
-  deriving (Eq)
+  deriving (Eq, Show)

 data LogicOperator
  = And
  | Or
-  deriving Eq
+  deriving (Eq, Show)

 data Filter
  = Filter
  { field  :: Field
  , opExpr :: OpExpr
  }
-  | FilterNullEmbed Bool FieldName
-  deriving (Eq)
+  deriving (Eq, Show)

 data OpExpr
  = OpExpr Bool Operation
  | NoOpExpr Text
-  deriving (Eq)
+  deriving (Eq, Show)

 data OpQuantifier = QuantAny | QuantAll
-  deriving Eq
+  deriving (Eq, Show)

 data Operation
  = Op SimpleOperator SingleVal
@@ -199,7 +214,7 @@ data Operation
  | Is TrileanVal
  | IsDistinctFrom SingleVal
  | Fts FtsOperator (Maybe Language) SingleVal
-  deriving (Eq)
+  deriving (Eq, Show)

 type Language = Text

@@ -215,7 +230,7 @@ data TrileanVal
  | TriFalse
  | TriNull
  | TriUnknown
-  deriving Eq
+  deriving (Eq, Show)

 -- Operators that are quantifiable, i.e. they can be used with the any/all modifiers
 data QuantOperator
@@ -228,7 +243,7 @@ data QuantOperator
  | OpILike
  | OpMatch
  | OpIMatch
-  deriving Eq
+  deriving (Eq, Show)

 data SimpleOperator
  = OpNotEqual
@@ -240,7 +255,7 @@ data SimpleOperator
  | OpNotExtendsRight
  | OpNotExtendsLeft
  | OpAdjacent
-  deriving Eq
+  deriving (Eq, Show)

 --
 -- | Operators for full text search operators
@@ -249,4 +264,4 @@ data FtsOperator
  | FilterFtsPlain
  | FilterFtsPhrase
  | FilterFtsWebsearch
-  deriving Eq
+  deriving (Eq, Show)
@@ -9,30 +9,28 @@ Some of its functionality includes:
 - Producing HTTP Headers according to RFCs.
 - Content Negotiation
 -}
-{-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 module PostgREST.App
-  ( SignalHandlerInstaller
-  , SocketRunner
-  , postgrest
+  ( postgrest
  , run
  ) where


 import Control.Monad.Except     (liftEither)
-import Data.Either.Combinators  (mapLeft, whenLeft)
+import Data.Either.Combinators  (mapLeft)
 import Data.Maybe               (fromJust)
 import Data.String              (IsString (..))
 import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
                                 setServerName)
-import System.Posix.Types       (FileMode)

 import qualified Data.HashMap.Strict        as HM
-import qualified Hasql.Pool                 as SQL
+import qualified Data.Text.Encoding         as T
 import qualified Hasql.Transaction.Sessions as SQL
 import qualified Network.Wai                as Wai
 import qualified Network.Wai.Handler.Warp   as Warp

+import qualified PostgREST.Admin            as Admin
 import qualified PostgREST.ApiRequest       as ApiRequest
 import qualified PostgREST.ApiRequest.Types as ApiRequestTypes
 import qualified PostgREST.AppState         as AppState
@@ -43,53 +41,51 @@ import qualified PostgREST.Logger           as Logger
 import qualified PostgREST.Plan             as Plan
 import qualified PostgREST.Query            as Query
 import qualified PostgREST.Response         as Response
-import qualified PostgREST.Workers          as Workers
+import qualified PostgREST.Unix             as Unix (installSignalHandlers)

-import PostgREST.ApiRequest          (Action (..), ApiRequest (..),
-                                      Mutation (..), Target (..))
-import PostgREST.AppState            (AppState)
-import PostgREST.Auth                (AuthResult (..))
-import PostgREST.Config              (AppConfig (..))
-import PostgREST.Config.PgVersion    (PgVersion (..))
-import PostgREST.Error               (Error)
-import PostgREST.Query               (DbHandler)
-import PostgREST.SchemaCache         (SchemaCache (..))
-import PostgREST.SchemaCache.Routine (Routine (..))
-import PostgREST.Version             (prettyVersion)
+import PostgREST.ApiRequest           (Action (..), ApiRequest (..),
+                                       Mutation (..), Target (..))
+import PostgREST.AppState             (AppState)
+import PostgREST.Auth                 (AuthResult (..))
+import PostgREST.Config               (AppConfig (..))
+import PostgREST.Config.PgVersion     (PgVersion (..))
+import PostgREST.Error                (Error)
+import PostgREST.Query                (DbHandler)
+import PostgREST.Response.Performance (ServerTiming (..),
+                                       serverTimingHeader)
+import PostgREST.SchemaCache          (SchemaCache (..))
+import PostgREST.SchemaCache.Routine  (Routine (..))
+import PostgREST.Version              (docsVersion, prettyVersion)

-import Protolude hiding (Handler)
+import qualified Data.ByteString.Char8 as BS
+import qualified Data.List             as L
+import qualified Network.HTTP.Types    as HTTP
+import qualified Network.Socket        as NS
+import           Protolude             hiding (Handler)
+import           System.TimeIt         (timeItT)

 type Handler = ExceptT Error

-type SignalHandlerInstaller = AppState -> IO()
-
-type SocketRunner = Warp.Settings -> Wai.Application -> FileMode -> FilePath -> IO()
-
-run :: SignalHandlerInstaller -> Maybe SocketRunner -> AppState -> IO ()
-run installHandlers maybeRunWithSocket appState = do
+run :: AppState -> IO ()
+run appState = do
  conf@AppConfig{..} <- AppState.getConfig appState
-  Workers.connectionWorker appState -- Loads the initial SchemaCache
-  installHandlers appState
+  AppState.connectionWorker appState -- Loads the initial SchemaCache
+  Unix.installSignalHandlers (AppState.getMainThreadId appState) (AppState.connectionWorker appState) (AppState.reReadConfig False appState)
  -- reload schema cache + config on NOTIFY
-  Workers.runListener conf appState
+  AppState.runListener conf appState

-  Workers.runAdmin conf appState $ serverSettings conf
+  Admin.runAdmin conf appState $ serverSettings conf

-  let app = postgrest conf appState (Workers.connectionWorker appState)
+  let app = postgrest conf appState (AppState.connectionWorker appState)

-  case configServerUnixSocket of
-    Just socket ->
-      -- run the postgrest application with user defined socket. Only for UNIX systems
-      case maybeRunWithSocket of
-        Just runWithSocket -> do
-          AppState.logWithZTime appState $ "Listening on unix socket " <> show socket
-          runWithSocket (serverSettings conf) app configServerUnixSocketMode socket
-        Nothing ->
-          panic "Cannot run with unix socket on non-unix platforms."
-    Nothing ->
-      do
-        AppState.logWithZTime appState $ "Listening on port " <> show configServerPort
-        Warp.runSettings (serverSettings conf) app
+  what <- case configServerUnixSocket of
+    Just path -> pure $ "unix socket " <> show path
+    Nothing   -> do
+      port <- NS.socketPort $ AppState.getSocketREST appState
+      pure $ "port " <> show port
+  AppState.logWithZTime appState $ "Listening on " <> what
+
+  Warp.runSettingsSocket (serverSettings conf) (AppState.getSocketREST appState) app

 serverSettings :: AppConfig -> Warp.Settings
 serverSettings AppConfig{..} =
@@ -101,8 +97,8 @@ serverSettings AppConfig{..} =
 -- | PostgREST application
 postgrest :: AppConfig -> AppState.AppState -> IO () -> Wai.Application
 postgrest conf appState connWorker =
-  Response.traceHeaderMiddleware conf .
-  Cors.middleware .
+  traceHeaderMiddleware conf .
+  Cors.middleware (configServerCorsAllowedOrigins conf) .
  Auth.middleware appState .
  Logger.middleware (configLogLevel conf) $
    -- fromJust can be used, because the auth middleware will **always** add
@@ -123,10 +119,10 @@ postgrest conf appState connWorker =
        -- Launch the connWorker when the connection is down.  The postgrest
        -- function can respond successfully (with a stale schema cache) before
        -- the connWorker is done.
-        when (Response.isServiceUnavailable response) connWorker
+        when (isServiceUnavailable response) connWorker
        resp <- do
          delay <- AppState.getRetryNextIn appState
-          return $ Response.addRetryHint delay response
+          return $ addRetryHint delay response
        respond resp

 postgrestResponse
@@ -147,81 +143,83 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@

  body <- lift $ Wai.strictRequestBody req

-  apiRequest <-
-    liftEither . mapLeft Error.ApiRequestError $
-      ApiRequest.userApiRequest conf req body
+  (parseTime, apiRequest) <-
+    calcTiming configServerTimingEnabled $
+      liftEither . mapLeft Error.ApiRequestError $
+        ApiRequest.userApiRequest conf req body sCache

-  Response.optionalRollback conf apiRequest $
-    handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache
+  let jwtTime = if configServerTimingEnabled then Auth.getJwtDur req else Nothing
+  handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache jwtTime parseTime

-runDbHandler :: AppState.AppState -> Maybe Text -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
-runDbHandler appState isoLvl mode authenticated prepared handler = do
+runDbHandler :: AppState.AppState -> AppConfig -> SQL.IsolationLevel -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
+runDbHandler appState config isoLvl mode authenticated prepared handler = do
  dbResp <- lift $ do
    let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction
-    res <- AppState.usePool appState . transaction (toIsolationLevel isoLvl) mode $ runExceptT handler
-    whenLeft res (\case
-      SQL.AcquisitionTimeoutUsageError -> AppState.debounceLogAcquisitionTimeout appState -- this can happen rapidly for many requests, so we debounce
-      _ -> pure ())
-    return res
+    AppState.usePool appState config . transaction isoLvl mode $ runExceptT handler

  resp <-
    liftEither . mapLeft Error.PgErr $
      mapLeft (Error.PgError authenticated) dbResp

  liftEither resp
-  where
-    toIsolationLevel = \case
-      Nothing                -> SQL.ReadCommitted
-      Just "repeatable read" -> SQL.RepeatableRead
-      Just "serializable"    -> SQL.Serializable
-      _                      -> SQL.ReadCommitted

-handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> Bool -> Bool -> PgVersion -> ApiRequest -> SchemaCache -> Handler IO Wai.Response
-handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@ApiRequest{..} sCache =
+handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> Bool -> Bool -> PgVersion -> ApiRequest -> SchemaCache -> Maybe Double -> Maybe Double -> Handler IO Wai.Response
+handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@ApiRequest{..} sCache jwtTime parseTime =
  case (iAction, iTarget) of
    (ActionRead headersOnly, TargetIdent identifier) -> do
-      wrPlan <- liftEither $ Plan.wrappedReadPlan identifier conf sCache apiReq
-      resultSet <- runQuery roleIsoLvl (Plan.wrTxMode wrPlan) $ Query.readQuery wrPlan conf apiReq
-      return $ Response.readResponse headersOnly identifier apiReq resultSet
+      (planTime', wrPlan) <- withTiming $ liftEither $ Plan.wrappedReadPlan identifier conf sCache apiReq
+      (txTime', resultSet) <- withTiming $ runQuery roleIsoLvl Nothing (Plan.wrTxMode wrPlan) $ Query.readQuery wrPlan conf apiReq
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.readResponse wrPlan headersOnly identifier apiReq resultSet
+      return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst

    (ActionMutate MutationCreate, TargetIdent identifier) -> do
-      mrPlan <- liftEither $ Plan.mutateReadPlan MutationCreate apiReq identifier conf sCache
-      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.createQuery mrPlan apiReq conf
-      return $ Response.createResponse identifier mrPlan apiReq resultSet
+      (planTime', mrPlan) <- withTiming $ liftEither $ Plan.mutateReadPlan MutationCreate apiReq identifier conf sCache
+      (txTime', resultSet) <- withTiming $ runQuery roleIsoLvl Nothing (Plan.mrTxMode mrPlan) $ Query.createQuery mrPlan apiReq conf
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.createResponse identifier mrPlan apiReq resultSet
+      return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst

    (ActionMutate MutationUpdate, TargetIdent identifier) -> do
-      mrPlan <- liftEither $ Plan.mutateReadPlan MutationUpdate apiReq identifier conf sCache
-      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.updateQuery mrPlan apiReq conf
-      return $ Response.updateResponse apiReq resultSet
+      (planTime', mrPlan) <- withTiming $ liftEither $ Plan.mutateReadPlan MutationUpdate apiReq identifier conf sCache
+      (txTime', resultSet) <- withTiming $ runQuery roleIsoLvl Nothing (Plan.mrTxMode mrPlan) $ Query.updateQuery mrPlan apiReq conf
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.updateResponse mrPlan apiReq resultSet
+      return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst

    (ActionMutate MutationSingleUpsert, TargetIdent identifier) -> do
-      mrPlan <- liftEither $ Plan.mutateReadPlan MutationSingleUpsert apiReq identifier conf sCache
-      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.singleUpsertQuery mrPlan apiReq conf
-      return $ Response.singleUpsertResponse apiReq resultSet
+      (planTime', mrPlan) <- withTiming $ liftEither $ Plan.mutateReadPlan MutationSingleUpsert apiReq identifier conf sCache
+      (txTime', resultSet) <- withTiming $ runQuery roleIsoLvl Nothing (Plan.mrTxMode mrPlan) $ Query.singleUpsertQuery mrPlan apiReq conf
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.singleUpsertResponse mrPlan apiReq resultSet
+      return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst

    (ActionMutate MutationDelete, TargetIdent identifier) -> do
-      mrPlan <- liftEither $ Plan.mutateReadPlan MutationDelete apiReq identifier conf sCache
-      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.deleteQuery mrPlan apiReq conf
-      return $ Response.deleteResponse apiReq resultSet
+      (planTime', mrPlan) <- withTiming $ liftEither $ Plan.mutateReadPlan MutationDelete apiReq identifier conf sCache
+      (txTime', resultSet) <- withTiming $ runQuery roleIsoLvl Nothing (Plan.mrTxMode mrPlan) $ Query.deleteQuery mrPlan apiReq conf
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.deleteResponse mrPlan apiReq resultSet
+      return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst

    (ActionInvoke invMethod, TargetProc identifier _) -> do
-      cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq invMethod
-      resultSet <- runQuery (roleIsoLvl <|> pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
-      return $ Response.invokeResponse invMethod (Plan.crProc cPlan) apiReq resultSet
+      (planTime', cPlan) <- withTiming $ liftEither $ Plan.callReadPlan identifier conf sCache apiReq invMethod
+      (txTime', resultSet) <- withTiming $ runQuery (fromMaybe roleIsoLvl $ pdIsoLvl (Plan.crProc cPlan)) (pdTimeout $ Plan.crProc cPlan) (Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.invokeResponse cPlan invMethod (Plan.crProc cPlan) apiReq resultSet
+      return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst

    (ActionInspect headersOnly, TargetDefaultSpec tSchema) -> do
-      oaiResult <- runQuery roleIsoLvl Plan.inspectPlanTxMode $ Query.openApiQuery sCache pgVer conf tSchema
-      return $ Response.openApiResponse headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
+      (planTime', iPlan) <- withTiming $ liftEither $ Plan.inspectPlan apiReq
+      (txTime', oaiResult) <- withTiming $ runQuery roleIsoLvl Nothing (Plan.ipTxmode iPlan) $ Query.openApiQuery sCache pgVer conf tSchema
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.openApiResponse (T.decodeUtf8 prettyVersion, docsVersion) headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
+      return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst

-    (ActionInfo, TargetIdent identifier) ->
-      return $ Response.infoIdentResponse identifier sCache
+    (ActionInfo, TargetIdent identifier) -> do
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.infoIdentResponse identifier sCache
+      return $ pgrstResponse (ServerTiming jwtTime parseTime Nothing Nothing respTime') pgrst

    (ActionInfo, TargetProc identifier _) -> do
-      cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq ApiRequest.InvHead
-      return $ Response.infoProcResponse (Plan.crProc cPlan)
+      (planTime', cPlan) <- withTiming $ liftEither $ Plan.callReadPlan identifier conf sCache apiReq ApiRequest.InvHead
+      (respTime', pgrst) <- withTiming $ liftEither $ Response.infoProcResponse (Plan.crProc cPlan)
+      return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' Nothing respTime') pgrst

-    (ActionInfo, TargetDefaultSpec _) ->
-      return Response.infoRootResponse
+    (ActionInfo, TargetDefaultSpec _) -> do
+      (respTime', pgrst) <- withTiming $ liftEither Response.infoRootResponse
+      return $ pgrstResponse (ServerTiming jwtTime parseTime Nothing Nothing respTime') pgrst

    _ ->
      -- This is unreachable as the ApiRequest.hs rejects it before
@@ -229,9 +227,39 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A
      throwError $ Error.ApiRequestError ApiRequestTypes.NotFound
  where
    roleSettings = fromMaybe mempty (HM.lookup authRole $ configRoleSettings conf)
-    roleIsoLvl = decodeUtf8 <$> HM.lookup "default_transaction_isolation" roleSettings
-    runQuery isoLvl mode query =
-      runDbHandler appState isoLvl mode authenticated prepared $ do
-        Query.setPgLocals conf authClaims authRole (HM.toList roleSettings) apiReq pgVer
+    roleIsoLvl = HM.findWithDefault SQL.ReadCommitted authRole $ configRoleIsoLvl conf
+    runQuery isoLvl timeout mode query =
+      runDbHandler appState conf isoLvl mode authenticated prepared $ do
+        Query.setPgLocals conf authClaims authRole (HM.toList roleSettings) apiReq timeout
        Query.runPreReq conf
        query
+
+    pgrstResponse :: ServerTiming -> Response.PgrstResponse -> Wai.Response
+    pgrstResponse timing (Response.PgrstResponse st hdrs bod) = Wai.responseLBS st (hdrs ++ ([serverTimingHeader timing | configServerTimingEnabled conf])) bod
+
+    withTiming = calcTiming $ configServerTimingEnabled conf
+
+calcTiming :: Bool -> Handler IO a -> Handler IO (Maybe Double, a)
+calcTiming timingEnabled f = if timingEnabled
+    then do
+      (t, r) <- timeItT f
+      pure (Just t, r)
+    else do
+      r <- f
+      pure (Nothing, r)
+
+traceHeaderMiddleware :: AppConfig -> Wai.Middleware
+traceHeaderMiddleware AppConfig{configServerTraceHeader} app req respond =
+  case configServerTraceHeader of
+    Nothing -> app req respond
+    Just hdr ->
+      let hdrVal = L.lookup hdr $ Wai.requestHeaders req in
+      app req (respond . Wai.mapResponseHeaders ([(hdr, fromMaybe mempty hdrVal)] ++))
+
+addRetryHint :: Int -> Wai.Response -> Wai.Response
+addRetryHint delay response = do
+  let h = ("Retry-After", BS.pack $ show delay)
+  Wai.mapResponseHeaders (\hs -> if isServiceUnavailable response then h:hs else hs) response
+
+isServiceUnavailable :: Wai.Response -> Bool
+isServiceUnavailable response = Wai.responseStatus response == HTTP.status503
@@ -1,9 +1,11 @@
+{-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}

 module PostgREST.AppState
  ( AppState
+  , AuthResult(..)
  , destroy
-  , flushPool
  , getConfig
  , getSchemaCache
  , getIsListenerOn
@@ -11,43 +13,73 @@ module PostgREST.AppState
  , getPgVersion
  , getRetryNextIn
  , getTime
-  , getWorkerSem
+  , getJwtCache
+  , getSocketREST
+  , getSocketAdmin
  , init
+  , initSockets
  , initWithPool
  , logWithZTime
-  , logPgrstError
-  , putConfig
  , putSchemaCache
-  , putIsListenerOn
  , putPgVersion
-  , putRetryNextIn
-  , signalListener
  , usePool
-  , waitListener
-  , debounceLogAcquisitionTimeout
+  , loadSchemaCache
+  , reReadConfig
+  , connectionWorker
+  , runListener
  ) where

-import qualified Data.ByteString.Lazy as LBS
-import qualified Data.Text.Encoding   as T
-import qualified Hasql.Pool           as SQL
-import qualified Hasql.Session        as SQL
-import qualified PostgREST.Error      as Error
+import qualified Data.Aeson                 as JSON
+import qualified Data.Aeson.KeyMap          as KM
+import qualified Data.ByteString.Char8      as BS
+import qualified Data.ByteString.Lazy       as LBS
+import qualified Data.Cache                 as C
+import           Data.Either.Combinators    (whenLeft)
+import qualified Data.Text                  as T (unpack)
+import qualified Data.Text.Encoding         as T
+import           Hasql.Connection           (acquire)
+import qualified Hasql.Notifications        as SQL
+import qualified Hasql.Pool                 as SQL
+import qualified Hasql.Session              as SQL
+import qualified Hasql.Transaction.Sessions as SQL
+import qualified Network.HTTP.Types.Status  as HTTP
+import qualified Network.Socket             as NS
+import qualified PostgREST.Error            as Error
+import           PostgREST.Version          (prettyVersion)

 import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
                           updateAction)
 import Control.Debounce
+import Control.Retry      (RetryStatus, capDelay, exponentialBackoff,
+                           retrying, rsPreviousDelay)
 import Data.IORef         (IORef, atomicWriteIORef, newIORef,
                           readIORef)
 import Data.Time          (ZonedTime, defaultTimeLocale, formatTime,
                           getZonedTime)
 import Data.Time.Clock    (UTCTime, getCurrentTime)

-import PostgREST.Config           (AppConfig (..))
-import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
-import PostgREST.SchemaCache      (SchemaCache)
+import PostgREST.Config                  (AppConfig (..),
+                                          LogLevel (..),
+                                          addFallbackAppName,
+                                          readAppConfig)
+import PostgREST.Config.Database         (queryDbSettings,
+                                          queryPgVersion,
+                                          queryRoleSettings)
+import PostgREST.Config.PgVersion        (PgVersion (..),
+                                          minimumPgVersion)
+import PostgREST.SchemaCache             (SchemaCache,
+                                          querySchemaCache)
+import PostgREST.SchemaCache.Identifiers (dumpQi)
+import PostgREST.Unix                    (createAndBindDomainSocket)

+import Data.Streaming.Network (bindPortTCP, bindRandomPortTCP)
+import Data.String            (IsString (..))
 import Protolude

+data AuthResult = AuthResult
+  { authClaims :: KM.KeyMap JSON.Value
+  , authRole   :: BS.ByteString
+  }

 data AppState = AppState
  -- | Database connection pool
@@ -56,8 +88,8 @@ data AppState = AppState
  , statePgVersion                :: IORef PgVersion
  -- | No schema cache at the start. Will be filled in by the connectionWorker
  , stateSchemaCache              :: IORef (Maybe SchemaCache)
-  -- | Binary semaphore to make sure just one connectionWorker can run at a time
-  , stateWorkerSem                :: MVar ()
+  -- | starts the connection worker with a debounce
+  , debouncedConnectionWorker     :: IO ()
  -- | Binary semaphore used to sync the listener(NOTIFY reload) with the connectionWorker.
  , stateListener                 :: MVar ()
  -- | State of the LISTEN channel, used for the admin server checks
@@ -74,19 +106,29 @@ data AppState = AppState
  , stateRetryNextIn              :: IORef Int
  -- | Logs a pool error with a debounce
  , debounceLogAcquisitionTimeout :: IO ()
+  -- | JWT Cache
+  , jwtCache                      :: C.Cache ByteString AuthResult
+  -- | Network socket for REST API
+  , stateSocketREST               :: NS.Socket
+  -- | Network socket for the admin UI
+  , stateSocketAdmin              :: Maybe NS.Socket
  }

+type AppSockets = (NS.Socket, Maybe NS.Socket)
+
 init :: AppConfig -> IO AppState
 init conf = do
  pool <- initPool conf
-  initWithPool pool conf
+  (sock, adminSock) <- initSockets conf
+  state' <- initWithPool (sock, adminSock) pool conf
+  pure state' { stateSocketREST = sock, stateSocketAdmin = adminSock }

-initWithPool :: SQL.Pool -> AppConfig -> IO AppState
-initWithPool pool conf = do
+initWithPool :: AppSockets -> SQL.Pool -> AppConfig -> IO AppState
+initWithPool (sock, adminSock) pool conf = do
  appState <- AppState pool
    <$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step
    <*> newIORef Nothing
-    <*> newEmptyMVar
+    <*> pure (pure ())
    <*> newEmptyMVar
    <*> newIORef False
    <*> newIORef conf
@@ -95,8 +137,12 @@ initWithPool pool conf = do
    <*> myThreadId
    <*> newIORef 0
    <*> pure (pure ())
+    <*> C.newCache Nothing
+    <*> pure sock
+    <*> pure adminSock

-  deb <-
+
+  debLogTimeout <-
    let oneSecond = 1000000 in
    mkDebounce defaultDebounceSettings
       { debounceAction = logPgrstError appState SQL.AcquisitionTimeoutUsageError
@@ -104,22 +150,75 @@ initWithPool pool conf = do
       , debounceEdge = leadingEdge -- logs at the start and the end
       }

-  return appState { debounceLogAcquisitionTimeout = deb }
+  debWorker <-
+    let decisecond = 100000 in
+    mkDebounce defaultDebounceSettings
+       { debounceAction = internalConnectionWorker appState
+       , debounceFreq = decisecond
+       , debounceEdge = leadingEdge -- runs the worker at the start and the end
+       }
+
+  return appState { debounceLogAcquisitionTimeout = debLogTimeout, debouncedConnectionWorker = debWorker }

 destroy :: AppState -> IO ()
 destroy = destroyPool

+initSockets :: AppConfig -> IO AppSockets
+initSockets AppConfig{..} = do
+  let
+    cfg'usp = configServerUnixSocket
+    cfg'uspm = configServerUnixSocketMode
+    cfg'host = configServerHost
+    cfg'port = configServerPort
+    cfg'adminport = configAdminServerPort
+
+  sock <- case cfg'usp of
+    -- I'm not using `streaming-commons`' bindPath function here because it's not defined for Windows,
+    -- but we need to have runtime error if we try to use it in Windows, not compile time error
+    Just path -> createAndBindDomainSocket path cfg'uspm
+    Nothing -> do
+      (_, sock) <-
+        if cfg'port /= 0
+          then do
+            sock <- bindPortTCP cfg'port (fromString $ T.unpack cfg'host)
+            pure (cfg'port, sock)
+          else do
+            -- explicitly bind to a random port, returning bound port number
+            (num, sock) <- bindRandomPortTCP (fromString $ T.unpack cfg'host)
+            pure (num, sock)
+      pure sock
+
+  adminSock <- case cfg'adminport of
+    Just adminPort -> do
+      adminSock <- bindPortTCP adminPort (fromString $ T.unpack cfg'host)
+      pure $ Just adminSock
+    Nothing -> pure Nothing
+
+  pure (sock, adminSock)
+
 initPool :: AppConfig -> IO SQL.Pool
 initPool AppConfig{..} =
  SQL.acquire
    configDbPoolSize
    (fromIntegral configDbPoolAcquisitionTimeout)
    (fromIntegral configDbPoolMaxLifetime)
-    (toUtf8 configDbUri)
+    (fromIntegral configDbPoolMaxIdletime)
+    (toUtf8 $ addFallbackAppName prettyVersion configDbUri)

 -- | Run an action with a database connection.
-usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a)
-usePool AppState{..} = SQL.use statePool
+usePool :: AppState -> AppConfig -> SQL.Session a -> IO (Either SQL.UsageError a)
+usePool appState@AppState{..} AppConfig{configLogLevel} x = do
+  res <- SQL.use statePool x
+
+  when (configLogLevel > LogCrit) $ do
+    whenLeft res (\case
+      SQL.AcquisitionTimeoutUsageError -> debounceLogAcquisitionTimeout -- this can happen rapidly for many requests, so we debounce
+      error
+        -- TODO We're using the 500 HTTP status for getting all internal db errors but there's no response here. We need a new intermediate type to not rely on the HTTP status.
+        | Error.status (Error.PgError False error) >= HTTP.status500 -> logPgrstError appState error
+        | otherwise -> pure ())
+
+  return res

 -- | Flush the connection pool so that any future use of the pool will
 -- use connections freshly established after this call.
@@ -142,8 +241,8 @@ getSchemaCache = readIORef . stateSchemaCache
 putSchemaCache :: AppState -> Maybe SchemaCache -> IO ()
 putSchemaCache appState = atomicWriteIORef (stateSchemaCache appState)

-getWorkerSem :: AppState -> MVar ()
-getWorkerSem = stateWorkerSem
+connectionWorker :: AppState -> IO ()
+connectionWorker = debouncedConnectionWorker

 getRetryNextIn :: AppState -> IO Int
 getRetryNextIn = readIORef . stateRetryNextIn
@@ -160,6 +259,15 @@ putConfig = atomicWriteIORef . stateConf
 getTime :: AppState -> IO UTCTime
 getTime = stateGetTime

+getJwtCache :: AppState -> C.Cache ByteString AuthResult
+getJwtCache = jwtCache
+
+getSocketREST :: AppState -> NS.Socket
+getSocketREST = stateSocketREST
+
+getSocketAdmin :: AppState -> Maybe NS.Socket
+getSocketAdmin = stateSocketAdmin
+
 -- | Log to stderr with local time
 logWithZTime :: AppState -> Text -> IO ()
 logWithZTime appState txt = do
@@ -188,3 +296,268 @@ getIsListenerOn = readIORef . stateIsListenerOn

 putIsListenerOn :: AppState -> Bool -> IO ()
 putIsListenerOn = atomicWriteIORef . stateIsListenerOn
+
+-- | Schema cache status
+data SCacheStatus
+  = SCLoaded
+  | SCOnRetry
+  | SCFatalFail
+
+-- | Load the SchemaCache by using a connection from the pool.
+loadSchemaCache :: AppState -> IO SCacheStatus
+loadSchemaCache appState = do
+  conf@AppConfig{..} <- getConfig appState
+  result <-
+    let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
+    usePool appState conf . transaction SQL.ReadCommitted SQL.Read $
+      querySchemaCache conf
+  case result of
+    Left e -> do
+      case checkIsFatal e of
+        Just hint -> do
+          logWithZTime appState "A fatal error ocurred when loading the schema cache"
+          logPgrstError appState e
+          logWithZTime appState hint
+          return SCFatalFail
+        Nothing -> do
+          putSchemaCache appState Nothing
+          logWithZTime appState "An error ocurred when loading the schema cache"
+          logPgrstError appState e
+          return SCOnRetry
+
+    Right sCache -> do
+      putSchemaCache appState (Just sCache)
+      logWithZTime appState "Schema cache loaded"
+      return SCLoaded
+
+-- | Current database connection status data ConnectionStatus
+data ConnectionStatus
+  = NotConnected
+  | Connected PgVersion
+  | FatalConnectionError Text
+  deriving (Eq)
+
+-- | The purpose of this worker is to obtain a healthy connection to pg and an
+-- up-to-date schema cache(SchemaCache).  This method is meant to be called
+-- multiple times by the same thread, but does nothing if the previous
+-- invocation has not terminated. In all cases this method does not halt the
+-- calling thread, the work is performed in a separate thread.
+--
+-- Background thread that does the following :
+--  1. Tries to connect to pg server and will keep trying until success.
+--  2. Checks if the pg version is supported and if it's not it kills the main
+--     program.
+--  3. Obtains the sCache. If this fails, it goes back to 1.
+internalConnectionWorker :: AppState -> IO ()
+internalConnectionWorker appState = work
+  where
+    work = do
+      config@AppConfig{..} <- getConfig appState
+      logWithZTime appState $ "Starting PostgREST " <> T.decodeUtf8 prettyVersion <> "..."
+      logWithZTime appState "Attempting to connect to the database..."
+      connected <- establishConnection appState config
+      case connected of
+        FatalConnectionError reason ->
+          -- Fatal error when connecting
+          logWithZTime appState reason >> killThread (getMainThreadId appState)
+        NotConnected ->
+          -- Unreachable because establishConnection will keep trying to connect, unless disable-recovery is turned on
+          unless configDbPoolAutomaticRecovery
+              $ logWithZTime appState "Automatic recovery disabled, exiting." >> killThread (getMainThreadId appState)
+        Connected actualPgVersion -> do
+          -- Procede with initialization
+          putPgVersion appState actualPgVersion
+          when configDbChannelEnabled $
+            signalListener appState
+          logWithZTime appState "Connection successful"
+          -- this could be fail because the connection drops, but the loadSchemaCache will pick the error and retry again
+          -- We cannot retry after it fails immediately, because db-pre-config could have user errors. We just log the error and continue.
+          when configDbConfig $ reReadConfig False appState
+          scStatus <- loadSchemaCache appState
+          case scStatus of
+            SCLoaded ->
+              -- do nothing and proceed if the load was successful
+              return ()
+            SCOnRetry ->
+              -- retry reloading the schema cache
+              work
+            SCFatalFail ->
+              -- die if our schema cache query has an error
+              killThread $ getMainThreadId appState
+
+-- | Repeatedly flush the pool, and check if a connection from the
+-- pool allows access to the PostgreSQL database.
+--
+-- Releasing the pool is key for rapid recovery. Otherwise, the pool
+-- timeout would have to be reached for new healthy connections to be acquired.
+-- Which might not happen if the server is busy with requests. No idle
+-- connection, no pool timeout.
+--
+-- The connection tries are capped, but if the connection times out no error is
+-- thrown, just 'False' is returned.
+establishConnection :: AppState -> AppConfig -> IO ConnectionStatus
+establishConnection appState config =
+  retrying retrySettings shouldRetry $
+    const $ flushPool appState >> getConnectionStatus
+  where
+    retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
+    delayMicroseconds = 32000000 -- 32 seconds
+    backoffMicroseconds = 1000000 -- 1 second
+
+    getConnectionStatus :: IO ConnectionStatus
+    getConnectionStatus = do
+      pgVersion <- usePool appState config $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
+      case pgVersion of
+        Left e -> do
+          logPgrstError appState e
+          case checkIsFatal e of
+            Just reason ->
+              return $ FatalConnectionError reason
+            Nothing ->
+              return NotConnected
+        Right version ->
+          if version < minimumPgVersion then
+            return . FatalConnectionError $
+              "Cannot run in this PostgreSQL version, PostgREST needs at least "
+              <> pgvName minimumPgVersion
+          else
+            return . Connected  $ version
+
+    shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
+    shouldRetry rs isConnSucc = do
+      AppConfig{..} <- getConfig appState
+      let
+        delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
+        itShould = NotConnected == isConnSucc && configDbPoolAutomaticRecovery
+      when itShould . logWithZTime appState $
+        "Attempting to reconnect to the database in "
+        <> (show delay::Text)
+        <> " seconds..."
+      when itShould $ putRetryNextIn appState delay
+      return itShould
+
+-- | Re-reads the config plus config options from the db
+reReadConfig :: Bool -> AppState -> IO ()
+reReadConfig startingUp appState = do
+  config@AppConfig{..} <- getConfig appState
+  pgVer <- getPgVersion appState
+  dbSettings <-
+    if configDbConfig then do
+      qDbSettings <- usePool appState config $ queryDbSettings (dumpQi <$> configDbPreConfig) configDbPreparedStatements
+      case qDbSettings of
+        Left e -> do
+          logWithZTime appState
+            "An error ocurred when trying to query database settings for the config parameters"
+          case checkIsFatal e of
+            Just hint -> do
+              logPgrstError appState e
+              logWithZTime appState hint
+              killThread (getMainThreadId appState)
+            Nothing -> do
+              logPgrstError appState e
+          pure mempty
+        Right x -> pure x
+    else
+      pure mempty
+  (roleSettings, roleIsolationLvl) <-
+    if configDbConfig then do
+      rSettings <- usePool appState config $ queryRoleSettings pgVer configDbPreparedStatements
+      case rSettings of
+        Left e -> do
+          logWithZTime appState "An error ocurred when trying to query the role settings"
+          logPgrstError appState e
+          pure (mempty, mempty)
+        Right x -> pure x
+    else
+      pure mempty
+  readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings roleIsolationLvl >>= \case
+    Left err   ->
+      if startingUp then
+        panic err -- die on invalid config if the program is starting up
+      else
+        logWithZTime appState $ "Failed reloading config: " <> err
+    Right newConf -> do
+      putConfig appState newConf
+      if startingUp then
+        pass
+      else
+        logWithZTime appState "Config reloaded"
+
+
+runListener :: AppConfig -> AppState -> IO ()
+runListener AppConfig{configDbChannelEnabled} appState =
+  when configDbChannelEnabled $ listener appState
+
+-- | Starts a dedicated pg connection to LISTEN for notifications.  When a
+-- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
+-- cache.  It uses the connectionWorker in case the LISTEN connection dies.
+listener :: AppState -> IO ()
+listener appState = do
+  AppConfig{..} <- getConfig appState
+  let dbChannel = toS configDbChannel
+
+  -- The listener has to wait for a signal from the connectionWorker.
+  -- This is because when the connection to the db is lost, the listener also
+  -- tries to recover the connection, but not with the same pace as the connectionWorker.
+  -- Not waiting makes stderr quickly fill with connection retries messages from the listener.
+  waitListener appState
+
+  -- forkFinally allows to detect if the thread dies
+  void . flip forkFinally (handleFinally dbChannel configDbPoolAutomaticRecovery) $ do
+    dbOrError <- acquire $ toUtf8 (addFallbackAppName prettyVersion configDbUri)
+    case dbOrError of
+      Right db -> do
+        logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
+        putIsListenerOn appState True
+        SQL.listen db $ SQL.toPgIdentifier dbChannel
+        SQL.waitForNotifications handleNotification db
+      _ ->
+        die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
+  where
+    handleFinally _ False _ =
+      logWithZTime appState "Automatic recovery disabled, exiting." >> killThread (getMainThreadId appState)
+    handleFinally dbChannel True _ = do
+      -- if the thread dies, we try to recover
+      logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
+      putIsListenerOn appState False
+      -- assume the pool connection was also lost, call the connection worker
+      connectionWorker appState
+      -- retry the listener
+      listener appState
+
+    handleNotification _ msg
+      | BS.null msg            = cacheReloader
+      | msg == "reload schema" = cacheReloader
+      | msg == "reload config" = reReadConfig False appState
+      | otherwise              = pure () -- Do nothing if anything else than an empty message is sent
+
+    cacheReloader =
+      -- reloads the schema cache + restarts pool connections
+      -- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
+      connectionWorker appState
+
+checkIsFatal :: SQL.UsageError -> Maybe Text
+checkIsFatal (SQL.ConnectionUsageError e)
+  | isAuthFailureMessage = Just $ toS failureMessage
+  | otherwise = Nothing
+  where isAuthFailureMessage =
+          ("FATAL:  password authentication failed" `isInfixOf` failureMessage) ||
+          ("no password supplied" `isInfixOf` failureMessage)
+        failureMessage = BS.unpack $ fromMaybe mempty e
+checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
+  = case serverError of
+      -- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
+      SQL.ServerError "42601" _ _ _ _
+        -> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
+      -- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
+      -- This would mean that a connection pooler in transaction mode is being used
+      -- while prepared statements are enabled in the PostgREST configuration,
+      -- both of which are incompatible with each other.
+      SQL.ServerError "42P05" _ _ _ _
+        -> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
+      -- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
+      -- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
+      SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
+        -> Just "Hint: Connection poolers in statement mode are not supported."
+      _ -> Nothing
+checkIsFatal _ = Nothing
@@ -14,6 +14,7 @@ very simple authentication system inside the PostgreSQL database.
 module PostgREST.Auth
  ( AuthResult (..)
  , getResult
+  , getJwtDur
  , getRole
  , middleware
  ) where
@@ -25,6 +26,8 @@ import qualified Data.Aeson.KeyMap               as KM
 import qualified Data.Aeson.Types                as JSON
 import qualified Data.ByteString                 as BS
 import qualified Data.ByteString.Lazy.Char8      as LBS
+import qualified Data.Cache                      as C
+import qualified Data.Scientific                 as Sci
 import qualified Data.Vault.Lazy                 as Vault
 import qualified Data.Vector                     as V
 import qualified Network.HTTP.Types.Header       as HTTP
@@ -35,21 +38,20 @@ import Control.Lens            (set)
 import Control.Monad.Except    (liftEither)
 import Data.Either.Combinators (mapLeft)
 import Data.List               (lookup)
-import Data.Time.Clock         (UTCTime)
+import Data.Time.Clock         (UTCTime, nominalDiffTimeToSeconds)
+import Data.Time.Clock.POSIX   (utcTimeToPOSIXSeconds)
+import System.Clock            (TimeSpec (..))
 import System.IO.Unsafe        (unsafePerformIO)
+import System.TimeIt           (timeItT)

-import PostgREST.AppState (AppState, getConfig, getTime)
+import PostgREST.AppState (AppState, AuthResult (..), getConfig,
+                           getJwtCache, getTime)
 import PostgREST.Config   (AppConfig (..), JSPath, JSPathExp (..))
 import PostgREST.Error    (Error (..))

 import Protolude


-data AuthResult = AuthResult
-  { authClaims :: KM.KeyMap JSON.Value
-  , authRole   :: BS.ByteString
-  }
-
 -- | Receives the JWT secret and audience (from config) and a JWT and returns a
 -- JSON object of JWT claims.
 parseToken :: Monad m =>
@@ -102,14 +104,52 @@ middleware appState app req respond = do
  conf <- getConfig appState
  time <- getTime appState

-  let token = fromMaybe "" $ Wai.extractBearerAuth =<< lookup HTTP.hAuthorization (Wai.requestHeaders req)
-  authResult <- runExceptT $
-    parseToken conf (LBS.fromStrict token) time >>=
-    parseClaims conf
+  let token  = fromMaybe "" $ Wai.extractBearerAuth =<< lookup HTTP.hAuthorization (Wai.requestHeaders req)
+      parseJwt = runExceptT $ parseToken conf (LBS.fromStrict token) time >>= parseClaims conf
+
+-- If DbPlanEnabled       -> calculate JWT validation time
+-- If JwtCacheMaxLifetime -> cache JWT validation result
+  req' <- case (configServerTimingEnabled conf, configJwtCacheMaxLifetime conf) of
+    (True, 0)            -> do
+          (dur, authResult) <- timeItT parseJwt
+          return $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult & Vault.insert jwtDurKey dur }
+
+    (True, maxLifetime)  -> do
+          (dur, authResult) <- timeItT $ getJWTFromCache appState token maxLifetime parseJwt time
+          return $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult & Vault.insert jwtDurKey dur }
+
+    (False, 0)           -> do
+          authResult <- parseJwt
+          return $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult }
+
+    (False, maxLifetime) -> do
+          authResult <- getJWTFromCache appState token maxLifetime parseJwt time
+          return $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult }

-  let req' = req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult }
  app req' respond

+-- | Used to retrieve and insert JWT to JWT Cache
+getJWTFromCache :: AppState -> ByteString -> Int -> IO (Either Error AuthResult) -> UTCTime -> IO (Either Error AuthResult)
+getJWTFromCache appState token maxLifetime parseJwt utc = do
+  checkCache <- C.lookup (getJwtCache appState) token
+  authResult <- maybe parseJwt (pure . Right) checkCache
+
+  case (authResult,checkCache) of
+    (Right res, Nothing) -> C.insert' (getJwtCache appState) (getTimeSpec res maxLifetime utc) token res
+    _                    -> pure ()
+
+  return authResult
+
+-- Used to extract JWT exp claim and add to JWT Cache
+getTimeSpec :: AuthResult -> Int -> UTCTime -> Maybe TimeSpec
+getTimeSpec res maxLifetime utc = do
+  let expireJSON = KM.lookup "exp" (authClaims res)
+      utcToSecs = floor . nominalDiffTimeToSeconds . utcTimeToPOSIXSeconds
+      sciToInt = fromMaybe 0 . Sci.toBoundedInteger
+  case expireJSON of
+    Just (JSON.Number seconds) -> Just $ TimeSpec (sciToInt seconds - utcToSecs utc) 0
+    _                          -> Just $ TimeSpec (fromIntegral maxLifetime :: Int64) 0
+
 authResultKey :: Vault.Key (Either Error AuthResult)
 authResultKey = unsafePerformIO Vault.newKey
 {-# NOINLINE authResultKey #-}
@@ -117,5 +157,12 @@ authResultKey = unsafePerformIO Vault.newKey
 getResult :: Wai.Request -> Maybe (Either Error AuthResult)
 getResult = Vault.lookup authResultKey . Wai.vault

+jwtDurKey :: Vault.Key Double
+jwtDurKey = unsafePerformIO Vault.newKey
+{-# NOINLINE jwtDurKey #-}
+
+getJwtDur :: Wai.Request -> Maybe Double
+getJwtDur =  Vault.lookup jwtDurKey . Wai.vault
+
 getRole :: Wai.Request -> Maybe BS.ByteString
 getRole req = authRole <$> (rightToMaybe =<< getResult req)
@@ -21,7 +21,6 @@ import PostgREST.AppState    (AppState)
 import PostgREST.Config      (AppConfig (..))
 import PostgREST.SchemaCache (querySchemaCache)
 import PostgREST.Version     (prettyVersion)
-import PostgREST.Workers     (reReadConfig)

 import qualified PostgREST.App      as App
 import qualified PostgREST.AppState as AppState
@@ -30,10 +29,10 @@ import qualified PostgREST.Config   as Config
 import Protolude hiding (hPutStrLn)


-main :: App.SignalHandlerInstaller -> Maybe App.SocketRunner -> CLI -> IO ()
-main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
+main :: CLI -> IO ()
+main CLI{cliCommand, cliPath} = do
  conf@AppConfig{..} <-
-    either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty
+    either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty mempty

  -- Per https://github.com/PostgREST/postgrest/issues/268, we want to
  -- explicitly close the connections to PostgreSQL on shutdown.
@@ -43,23 +42,20 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
    AppState.destroy
    (\appState -> case cliCommand of
      CmdDumpConfig -> do
-        when configDbConfig $ reReadConfig True appState
+        when configDbConfig $ AppState.reReadConfig True appState
        putStr . Config.toText =<< AppState.getConfig appState
      CmdDumpSchema -> putStrLn =<< dumpSchema appState
-      CmdRun -> App.run installSignalHandlers runAppWithSocket appState)
+      CmdRun -> App.run appState)

 -- | Dump SchemaCache schema to JSON
 dumpSchema :: AppState -> IO LBS.ByteString
 dumpSchema appState = do
-  AppConfig{..} <- AppState.getConfig appState
+  conf@AppConfig{..} <- AppState.getConfig appState
  result <-
    let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
-    AppState.usePool appState $
+    AppState.usePool appState conf $
      transaction SQL.ReadCommitted SQL.Read $
-        querySchemaCache
-          (toList configDbSchemas)
-          configDbExtraSearchPath
-          configDbPreparedStatements
+        querySchemaCache conf
  case result of
    Left e -> do
      hPutStrLn stderr $ "An error ocurred when loading the schema cache:\n" <> show e
@@ -84,7 +80,7 @@ readCLIShowHelp =
  where
    prefs = O.prefs $ O.showHelpOnError <> O.showHelpOnEmpty
    opts = O.info parser $ O.fullDesc <> progDesc
-    parser = O.helper <*> exampleParser <*> cliParser
+    parser = O.helper <*> versionFlag <*> exampleParser <*> cliParser

    progDesc =
      O.progDesc $
@@ -92,6 +88,12 @@ readCLIShowHelp =
        <> BS.unpack prettyVersion
        <> " / create a REST API to an existing Postgres database"

+    versionFlag =
+      O.infoOption ("PostgREST " <> BS.unpack prettyVersion) $
+        O.long "version"
+        <> O.short 'v'
+        <> O.help "Show the version information"
+
    exampleParser =
      O.infoOption exampleConfigFile $
        O.long "example"
@@ -136,6 +138,9 @@ exampleConfigFile =
      |## Enable in-database configuration
      |db-config = true
      |
+      |## Function for in-database configuration
+      |## db-pre-config = "postgrest.pre_config"
+      |
      |## Extra schemas to add to the search_path of every request
      |db-extra-search-path = "public"
      |
@@ -154,6 +159,12 @@ exampleConfigFile =
      |## Time in seconds after which to recycle pool connections
      |# db-pool-max-lifetime = 1800
      |
+      |## Time in seconds after which to recycle unused pool connections
+      |# db-pool-max-idletime = 30
+      |
+      |## Allow automatic database connection retrying
+      |# db-pool-automatic-recovery = true
+      |
      |## Stored proc to exec immediately after auth
      |# db-pre-request = "stored_proc_name"
      |
@@ -180,10 +191,6 @@ exampleConfigFile =
      |## https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
      |db-uri = "postgresql://"
      |
-      |## Determine if GUC request settings for headers, cookies and jwt claims use the legacy names (string with dashes, invalid starting from PostgreSQL v14) with text values instead of the new names (string without dashes, valid on all PostgreSQL versions) with json values.
-      |## For PostgreSQL v14 and up, this setting will be ignored.
-      |db-use-legacy-gucs = true
-      |
      |# jwt-aud = "your_audience_claim"
      |
      |## Jspath to the role claim key
@@ -194,6 +201,9 @@ exampleConfigFile =
      |# jwt-secret = "secret_with_at_least_32_characters"
      |jwt-secret-is-base64 = false
      |
+      |## Enables and set JWT Cache max lifetime, disables caching with 0
+      |# jwt-cache-max-lifetime = 0
+      |
      |## Logging level, the admitted values are: crit, error, warn and info.
      |log-level = "error"
      |
@@ -204,12 +214,15 @@ exampleConfigFile =
      |## Base url for the OpenAPI output
      |openapi-server-proxy-uri = ""
      |
-      |## Content types to produce raw output
-      |# raw-media-types="image/png, image/jpg"
+      |## Configurable CORS origins
+      |# server-cors-allowed-origins = ""
      |
      |server-host = "!4"
      |server-port = 3000
      |
+      |## Allow getting the request-response timing information through the `Server-Timing` header
+      |server-timing-enabled = false
+      |
      |## Unix socket location
      |## if specified it takes precedence over server-port
      |# server-unix-socket = "/tmp/pgrst.sock"
@@ -24,6 +24,7 @@ module PostgREST.Config
  , readPGRSTEnvironment
  , toURI
  , parseSecret
+  , addFallbackAppName
  ) where

 import qualified Crypto.JOSE.Types      as JOSE
@@ -47,16 +48,19 @@ import Data.List               (lookup)
 import Data.List.NonEmpty      (fromList, toList)
 import Data.Maybe              (fromJust)
 import Data.Scientific         (floatingOrInteger)
+import Network.URI             (escapeURIString,
+                                isUnescapedInURIComponent, parseURI,
+                                uriQuery)
 import Numeric                 (readOct, showOct)
 import System.Environment      (getEnvironment)
 import System.Posix.Types      (FileMode)

-import PostgREST.Config.Database         (RoleSettings)
+import PostgREST.Config.Database         (RoleIsolationLvl,
+                                          RoleSettings)
 import PostgREST.Config.JSPath           (JSPath, JSPathExp (..),
                                          dumpJSPath, pRoleClaimKey)
 import PostgREST.Config.Proxy            (Proxy (..),
                                          isMalformedProxyUri, toURI)
-import PostgREST.MediaType               (MediaType (..), toMime)
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier, dumpQi,
                                          toQi)

@@ -65,6 +69,7 @@ import Protolude hiding (Proxy, toList)

 data AppConfig = AppConfig
  { configAppSettings              :: [(Text, Text)]
+  , configDbAggregates             :: Bool
  , configDbAnonRole               :: Maybe BS.ByteString
  , configDbChannel                :: Text
  , configDbChannelEnabled         :: Bool
@@ -74,36 +79,43 @@ data AppConfig = AppConfig
  , configDbPoolSize               :: Int
  , configDbPoolAcquisitionTimeout :: Int
  , configDbPoolMaxLifetime        :: Int
+  , configDbPoolMaxIdletime        :: Int
+  , configDbPoolAutomaticRecovery  :: Bool
  , configDbPreRequest             :: Maybe QualifiedIdentifier
  , configDbPreparedStatements     :: Bool
  , configDbRootSpec               :: Maybe QualifiedIdentifier
  , configDbSchemas                :: NonEmpty Text
  , configDbConfig                 :: Bool
+  , configDbPreConfig              :: Maybe QualifiedIdentifier
  , configDbTxAllowOverride        :: Bool
  , configDbTxRollbackAll          :: Bool
  , configDbUri                    :: Text
-  , configDbUseLegacyGucs          :: Bool
  , configFilePath                 :: Maybe FilePath
  , configJWKS                     :: Maybe JWKSet
  , configJwtAudience              :: Maybe StringOrURI
  , configJwtRoleClaimKey          :: JSPath
  , configJwtSecret                :: Maybe BS.ByteString
  , configJwtSecretIsBase64        :: Bool
+  , configJwtCacheMaxLifetime      :: Int
  , configLogLevel                 :: LogLevel
  , configOpenApiMode              :: OpenAPIMode
  , configOpenApiSecurityActive    :: Bool
  , configOpenApiServerProxyUri    :: Maybe Text
-  , configRawMediaTypes            :: [MediaType]
+  , configServerCorsAllowedOrigins :: Maybe [Text]
  , configServerHost               :: Text
  , configServerPort               :: Int
  , configServerTraceHeader        :: Maybe (CI.CI BS.ByteString)
+  , configServerTimingEnabled      :: Bool
  , configServerUnixSocket         :: Maybe FilePath
  , configServerUnixSocketMode     :: FileMode
  , configAdminServerPort          :: Maybe Int
  , configRoleSettings             :: RoleSettings
+  , configRoleIsoLvl               :: RoleIsolationLvl
+  , configInternalSCSleep          :: Maybe Int32
  }

 data LogLevel = LogCrit | LogError | LogWarn | LogInfo
+  deriving (Eq, Ord)

 dumpLogLevel :: LogLevel -> Text
 dumpLogLevel = \case
@@ -128,7 +140,8 @@ toText conf =
  where
    -- apply conf to all pgrst settings
    pgrstSettings = (\(k, v) -> (k, v conf)) <$>
-      [("db-anon-role",              q . T.decodeUtf8 . fromMaybe "" . configDbAnonRole)
+      [("db-aggregates-enabled",         T.toLower . show . configDbAggregates)
+      ,("db-anon-role",              q . T.decodeUtf8 . fromMaybe "" . configDbAnonRole)
      ,("db-channel",                q . configDbChannel)
      ,("db-channel-enabled",            T.toLower . show . configDbChannelEnabled)
      ,("db-extra-search-path",      q . T.intercalate "," . configDbExtraSearchPath)
@@ -137,26 +150,30 @@ toText conf =
      ,("db-pool",                       show . configDbPoolSize)
      ,("db-pool-acquisition-timeout",   show . configDbPoolAcquisitionTimeout)
      ,("db-pool-max-lifetime",          show . configDbPoolMaxLifetime)
+      ,("db-pool-max-idletime",          show . configDbPoolMaxIdletime)
+      ,("db-pool-automatic-recovery",    T.toLower . show . configDbPoolAutomaticRecovery)
      ,("db-pre-request",            q . maybe mempty dumpQi . configDbPreRequest)
      ,("db-prepared-statements",        T.toLower . show . configDbPreparedStatements)
      ,("db-root-spec",              q . maybe mempty dumpQi . configDbRootSpec)
      ,("db-schemas",                q . T.intercalate "," . toList . configDbSchemas)
      ,("db-config",                     T.toLower . show . configDbConfig)
+      ,("db-pre-config",             q . maybe mempty dumpQi . configDbPreConfig)
      ,("db-tx-end",                 q . showTxEnd)
      ,("db-uri",                    q . configDbUri)
-      ,("db-use-legacy-gucs",            T.toLower . show . configDbUseLegacyGucs)
      ,("jwt-aud",                       T.decodeUtf8 . LBS.toStrict . JSON.encode . maybe "" toJSON . configJwtAudience)
      ,("jwt-role-claim-key",        q . T.intercalate mempty . fmap dumpJSPath . configJwtRoleClaimKey)
      ,("jwt-secret",                q . T.decodeUtf8 . showJwtSecret)
      ,("jwt-secret-is-base64",          T.toLower . show . configJwtSecretIsBase64)
+      ,("jwt-cache-max-lifetime",                   show . configJwtCacheMaxLifetime)
      ,("log-level",                 q . dumpLogLevel . configLogLevel)
      ,("openapi-mode",              q . dumpOpenApiMode . configOpenApiMode)
      ,("openapi-security-active",       T.toLower . show . configOpenApiSecurityActive)
      ,("openapi-server-proxy-uri",  q . fromMaybe mempty . configOpenApiServerProxyUri)
-      ,("raw-media-types",           q . T.decodeUtf8 . BS.intercalate "," . fmap toMime . configRawMediaTypes)
+      ,("server-cors-allowed-origins",      q . maybe "" (T.intercalate ",") . configServerCorsAllowedOrigins)
      ,("server-host",               q . configServerHost)
      ,("server-port",                   show . configServerPort)
      ,("server-trace-header",       q . T.decodeUtf8 . maybe mempty CI.original . configServerTraceHeader)
+      ,("server-timing-enabled",         T.toLower . show . configServerTimingEnabled)
      ,("server-unix-socket",        q . maybe mempty T.pack . configServerUnixSocket)
      ,("server-unix-socket-mode",   q . T.pack . showSocketMode)
      ,("admin-server-port",             maybe "\"\"" show . configAdminServerPort)
@@ -193,13 +210,13 @@ instance JustIfMaybe a (Maybe a) where

 -- | Reads and parses the config and overrides its parameters from env vars,
 -- files or db settings.
-readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> IO (Either Text AppConfig)
-readAppConfig dbSettings optPath prevDbUri roleSettings = do
+readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> RoleIsolationLvl -> IO (Either Text AppConfig)
+readAppConfig dbSettings optPath prevDbUri roleSettings roleIsolationLvl = do
  env <- readPGRSTEnvironment
  -- if no filename provided, start with an empty map to read config from environment
  conf <- maybe (return $ Right M.empty) loadConfig optPath

-  case C.runParser (parser optPath env dbSettings roleSettings) =<< mapLeft show conf of
+  case C.runParser (parser optPath env dbSettings roleSettings roleIsolationLvl) =<< mapLeft show conf of
    Left err ->
      return . Left $ "Error in config " <> err
    Right parsedConfig ->
@@ -214,10 +231,11 @@ readAppConfig dbSettings optPath prevDbUri roleSettings = do
      decodeJWKS <$>
        (decodeSecret =<< readSecretFile =<< readDbUriFile prevDbUri parsedConfig)

-parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> C.Parser C.Config AppConfig
-parser optPath env dbSettings roleSettings =
+parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> RoleIsolationLvl -> C.Parser C.Config AppConfig
+parser optPath env dbSettings roleSettings roleIsolationLvl =
  AppConfig
    <$> parseAppSettings "app.settings"
+    <*> (fromMaybe False <$> optBool "db-aggregates-enabled")
    <*> (fmap encodeUtf8 <$> optString "db-anon-role")
    <*> (fromMaybe "pgrst" <$> optString "db-channel")
    <*> (fromMaybe True <$> optBool "db-channel-enabled")
@@ -228,6 +246,9 @@ parser optPath env dbSettings roleSettings =
    <*> (fromMaybe 10 <$> optInt "db-pool")
    <*> (fromMaybe 10 <$> optInt "db-pool-acquisition-timeout")
    <*> (fromMaybe 1800 <$> optInt "db-pool-max-lifetime")
+    <*> (fromMaybe 30 <$> optWithAlias (optInt "db-pool-timeout")
+                                       (optInt "db-pool-max-idletime"))
+    <*> (fromMaybe True <$> optBool "db-pool-automatic-recovery")
    <*> (fmap toQi <$> optWithAlias (optString "db-pre-request")
                                    (optString "pre-request"))
    <*> (fromMaybe True <$> optBool "db-prepared-statements")
@@ -236,10 +257,10 @@ parser optPath env dbSettings roleSettings =
    <*> (fromList . maybe ["public"] splitOnCommas <$> optWithAlias (optValue "db-schemas")
                                                                    (optValue "db-schema"))
    <*> (fromMaybe True <$> optBool "db-config")
+    <*> (fmap toQi <$> optString "db-pre-config")
    <*> parseTxEnd "db-tx-end" snd
    <*> parseTxEnd "db-tx-end" fst
    <*> (fromMaybe "postgresql://" <$> optString "db-uri")
-    <*> (fromMaybe True <$> optBool "db-use-legacy-gucs")
    <*> pure optPath
    <*> pure Nothing
    <*> parseJwtAudience "jwt-aud"
@@ -248,18 +269,22 @@ parser optPath env dbSettings roleSettings =
    <*> (fromMaybe False <$> optWithAlias
          (optBool "jwt-secret-is-base64")
          (optBool "secret-is-base64"))
+    <*> (fromMaybe 0 <$> optInt "jwt-cache-max-lifetime")
    <*> parseLogLevel "log-level"
    <*> parseOpenAPIMode "openapi-mode"
    <*> (fromMaybe False <$> optBool "openapi-security-active")
    <*> parseOpenAPIServerProxyURI "openapi-server-proxy-uri"
-    <*> (maybe [] (fmap (MTOther . encodeUtf8) . splitOnCommas) <$> optValue "raw-media-types")
+    <*> parseCORSAllowedOrigins "server-cors-allowed-origins"
    <*> (fromMaybe "!4" <$> optString "server-host")
    <*> (fromMaybe 3000 <$> optInt "server-port")
    <*> (fmap (CI.mk . encodeUtf8) <$> optString "server-trace-header")
+    <*> (fromMaybe False <$> optBool "server-timing-enabled")
    <*> (fmap T.unpack <$> optString "server-unix-socket")
    <*> parseSocketFileMode "server-unix-socket-mode"
    <*> optInt "admin-server-port"
    <*> pure roleSettings
+    <*> pure roleIsolationLvl
+    <*> optInt "internal-schema-cache-sleep"
  where
    parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
    parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
@@ -332,6 +357,11 @@ parser optPath env dbSettings roleSettings =
        Nothing  -> pure [JSPKey "role"]
        Just rck -> either (fail . show) pure $ pRoleClaimKey rck

+    parseCORSAllowedOrigins k =
+      optString k >>= \case
+        Nothing   -> pure Nothing
+        Just orig -> pure $ Just (T.strip <$> T.splitOn "," orig)
+
    optWithAlias :: C.Parser C.Config (Maybe a) -> C.Parser C.Config (Maybe a) -> C.Parser C.Config (Maybe a)
    optWithAlias orig alias =
      orig >>= \case
@@ -354,21 +384,14 @@ parser optPath env dbSettings roleSettings =
                               (C.Key -> C.Parser C.Value a -> C.Parser C.Config b) ->
                               C.Key -> (C.Value -> a) -> C.Parser C.Config b
    overrideFromDbOrEnvironment necessity key coercion =
-      case reloadableDbSetting <|> M.lookup envVarName env of
+      case dbConf <|> M.lookup envVarName env of
        Just dbOrEnvVal -> pure $ justIfMaybe $ coercion $ C.String dbOrEnvVal
-        Nothing  -> necessity key (coercion <$> C.value)
+        Nothing         -> necessity key (coercion <$> C.value)
      where
        dashToUnderscore '-' = '_'
        dashToUnderscore c   = c
        envVarName = "PGRST_" <> (toUpper . dashToUnderscore <$> toS key)
-        reloadableDbSetting =
-          let dbSettingName = T.pack $ dashToUnderscore <$> toS key in
-          if dbSettingName `notElem` [
-            "server_host", "server_port", "server_unix_socket", "server_unix_socket_mode", "admin_server_port", "log_level",
-            "db_uri", "db_channel_enabled", "db_channel", "db_pool", "db_pool_acquisition_timeout",
-            "db_pool_max_lifetime", "db_config"]
-          then lookup dbSettingName dbSettings
-          else Nothing
+        dbConf = lookup (T.pack $ dashToUnderscore <$> toS key) dbSettings

    coerceText :: C.Value -> Text
    coerceText (C.String s) = s
@@ -455,3 +478,41 @@ type Environment = M.Map [Char] Text
 readPGRSTEnvironment :: IO Environment
 readPGRSTEnvironment =
  M.map T.pack . M.fromList . filter (isPrefixOf "PGRST_" . fst) <$> getEnvironment
+
+-- | Adds a `fallback_application_name` value to the connection string. This allows querying the PostgREST version on pg_stat_activity.
+--
+-- >>> let ver = "11.1.0 (5a04ec7)"::ByteString
+-- >>> let strangeVer = "11'1&0@#$%,.:\"[]{}?+^()=asdfqwer"::ByteString
+--
+-- >>> addFallbackAppName ver "postgres://user:pass@host:5432/postgres"
+-- "postgres://user:pass@host:5432/postgres?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgres://user:pass@host:5432/postgres?"
+-- "postgres://user:pass@host:5432/postgres?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgres:///postgres?host=server&port=5432"
+-- "postgres:///postgres?host=server&port=5432&fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgresql://"
+-- "postgresql://?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName strangeVer "postgres:///postgres?host=server&port=5432"
+-- "postgres:///postgres?host=server&port=5432&fallback_application_name=PostgREST%2011%271%260%40%23%24%25%2C.%3A%22%5B%5D%7B%7D%3F%2B%5E%28%29%3Dasdfqwer"
+--
+-- >>> addFallbackAppName ver "postgres://user:invalid_chars[]#@host:5432/postgres"
+-- "postgres://user:invalid_chars[]#@host:5432/postgres"
+--
+-- >>> addFallbackAppName ver "invalid_uri1=val1 invalid_uri2=val2"
+-- "invalid_uri1=val1 invalid_uri2=val2"
+addFallbackAppName :: ByteString -> Text -> Text
+addFallbackAppName version dbUri = dbUri <>
+  case uriQuery <$> parseURI (toS dbUri) of
+    -- Does not add the application name to key=val connection strings or invalid URIs
+    Nothing  -> mempty
+    Just ""  -> "?" <> uriFmt
+    Just "?" -> uriFmt
+    _        -> "&" <> uriFmt
+  where
+    uriFmt = pKeyWord <> toS (escapeURIString isUnescapedInURIComponent $ toS pgrstVer)
+    pKeyWord = "fallback_application_name="
+    pgrstVer = "PostgREST " <> T.decodeUtf8 version
@@ -3,14 +3,17 @@
 module PostgREST.Config.Database
  ( pgVersionStatement
  , queryDbSettings
-  , queryRoleSettings
  , queryPgVersion
+  , queryRoleSettings
  , RoleSettings
+  , RoleIsolationLvl
+  , TimezoneNames
+  , toIsolationLevel
  ) where

 import Control.Arrow ((***))

-import PostgREST.Config.PgVersion (PgVersion (..))
+import PostgREST.Config.PgVersion (PgVersion (..), pgVersion150)

 import qualified Data.HashMap.Strict as HM

@@ -21,11 +24,49 @@ import qualified Hasql.Statement            as SQL
 import qualified Hasql.Transaction          as SQL
 import qualified Hasql.Transaction.Sessions as SQL

-import Text.InterpolatedString.Perl6 (q)
+import Text.InterpolatedString.Perl6 (q, qc)

 import Protolude

-type RoleSettings = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
+type RoleSettings     = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
+type RoleIsolationLvl = HM.HashMap ByteString SQL.IsolationLevel
+type TimezoneNames    = Set ByteString -- cache timezone names for prefer timezone=
+
+toIsolationLevel :: (Eq a, IsString a) => a -> SQL.IsolationLevel
+toIsolationLevel a = case a of
+  "repeatable read" -> SQL.RepeatableRead
+  "serializable"    -> SQL.Serializable
+  _                 -> SQL.ReadCommitted
+
+prefix :: Text
+prefix = "pgrst."
+
+-- | In-db settings names
+dbSettingsNames :: [Text]
+dbSettingsNames =
+  (prefix <>) <$>
+  ["db_aggregates_enabled"
+  ,"db_anon_role"
+  ,"db_pre_config"
+  ,"db_extra_search_path"
+  ,"db_max_rows"
+  ,"db_plan_enabled"
+  ,"db_pre_request"
+  ,"db_prepared_statements"
+  ,"db_root_spec"
+  ,"db_schemas"
+  ,"db_tx_end"
+  ,"jwt_aud"
+  ,"jwt_role_claim_key"
+  ,"jwt_secret"
+  ,"jwt_secret_is_base64"
+  ,"openapi_mode"
+  ,"openapi_security_active"
+  ,"openapi_server_proxy_uri"
+  ,"raw_media_types"
+  ,"server_trace_header"
+  ,"server_timing_enabled"
+  ]

 queryPgVersion :: Bool -> Session PgVersion
 queryPgVersion prepared = statement mempty $ pgVersionStatement prepared
@@ -36,46 +77,62 @@ pgVersionStatement = SQL.Statement sql HE.noParams versionRow
    sql = "SELECT current_setting('server_version_num')::integer, current_setting('server_version')"
    versionRow = HD.singleRow $ PgVersion <$> column HD.int4 <*> column HD.text

-queryDbSettings :: Bool -> Session [(Text, Text)]
-queryDbSettings prepared =
+-- | Query the in-database configuration. The settings have the following priorities:
+--
+-- 1. Role + with database-specific settings:
+--    ALTER ROLE authenticator IN DATABASE postgres SET <prefix>jwt_aud = 'val';
+-- 2. Role + with settings:
+--    ALTER ROLE authenticator SET <prefix>jwt_aud = 'overridden';
+-- 3. pre-config function:
+--    CREATE FUNCTION pre_config() .. PERFORM set_config(<prefix>jwt_aud, 'pre_config_aud'..)
+--
+-- The example above will result in <prefix>jwt_aud = 'val'
+-- A setting on the database only will have no effect: ALTER DATABASE postgres SET <prefix>jwt_aud = 'xx'
+queryDbSettings :: Maybe Text -> Bool -> Session [(Text, Text)]
+queryDbSettings preConfFunc prepared =
  let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
-  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ dbSettingsStatement prepared
-
-- | Get db settings from the connection role. Global settings will be overridden by database specific settings.
-dbSettingsStatement :: Bool -> SQL.Statement () [(Text, Text)]
-dbSettingsStatement = SQL.Statement sql HE.noParams decodeSettings
+  transaction SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared
  where
-    sql = [q|
+    sql = [qc|
      WITH
-      role_setting (database, setting) AS (
-        SELECT setdatabase,
-               unnest(setconfig)
-          FROM pg_catalog.pg_db_role_setting
-         WHERE setrole = CURRENT_USER::regrole::oid
-           AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
+      role_setting AS (
+        SELECT setdatabase as database,
+               unnest(setconfig) as setting
+        FROM pg_catalog.pg_db_role_setting
+        WHERE setrole = CURRENT_USER::regrole::oid
+          AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
      ),
-      kv_settings (database, k, v) AS (
+      kv_settings AS (
        SELECT database,
-               substr(setting, 1, strpos(setting, '=') - 1),
-               substr(setting, strpos(setting, '=') + 1)
-          FROM role_setting
-         WHERE setting LIKE 'pgrst.%'
+               substr(setting, 1, strpos(setting, '=') - 1) as k,
+               substr(setting, strpos(setting, '=') + 1) as v
+        FROM role_setting
+        {preConfigF}
      )
      SELECT DISTINCT ON (key)
-             replace(k, 'pgrst.', '') AS key,
+             replace(k, '{prefix}', '') AS key,
             v AS value
-        FROM kv_settings
-       ORDER BY key, database DESC;
+      FROM kv_settings
+      WHERE k = ANY($1) AND v IS NOT NULL
+      ORDER BY key, database DESC NULLS LAST;
    |]
+    preConfigF = case preConfFunc of
+      Nothing   -> mempty
+      Just func -> [qc|
+          UNION
+          SELECT
+            null as database,
+            x as k,
+            current_setting(x, true) as v
+          FROM unnest($1) x
+          JOIN {func}() _ ON TRUE
+      |]::Text
    decodeSettings = HD.rowList $ (,) <$> column HD.text <*> column HD.text

-queryRoleSettings :: Bool -> Session RoleSettings
-queryRoleSettings prepared =
+queryRoleSettings :: PgVersion -> Bool -> Session (RoleSettings, RoleIsolationLvl)
+queryRoleSettings pgVer prepared =
  let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
-  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ roleSettingsStatement prepared
-
-roleSettingsStatement :: Bool -> SQL.Statement () RoleSettings
-roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
+  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared
  where
    sql = [q|
      with
@@ -91,18 +148,44 @@ roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
          substr(setting, 1, strpos(setting, '=') - 1) as key,
          lower(substr(setting, strpos(setting, '=') + 1)) as value
        FROM role_setting
+      ),
+      iso_setting AS (
+        SELECT rolname, value
+        FROM kv_settings
+        WHERE key = 'default_transaction_isolation'
      )
-      select rolname, array_agg(row(key, value))
-      from kv_settings
-      group by rolname;
+      select
+        kv.rolname,
+        i.value as iso_lvl,
+        coalesce(array_agg(row(kv.key, kv.value)) filter (where key <> 'default_transaction_isolation'), '{}') as role_settings
+      from kv_settings kv
+      join pg_settings ps on ps.name = kv.key |] <>
+      (if pgVer >= pgVersion150
+        then "and (ps.context = 'user' or has_parameter_privilege(current_user::regrole::oid, ps.name, 'set')) "
+        else "and ps.context = 'user' ") <> [q|
+      left join iso_setting i on i.rolname = kv.rolname
+      group by kv.rolname, i.value;
    |]
-    decodeRoleSettings = HM.fromList . map (bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>))) <$> HD.rowList aRow
-    aRow :: HD.Row (Text, [(Text, Text)])
-    aRow = (,) <$> column HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
+
+    processRows :: [(Text, Maybe Text, [(Text, Text)])] -> (RoleSettings, RoleIsolationLvl)
+    processRows rs =
+      let
+        rowsWRoleSettings = [ (x, z) | (x, _, z) <- rs ]
+        rowsWIsolation    = [ (x, y) | (x, Just y, _) <- rs ]
+      in
+      ( HM.fromList $ bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>)) <$> rowsWRoleSettings
+      , HM.fromList $ (encodeUtf8 *** toIsolationLevel) <$> rowsWIsolation
+      )
+
+    rows :: HD.Result [(Text, Maybe Text, [(Text, Text)])]
+    rows = HD.rowList $ (,,) <$> column HD.text <*> nullableColumn HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)

 column :: HD.Value a -> HD.Row a
 column = HD.column . HD.nonNullable

+nullableColumn :: HD.Value a -> HD.Row (Maybe a)
+nullableColumn = HD.column . HD.nullable
+
 compositeField :: HD.Value a -> HD.Composite a
 compositeField = HD.field . HD.nonNullable

@@ -111,3 +194,9 @@ compositeArrayColumn = arrayColumn . HD.composite

 arrayColumn :: HD.Value a -> HD.Row [a]
 arrayColumn = column . HD.listArray . HD.nonNullable
+
+param :: HE.Value a -> HE.Params a
+param = HE.param . HE.nonNullable
+
+arrayParam :: HE.Value a -> HE.Params [a]
+arrayParam = param . HE.foldableArray . HE.nonNullable
@@ -13,6 +13,7 @@ module PostgREST.Config.PgVersion
  , pgVersion121
  , pgVersion130
  , pgVersion140
+  , pgVersion150
  ) where

 import qualified Data.Aeson as JSON
@@ -62,3 +63,6 @@ pgVersion130 = PgVersion 130000 "13.0"

 pgVersion140 :: PgVersion
 pgVersion140 = PgVersion 140000 "14.0"
+
+pgVersion150 :: PgVersion
+pgVersion150 = PgVersion 150000 "15.0"
@@ -2,10 +2,14 @@
 Module      : PostgREST.Cors
 Description : Wai Middleware to set cors policy.
 -}
+
+{-# LANGUAGE TupleSections #-}
+
 module PostgREST.Cors (middleware) where

 import qualified Data.ByteString.Char8       as BS
 import qualified Data.CaseInsensitive        as CI
+import qualified Data.Text.Encoding          as T
 import qualified Network.Wai                 as Wai
 import qualified Network.Wai.Middleware.Cors as Wai

@@ -13,15 +17,15 @@ import Data.List (lookup)

 import Protolude

-middleware :: Wai.Middleware
-middleware = Wai.cors corsPolicy
+middleware :: Maybe [Text] -> Wai.Middleware
+middleware corsAllowedOrigins = Wai.cors $ corsPolicy corsAllowedOrigins

 -- | CORS policy to be used in by Wai Cors middleware
-corsPolicy :: Wai.Request -> Maybe Wai.CorsResourcePolicy
-corsPolicy req = case lookup "origin" headers of
-  Just origin ->
+corsPolicy :: Maybe [Text] -> Wai.Request -> Maybe Wai.CorsResourcePolicy
+corsPolicy corsAllowedOrigins req = case lookup "origin" headers of
+  Just _ ->
    Just Wai.CorsResourcePolicy
-    { Wai.corsOrigins = Just ([origin], True)
+    { Wai.corsOrigins = (, True) . map T.encodeUtf8 <$> corsAllowedOrigins
    , Wai.corsMethods = ["GET", "POST", "PATCH", "PUT", "DELETE", "OPTIONS"]
    , Wai.corsRequestHeaders = "Authorization" : accHeaders
    , Wai.corsExposedHeaders = Just
@@ -11,14 +11,15 @@ module PostgREST.Error
  , PgError(..)
  , Error(..)
  , errorPayload
-  , checkIsFatal
-  , singularityError
+  , status
  ) where

 import qualified Data.Aeson                as JSON
 import qualified Data.ByteString.Char8     as BS
+import qualified Data.CaseInsensitive      as CI
 import qualified Data.FuzzySet             as Fuzzy
 import qualified Data.HashMap.Strict       as HM
+import qualified Data.Map.Internal         as M
 import qualified Data.Text                 as T
 import qualified Data.Text.Encoding        as T
 import qualified Data.Text.Encoding.Error  as T
@@ -26,7 +27,7 @@ import qualified Hasql.Pool                as SQL
 import qualified Hasql.Session             as SQL
 import qualified Network.HTTP.Types.Status as HTTP

-import Data.Aeson  ((.=))
+import Data.Aeson  ((.:), (.:?), (.=))
 import Network.Wai (Response, responseLBS)

 import Network.HTTP.Types.Header (Header)
@@ -56,15 +57,18 @@ class (JSON.ToJSON a) => PgrstError a where
  errorPayload = JSON.encode

  errorResponseFor :: a -> Response
-  errorResponseFor err = responseLBS (status err) (headers err) $ errorPayload err
+  errorResponseFor err =
+    let baseHeader = MediaType.toContentType MTApplicationJSON in
+    responseLBS (status err) (baseHeader : headers err) $ errorPayload err

 instance PgrstError ApiRequestError where
+  status AggregatesNotAllowed{}  = HTTP.status400
  status AmbiguousRelBetween{}   = HTTP.status300
  status AmbiguousRpc{}          = HTTP.status300
-  status BinaryFieldError{}      = HTTP.status406
  status MediaTypeError{}        = HTTP.status415
  status InvalidBody{}           = HTTP.status400
  status InvalidFilters          = HTTP.status405
+  status InvalidPreferences{}    = HTTP.status400
  status InvalidRpcMethod{}      = HTTP.status405
  status InvalidRange{}          = HTTP.status416
  status NotFound                = HTTP.status404
@@ -81,108 +85,132 @@ instance PgrstError ApiRequestError where
  status UnsupportedMethod{}     = HTTP.status405
  status LimitNoOrderError       = HTTP.status400
  status ColumnNotFound{}        = HTTP.status400
+  status GucHeadersError         = HTTP.status500
+  status GucStatusError          = HTTP.status500
+  status OffLimitsChangesError{} = HTTP.status400
+  status PutMatchingPkError      = HTTP.status400
+  status SingularityError{}      = HTTP.status406
+  status PGRSTParseError         = HTTP.status500

-  headers _ = [MediaType.toContentType MTApplicationJSON]
+  headers SingularityError{}     = [MediaType.toContentType $ MTVndSingularJSON False]
+  headers _ = mempty
+
+toJsonPgrstError :: ErrorCode -> Text -> Maybe JSON.Value -> Maybe JSON.Value -> JSON.Value
+toJsonPgrstError code msg details hint = JSON.object [
+    "code"     .= code
+  , "message"  .= msg
+  , "details"  .= details
+  , "hint"     .= hint
+  ]

 instance JSON.ToJSON ApiRequestError where
-  toJSON (QueryParamError (QPError message details)) = JSON.object [
-    "code"    .= ApiRequestErrorCode00,
-    "message" .= message,
-    "details" .= details,
-    "hint"    .= JSON.Null]
-  toJSON (InvalidRpcMethod method) = JSON.object [
-    "code"    .= ApiRequestErrorCode01,
-    "message" .= ("Cannot use the " <> T.decodeUtf8 method <> " method on RPC"),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
-  toJSON (InvalidBody errorMessage) = JSON.object [
-    "code"    .= ApiRequestErrorCode02,
-    "message" .= T.decodeUtf8 errorMessage,
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
-  toJSON (InvalidRange rangeError) = JSON.object [
-    "code"    .= ApiRequestErrorCode03,
-    "message" .= ("Requested range not satisfiable" :: Text),
-    "details" .= (case rangeError of
-                   NegativeLimit           -> "Limit should be greater than or equal to zero."
-                   LowerGTUpper            -> "The lower boundary must be lower than or equal to the upper boundary in the Range header."
-                   OutOfBounds lower total -> "An offset of " <> lower <> " was requested, but there are only " <> total <> " rows."),
-    "hint"    .= JSON.Null]
-  toJSON InvalidFilters = JSON.object [
-    "code"    .= ApiRequestErrorCode05,
-    "message" .= ("Filters must include all and only primary key columns with 'eq' operators" :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
-  toJSON (UnacceptableSchema schemas) = JSON.object [
-    "code"    .= ApiRequestErrorCode06,
-    "message" .= ("The schema must be one of the following: " <> T.intercalate ", " schemas),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
-  toJSON (MediaTypeError cts)    = JSON.object [
-    "code"    .= ApiRequestErrorCode07,
-    "message" .= ("None of these media types are available: " <> T.intercalate ", " (map T.decodeUtf8 cts)),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
+  toJSON (QueryParamError (QPError message details)) = toJsonPgrstError
+    ApiRequestErrorCode00 message (Just (JSON.String details)) Nothing
+
+  toJSON (InvalidRpcMethod method) = toJsonPgrstError
+    ApiRequestErrorCode01 ("Cannot use the " <> T.decodeUtf8 method <> " method on RPC") Nothing Nothing
+
+  toJSON (InvalidBody errorMessage) = toJsonPgrstError
+    ApiRequestErrorCode02 (T.decodeUtf8 errorMessage) Nothing Nothing
+
+  toJSON (InvalidRange rangeError) = toJsonPgrstError
+    ApiRequestErrorCode03
+    "Requested range not satisfiable"
+    (Just $ case rangeError of
+       NegativeLimit           -> "Limit should be greater than or equal to zero."
+       LowerGTUpper            -> "The lower boundary must be lower than or equal to the upper boundary in the Range header."
+       OutOfBounds lower total -> JSON.String $ "An offset of " <> lower <> " was requested, but there are only " <> total <> " rows.")
+    Nothing
+
+  toJSON InvalidFilters = toJsonPgrstError
+    ApiRequestErrorCode05 "Filters must include all and only primary key columns with 'eq' operators" Nothing Nothing
+
+  toJSON (UnacceptableSchema schemas) = toJsonPgrstError
+    ApiRequestErrorCode06 ("The schema must be one of the following: " <> T.intercalate ", " schemas) Nothing Nothing
+
+  toJSON (MediaTypeError cts) = toJsonPgrstError
+    ApiRequestErrorCode07 ("None of these media types are available: " <> T.intercalate ", " (map T.decodeUtf8 cts)) Nothing Nothing
+
  toJSON NotFound = JSON.object []
-  toJSON (NotEmbedded resource) = JSON.object [
-    "code"    .= ApiRequestErrorCode08,
-    "message" .= ("'" <> resource <> "' is not an embedded resource in this request" :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= ("Verify that '" <> resource <> "' is included in the 'select' query parameter." :: Text)]

-  toJSON LimitNoOrderError = JSON.object [
-    "code"    .= ApiRequestErrorCode09,
-    "message" .= ("A 'limit' was applied without an explicit 'order'":: Text),
-    "details" .= JSON.Null,
-    "hint"    .= ("Apply an 'order' using unique column(s)" :: Text)]
+  toJSON (NotEmbedded resource) = toJsonPgrstError
+    ApiRequestErrorCode08
+    ("'" <> resource <> "' is not an embedded resource in this request")
+    Nothing
+    (Just $ JSON.String $ "Verify that '" <> resource <> "' is included in the 'select' query parameter.")

-  toJSON (BinaryFieldError ct) = JSON.object [
-    "code"    .= ApiRequestErrorCode13,
-    "message" .= ((T.decodeUtf8 (MediaType.toMime ct) <> " requested but more than one column was selected") :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
+  toJSON LimitNoOrderError = toJsonPgrstError
+    ApiRequestErrorCode09 "A 'limit' was applied without an explicit 'order'" Nothing (Just "Apply an 'order' using unique column(s)")

-  toJSON PutLimitNotAllowedError = JSON.object [
-    "code"    .= ApiRequestErrorCode14,
-    "message" .= ("limit/offset querystring parameters are not allowed for PUT" :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
+  toJSON (OffLimitsChangesError n maxs) = toJsonPgrstError
+    ApiRequestErrorCode10
+    "The maximum number of rows allowed to change was surpassed"
+    (Just $ JSON.String $ T.unwords ["Results contain", show n, "rows changed but the maximum number allowed is", show maxs])
+    Nothing

-  toJSON (UnsupportedMethod method) = JSON.object [
-    "code"    .= ApiRequestErrorCode17,
-    "message" .= ("Unsupported HTTP method: " <> T.decodeUtf8 method),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
+  toJSON GucHeadersError = toJsonPgrstError
+    ApiRequestErrorCode11 "response.headers guc must be a JSON array composed of objects with a single key and a string value" Nothing Nothing

-  toJSON (RelatedOrderNotToOne origin target) = JSON.object [
-    "code"    .= ApiRequestErrorCode18,
-    "message" .= ("A related order on '" <> target <> "' is not possible" :: Text),
-    "details" .= ("'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship" :: Text),
-    "hint"    .= JSON.Null]
+  toJSON GucStatusError = toJsonPgrstError
+    ApiRequestErrorCode12 "response.status guc must be a valid status code" Nothing Nothing

-  toJSON (SpreadNotToOne origin target) = JSON.object [
-    "code"    .= ApiRequestErrorCode19,
-    "message" .= ("A spread operation on '" <> target <> "' is not possible" :: Text),
-    "details" .= ("'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship" :: Text),
-    "hint"    .= JSON.Null]
+  toJSON PutLimitNotAllowedError = toJsonPgrstError
+    ApiRequestErrorCode14 "limit/offset querystring parameters are not allowed for PUT" Nothing Nothing

-  toJSON (UnacceptableFilter target) = JSON.object [
-    "code"    .= ApiRequestErrorCode20,
-    "message" .= ("Bad operator on the '" <> target <> "' embedded resource":: Text),
-    "details" .= ("Only is null or not is null filters are allowed on embedded resources":: Text),
-    "hint"    .= JSON.Null]
+  toJSON PutMatchingPkError = toJsonPgrstError
+    ApiRequestErrorCode15 "Payload values do not match URL in primary key column(s)" Nothing Nothing

-  toJSON (NoRelBetween parent child embedHint schema allRels) = JSON.object [
-    "code"    .= SchemaCacheErrorCode00,
-    "message" .= ("Could not find a relationship between '" <> parent <> "' and '" <> child <> "' in the schema cache" :: Text),
-    "details" .= ("Searched for a foreign key relationship between '" <> parent <> "' and '" <> child <> maybe mempty ("' using the hint '" <>) embedHint <> "' in the schema '" <> schema <> "', but no matches were found."),
-    "hint"    .= noRelBetweenHint parent child schema allRels]
+  toJSON (SingularityError n) = toJsonPgrstError
+    ApiRequestErrorCode16
+    "JSON object requested, multiple (or no) rows returned"
+    (Just $ JSON.String $ T.unwords ["The result contains", show n, "rows"])
+    Nothing
+
+  toJSON (UnsupportedMethod method) = toJsonPgrstError
+    ApiRequestErrorCode17 ("Unsupported HTTP method: " <> T.decodeUtf8 method) Nothing Nothing
+
+  toJSON (RelatedOrderNotToOne origin target) = toJsonPgrstError
+    ApiRequestErrorCode18
+    ("A related order on '" <> target <> "' is not possible")
+    (Just $ JSON.String $ "'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship")
+    Nothing
+
+  toJSON (SpreadNotToOne origin target) = toJsonPgrstError
+    ApiRequestErrorCode19
+    ("A spread operation on '" <> target <> "' is not possible")
+    (Just $ JSON.String $ "'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship")
+    Nothing
+
+  toJSON (UnacceptableFilter target) = toJsonPgrstError
+    ApiRequestErrorCode20
+    ("Bad operator on the '" <> target <> "' embedded resource")
+    (Just "Only is null or not is null filters are allowed on embedded resources")
+    Nothing
+
+  toJSON PGRSTParseError = toJsonPgrstError
+    ApiRequestErrorCode21 "The message and detail field of RAISE 'PGRST' error expects JSON" Nothing Nothing
+
+  toJSON (InvalidPreferences prefs) = toJsonPgrstError
+    ApiRequestErrorCode22
+    "Invalid preferences given with handling=strict"
+    (Just $ JSON.String $ T.decodeUtf8 ("Invalid preferences: " <> BS.intercalate ", " prefs))
+    Nothing
+
+  toJSON AggregatesNotAllowed = toJsonPgrstError
+    ApiRequestErrorCode23 "Use of aggregate functions is not allowed" Nothing Nothing
+
+  toJSON (NoRelBetween parent child embedHint schema allRels) = toJsonPgrstError
+    SchemaCacheErrorCode00
+    ("Could not find a relationship between '" <> parent <> "' and '" <> child <> "' in the schema cache")
+    (Just $ JSON.String $ "Searched for a foreign key relationship between '" <> parent <> "' and '" <> child <> maybe mempty ("' using the hint '" <>) embedHint <> "' in the schema '" <> schema <> "', but no matches were found.")
+    (JSON.String <$> noRelBetweenHint parent child schema allRels)
+
+  toJSON (AmbiguousRelBetween parent child rels) = toJsonPgrstError
+    SchemaCacheErrorCode01
+    ("Could not embed because more than one relationship was found for '" <> parent <> "' and '" <> child <> "'")
+    (Just $ JSON.toJSONList (compressedRel <$> rels))
+    (Just $ JSON.String $ "Try changing '" <> child <> "' to one of the following: " <> relHint rels <> ". Find the desired relationship in the 'details' key.")

-  toJSON (AmbiguousRelBetween parent child rels) = JSON.object [
-    "code"    .= SchemaCacheErrorCode01,
-    "message" .= ("Could not embed because more than one relationship was found for '" <> parent <> "' and '" <> child <> "'" :: Text),
-    "details" .= (compressedRel <$> rels),
-    "hint"    .= ("Try changing '" <> child <> "' to one of the following: " <> relHint rels <> ". Find the desired relationship in the 'details' key." :: Text)]
  toJSON (NoRpc schema procName argumentKeys hasPreferSingleObject contentType isInvPost allProcs overloadedProcs)  =
    let func = schema <> "." <> procName
        prms = T.intercalate ", " argumentKeys
@@ -190,10 +218,10 @@ instance JSON.ToJSON ApiRequestError where
        prmsDet = " with parameter" <> (if length argumentKeys > 1 then "s " else " ") <> prms
        fmtPrms p = if null argumentKeys then " without parameters" else p
        onlySingleParams = hasPreferSingleObject || (isInvPost && contentType `elem` [MTTextPlain, MTTextXML, MTOctetStream])
-    in JSON.object [
-    "code"    .= SchemaCacheErrorCode02,
-    "message" .= ("Could not find the function " <> func <> (if onlySingleParams then "" else fmtPrms prmsMsg) <> " in the schema cache"),
-    "details" .= ("Searched for the function " <> func <>
+    in toJsonPgrstError
+    SchemaCacheErrorCode02
+    ("Could not find the function " <> func <> (if onlySingleParams then "" else fmtPrms prmsMsg) <> " in the schema cache")
+    (Just $ JSON.String $ "Searched for the function " <> func <>
      (case (hasPreferSingleObject, isInvPost, contentType) of
        (True, _, _)                 -> " with a single json/jsonb parameter"
        (_, True, MTTextPlain)       -> " with a single unnamed text parameter"
@@ -201,21 +229,20 @@ instance JSON.ToJSON ApiRequestError where
        (_, True, MTOctetStream)     -> " with a single unnamed bytea parameter"
        (_, True, MTApplicationJSON) -> fmtPrms prmsDet <> " or with a single unnamed json/jsonb parameter"
        _                            -> fmtPrms prmsDet) <>
-      ", but no matches were found in the schema cache."),
+      ", but no matches were found in the schema cache.")
    -- The hint will be null in the case of single unnamed parameter functions
-    "hint"    .= if onlySingleParams
-                 then Nothing
-                 else noRpcHint schema procName argumentKeys allProcs overloadedProcs ]
-  toJSON (AmbiguousRpc procs)  = JSON.object [
-    "code"    .= SchemaCacheErrorCode03,
-    "message" .= ("Could not choose the best candidate function between: " <> T.intercalate ", " [pdSchema p <> "." <> pdName p <> "(" <> T.intercalate ", " [ppName a <> " => " <> ppType a | a <- pdParams p] <> ")" | p <- procs]),
-    "details" .= JSON.Null,
-    "hint"    .= ("Try renaming the parameters or the function itself in the database so function overloading can be resolved" :: Text)]
-  toJSON (ColumnNotFound relName colName) = JSON.object [
-    "code"    .= SchemaCacheErrorCode04,
-    "message" .= ("Column '" <> colName <> "' of relation '" <> relName <> "' does not exist" :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
+    (if onlySingleParams
+      then Nothing
+      else JSON.String <$> noRpcHint schema procName argumentKeys allProcs overloadedProcs)
+
+  toJSON (AmbiguousRpc procs)  = toJsonPgrstError
+    SchemaCacheErrorCode03
+    ("Could not choose the best candidate function between: " <> T.intercalate ", " [pdSchema p <> "." <> pdName p <> "(" <> T.intercalate ", " [ppName a <> " => " <> ppType a | a <- pdParams p] <> ")" | p <- procs])
+    Nothing
+    (Just "Try renaming the parameters or the function itself in the database so function overloading can be resolved")
+
+  toJSON (ColumnNotFound relName colName) = toJsonPgrstError
+    SchemaCacheErrorCode04 ("Column '" <> colName <> "' of relation '" <> relName <> "' does not exist") Nothing Nothing

 -- |
 -- If no relationship is found then:
@@ -360,49 +387,60 @@ type Authenticated = Bool
 instance PgrstError PgError where
  status (PgError authed usageError) = pgErrorStatus authed usageError

+  headers (PgError _ (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError (SQL.ServerError "PGRST" m d _ _p))))) =
+    case (parseMessage m, parseDetails d) of
+      (Just _, Just r) -> headers PGRSTParseError ++ map intoHeader (M.toList $ getHeaders r)
+      _                -> headers PGRSTParseError
+    where
+      intoHeader (k,v) = (CI.mk $ T.encodeUtf8 k, T.encodeUtf8 v)
+
  headers err =
    if status err == HTTP.status401
-       then [MediaType.toContentType MTApplicationJSON, ("WWW-Authenticate", "Bearer") :: Header]
-       else [MediaType.toContentType MTApplicationJSON]
+       then [("WWW-Authenticate", "Bearer") :: Header]
+       else mempty

 instance JSON.ToJSON PgError where
  toJSON (PgError _ usageError) = JSON.toJSON usageError

 instance JSON.ToJSON SQL.UsageError where
-  toJSON (SQL.ConnectionUsageError e) = JSON.object [
-    "code"    .= ConnectionErrorCode00,
-    "message" .= ("Database connection error. Retrying the connection." :: Text),
-    "details" .= (T.decodeUtf8With T.lenientDecode $ fromMaybe "" e :: Text),
-    "hint"    .= JSON.Null]
+  toJSON (SQL.ConnectionUsageError e) = toJsonPgrstError
+    ConnectionErrorCode00
+    "Database connection error. Retrying the connection."
+    (Just $ JSON.String $ T.decodeUtf8With T.lenientDecode $ fromMaybe "" e)
+    Nothing
+
  toJSON (SQL.SessionUsageError e) = JSON.toJSON e -- SQL.Error
-  toJSON SQL.AcquisitionTimeoutUsageError = JSON.object [
-    "code"    .= ConnectionErrorCode03,
-    "message" .= ("Timed out acquiring connection from connection pool." :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
+
+  toJSON SQL.AcquisitionTimeoutUsageError = toJsonPgrstError
+    ConnectionErrorCode03 "Timed out acquiring connection from connection pool." Nothing Nothing

 instance JSON.ToJSON SQL.QueryError where
  toJSON (SQL.QueryError _ _ e) = JSON.toJSON e

 instance JSON.ToJSON SQL.CommandError where
+  -- Special error raised with code PGRST, to allow full response control
+  toJSON (SQL.ResultError (SQL.ServerError "PGRST" m d _ _p)) =
+    case (parseMessage m, parseDetails d) of
+      (Just r, Just _) -> JSON.object [
+        "code"     .= getCode r,
+        "message"  .= getMessage r,
+        "details"  .= checkMaybe (getDetails r),
+        "hint"     .= checkMaybe (getHint r)]
+      _ -> JSON.toJSON PGRSTParseError
+    where
+      checkMaybe = maybe JSON.Null JSON.String
+
  toJSON (SQL.ResultError (SQL.ServerError c m d h _p)) = JSON.object [
    "code"     .= (T.decodeUtf8 c      :: Text),
    "message"  .= (T.decodeUtf8 m      :: Text),
    "details"  .= (fmap T.decodeUtf8 d :: Maybe Text),
    "hint"     .= (fmap T.decodeUtf8 h :: Maybe Text)]

-  toJSON (SQL.ResultError resultError) = JSON.object [
-    "code"    .= InternalErrorCode00,
-    "message" .= (show resultError :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
-
-  toJSON (SQL.ClientError d) = JSON.object [
-    "code"    .= ConnectionErrorCode01,
-    "message" .= ("Database client error. Retrying the connection." :: Text),
-    "details" .= (fmap T.decodeUtf8 d :: Maybe Text),
-    "hint"    .= JSON.Null]
+  toJSON (SQL.ResultError resultError) = toJsonPgrstError
+    InternalErrorCode00 (show resultError) Nothing Nothing

+  toJSON (SQL.ClientError d) = toJsonPgrstError
+    ConnectionErrorCode01 "Database client error. Retrying the connection." (JSON.String <$> fmap T.decodeUtf8 d) Nothing

 pgErrorStatus :: Bool -> SQL.UsageError -> HTTP.Status
 pgErrorStatus _      (SQL.ConnectionUsageError _) = HTTP.status503
@@ -410,7 +448,7 @@ pgErrorStatus _      SQL.AcquisitionTimeoutUsageError = HTTP.status504
 pgErrorStatus _      (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ClientError _)))      = HTTP.status503
 pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError rError))) =
  case rError of
-    (SQL.ServerError c m _ _ _) ->
+    (SQL.ServerError c m d _ _) ->
      case BS.unpack c of
        '0':'8':_ -> HTTP.status503 -- pg connection err
        '0':'9':_ -> HTTP.status500 -- triggered action exception
@@ -429,6 +467,7 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
        '5':'3':_ -> HTTP.status503 -- insufficient resources
        '5':'4':_ -> HTTP.status413 -- too complex
        '5':'5':_ -> HTTP.status500 -- obj not on prereq state
+        '5':'7':'P':'0':'1':_ -> HTTP.status503 -- terminating connection due to administrator command
        '5':'7':_ -> HTTP.status500 -- operator intervention
        '5':'8':_ -> HTTP.status500 -- system error
        'F':'0':_ -> HTTP.status500 -- conf file error
@@ -442,119 +481,49 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
        "42P01"   -> HTTP.status404 -- undefined table
        "42501"   -> if authed then HTTP.status403 else HTTP.status401 -- insufficient privilege
        'P':'T':n -> fromMaybe HTTP.status500 (HTTP.mkStatus <$> readMaybe n <*> pure m)
+        "PGRST"   ->
+          case (parseMessage m, parseDetails d) of
+            (Just _, Just r) -> maybe (toEnum $ getStatus r) (HTTP.mkStatus (getStatus r) . T.encodeUtf8) (getStatusText r)
+            _                -> status PGRSTParseError
        _         -> HTTP.status400

    _                       -> HTTP.status500

-checkIsFatal :: SQL.UsageError -> Maybe Text
-checkIsFatal (SQL.ConnectionUsageError e)
-  | isAuthFailureMessage = Just $ toS failureMessage
-  | otherwise = Nothing
-  where isAuthFailureMessage = "FATAL:  password authentication failed" `isInfixOf` failureMessage
-        failureMessage = BS.unpack $ fromMaybe mempty e
-checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
-  = case serverError of
-      -- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
-      SQL.ServerError "42601" _ _ _ _
-        -> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
-      -- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
-      -- This would mean that a connection pooler in transaction mode is being used
-      -- while prepared statements are enabled in the PostgREST configuration,
-      -- both of which are incompatible with each other.
-      SQL.ServerError "42P05" _ _ _ _
-        -> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
-      -- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
-      -- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
-      SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
-        -> Just "Hint: Connection poolers in statement mode are not supported."
-      _ -> Nothing
-checkIsFatal _ = Nothing
-

 data Error
  = ApiRequestError ApiRequestError
-  | GucHeadersError
-  | GucStatusError
  | JwtTokenInvalid Text
  | JwtTokenMissing
  | JwtTokenRequired
  | NoSchemaCacheError
-  | OffLimitsChangesError Int64 Integer
  | PgErr PgError
-  | PutMatchingPkError
-  | SingularityError Integer

 instance PgrstError Error where
-  status (ApiRequestError err)   = status err
-  status GucHeadersError         = HTTP.status500
-  status GucStatusError          = HTTP.status500
-  status JwtTokenInvalid{}       = HTTP.unauthorized401
-  status JwtTokenMissing         = HTTP.status500
-  status JwtTokenRequired        = HTTP.unauthorized401
-  status NoSchemaCacheError      = HTTP.status503
-  status OffLimitsChangesError{} = HTTP.status400
-  status (PgErr err)             = status err
-  status PutMatchingPkError      = HTTP.status400
-  status SingularityError{}      = HTTP.status406
+  status (ApiRequestError err) = status err
+  status JwtTokenInvalid{}     = HTTP.unauthorized401
+  status JwtTokenMissing       = HTTP.status500
+  status JwtTokenRequired      = HTTP.unauthorized401
+  status NoSchemaCacheError    = HTTP.status503
+  status (PgErr err)           = status err

-  headers (ApiRequestError err)  = headers err
-  headers (JwtTokenInvalid m)    = [MediaType.toContentType MTApplicationJSON, invalidTokenHeader m]
-  headers JwtTokenRequired       = [MediaType.toContentType MTApplicationJSON, requiredTokenHeader]
-  headers (PgErr err)            = headers err
-  headers SingularityError{}     = [MediaType.toContentType MTSingularJSON]
-  headers _                      = [MediaType.toContentType MTApplicationJSON]
+  headers (ApiRequestError err) = headers err
+  headers (JwtTokenInvalid m)   = [invalidTokenHeader m]
+  headers JwtTokenRequired      = [requiredTokenHeader]
+  headers (PgErr err)           = headers err
+  headers _                     = mempty

 instance JSON.ToJSON Error where
-  toJSON NoSchemaCacheError = JSON.object [
-      "code"    .= ConnectionErrorCode02,
-      "message" .= ("Could not query the database for the schema cache. Retrying." :: Text),
-      "details" .= JSON.Null,
-      "hint"    .= JSON.Null]
+  toJSON NoSchemaCacheError = toJsonPgrstError
+      ConnectionErrorCode02 "Could not query the database for the schema cache. Retrying." Nothing Nothing

-  toJSON JwtTokenMissing = JSON.object [
-      "code"    .= JWTErrorCode00,
-      "message" .= ("Server lacks JWT secret" :: Text),
-      "details" .= JSON.Null,
-      "hint"    .= JSON.Null]
-  toJSON (JwtTokenInvalid message) = JSON.object [
-      "code"    .= JWTErrorCode01,
-      "message" .= (message :: Text),
-      "details" .= JSON.Null,
-      "hint"    .= JSON.Null]
-  toJSON JwtTokenRequired = JSON.object [
-      "code"    .= JWTErrorCode02,
-      "message" .= ("Anonymous access is disabled" :: Text),
-      "details" .= JSON.Null,
-      "hint"    .= JSON.Null]
+  toJSON JwtTokenMissing = toJsonPgrstError
+      JWTErrorCode00 "Server lacks JWT secret" Nothing Nothing

-  toJSON (OffLimitsChangesError n maxs) = JSON.object [
-      "code"    .= ApiRequestErrorCode10,
-      "message" .= ("The maximum number of rows allowed to change was surpassed" :: Text),
-      "details" .= T.unwords ["Results contain", show n, "rows changed but the maximum number allowed is", show maxs],
-      "hint"    .= JSON.Null]
+  toJSON (JwtTokenInvalid message) = toJsonPgrstError
+      JWTErrorCode01 message Nothing Nothing

-  toJSON GucHeadersError = JSON.object [
-    "code"    .= ApiRequestErrorCode11,
-    "message" .= ("response.headers guc must be a JSON array composed of objects with a single key and a string value" :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
-  toJSON GucStatusError = JSON.object [
-    "code"    .= ApiRequestErrorCode12,
-    "message" .= ("response.status guc must be a valid status code" :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
-
-  toJSON PutMatchingPkError = JSON.object [
-    "code"    .= ApiRequestErrorCode15,
-    "message" .= ("Payload values do not match URL in primary key column(s)" :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= JSON.Null]
-
-  toJSON (SingularityError n) = JSON.object [
-    "code"    .= ApiRequestErrorCode16,
-    "message" .= ("JSON object requested, multiple (or no) rows returned" :: Text),
-    "details" .= T.unwords ["Results contain", show n, "rows,", T.decodeUtf8 (MediaType.toMime MTSingularJSON), "requires 1 row"],
-    "hint"    .= JSON.Null]
+  toJSON JwtTokenRequired = toJsonPgrstError
+      JWTErrorCode02 "Anonymous access is disabled" Nothing Nothing

  toJSON (PgErr err) = JSON.toJSON err
  toJSON (ApiRequestError err) = JSON.toJSON err
@@ -566,8 +535,44 @@ invalidTokenHeader m =
 requiredTokenHeader :: Header
 requiredTokenHeader = ("WWW-Authenticate", "Bearer")

-singularityError :: (Integral a) => a -> Error
-singularityError = SingularityError . toInteger
+-- For parsing byteString to JSON Object, used for allowing full response control
+data PgRaiseErrMessage = PgRaiseErrMessage {
+  getCode    :: Text,
+  getMessage :: Text,
+  getDetails :: Maybe Text,
+  getHint    :: Maybe Text
+}
+
+data PgRaiseErrDetails = PgRaiseErrDetails {
+  getStatus     :: Int,
+  getStatusText :: Maybe Text,
+  getHeaders    :: Map Text Text
+}
+
+instance JSON.FromJSON PgRaiseErrMessage where
+  parseJSON (JSON.Object m) =
+    PgRaiseErrMessage
+      <$> m .: "code"
+      <*> m .: "message"
+      <*> m .:? "details"
+      <*> m .:? "hint"
+
+  parseJSON _ = mzero
+
+instance JSON.FromJSON PgRaiseErrDetails where
+  parseJSON (JSON.Object d) =
+    PgRaiseErrDetails
+      <$> d .: "status"
+      <*> d .:? "status_text"
+      <*> d .: "headers"
+
+  parseJSON _ = mzero
+
+parseMessage :: ByteString -> Maybe PgRaiseErrMessage
+parseMessage = JSON.decodeStrict
+
+parseDetails :: Maybe ByteString -> Maybe PgRaiseErrDetails
+parseDetails d = JSON.decodeStrict =<< d

 -- Error codes are grouped by common modules or characteristics
 data ErrorCode
@@ -581,7 +586,7 @@ data ErrorCode
  | ApiRequestErrorCode01
  | ApiRequestErrorCode02
  | ApiRequestErrorCode03
-  | ApiRequestErrorCode04 -- no longer used (used to be mapped to ParseRequestError)
+  -- | ApiRequestErrorCode04 -- no longer used (used to be mapped to ParseRequestError)
  | ApiRequestErrorCode05
  | ApiRequestErrorCode06
  | ApiRequestErrorCode07
@@ -589,8 +594,8 @@ data ErrorCode
  | ApiRequestErrorCode09
  | ApiRequestErrorCode10
  | ApiRequestErrorCode11
+  -- | ApiRequestErrorCode13 -- no longer used (used to be mapped to BinaryFieldError)
  | ApiRequestErrorCode12
-  | ApiRequestErrorCode13
  | ApiRequestErrorCode14
  | ApiRequestErrorCode15
  | ApiRequestErrorCode16
@@ -598,6 +603,9 @@ data ErrorCode
  | ApiRequestErrorCode18
  | ApiRequestErrorCode19
  | ApiRequestErrorCode20
+  | ApiRequestErrorCode21
+  | ApiRequestErrorCode22
+  | ApiRequestErrorCode23
  -- Schema Cache errors
  | SchemaCacheErrorCode00
  | SchemaCacheErrorCode01
@@ -627,7 +635,6 @@ buildErrorCode code = "PGRST" <> case code of
  ApiRequestErrorCode01  -> "101"
  ApiRequestErrorCode02  -> "102"
  ApiRequestErrorCode03  -> "103"
-  ApiRequestErrorCode04  -> "104"
  ApiRequestErrorCode05  -> "105"
  ApiRequestErrorCode06  -> "106"
  ApiRequestErrorCode07  -> "107"
@@ -636,7 +643,6 @@ buildErrorCode code = "PGRST" <> case code of
  ApiRequestErrorCode10  -> "110"
  ApiRequestErrorCode11  -> "111"
  ApiRequestErrorCode12  -> "112"
-  ApiRequestErrorCode13  -> "113"
  ApiRequestErrorCode14  -> "114"
  ApiRequestErrorCode15  -> "115"
  ApiRequestErrorCode16  -> "116"
@@ -644,6 +650,9 @@ buildErrorCode code = "PGRST" <> case code of
  ApiRequestErrorCode18  -> "118"
  ApiRequestErrorCode19  -> "119"
  ApiRequestErrorCode20  -> "120"
+  ApiRequestErrorCode21  -> "121"
+  ApiRequestErrorCode22  -> "122"
+  ApiRequestErrorCode23  -> "123"

  SchemaCacheErrorCode00 -> "200"
  SchemaCacheErrorCode01 -> "201"
@@ -1,19 +1,17 @@
+{-# LANGUAGE DeriveGeneric         #-}
 {-# LANGUAGE DuplicateRecordFields #-}

 module PostgREST.MediaType
  ( MediaType(..)
-  , MTPlanOption (..)
-  , MTPlanFormat (..)
-  , MTPlanAttrs(..)
+  , MTVndPlanOption (..)
+  , MTVndPlanFormat (..)
  , toContentType
  , toMime
  , decodeMediaType
-  , getMediaType
  ) where

 import qualified Data.ByteString          as BS
 import qualified Data.ByteString.Internal as BS (c2w)
-import           Data.Maybe               (fromJust)

 import Network.HTTP.Types.Header (Header, hContentType)

@@ -22,7 +20,6 @@ import Protolude
 -- | Enumeration of currently supported media types
 data MediaType
  = MTApplicationJSON
-  | MTSingularJSON
  | MTGeoJSON
  | MTTextCSV
  | MTTextPlain
@@ -32,18 +29,23 @@ data MediaType
  | MTOctetStream
  | MTAny
  | MTOther ByteString
-  | MTPlan MTPlanAttrs
-  deriving Eq
+  -- vendored media types
+  | MTVndArrayJSONStrip
+  | MTVndSingularJSON Bool
+  -- TODO MTVndPlan should only have its options as [Text]. Its ResultAggregate should have the typed attributes.
+  | MTVndPlan MediaType MTVndPlanFormat [MTVndPlanOption]
+  deriving (Eq, Show, Generic)
+instance Hashable MediaType

-data MTPlanAttrs = MTPlanAttrs (Maybe MediaType) MTPlanFormat [MTPlanOption]
-instance Eq MTPlanAttrs where
-  MTPlanAttrs {} == MTPlanAttrs {} = True -- we don't care about the attributes when comparing two MTPlan media types
-
-data MTPlanOption
+data MTVndPlanOption
  = PlanAnalyze | PlanVerbose | PlanSettings | PlanBuffers | PlanWAL
+  deriving (Eq, Show, Generic)
+instance Hashable MTVndPlanOption

-data MTPlanFormat
+data MTVndPlanFormat
  = PlanJSON | PlanText
+  deriving (Eq, Show, Generic)
+instance Hashable MTVndPlanFormat

 -- | Convert MediaType to a Content-Type HTTP Header
 toContentType :: MediaType -> Header
@@ -56,69 +58,102 @@ toContentType ct = (hContentType, toMime ct <> charset)

 -- | Convert from MediaType to a ByteString representing the mime type
 toMime :: MediaType -> ByteString
-toMime MTApplicationJSON = "application/json"
-toMime MTGeoJSON         = "application/geo+json"
-toMime MTTextCSV         = "text/csv"
-toMime MTTextPlain       = "text/plain"
-toMime MTTextXML         = "text/xml"
-toMime MTOpenAPI         = "application/openapi+json"
-toMime MTSingularJSON    = "application/vnd.pgrst.object+json"
-toMime MTUrlEncoded      = "application/x-www-form-urlencoded"
-toMime MTOctetStream     = "application/octet-stream"
-toMime MTAny             = "*/*"
-toMime (MTOther ct)      = ct
-toMime (MTPlan (MTPlanAttrs mt fmt opts)) =
+toMime MTApplicationJSON      = "application/json"
+toMime MTVndArrayJSONStrip    = "application/vnd.pgrst.array+json;nulls=stripped"
+toMime MTGeoJSON              = "application/geo+json"
+toMime MTTextCSV              = "text/csv"
+toMime MTTextPlain            = "text/plain"
+toMime MTTextXML              = "text/xml"
+toMime MTOpenAPI              = "application/openapi+json"
+toMime (MTVndSingularJSON True)  = "application/vnd.pgrst.object+json;nulls=stripped"
+toMime (MTVndSingularJSON False) = "application/vnd.pgrst.object+json"
+toMime MTUrlEncoded           = "application/x-www-form-urlencoded"
+toMime MTOctetStream          = "application/octet-stream"
+toMime MTAny                  = "*/*"
+toMime (MTOther ct)           = ct
+toMime (MTVndPlan mt fmt opts)   =
  "application/vnd.pgrst.plan+" <> toMimePlanFormat fmt <>
-  (if isNothing mt then mempty else "; for=\"" <> toMime (fromJust mt) <> "\"") <>
+  ("; for=\"" <> toMime mt <> "\"") <>
  (if null opts then mempty else "; options=" <> BS.intercalate "|" (toMimePlanOption <$> opts))

-toMimePlanOption :: MTPlanOption -> ByteString
+toMimePlanOption :: MTVndPlanOption -> ByteString
 toMimePlanOption PlanAnalyze  = "analyze"
 toMimePlanOption PlanVerbose  = "verbose"
 toMimePlanOption PlanSettings = "settings"
 toMimePlanOption PlanBuffers  = "buffers"
 toMimePlanOption PlanWAL      = "wal"

-toMimePlanFormat :: MTPlanFormat -> ByteString
+toMimePlanFormat :: MTVndPlanFormat -> ByteString
 toMimePlanFormat PlanJSON = "json"
 toMimePlanFormat PlanText = "text"

-- | Convert from ByteString to MediaType. Warning: discards MIME parameters
+-- | Convert from ByteString to MediaType.
+--
+-- >>> decodeMediaType "application/json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan;"
+-- MTVndPlan MTApplicationJSON PlanText []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan;for=\"application/json\""
+-- MTVndPlan MTApplicationJSON PlanText []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan+json;for=\"text/csv\""
+-- MTVndPlan MTTextCSV PlanJSON []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json;nulls=stripped"
+-- MTVndArrayJSONStrip
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json;nulls=stripped"
+-- MTVndSingularJSON True
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json"
+-- MTVndSingularJSON False
+
 decodeMediaType :: BS.ByteString -> MediaType
 decodeMediaType mt =
  case BS.split (BS.c2w ';') mt of
-    "application/json":_                   -> MTApplicationJSON
-    "application/geo+json":_               -> MTGeoJSON
-    "text/csv":_                           -> MTTextCSV
-    "text/plain":_                         -> MTTextPlain
-    "text/xml":_                           -> MTTextXML
-    "application/openapi+json":_           -> MTOpenAPI
-    "application/vnd.pgrst.object+json":_  -> MTSingularJSON
-    "application/vnd.pgrst.object":_       -> MTSingularJSON
-    "application/x-www-form-urlencoded":_  -> MTUrlEncoded
-    "application/octet-stream":_           -> MTOctetStream
-    "application/vnd.pgrst.plan":rest      -> getPlan PlanText rest
-    "application/vnd.pgrst.plan+text":rest -> getPlan PlanText rest
-    "application/vnd.pgrst.plan+json":rest -> getPlan PlanJSON rest
-    "*/*":_                                -> MTAny
-    other:_                                -> MTOther other
-    _                                      -> MTAny
+    "application/json":_                     -> MTApplicationJSON
+    "application/geo+json":_                 -> MTGeoJSON
+    "text/csv":_                             -> MTTextCSV
+    "text/plain":_                           -> MTTextPlain
+    "text/xml":_                             -> MTTextXML
+    "application/openapi+json":_             -> MTOpenAPI
+    "application/x-www-form-urlencoded":_    -> MTUrlEncoded
+    "application/octet-stream":_             -> MTOctetStream
+    "application/vnd.pgrst.plan":rest        -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+text":rest   -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+json":rest   -> getPlan PlanJSON rest
+    "application/vnd.pgrst.object+json":rest -> checkSingularNullStrip rest
+    "application/vnd.pgrst.object":rest      -> checkSingularNullStrip rest
+    "application/vnd.pgrst.array+json":rest  -> checkArrayNullStrip rest
+    "application/vnd.pgrst.array":rest       -> checkArrayNullStrip rest
+    "*/*":_                                  -> MTAny
+    other:_                                  -> MTOther other
+    _                                        -> MTAny
  where
-    getPlan fmt rest =
-     let
-       opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)
-       inOpts str   = str `elem` opts
-       mtFor        = decodeMediaType . dropAround (== BS.c2w '"') <$> (BS.stripPrefix "for=" =<< find (BS.isPrefixOf "for=") rest)
-       dropAround p = BS.dropWhile p . BS.dropWhileEnd p in
-     MTPlan $ MTPlanAttrs mtFor fmt $
-      [PlanAnalyze  | inOpts "analyze" ] ++
-      [PlanVerbose  | inOpts "verbose" ] ++
-      [PlanSettings | inOpts "settings"] ++
-      [PlanBuffers  | inOpts "buffers" ] ++
-      [PlanWAL      | inOpts "wal"     ]
+    checkArrayNullStrip ["nulls=stripped"] = MTVndArrayJSONStrip
+    checkArrayNullStrip  _                 = MTApplicationJSON

-getMediaType :: MediaType -> MediaType
-getMediaType mt = case mt of
-  MTPlan (MTPlanAttrs (Just mType) _ _) -> mType
-  MTPlan (MTPlanAttrs Nothing _ _)      -> MTApplicationJSON
-  other                                 -> other
+    checkSingularNullStrip ["nulls=stripped"] = MTVndSingularJSON True
+    checkSingularNullStrip  _                 = MTVndSingularJSON False
+
+    getPlan fmt rest =
+      let
+        opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)
+        inOpts str   = str `elem` opts
+        dropAround p = BS.dropWhile p . BS.dropWhileEnd p
+        mtFor        = fromMaybe MTApplicationJSON $ do
+          foundFor    <- find (BS.isPrefixOf "for=") rest
+          strippedFor <- BS.stripPrefix "for=" foundFor
+          pure . decodeMediaType $ dropAround (== BS.c2w '"') strippedFor
+      in
+      MTVndPlan mtFor fmt $
+        [PlanAnalyze  | inOpts "analyze" ] ++
+        [PlanVerbose  | inOpts "verbose" ] ++
+        [PlanSettings | inOpts "settings"] ++
+        [PlanBuffers  | inOpts "buffers" ] ++
+        [PlanWAL      | inOpts "wal"     ]
@@ -13,59 +13,66 @@ resource.
 {-# LANGUAGE DuplicateRecordFields #-}
 {-# LANGUAGE LambdaCase            #-}
 {-# LANGUAGE NamedFieldPuns        #-}
-{-# LANGUAGE OverloadedRecordDot   #-}
 {-# LANGUAGE RecordWildCards       #-}

 module PostgREST.Plan
  ( wrappedReadPlan
  , mutateReadPlan
  , callReadPlan
+  , inspectPlan
  , WrappedReadPlan(..)
  , MutateReadPlan(..)
  , CallReadPlan(..)
-  , inspectPlanTxMode
+  , InspectPlan(..)
  ) where

-
 import qualified Data.ByteString.Lazy          as LBS
 import qualified Data.HashMap.Strict           as HM
+import qualified Data.HashMap.Strict.InsOrd    as HMI
 import qualified Data.List                     as L
 import qualified Data.Set                      as S
 import qualified PostgREST.SchemaCache.Routine as Routine

 import Data.Either.Combinators (mapLeft, mapRight)
-import Data.List               (delete)
+import Data.List               (delete, lookup)
 import Data.Tree               (Tree (..))

-import PostgREST.ApiRequest               (Action (..),
-                                           ApiRequest (..),
-                                           InvokeMethod (..),
-                                           Mutation (..),
-                                           Payload (..))
-import PostgREST.Config                   (AppConfig (..))
-import PostgREST.Error                    (Error (..))
-import PostgREST.MediaType                (MTPlanAttrs (..),
-                                           MediaType (..))
-import PostgREST.Query.SqlFragment        (sourceCTEName)
-import PostgREST.RangeQuery               (NonnegRange, allRange,
-                                           convertToLimitZeroRange,
-                                           restrictRange)
-import PostgREST.SchemaCache              (SchemaCache (..))
-import PostgREST.SchemaCache.Identifiers  (FieldName,
-                                           QualifiedIdentifier (..),
-                                           Schema)
-import PostgREST.SchemaCache.Relationship (Cardinality (..),
-                                           Junction (..),
-                                           Relationship (..),
-                                           RelationshipsMap,
-                                           relIsToOne)
-import PostgREST.SchemaCache.Routine      (Routine (..), RoutineMap,
-                                           RoutineParam (..),
-                                           funcReturnsCompositeAlias,
-                                           funcReturnsScalar,
-                                           funcReturnsSetOfScalar)
-import PostgREST.SchemaCache.Table        (Table (tableName),
-                                           tablePKCols)
+import PostgREST.ApiRequest                  (Action (..),
+                                              ApiRequest (..),
+                                              InvokeMethod (..),
+                                              Mutation (..),
+                                              Payload (..))
+import PostgREST.Config                      (AppConfig (..))
+import PostgREST.Error                       (Error (..))
+import PostgREST.MediaType                   (MediaType (..))
+import PostgREST.Query.SqlFragment           (sourceCTEName)
+import PostgREST.RangeQuery                  (NonnegRange, allRange,
+                                              convertToLimitZeroRange,
+                                              restrictRange)
+import PostgREST.SchemaCache                 (SchemaCache (..))
+import PostgREST.SchemaCache.Identifiers     (FieldName,
+                                              QualifiedIdentifier (..),
+                                              RelIdentifier (..),
+                                              Schema)
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
+                                              Junction (..),
+                                              Relationship (..),
+                                              RelationshipsMap,
+                                              relIsToOne)
+import PostgREST.SchemaCache.Representations (DataRepresentation (..),
+                                              RepresentationsMap)
+import PostgREST.SchemaCache.Routine         (MediaHandler (..),
+                                              MediaHandlerMap,
+                                              Routine (..),
+                                              RoutineMap,
+                                              RoutineParam (..),
+                                              funcReturnsCompositeAlias,
+                                              funcReturnsScalar,
+                                              funcReturnsSetOfScalar)
+import PostgREST.SchemaCache.Table           (Column (..), Table (..),
+                                              TablesMap,
+                                              tableColumnsList,
+                                              tablePKCols)

 import PostgREST.ApiRequest.Preferences
 import PostgREST.ApiRequest.Types
@@ -76,19 +83,29 @@ import PostgREST.Plan.Types

 import qualified Hasql.Transaction.Sessions       as SQL
 import qualified PostgREST.ApiRequest.QueryParams as QueryParams
+import qualified PostgREST.MediaType              as MediaType

 import Protolude hiding (from)

+-- $setup
+-- Setup for doctests
+-- >>> import Data.Ranged.Ranges (fullRange)
+
 data WrappedReadPlan = WrappedReadPlan {
  wrReadPlan :: ReadPlanTree
 , wrTxMode   :: SQL.Mode
-, wrBinField :: Maybe FieldName
+, wrHandler  :: MediaHandler
+, wrMedia    :: MediaType
+, wrIdent    :: QualifiedIdentifier
 }

 data MutateReadPlan = MutateReadPlan {
  mrReadPlan   :: ReadPlanTree
 , mrMutatePlan :: MutatePlan
 , mrTxMode     :: SQL.Mode
+, mrHandler    :: MediaHandler
+, mrMedia      :: MediaType
+, mrIdent      :: QualifiedIdentifier
 }

 data CallReadPlan = CallReadPlan {
@@ -96,36 +113,46 @@ data CallReadPlan = CallReadPlan {
 , crCallPlan :: CallPlan
 , crTxMode   :: SQL.Mode
 , crProc     :: Routine
-, crBinField :: Maybe FieldName
+, crHandler  :: MediaHandler
+, crMedia    :: MediaType
+, crIdent    :: QualifiedIdentifier
+}
+
+data InspectPlan = InspectPlan {
+  ipMedia  :: MediaType
+, ipTxmode :: SQL.Mode
 }

 wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error WrappedReadPlan
-wrappedReadPlan  identifier conf sCache apiRequest = do
+wrappedReadPlan  identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} = do
  rPlan <- readPlan identifier conf sCache apiRequest
-  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) Nothing rPlan
-  return $ WrappedReadPlan rPlan SQL.Read binField
+  (hdler, mediaType)  <- mapLeft ApiRequestError $ negotiateContent conf apiRequest identifier iAcceptMediaType (dbMediaHandlers sCache)
+  if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
+  return $ WrappedReadPlan rPlan SQL.Read hdler mediaType identifier

 mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error MutateReadPlan
-mutateReadPlan  mutation apiRequest identifier conf sCache = do
+mutateReadPlan  mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf sCache = do
  rPlan <- readPlan identifier conf sCache apiRequest
  mPlan <- mutatePlan mutation identifier apiRequest sCache rPlan
-  return $ MutateReadPlan rPlan mPlan SQL.Write
+  if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
+  (hdler, mediaType)  <- mapLeft ApiRequestError $ negotiateContent conf apiRequest identifier iAcceptMediaType (dbMediaHandlers sCache)
+  return $ MutateReadPlan rPlan mPlan SQL.Write hdler mediaType identifier

 callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CallReadPlan
-callReadPlan identifier conf sCache apiRequest invMethod = do
+callReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} invMethod = do
  let paramKeys = case invMethod of
        InvGet  -> S.fromList $ fst <$> qsParams'
        InvHead -> S.fromList $ fst <$> qsParams'
-        InvPost -> iColumns apiRequest
+        InvPost -> iColumns
  proc@Function{..} <- mapLeft ApiRequestError $
-    findProc identifier paramKeys (preferParameters == Just SingleObject) (dbRoutines sCache) (iContentMediaType apiRequest) (invMethod == InvPost)
+    findProc identifier paramKeys (preferParameters == Just SingleObject) (dbRoutines sCache) iContentMediaType (invMethod == InvPost)
  let relIdentifier = QualifiedIdentifier pdSchema (fromMaybe pdName $ Routine.funcTableName proc) -- done so a set returning function can embed other relations
  rPlan <- readPlan relIdentifier conf sCache apiRequest
-  let args = case (invMethod, iContentMediaType apiRequest) of
+  let args = case (invMethod, iContentMediaType) of
        (InvGet, _)             -> jsonRpcParams proc qsParams'
        (InvHead, _)            -> jsonRpcParams proc qsParams'
-        (InvPost, MTUrlEncoded) -> maybe mempty (jsonRpcParams proc . payArray) $ iPayload apiRequest
-        (InvPost, _)            -> maybe mempty payRaw $ iPayload apiRequest
+        (InvPost, MTUrlEncoded) -> maybe mempty (jsonRpcParams proc . payArray) iPayload
+        (InvPost, _)            -> maybe mempty payRaw iPayload
      txMode = case (invMethod, pdVolatility) of
          (InvGet,  _)                 -> SQL.Read
          (InvHead, _)                 -> SQL.Read
@@ -133,11 +160,20 @@ callReadPlan identifier conf sCache apiRequest invMethod = do
          (InvPost, Routine.Immutable) -> SQL.Read
          (InvPost, Routine.Volatile)  -> SQL.Write
      cPlan = callPlan proc apiRequest paramKeys args rPlan
-  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) (Just proc) rPlan
-  return $ CallReadPlan rPlan cPlan txMode proc binField
+  (hdler, mediaType)  <- mapLeft ApiRequestError $ negotiateContent conf apiRequest relIdentifier iAcceptMediaType (dbMediaHandlers sCache)
+  if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
+  return $ CallReadPlan rPlan cPlan txMode proc hdler mediaType relIdentifier
  where
-    Preferences{..} = iPreferences apiRequest
-    qsParams' = QueryParams.qsParams (iQueryParams apiRequest)
+    qsParams' = QueryParams.qsParams iQueryParams
+
+inspectPlan :: ApiRequest -> Either Error InspectPlan
+inspectPlan apiRequest = do
+  let producedMTs = [MTOpenAPI, MTApplicationJSON, MTAny]
+      accepts     = iAcceptMediaType apiRequest
+  mediaType <- if not . null $ L.intersect accepts producedMTs
+    then Right MTOpenAPI
+    else Left . ApiRequestError . MediaTypeError $ MediaType.toMime <$> accepts
+  return $ InspectPlan mediaType SQL.Read

 {-|
  Search a pg proc by matching name and arguments keys to parameters. Since a function can be overloaded,
@@ -196,32 +232,99 @@ findProc qi argumentsKeys paramsAsSingleObject allProcs contentMediaType isInvPo
      -- and can match any or none of the default parameters.
        (reqParams, optParams) -> argumentsKeys `S.difference` S.fromList (ppName <$> optParams) == S.fromList (ppName <$> reqParams)

-inspectPlanTxMode :: SQL.Mode
-inspectPlanTxMode = SQL.Read
+-- | During planning we need to resolve Field -> CoercibleField (finding the context specific target type and map function).
+-- | ResolverContext facilitates this without the need to pass around a laundry list of parameters.
+data ResolverContext = ResolverContext
+  { tables          :: TablesMap
+  , representations :: RepresentationsMap
+  , qi              :: QualifiedIdentifier  -- ^ The table we're currently attending; changes as we recurse into joins etc.
+  , outputType      :: Text                 -- ^ The output type for the response payload; e.g. "csv", "json", "binary".
+  }
+
+resolveColumnField :: Column -> CoercibleField
+resolveColumnField col = CoercibleField (colName col) mempty False (colNominalType col) Nothing (colDefault col)
+
+resolveTableFieldName :: Table -> FieldName -> CoercibleField
+resolveTableFieldName table fieldName =
+  fromMaybe (unknownField fieldName []) $ HMI.lookup fieldName (tableColumns table) >>=
+    Just . resolveColumnField
+
+-- | Resolve a type within the context based on the given field name and JSON path. Although there are situations where failure to resolve a field is considered an error (see `resolveOrError`), there are also situations where we allow it (RPC calls). If it should be an error and `resolveOrError` doesn't fit, ensure to check the `cfIRType` isn't empty.
+resolveTypeOrUnknown :: ResolverContext -> Field -> CoercibleField
+resolveTypeOrUnknown ResolverContext{..} (fn, jp) =
+  case res of
+    -- types that are already json/jsonb don't need to be converted with `to_jsonb` for using arrow operators `data->attr`
+    -- this prevents indexes not applying https://github.com/PostgREST/postgrest/issues/2594
+    cf@CoercibleField{cfIRType="json"}  -> cf{cfJsonPath=jp, cfToJson=False}
+    cf@CoercibleField{cfIRType="jsonb"} -> cf{cfJsonPath=jp, cfToJson=False}
+    -- other types will get converted `to_jsonb(col)->attr`, even unknown types
+    cf                                  -> cf{cfJsonPath=jp, cfToJson=True}
+  where
+    res = fromMaybe (unknownField fn jp) $ HM.lookup qi tables >>=
+          Just . flip resolveTableFieldName fn
+
+-- | Install any pre-defined data representation from source to target to coerce this reference.
+--
+-- Note that we change the IR type here. This might seem unintuitive. The short of it is that for a CoercibleField without a transformer, input type == output type. A transformer maps from a -> b, so by definition the input type will be a and the output type b after. And cfIRType is the *input* type.
+--
+-- It might feel odd that once a transformer is added we 'forget' the target type (because now a /= b). You might also note there's no obvious way to stack transforms (even if there was a stack, you erased what type you're working with so it's awkward). Alas as satisfying as it would be to engineer a layered mapping system with full type information, we just don't need it.
+withTransformer :: ResolverContext -> Text -> Text -> CoercibleField -> CoercibleField
+withTransformer ResolverContext{representations} sourceType targetType field =
+  fromMaybe field $ HM.lookup (sourceType, targetType) representations >>=
+    (\fieldRepresentation -> Just field{cfIRType=sourceType, cfTransform=Just (drFunction fieldRepresentation)})
+
+-- | Map the intermediate representation type to the output type, if available.
+withOutputFormat :: ResolverContext -> CoercibleField -> CoercibleField
+withOutputFormat ctx@ResolverContext{outputType} field@CoercibleField{cfIRType} = withTransformer ctx cfIRType outputType field
+
+-- | Map text into the intermediate representation type, if available.
+withTextParse :: ResolverContext -> CoercibleField -> CoercibleField
+withTextParse ctx field@CoercibleField{cfIRType} = withTransformer ctx "text" cfIRType field
+
+-- | Map json into the intermediate representation type, if available.
+withJsonParse :: ResolverContext -> CoercibleField -> CoercibleField
+withJsonParse ctx field@CoercibleField{cfIRType} = withTransformer ctx "json" cfIRType field
+
+-- | Map the intermediate representation type to the output type defined by the resolver context (normally json), if available.
+resolveOutputField :: ResolverContext -> Field -> CoercibleField
+resolveOutputField ctx field = withOutputFormat ctx $ resolveTypeOrUnknown ctx field
+
+-- | Map the query string format of a value (text) into the intermediate representation type, if available.
+resolveQueryInputField :: ResolverContext -> Field -> CoercibleField
+resolveQueryInputField ctx field = withTextParse ctx $ resolveTypeOrUnknown ctx field

 -- | Builds the ReadPlan tree on a number of stages.
 -- | Adds filters, order, limits on its respective nodes.
 -- | Adds joins conditions obtained from resource embedding.
 readPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error ReadPlanTree
-readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows} SchemaCache{dbRelationships} apiRequest  =
-  mapLeft ApiRequestError $
-  treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
-  addNullEmbedFilters =<<
-  validateSpreadEmbeds =<<
-  addRelatedOrders =<<
-  addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
-  addLogicTrees apiRequest =<<
-  addRanges apiRequest =<<
-  addOrders apiRequest =<<
-  addFilters apiRequest (initReadRequest qi $ QueryParams.qsSelect $ iQueryParams apiRequest)
+readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows, configDbAggregates} SchemaCache{dbTables, dbRelationships, dbRepresentations} apiRequest  =
+  let
+    -- JSON output format hardcoded for now. In the future we might want to support other output mappings such as CSV.
+    ctx = ResolverContext dbTables dbRepresentations qi "json"
+  in
+    mapLeft ApiRequestError $
+    treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
+    validateAggFunctions configDbAggregates =<<
+    hoistSpreadAggFunctions =<<
+    addRelSelects =<<
+    addNullEmbedFilters =<<
+    validateSpreadEmbeds =<<
+    addRelatedOrders =<<
+    addAliases =<<
+    expandStars ctx =<<
+    addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
+    addLogicTrees ctx apiRequest =<<
+    addRanges apiRequest =<<
+    addOrders ctx apiRequest =<<
+    addFilters ctx apiRequest (initReadRequest ctx $ QueryParams.qsSelect $ iQueryParams apiRequest)

 -- Build the initial read plan tree
-initReadRequest :: QualifiedIdentifier -> [Tree SelectItem] -> ReadPlanTree
-initReadRequest qi@QualifiedIdentifier{..} =
-  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi, relName=qiName, depth=rootDepth} []
+initReadRequest :: ResolverContext -> [Tree SelectItem] -> ReadPlanTree
+initReadRequest ctx@ResolverContext{qi=QualifiedIdentifier{..}} =
+  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi ctx, relName=qiName, depth=rootDepth} []
  where
    rootDepth = 0
-    defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing False rootDepth
+    defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing False [] rootDepth
    treeEntry :: Depth -> Tree SelectItem -> ReadPlanTree -> ReadPlanTree
    treeEntry depth (Node si fldForest) (Node q rForest) =
      let nxtDepth = succ depth in
@@ -237,7 +340,86 @@ initReadRequest qi@QualifiedIdentifier{..} =
            (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth, relIsSpread=True} [])
            fldForest:rForest
        SelectField{..} ->
-          Node q{select=(selField, selCast, selAlias):select q} rForest
+          Node q{select=CoercibleSelectField (resolveOutputField ctx{qi=from q} selField) selAggregateFunction selAggregateCast selCast selAlias:select q} rForest
+
+-- If an alias is explicitly specified, it is always respected. However, an alias may be
+-- determined automatically in the case of a select term with a JSON path, or in the case
+-- of domain representations.
+addAliases :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addAliases = Right . fmap addAliasToPlan
+  where
+    addAliasToPlan rp@ReadPlan{select=sel} = rp{select=map aliasSelectField sel}
+
+    aliasSelectField :: CoercibleSelectField -> CoercibleSelectField
+    aliasSelectField field@CoercibleSelectField{csField=fieldDetails, csAggFunction=aggFun, csAlias=alias}
+      | isJust alias || isJust aggFun = field
+      | isJsonKeyPath fieldDetails, Just key <- lastJsonKey fieldDetails = field { csAlias = Just key }
+      | isTransformPath fieldDetails = field { csAlias = Just (cfName fieldDetails) }
+      | otherwise = field
+
+    isJsonKeyPath CoercibleField{cfJsonPath=(_: _)} = True
+    isJsonKeyPath _                                 = False
+
+    isTransformPath CoercibleField{cfTransform=(Just _), cfName=_} = True
+    isTransformPath _ = False
+
+    lastJsonKey CoercibleField{cfName=fieldName, cfJsonPath=jsonPath} =
+      case jOp <$> lastMay jsonPath of
+            Just (JKey key) -> Just key
+            Just (JIdx _)   -> Just $ fromMaybe fieldName lastKey
+              -- We get the lastKey because on:
+              -- `select=data->1->mycol->>2`, we need to show the result as [ {"mycol": ..}, {"mycol": ..} ]
+              -- `select=data->3`, we need to show the result as [ {"data": ..}, {"data": ..} ]
+              where lastKey = jVal <$> find (\case JKey{} -> True; _ -> False) (jOp <$> reverse jsonPath)
+            Nothing -> Nothing
+
+knownColumnsInContext :: ResolverContext -> [Column]
+knownColumnsInContext ResolverContext{..} =
+  fromMaybe [] $ HM.lookup qi tables >>=
+  Just . tableColumnsList
+
+-- | Expand "select *" into explicit field names of the table in the following situations:
+-- * When there are data representations present.
+-- * When there is an aggregate function in a given ReadPlan or its parent.
+expandStars :: ResolverContext -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+expandStars ctx rPlanTree = Right $ expandStarsForReadPlan False rPlanTree
+  where
+    expandStarsForReadPlan :: Bool -> ReadPlanTree -> ReadPlanTree
+    expandStarsForReadPlan hasAgg (Node rp@ReadPlan{select, from=fromQI, fromAlias=alias} children) =
+      let
+        newHasAgg = hasAgg || any (isJust . csAggFunction) select
+        newCtx = adjustContext ctx fromQI alias
+        newRPlan = expandStarsForTable newCtx newHasAgg rp
+      in Node newRPlan (map (expandStarsForReadPlan newHasAgg) children)
+
+    -- Choose the appropriate context based on whether we're dealing with "pgrst_source"
+    adjustContext :: ResolverContext -> QualifiedIdentifier -> Maybe Text -> ResolverContext
+    -- When the schema is "" and the table is the source CTE, we assume the true source table is given in the from
+    -- alias and belongs to the request schema. See the bit in `addRels` with `newFrom = ...`.
+    adjustContext context@ResolverContext{qi=ctxQI} (QualifiedIdentifier "" "pgrst_source") (Just a) = context{qi=ctxQI{qiName=a}}
+    adjustContext context fromQI _ = context{qi=fromQI}
+
+expandStarsForTable :: ResolverContext -> Bool -> ReadPlan -> ReadPlan
+expandStarsForTable ctx@ResolverContext{representations, outputType} hasAgg rp@ReadPlan{select=selectFields}
+  -- We expand if either of the below are true:
+  -- * We have a '*' select AND there is an aggregate function in this ReadPlan's sub-tree.
+  -- * We have a '*' select AND the target table has at least one data representation.
+  -- We ignore any '*' selects that have an aggregate function attached (i.e for COUNT(*)).
+  | hasStarSelect && (hasAgg || hasDataRepresentation) = rp{select = concatMap (expandStarSelectField knownColumns) selectFields}
+  | otherwise = rp
+  where
+    hasStarSelect = "*" `elem` map (cfName . csField) filteredSelectFields
+    filteredSelectFields = filter (isNothing . csAggFunction) selectFields
+    hasDataRepresentation = any hasOutputRep knownColumns
+    knownColumns = knownColumnsInContext ctx
+
+    hasOutputRep :: Column -> Bool
+    hasOutputRep col = HM.member (colNominalType col, outputType) representations
+
+    expandStarSelectField :: [Column] -> CoercibleSelectField -> [CoercibleSelectField]
+    expandStarSelectField columns sel@CoercibleSelectField{csField=CoercibleField{cfName="*", cfJsonPath=[]}, csAggFunction=Nothing} =
+      map (\col -> sel { csField = withOutputFormat ctx $ resolveColumnField col }) columns
+    expandStarSelectField _ selectField = [selectField]

 -- | Enforces the `max-rows` config on the result
 treeRestrictRange :: Maybe Integer -> Action -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -261,7 +443,7 @@ addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,re
            Relationship{relCardinality=M2M _} -> -- m2m does internal implicit joins that don't need aliasing
              rPlan{from=relForeignTable r, relToParent=Just r, relAggAlias=aggAlias, relJoinConds=getJoinConditions Nothing parentAlias r}
            ComputedRelationship{} ->
-              rPlan{from=relForeignTable r, relToParent=Just r{relTable=maybe (relTable r) (QualifiedIdentifier mempty) parentAlias}, relAggAlias=aggAlias, fromAlias=newAlias}
+              rPlan{from=relForeignTable r, relToParent=Just r{relTableAlias=maybe (relTable r) (QualifiedIdentifier mempty) parentAlias}, relAggAlias=aggAlias, fromAlias=newAlias}
            _ ->
              rPlan{from=relForeignTable r, relToParent=Just r, relAggAlias=aggAlias, fromAlias=newAlias, relJoinConds=getJoinConditions newAlias parentAlias r}
          ) <$> rel
@@ -273,7 +455,7 @@ addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,re
      Node <$> newReadPlan <*> (updateForest . hush $ Node <$> newReadPlan <*> pure forest)
    Nothing -> -- root case
      let
-        newFrom  = QualifiedIdentifier mempty $ decodeUtf8 sourceCTEName
+        newFrom  = QualifiedIdentifier mempty sourceCTEName
        newAlias = Just (qiName $ from rPlan)
        newReadPlan = case action of
          -- the CTE for mutations/rpc is used as WITH sourceCTEName .. SELECT .. FROM sourceCTEName as alias,
@@ -363,7 +545,9 @@ findRel schema allRels origin target hint =
            target == qiName relForeignTable && isO2M relCardinality
            && matchFKRefSingleCol hnt relCardinality -- auditor
        else case hint of
+          -- DEPRECATED(remove after 2 major releases since v11.1.0): remove target
          -- target = table / view / constraint / column-from-origin (constraint/column-from-origin can only come from tables https://github.com/PostgREST/postgrest/issues/2277)
+          -- DEPRECATED(remove after 2 major releases since v11.1.0): remove hint as table/view/columns and only leave it as constraint
          -- hint   = table / view / constraint / column-from-origin / column-from-target (hint can take table / view values to aid in finding the junction in an m2m relationship)
          Nothing ->
              -- /projects?select=clients(*)
@@ -392,8 +576,125 @@ findRel schema allRels origin target hint =
            )
      ) $ fromMaybe mempty $ HM.lookup (QualifiedIdentifier schema origin, schema) allRels

-addFilters :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addFilters ApiRequest{..} rReq =
+
+addRelSelects :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addRelSelects node@(Node rp forest)
+  | null forest = Right node
+  | otherwise   =
+    let newForest     = rights $ addRelSelects <$> forest
+        newRelSelects = mapMaybe generateRelSelectField newForest
+    in Right $ Node rp { relSelect = newRelSelects } newForest
+
+generateRelSelectField :: ReadPlanTree -> Maybe RelSelectField
+generateRelSelectField (Node rp@ReadPlan{relToParent=Just _, relAggAlias, relIsSpread = True} _) =
+  Just $ Spread { rsSpreadSel = generateSpreadSelectFields rp, rsAggAlias = relAggAlias }
+generateRelSelectField (Node ReadPlan{relToParent=Just rel, select, relName, relAlias, relAggAlias, relIsSpread = False} forest) =
+  Just $ JsonEmbed { rsEmbedMode, rsSelName, rsAggAlias = relAggAlias, rsEmptyEmbed }
+  where
+    rsSelName = fromMaybe relName relAlias
+    rsEmbedMode = if relIsToOne rel then JsonObject else JsonArray
+    rsEmptyEmbed = null select && null forest
+generateRelSelectField _ = Nothing
+
+generateSpreadSelectFields :: ReadPlan -> [SpreadSelectField]
+generateSpreadSelectFields ReadPlan{select, relSelect} =
+  -- We combine the select and relSelect fields into a single list of SpreadSelectField.
+  selectSpread ++ relSelectSpread
+  where
+    selectSpread = map selectToSpread select
+    selectToSpread :: CoercibleSelectField -> SpreadSelectField
+    selectToSpread CoercibleSelectField{csField = CoercibleField{cfName}, csAlias} =
+      SpreadSelectField { ssSelName = fromMaybe cfName csAlias, ssSelAggFunction = Nothing, ssSelAggCast = Nothing, ssSelAlias = Nothing }
+
+    relSelectSpread = concatMap relSelectToSpread relSelect
+    relSelectToSpread :: RelSelectField -> [SpreadSelectField]
+    relSelectToSpread (JsonEmbed{rsSelName}) =
+      [SpreadSelectField { ssSelName = rsSelName, ssSelAggFunction = Nothing, ssSelAggCast = Nothing, ssSelAlias = Nothing }]
+    relSelectToSpread (Spread{rsSpreadSel}) =
+      rsSpreadSel
+
+-- When aggregates are present in a ReadPlan that will be spread, we "hoist"
+-- to the highest level possible so that their semantics make sense. For instance,
+-- imagine the user performs the following request:
+-- `GET /projects?select=client_id,...project_invoices(invoice_total.sum())`
+--
+-- In this case, it is sensible that we would expect to receive the sum of the
+-- `invoice_total`, grouped by the `client_id`. Without hoisting, the sum would
+-- be performed in the sub-query for the joined table `project_invoices`, thus
+-- making it essentially a no-op. With hoisting, we hoist the aggregate function
+-- so that the aggregate function is performed in a more sensible context.
+--
+-- We will try to hoist the aggregate function to the highest possible level,
+-- which means that we hoist until we reach the root node, or until we reach a
+-- ReadPlan that will be embedded a JSON object or JSON array.
+
+-- This type alias represents an aggregate that is to be hoisted to the next
+-- level up. The first tuple of `Alias` and `FieldName` contain the alias for
+-- the joined table and the original field name for the hoisted field.
+--
+-- The second tuple contains the aggregate function to be applied, the cast, and
+-- the alias, if it was supplied by the user or otherwise determined.
+type HoistedAgg = ((Alias, FieldName), (AggregateFunction, Maybe Cast, Maybe Alias))
+
+hoistSpreadAggFunctions :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+hoistSpreadAggFunctions tree = Right $ fst $ applySpreadAggHoistingToNode tree
+
+applySpreadAggHoistingToNode :: ReadPlanTree -> (ReadPlanTree, [HoistedAgg])
+applySpreadAggHoistingToNode (Node rp@ReadPlan{relAggAlias, relToParent, relIsSpread} children) =
+  let (newChildren, childAggLists) = unzip $ map applySpreadAggHoistingToNode children
+      allChildAggLists = concat childAggLists
+      (newSelects, aggList) = if depth rp == 0 || (isJust relToParent && not relIsSpread)
+                                then (select rp, [])
+                                else hoistFromSelectFields relAggAlias (select rp)
+
+      newRelSelects = if null children
+                      then relSelect rp
+                      else map (hoistIntoRelSelectFields allChildAggLists) $ relSelect rp
+  in  (Node rp { select = newSelects, relSelect = newRelSelects } newChildren, aggList)
+
+-- Hoist aggregate functions from the select list of a ReadPlan, and return the
+-- updated select list and the list of hoisted aggregates.
+hoistFromSelectFields :: Alias -> [CoercibleSelectField] -> ([CoercibleSelectField], [HoistedAgg])
+hoistFromSelectFields aggAlias fields =
+    let (newFields, maybeAggs) = foldr processField ([], []) fields
+    in (newFields, catMaybes maybeAggs)
+  where
+    processField field (newFields, aggList) =
+      let (modifiedField, maybeAgg) = modifyField field
+      in (modifiedField : newFields, maybeAgg : aggList)
+
+    modifyField field =
+      case csAggFunction field of
+        Just aggFunc ->
+          ( field { csAggFunction = Nothing, csAggCast = Nothing },
+            Just ((aggAlias, determineFieldName field), (aggFunc, csAggCast field, csAlias field)))
+        Nothing -> (field, Nothing)
+
+    determineFieldName field = fromMaybe (cfName $ csField field) (csAlias field)
+
+-- Taking the hoisted aggregates, modify the rel selects to apply the aggregates,
+-- and any applicable casts or aliases.
+hoistIntoRelSelectFields :: [HoistedAgg] -> RelSelectField -> RelSelectField
+hoistIntoRelSelectFields aggList r@(Spread {rsSpreadSel = spreadSelects, rsAggAlias = aggAlias}) =
+    r { rsSpreadSel = map updateSelect spreadSelects }
+  where
+    updateSelect s =
+        case lookup (aggAlias, ssSelName s) aggList of
+            Just (aggFunc, aggCast, fldAlias) ->
+                s { ssSelAggFunction = Just aggFunc,
+                    ssSelAggCast     = aggCast,
+                    ssSelAlias       = fldAlias }
+            Nothing -> s
+hoistIntoRelSelectFields _ r = r
+
+validateAggFunctions :: Bool -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+validateAggFunctions aggFunctionsAllowed (Node rp@ReadPlan {select} forest)
+  | aggFunctionsAllowed = Node rp <$> traverse (validateAggFunctions aggFunctionsAllowed) forest
+  | any (isJust . csAggFunction) select = Left AggregatesNotAllowed
+  | otherwise = Node rp <$> traverse (validateAggFunctions aggFunctionsAllowed) forest
+
+addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addFilters ctx ApiRequest{..} rReq =
  foldr addFilterToNode (Right rReq) flts
  where
    QueryParams.QueryParams{..} = iQueryParams
@@ -405,10 +706,10 @@ addFilters ApiRequest{..} rReq =

    addFilterToNode :: (EmbedPath, Filter) -> Either ApiRequestError ReadPlanTree ->  Either ApiRequestError ReadPlanTree
    addFilterToNode =
-      updateNode (\flt (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest flt lf}  f)
+      updateNode (\flt (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest (resolveFilter ctx{qi=fromTable} flt) lf}  f)

-addOrders :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addOrders ApiRequest{..} rReq =
+addOrders :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addOrders ctx ApiRequest{..} rReq =
  case iAction of
    ActionMutate _ -> Right rReq
    _              -> foldr addOrderToNode (Right rReq) qsOrder
@@ -416,48 +717,117 @@ addOrders ApiRequest{..} rReq =
    QueryParams.QueryParams{..} = iQueryParams

    addOrderToNode :: (EmbedPath, [OrderTerm]) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
-    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=o} f)
+    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=resolveOrder ctx <$> o} f)
+
+resolveOrder :: ResolverContext -> OrderTerm -> CoercibleOrderTerm
+resolveOrder _ (OrderRelationTerm a b c d) = CoercibleOrderRelationTerm a b c d
+resolveOrder ctx (OrderTerm fld dir nulls) = CoercibleOrderTerm (resolveTypeOrUnknown ctx fld) dir nulls

 -- Validates that the related resource on the order is an embedded resource,
 -- e.g. if `clients` is inside the `select` in /projects?order=clients(id)&select=*,clients(*),
 -- and if it's a to-one relationship, it adds the right alias to the OrderRelationTerm so the generated query can succeed.
-- TODO might be clearer if there's an additional intermediate type
 addRelatedOrders :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
-  newOrder <- getRelOrder `traverse` order
+  newOrder <- newRelOrder `traverse` order
  Node rp{order=newOrder} <$> addRelatedOrders `traverse` forest
  where
-    getRelOrder ot@OrderTerm{}                   = Right ot
-    getRelOrder ot@OrderRelationTerm{otRelation} =
-      let foundRP = rootLabel <$> find (\(Node ReadPlan{relName, relAlias} _) -> otRelation == fromMaybe relName relAlias) forest in
+    newRelOrder cot@CoercibleOrderTerm{}                   = Right cot
+    newRelOrder cot@CoercibleOrderRelationTerm{coRelation} =
+      let foundRP = rootLabel <$> find (\(Node ReadPlan{relName, relAlias} _) -> coRelation == fromMaybe relName relAlias) forest in
      case foundRP of
        Just ReadPlan{relName,relAlias,relAggAlias,relToParent} ->
          let isToOne = relIsToOne <$> relToParent
              name    = fromMaybe relName relAlias in
          if isToOne == Just True
-            then Right $ ot{otRelation=relAggAlias}
+            then Right $ cot{coRelation=relAggAlias}
            else Left $ RelatedOrderNotToOne (qiName from) name
        Nothing ->
-          Left $ NotEmbedded otRelation
+          Left $ NotEmbedded coRelation

-- Searches for null filters on embeds, e.g. `clients` on /projects?select=*,clients()&clients=not.is.null.
-- If these are found, it changes the filter to use the internal aggregate name(`projects_clients_1`) so the filter can succeed.
-- It fails if operators other than is.null or not.is.null are used.
+-- | Searches for null filters on embeds, e.g. `projects=not.is.null` on `GET /clients?select=*,projects(*)&projects=not.is.null`
+--
+-- (It doesn't err but uses an Either ApiRequestError type so it can combine with the other functions that modify the read plan tree)
+--
+-- Setup:
+--
+-- >>> let nullOp    = OpExpr True (Is TriNull)
+-- >>> let nonNullOp = OpExpr False (Is TriNull)
+-- >>> let notEqOp   = OpExpr True (Op OpNotEqual "val")
+-- >>> :{
+-- -- this represents the `projects(*)` part on `/clients?select=*,projects(*)`
+-- let
+-- subForestPlan =
+--   [
+--     Node {
+--       rootLabel = ReadPlan {
+--         select = [], -- there will be fields at this stage but we just omit them for brevity
+--         from = QualifiedIdentifier {qiSchema = "test", qiName = "projects"},
+--         fromAlias = Just "projects_1", where_ = [], order = [], range_ = fullRange,
+--         relName = "projects",
+--         relToParent = Nothing,
+--         relJoinConds = [],
+--         relAlias = Nothing, relAggAlias = "clients_projects_1", relHint = Nothing, relJoinType = Nothing, relIsSpread = False, depth = 1,
+--         relSelect = []
+--       },
+--       subForest = []
+--     }
+--   ]
+-- :}
+--
+-- >>> :{
+-- -- this represents the full URL `/clients?select=*,projects(*)&projects=not.is.null`, if subForst takes the above subForestPlan and nullOp
+-- let
+-- readPlanTree op subForst =
+--   Node {
+--     rootLabel = ReadPlan {
+--       select = [], -- there will be fields at this stage but we just omit them for brevity
+--       from = QualifiedIdentifier { qiSchema = "test", qiName = "clients"},
+--       fromAlias = Nothing,
+--       where_ = [
+--         CoercibleStmnt (
+--           CoercibleFilter {
+--            field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson=False, cfIRType = "", cfTransform = Nothing, cfDefault = Nothing},
+--            opExpr = op
+--           }
+--         )
+--       ],
+--       order = [], range_ = fullRange, relName = "clients", relToParent = Nothing, relJoinConds = [], relAlias = Nothing, relAggAlias = "", relHint = Nothing,
+--       relJoinType = Nothing, relIsSpread = False, depth = 0,
+--       relSelect = []
+--     },
+--     subForest = subForst
+--   }
+-- :}
+--
+-- Don't do anything to the filter if there's no embedding (a subtree) on projects. Assume it's a normal filter.
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp [])
+-- Right [CoercibleStmnt (CoercibleFilter {field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson = False, cfIRType = "", cfTransform = Nothing, cfDefault = Nothing}, opExpr = OpExpr True (Is TriNull)})]
+--
+-- If there's an embedding on projects, then change the filter to use the internal aggregate name (`clients_projects_1`) so the filter can succeed later.
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp subForestPlan)
+-- Right [CoercibleStmnt (CoercibleFilterNullEmbed True "clients_projects_1")]
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nonNullOp subForestPlan)
+-- Right [CoercibleStmnt (CoercibleFilterNullEmbed False "clients_projects_1")]
 addNullEmbedFilters :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addNullEmbedFilters (Node rp@ReadPlan{where_=oldLogic} forest) = do
-  let readPlans = rootLabel <$> forest
-  newLogic <- getFilters readPlans `traverse` oldLogic
+addNullEmbedFilters (Node rp@ReadPlan{where_=curLogic} forest) = do
+  let forestReadPlans = rootLabel <$> forest
+  newLogic <- newNullFilters forestReadPlans `traverse` curLogic
  Node rp{ReadPlan.where_= newLogic} <$> (addNullEmbedFilters `traverse` forest)
  where
-    getFilters :: [ReadPlan] -> LogicTree -> Either ApiRequestError LogicTree
-    getFilters rPlans (Expr b lOp trees) = Expr b lOp <$> (getFilters rPlans `traverse` trees)
-    getFilters rPlans flt@(Stmnt (Filter (fld, []) opExpr)) =
-      let foundRP = find (\ReadPlan{relName, relAlias} -> fld == fromMaybe relName relAlias) rPlans in
-      case (foundRP, opExpr) of
-        (Just ReadPlan{relAggAlias}, OpExpr b (Is TriNull)) -> Right $ Stmnt $ FilterNullEmbed b relAggAlias
-        (Just ReadPlan{relName}, _)                         -> Left $ UnacceptableFilter relName
-        _                                                   -> Right flt
-    getFilters _ flt@(Stmnt _)        = Right flt
+    newNullFilters :: [ReadPlan] -> CoercibleLogicTree -> Either ApiRequestError CoercibleLogicTree
+    newNullFilters rPlans = \case
+      (CoercibleExpr b lOp trees) ->
+        CoercibleExpr b lOp <$> (newNullFilters rPlans `traverse` trees)
+      flt@(CoercibleStmnt (CoercibleFilter (CoercibleField fld [] _ _ _ _) opExpr)) ->
+        let foundRP = find (\ReadPlan{relName, relAlias} -> fld == fromMaybe relName relAlias) rPlans in
+        case (foundRP, opExpr) of
+          (Just ReadPlan{relAggAlias}, OpExpr b (Is TriNull)) -> Right $ CoercibleStmnt $ CoercibleFilterNullEmbed b relAggAlias
+          _                                                   -> Right flt
+      flt@(CoercibleStmnt _) ->
+        Right flt

 addRanges :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRanges ApiRequest{..} rReq =
@@ -471,14 +841,21 @@ addRanges ApiRequest{..} rReq =
    addRangeToNode :: (EmbedPath, NonnegRange) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
    addRangeToNode = updateNode (\r (Node q f) -> Node q{range_=r} f)

-addLogicTrees :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addLogicTrees ApiRequest{..} rReq =
+addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addLogicTrees ctx ApiRequest{..} rReq =
  foldr addLogicTreeToNode (Right rReq) qsLogic
  where
    QueryParams.QueryParams{..} = iQueryParams

    addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
-    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=t:lf} f)
+    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)
+
+resolveLogicTree :: ResolverContext -> LogicTree -> CoercibleLogicTree
+resolveLogicTree ctx (Stmnt flt) = CoercibleStmnt $ resolveFilter ctx flt
+resolveLogicTree ctx (Expr b op lts) = CoercibleExpr b op (map (resolveLogicTree ctx) lts)
+
+resolveFilter :: ResolverContext -> Filter -> CoercibleFilter
+resolveFilter ctx (Filter fld opExpr) = CoercibleFilter{field=resolveQueryInputField ctx fld, opExpr=opExpr}

 -- Validates that spread embeds are only done on to-one relationships
 validateSpreadEmbeds :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -504,10 +881,10 @@ updateNode f (targetNodeName:remainingPath, a) (Right (Node rootNode forest)) =
    findNode = find (\(Node ReadPlan{relName, relAlias} _) -> relName == targetNodeName || relAlias == Just targetNodeName) forest

 mutatePlan :: Mutation -> QualifiedIdentifier -> ApiRequest -> SchemaCache -> ReadPlanTree -> Either Error MutatePlan
-mutatePlan mutation qi ApiRequest{iPreferences=preferences, ..} sCache readReq = mapLeft ApiRequestError $
+mutatePlan mutation qi ApiRequest{iPreferences=Preferences{..}, ..} SchemaCache{dbTables, dbRepresentations} readReq = mapLeft ApiRequestError $
  case mutation of
    MutationCreate ->
-      mapRight (\typedColumns -> Insert qi typedColumns body ((,) <$> preferences.preferResolution <*> Just confCols) [] returnings pkCols applyDefaults) typedColumnsOrError
+      mapRight (\typedColumns -> Insert qi typedColumns body ((,) <$> preferResolution <*> Just confCols) [] returnings pkCols applyDefaults) typedColumnsOrError
    MutationUpdate ->
      mapRight (\typedColumns -> Update qi typedColumns body combinedLogic iTopLevelRange rootOrder returnings applyDefaults) typedColumnsOrError
    MutationSingleUpsert ->
@@ -522,27 +899,28 @@ mutatePlan mutation qi ApiRequest{iPreferences=preferences, ..} sCache readReq =
          Left InvalidFilters
    MutationDelete -> Right $ Delete qi combinedLogic iTopLevelRange rootOrder returnings
  where
+    ctx = ResolverContext dbTables dbRepresentations qi "json"
    confCols = fromMaybe pkCols qsOnConflict
    QueryParams.QueryParams{..} = iQueryParams
    returnings =
-      if preferences.preferRepresentation == None
+      if preferRepresentation == Just None || isNothing preferRepresentation
        then []
        else inferColsEmbedNeeds readReq pkCols
-    pkCols = maybe mempty tablePKCols $ HM.lookup qi $ dbTables sCache
-    logic = map snd qsLogic
-    rootOrder = maybe [] snd $ find (\(x, _) -> null x) qsOrder
-    combinedLogic = foldr addFilterToLogicForest logic qsFiltersRoot
+    tbl = HM.lookup qi dbTables
+    pkCols = maybe mempty tablePKCols tbl
+    logic = map (resolveLogicTree ctx . snd) qsLogic
+    rootOrder = resolveOrder ctx <$> maybe [] snd (find (\(x, _) -> null x) qsOrder)
+    combinedLogic = foldr (addFilterToLogicForest . resolveFilter ctx) logic qsFiltersRoot
    body = payRaw <$> iPayload -- the body is assumed to be json at this stage(ApiRequest validates)
-    tbl = HM.lookup qi $ dbTables sCache
-    typedColumnsOrError = resolveOrError tbl `traverse` S.toList iColumns
-    applyDefaults = preferences.preferMissing == Just ApplyDefaults
+    applyDefaults = preferMissing == Just ApplyDefaults
+    typedColumnsOrError = resolveOrError ctx tbl `traverse` S.toList iColumns

-resolveOrError :: Maybe Table -> FieldName -> Either ApiRequestError TypedField
-resolveOrError Nothing _ = Left NotFound
-resolveOrError (Just table) field =
-  case resolveTableField table field of
-    Nothing         -> Left $ ColumnNotFound (tableName table) field
-    Just typedField -> Right typedField
+resolveOrError :: ResolverContext -> Maybe Table -> FieldName -> Either ApiRequestError CoercibleField
+resolveOrError _ Nothing _ = Left NotFound
+resolveOrError ctx (Just table) field =
+  case resolveTableFieldName table field of
+    CoercibleField{cfIRType=""} -> Left $ ColumnNotFound (tableName table) field
+    cf                          -> Right $ withJsonParse ctx cf

 callPlan :: Routine -> ApiRequest -> S.Set FieldName -> LBS.ByteString -> ReadPlanTree -> CallPlan
 callPlan proc ApiRequest{iPreferences=Preferences{..}} paramKeys args readReq = FunctionCall {
@@ -571,7 +949,7 @@ inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols
  | "*" `elem` fldNames = ["*"]
  | otherwise           = returnings
  where
-    fldNames = (\((fld, _), _, _) -> fld) <$> select
+    fldNames = cfName . csField <$> select
    -- Without fkCols, when a mutatePlan to
    -- /projects?select=name,clients(name) occurs, the RETURNING SQL part would
    -- be `RETURNING name`(see QueryBuilder).  This would make the embedding
@@ -610,36 +988,38 @@ inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols

 -- Traditional filters(e.g. id=eq.1) are added as root nodes of the LogicTree
 -- they are later concatenated with AND in the QueryBuilder
-addFilterToLogicForest :: Filter -> [LogicTree] -> [LogicTree]
-addFilterToLogicForest flt lf = Stmnt flt : lf
+addFilterToLogicForest :: CoercibleFilter -> [CoercibleLogicTree] -> [CoercibleLogicTree]
+addFilterToLogicForest flt lf = CoercibleStmnt flt : lf

-- | If raw(binary) output is requested, check that MediaType is one of the
-- admitted rawMediaTypes and that`?select=...` contains only one field other
-- than `*`
-binaryField :: AppConfig -> MediaType -> Maybe Routine -> ReadPlanTree -> Either ApiRequestError (Maybe FieldName)
-binaryField AppConfig{configRawMediaTypes} acceptMediaType proc rpTree
-  | isRawMediaType =
-    if (funcReturnsScalar <$> proc) == Just True ||
-       (funcReturnsSetOfScalar <$> proc) == Just True
-      then Right $ Just "pgrst_scalar"
-      else
-        let
-          fieldName = fstFieldName rpTree
-        in
-        case fieldName of
-          Just fld -> Right $ Just fld
-          Nothing  -> Left $ BinaryFieldError acceptMediaType
-  | otherwise =
-      Right Nothing
+-- | Do content negotiation. i.e. choose a media type based on the intersection of accepted/produced media types.
+negotiateContent :: AppConfig -> ApiRequest -> QualifiedIdentifier -> [MediaType] -> MediaHandlerMap -> Either ApiRequestError (MediaHandler, MediaType)
+negotiateContent conf ApiRequest{iAction=act, iPreferences=Preferences{preferRepresentation=rep}} identifier accepts produces =
+  defaultMTAnyToMTJSON $ case (act, firstAcceptedPick) of
+    (_, Nothing)                         -> Left . MediaTypeError $ map MediaType.toMime accepts
+    (ActionMutate _, Just (x, mt))       -> Right (if rep == Just Full then x else NoAgg, mt)
+    -- no need for an aggregate on HEAD https://github.com/PostgREST/postgrest/issues/2849
+    -- TODO: despite no aggregate, these are responding with a Content-Type, which is not correct.
+    (ActionRead True, Just (_, mt))      -> Right (NoAgg, mt)
+    (ActionInvoke InvHead, Just (_, mt)) -> Right (NoAgg, mt)
+    (_, Just (x, mt))                    -> Right (x, mt)
  where
-    isRawMediaType = acceptMediaType `elem` configRawMediaTypes `L.union` [MTOctetStream, MTTextPlain, MTTextXML] || isRawPlan acceptMediaType
-    isRawPlan mt = case mt of
-      MTPlan (MTPlanAttrs (Just MTOctetStream) _ _) -> True
-      MTPlan (MTPlanAttrs (Just MTTextPlain) _ _)   -> True
-      MTPlan (MTPlanAttrs (Just MTTextXML) _ _)     -> True
-      _                                             -> False
-
-    fstFieldName :: ReadPlanTree -> Maybe FieldName
-    fstFieldName (Node ReadPlan{select=(("*", []), _, _):_} [])  = Nothing
-    fstFieldName (Node ReadPlan{select=[((fld, []), _, _)]} []) = Just fld
-    fstFieldName _                                               = Nothing
+    -- the initial handler in the schema cache has a */* to BuiltinAggJson but it doesn't preserve the media type (application/json)
+    -- we just convert the default */* to application/json here
+    -- TODO resolving to "application/json" for "*/*" is not correct when using a "*/*" custom handler media type.
+    -- We should return "application/octet-stream" as the generic type instead.
+    defaultMTAnyToMTJSON = mapRight (\(x, y) -> (x, if y == MTAny then MTApplicationJSON else y))
+    firstAcceptedPick = listToMaybe $ mapMaybe matchMT accepts -- If there are multiple accepted media types, pick the first. This is usual in content negotiation.
+    matchMT mt = case mt of
+      -- all the vendored media types have special handling as they have media type parameters, they cannot be overridden
+      m@(MTVndSingularJSON strip)                 -> Just (BuiltinAggSingleJson strip, m)
+      m@MTVndArrayJSONStrip                       -> Just (BuiltinAggArrayJsonStrip, m)
+      m@(MTVndPlan (MTVndSingularJSON strip) _ _) -> mtPlanToNothing $ Just (BuiltinAggSingleJson strip, m)
+      m@(MTVndPlan MTVndArrayJSONStrip _ _)       -> mtPlanToNothing $ Just (BuiltinAggArrayJsonStrip, m)
+      -- all the other media types can be overridden
+      m@(MTVndPlan mType _ _)                     -> mtPlanToNothing $ (,) <$> lookupHandler mType <*> pure m
+      x                                           -> (,) <$> lookupHandler x <*> pure x
+    mtPlanToNothing x = if configDbPlanEnabled conf then x else Nothing -- don't find anything if the plan media type is not allowed
+    lookupHandler mt =
+      HM.lookup (RelId identifier, MTAny) produces <|> -- lookup handler that applies to `*/*` and identifier
+      HM.lookup (RelId identifier, mt) produces <|>    -- lookup handler that applies to a particular media type and identifier
+      HM.lookup (RelAnyElement, mt) produces           -- lookup handler that applies to a particular media type and anyelement
@@ -6,8 +6,9 @@ where
 import qualified Data.ByteString.Lazy as LBS

 import PostgREST.ApiRequest.Preferences  (PreferResolution)
-import PostgREST.ApiRequest.Types        (LogicTree, OrderTerm)
-import PostgREST.Plan.Types              (TypedField)
+import PostgREST.Plan.Types              (CoercibleField,
+                                          CoercibleLogicTree,
+                                          CoercibleOrderTerm)
 import PostgREST.RangeQuery              (NonnegRange)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                          QualifiedIdentifier)
@@ -18,28 +19,28 @@ import Protolude
 data MutatePlan
  = Insert
      { in_        :: QualifiedIdentifier
-      , insCols    :: [TypedField]
+      , insCols    :: [CoercibleField]
      , insBody    :: Maybe LBS.ByteString
      , onConflict :: Maybe (PreferResolution, [FieldName])
-      , where_     :: [LogicTree]
+      , where_     :: [CoercibleLogicTree]
      , returning  :: [FieldName]
      , insPkCols  :: [FieldName]
      , applyDefs  :: Bool
      }
  | Update
      { in_       :: QualifiedIdentifier
-      , updCols   :: [TypedField]
+      , updCols   :: [CoercibleField]
      , updBody   :: Maybe LBS.ByteString
-      , where_    :: [LogicTree]
+      , where_    :: [CoercibleLogicTree]
      , mutRange  :: NonnegRange
-      , mutOrder  :: [OrderTerm]
+      , mutOrder  :: [CoercibleOrderTerm]
      , returning :: [FieldName]
      , applyDefs :: Bool
      }
  | Delete
      { in_       :: QualifiedIdentifier
-      , where_    :: [LogicTree]
+      , where_    :: [CoercibleLogicTree]
      , mutRange  :: NonnegRange
-      , mutOrder  :: [OrderTerm]
+      , mutOrder  :: [CoercibleOrderTerm]
      , returning :: [FieldName]
      }
@@ -6,9 +6,12 @@ module PostgREST.Plan.ReadPlan

 import Data.Tree (Tree (..))

-import PostgREST.ApiRequest.Types         (Alias, Cast, Depth, Field,
-                                           Hint, JoinType, LogicTree,
-                                           NodeName, OrderTerm)
+import PostgREST.ApiRequest.Types         (Alias, Depth, Hint,
+                                           JoinType, NodeName)
+import PostgREST.Plan.Types               (CoercibleLogicTree,
+                                           CoercibleOrderTerm,
+                                           CoercibleSelectField (..),
+                                           RelSelectField (..))
 import PostgREST.RangeQuery               (NonnegRange)
 import PostgREST.SchemaCache.Identifiers  (FieldName,
                                           QualifiedIdentifier)
@@ -23,14 +26,14 @@ data JoinCondition =
  JoinCondition
    (QualifiedIdentifier, FieldName)
    (QualifiedIdentifier, FieldName)
-  deriving (Eq)
+  deriving (Eq, Show)

 data ReadPlan = ReadPlan
-  { select       :: [(Field, Maybe Cast, Maybe Alias)]
+  { select       :: [CoercibleSelectField]
  , from         :: QualifiedIdentifier
  , fromAlias    :: Maybe Alias
-  , where_       :: [LogicTree]
-  , order        :: [OrderTerm]
+  , where_       :: [CoercibleLogicTree]
+  , order        :: [CoercibleOrderTerm]
  , range_       :: NonnegRange
  , relName      :: NodeName
  , relToParent  :: Maybe Relationship
@@ -40,7 +43,8 @@ data ReadPlan = ReadPlan
  , relHint      :: Maybe Hint
  , relJoinType  :: Maybe JoinType
  , relIsSpread  :: Bool
+  , relSelect    :: [RelSelectField]
  , depth        :: Depth
  -- ^ used for aliasing
  }
-  deriving (Eq)
+  deriving (Eq, Show)
@@ -1,24 +1,106 @@
 module PostgREST.Plan.Types
-  ( TypedField(..)
-  , resolveTableField
+  ( CoercibleField(..)
+  , CoercibleSelectField(..)
+  , unknownField
+  , CoercibleLogicTree(..)
+  , CoercibleFilter(..)
+  , TransformerProc
+  , CoercibleOrderTerm(..)
+  , RelSelectField(..)
+  , RelJsonEmbedMode(..)
+  , SpreadSelectField(..)
  ) where

-import qualified Data.HashMap.Strict.InsOrd as HMI
+import PostgREST.ApiRequest.Types (AggregateFunction, Alias, Cast,
+                                   Field, JsonPath, LogicOperator,
+                                   OpExpr, OrderDirection, OrderNulls)

 import PostgREST.SchemaCache.Identifiers (FieldName)
-import PostgREST.SchemaCache.Table       (Column (..), Table (..))

 import Protolude

-- | A TypedField is a field with sufficient information to be read from JSON with `json_to_recordset`.
-data TypedField = TypedField
-   { tfName    :: FieldName
-   , tfIRType  :: Text -- ^ The initial type of the field, before any casting.
-   , tfDefault :: Maybe Text
-   } deriving (Eq)
+type TransformerProc = Text

-resolveTableField :: Table -> FieldName -> Maybe TypedField
-resolveTableField table fieldName =
-  case HMI.lookup fieldName (tableColumns table) of
-    Just column -> Just $ TypedField (colName column) (colNominalType column) (colDefault column)
-    Nothing     -> Nothing
+-- | A CoercibleField pairs the name of a query element with any type coercion information we need for some specific use case.
+-- |
+-- | As suggested by the name, it's often a reference to a field in a table but really it can be any nameable element (function parameter, calculation with an alias, etc) with a knowable type.
+-- |
+-- | In the simplest case, it allows us to parse JSON payloads with `json_to_recordset`, for which we need to know both the name and the type of each thing we'd like to extract. At a higher level, CoercibleField generalises to reflect that any value we work with in a query may need type specific handling.
+-- |
+-- | CoercibleField is the foundation for the Data Representations feature. This feature allow user-definable mappings between database types so that the same data can be presented or interpreted in various ways as needed. Sometimes the way Postgres coerces data implicitly isn't right for the job. Different mappings might be appropriate for different situations: parsing a filter from a query string requires one function (text -> field type) while parsing a payload from JSON takes another (json -> field type). And the reverse, outputting a field as JSON, requires yet a third (field type -> json). CoercibleField is that "job specific" reference to an element paired with the type we desire for that particular purpose and the function we'll use to get there, if any.
+-- |
+-- | In the planning phase, we "resolve" generic named elements into these specialised CoercibleFields. Again this is context specific: two different CoercibleFields both representing the exact same table column in the database, even in the same query, might have two different target types and mapping functions. For example, one might represent a column in a filter, and another the very same column in an output role to be sent in the response body.
+-- |
+-- | The type value is allowed to be the empty string. The analog here is soft type checking in programming languages: sometimes we don't need a variable to have a specified type and things will work anyhow. So the empty type variant is valid when we don't know and *don't need to know* about the specific type in some context. Note that this variation should not be used if it guarantees failure: in that case you should instead raise an error at the planning stage and bail out. For example, we can't parse JSON with `json_to_recordset` without knowing the types of each recipient field, and so error out. Using the empty string for the type would be incorrect and futile. On the other hand we use the empty type for RPC calls since type resolution isn't implemented for RPC, but it's fine because the query still works with Postgres' implicit coercion. In the future, hopefully we will support data representations across the board and then the empty type may be permanently retired.
+data CoercibleField = CoercibleField
+  { cfName      :: FieldName
+  , cfJsonPath  :: JsonPath
+  , cfToJson    :: Bool
+  , cfIRType    :: Text                  -- ^ The native Postgres type of the field, the intermediate (IR) type before mapping.
+  , cfTransform :: Maybe TransformerProc -- ^ The optional mapping from irType -> targetType.
+  , cfDefault   :: Maybe Text
+  } deriving (Eq, Show)
+
+unknownField :: FieldName -> JsonPath -> CoercibleField
+unknownField name path = CoercibleField name path False "" Nothing Nothing
+
+-- | Like an API request LogicTree, but with coercible field information.
+data CoercibleLogicTree
+  = CoercibleExpr Bool LogicOperator [CoercibleLogicTree]
+  | CoercibleStmnt CoercibleFilter
+  deriving (Eq, Show)
+
+data CoercibleFilter = CoercibleFilter
+  { field  :: CoercibleField
+  , opExpr :: OpExpr
+  }
+  | CoercibleFilterNullEmbed Bool FieldName
+  deriving (Eq, Show)
+
+data CoercibleOrderTerm
+  = CoercibleOrderTerm
+    { coField     :: CoercibleField
+    , coDirection :: Maybe OrderDirection
+    , coNullOrder :: Maybe OrderNulls
+    }
+  | CoercibleOrderRelationTerm
+    { coRelation  :: FieldName
+    , coRelTerm   :: Field
+    , coDirection :: Maybe OrderDirection
+    , coNullOrder :: Maybe OrderNulls
+    }
+  deriving (Eq, Show)
+
+data CoercibleSelectField = CoercibleSelectField
+  { csField       :: CoercibleField
+  , csAggFunction :: Maybe AggregateFunction
+  , csAggCast     :: Maybe Cast
+  , csCast        :: Maybe Cast
+  , csAlias       :: Maybe Alias
+  }
+  deriving (Eq, Show)
+
+data RelJsonEmbedMode = JsonObject | JsonArray
+  deriving (Show, Eq)
+
+data RelSelectField
+  = JsonEmbed
+      { rsSelName    :: FieldName
+      , rsAggAlias   :: Alias
+      , rsEmbedMode  :: RelJsonEmbedMode
+      , rsEmptyEmbed :: Bool
+      }
+  | Spread
+      { rsSpreadSel :: [SpreadSelectField]
+      , rsAggAlias  :: Alias
+      }
+  deriving (Eq, Show)
+
+data SpreadSelectField =
+  SpreadSelectField
+  { ssSelName        :: FieldName
+  , ssSelAggFunction :: Maybe AggregateFunction
+  , ssSelAggCast     :: Maybe Cast
+  , ssSelAlias       :: Maybe Alias
+  }
+  deriving (Eq, Show)
@@ -14,48 +14,43 @@ module PostgREST.Query
  ) where

 import qualified Data.Aeson                        as JSON
-import qualified Data.Aeson.Key                    as K
 import qualified Data.Aeson.KeyMap                 as KM
 import qualified Data.ByteString                   as BS
 import qualified Data.ByteString.Lazy.Char8        as LBS
 import qualified Data.HashMap.Strict               as HM
 import qualified Data.Set                          as S
-import qualified Data.Text.Encoding                as T
 import qualified Hasql.Decoders                    as HD
 import qualified Hasql.DynamicStatements.Snippet   as SQL (Snippet)
 import qualified Hasql.DynamicStatements.Statement as SQL
-import qualified Hasql.Encoders                    as HE
-import qualified Hasql.Statement                   as SQL
 import qualified Hasql.Transaction                 as SQL

-import qualified PostgREST.Error               as Error
-import qualified PostgREST.Query.QueryBuilder  as QueryBuilder
-import qualified PostgREST.Query.Statements    as Statements
-import qualified PostgREST.RangeQuery          as RangeQuery
-import qualified PostgREST.SchemaCache         as SchemaCache
-import qualified PostgREST.SchemaCache.Routine as Routine
-
-import Data.Scientific (FPFormat (..), formatScientific, isInteger)
+import qualified PostgREST.ApiRequest.Types   as ApiRequestTypes
+import qualified PostgREST.Error              as Error
+import qualified PostgREST.Query.QueryBuilder as QueryBuilder
+import qualified PostgREST.Query.Statements   as Statements
+import qualified PostgREST.RangeQuery         as RangeQuery
+import qualified PostgREST.SchemaCache        as SchemaCache

 import PostgREST.ApiRequest              (ApiRequest (..))
 import PostgREST.ApiRequest.Preferences  (PreferCount (..),
+                                          PreferTimezone (..),
                                          PreferTransaction (..),
                                          Preferences (..),
                                          shouldCount)
 import PostgREST.Config                  (AppConfig (..),
                                          OpenAPIMode (..))
-import PostgREST.Config.PgVersion        (PgVersion (..),
-                                          pgVersion140)
+import PostgREST.Config.PgVersion        (PgVersion (..))
 import PostgREST.Error                   (Error)
 import PostgREST.MediaType               (MediaType (..))
 import PostgREST.Plan                    (CallReadPlan (..),
                                          MutateReadPlan (..),
                                          WrappedReadPlan (..))
 import PostgREST.Plan.MutatePlan         (MutatePlan (..))
-import PostgREST.Query.SqlFragment       (fromQi, intercalateSnippet,
-                                          pgFmtIdentList,
-                                          setConfigLocal,
-                                          setConfigLocalJson)
+import PostgREST.Query.SqlFragment       (escapeIdentList, fromQi,
+                                          intercalateSnippet,
+                                          setConfigWithConstantName,
+                                          setConfigWithConstantNameJSON,
+                                          setConfigWithDynamicName)
 import PostgREST.Query.Statements        (ResultSet (..))
 import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
@@ -68,11 +63,12 @@ import Protolude hiding (Handler)
 type DbHandler = ExceptT Error SQL.Transaction

 readQuery :: WrappedReadPlan -> AppConfig -> ApiRequest -> DbHandler ResultSet
-readQuery WrappedReadPlan{wrReadPlan, wrBinField} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} = do
+readQuery WrappedReadPlan{..} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} = do
  let countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan
  resultSet <-
     lift . SQL.statement mempty $
      Statements.prepareRead
+        wrIdent
        (QueryBuilder.readPlanToQuery wrReadPlan)
        (if preferCount == Just EstimatedCount then
           -- LIMIT maxRows + 1 so we can determine below that maxRows was surpassed
@@ -81,10 +77,10 @@ readQuery WrappedReadPlan{wrReadPlan, wrBinField} conf@AppConfig{..} apiReq@ApiR
           countQuery
        )
        (shouldCount preferCount)
-        iAcceptMediaType
-        wrBinField
+        wrMedia
+        wrHandler
        configDbPreparedStatements
-  failNotSingular iAcceptMediaType resultSet
+  failNotSingular wrMedia resultSet
  optionalRollback conf apiReq
  resultSetWTotal conf apiReq resultSet countQuery

@@ -111,16 +107,16 @@ resultSetWTotal AppConfig{..} ApiRequest{iPreferences=Preferences{..}} rs@RSStan
        configDbPreparedStatements

 createQuery :: MutateReadPlan -> ApiRequest -> AppConfig -> DbHandler ResultSet
-createQuery mrPlan apiReq@ApiRequest{..} conf = do
+createQuery mrPlan@MutateReadPlan{mrMedia} apiReq conf = do
  resultSet <- writeQuery mrPlan apiReq conf
-  failNotSingular iAcceptMediaType resultSet
+  failNotSingular mrMedia resultSet
  optionalRollback conf apiReq
  pure resultSet

 updateQuery :: MutateReadPlan -> ApiRequest -> AppConfig -> DbHandler ResultSet
-updateQuery mrPlan apiReq@ApiRequest{..} conf = do
+updateQuery mrPlan@MutateReadPlan{mrMedia} apiReq@ApiRequest{..} conf = do
  resultSet <- writeQuery mrPlan apiReq conf
-  failNotSingular iAcceptMediaType resultSet
+  failNotSingular mrMedia resultSet
  failsChangesOffLimits (RangeQuery.rangeLimit iTopLevelRange) resultSet
  optionalRollback conf apiReq
  pure resultSet
@@ -142,34 +138,33 @@ failPut RSPlan{} = pure ()
 failPut RSStandard{rsQueryTotal=queryTotal} =
  when (queryTotal /= 1) $ do
    lift SQL.condemn
-    throwError Error.PutMatchingPkError
+    throwError $ Error.ApiRequestError ApiRequestTypes.PutMatchingPkError

 deleteQuery :: MutateReadPlan -> ApiRequest -> AppConfig -> DbHandler ResultSet
-deleteQuery mrPlan apiReq@ApiRequest{..} conf = do
+deleteQuery mrPlan@MutateReadPlan{mrMedia} apiReq@ApiRequest{..} conf = do
  resultSet <- writeQuery mrPlan apiReq conf
-  failNotSingular iAcceptMediaType resultSet
+  failNotSingular mrMedia resultSet
  failsChangesOffLimits (RangeQuery.rangeLimit iTopLevelRange) resultSet
  optionalRollback conf apiReq
  pure resultSet

 invokeQuery :: Routine -> CallReadPlan -> ApiRequest -> AppConfig -> PgVersion -> DbHandler ResultSet
-invokeQuery proc CallReadPlan{crReadPlan, crCallPlan, crBinField} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} conf@AppConfig{..} pgVer = do
+invokeQuery rout CallReadPlan{..} apiReq@ApiRequest{iPreferences=Preferences{..}} conf@AppConfig{..} pgVer = do
  resultSet <-
    lift . SQL.statement mempty $
      Statements.prepareCall
-        (Routine.funcReturnsScalar proc)
-        (Routine.funcReturnsSingleComposite proc)
-        (Routine.funcReturnsSetOfScalar proc)
+        crIdent
+        rout
        (QueryBuilder.callPlanToQuery crCallPlan pgVer)
        (QueryBuilder.readPlanToQuery crReadPlan)
        (QueryBuilder.readPlanToCountQuery crReadPlan)
        (shouldCount preferCount)
-        iAcceptMediaType
-        crBinField
+        crMedia
+        crHandler
        configDbPreparedStatements

  optionalRollback conf apiReq
-  failNotSingular iAcceptMediaType resultSet
+  failNotSingular crMedia resultSet
  pure resultSet

 openApiQuery :: SchemaCache -> PgVersion -> AppConfig -> Schema -> DbHandler (Maybe (TablesMap, RoutineMap, Maybe Text))
@@ -190,17 +185,21 @@ openApiQuery sCache pgVer AppConfig{..} tSchema =
      pure Nothing

 writeQuery :: MutateReadPlan -> ApiRequest -> AppConfig  -> DbHandler ResultSet
-writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferences=Preferences{..}} conf =
+writeQuery MutateReadPlan{..} ApiRequest{iPreferences=Preferences{..}} conf =
  let
-    (isInsert, pkCols) = case mrMutatePlan of {Insert{insPkCols} -> (True, insPkCols); _ -> (False, mempty);}
+    (isPut, isInsert, pkCols) = case mrMutatePlan of {Insert{where_,insPkCols} -> ((not . null) where_, True, insPkCols); _ -> (False,False, mempty);}
  in
  lift . SQL.statement mempty $
    Statements.prepareWrite
+      mrIdent
      (QueryBuilder.readPlanToQuery mrReadPlan)
      (QueryBuilder.mutatePlanToQuery mrMutatePlan)
      isInsert
-      (iAcceptMediaType apiReq)
+      isPut
+      mrMedia
+      mrHandler
      preferRepresentation
+      preferResolution
      pkCols
      (configDbPreparedStatements conf)

@@ -210,9 +209,9 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferenc
 failNotSingular :: MediaType -> ResultSet -> DbHandler ()
 failNotSingular _ RSPlan{} = pure ()
 failNotSingular mediaType RSStandard{rsQueryTotal=queryTotal} =
-  when (mediaType == MTSingularJSON && queryTotal /= 1) $ do
+  when (elem mediaType [MTVndSingularJSON True, MTVndSingularJSON False] && queryTotal /= 1) $ do
    lift SQL.condemn
-    throwError $ Error.singularityError queryTotal
+    throwError $ Error.ApiRequestError . ApiRequestTypes.SingularityError $ toInteger queryTotal

 failsChangesOffLimits :: Maybe Integer -> ResultSet -> DbHandler ()
 failsChangesOffLimits _ RSPlan{} = pure ()
@@ -220,7 +219,7 @@ failsChangesOffLimits Nothing _  = pure ()
 failsChangesOffLimits (Just maxChanges) RSStandard{rsQueryTotal=queryTotal} =
  when (queryTotal > fromIntegral maxChanges) $ do
    lift SQL.condemn
-    throwError $ Error.OffLimitsChangesError queryTotal maxChanges
+    throwError $ Error.ApiRequestError $ ApiRequestTypes.OffLimitsChangesError queryTotal maxChanges

 -- | Set a transaction to roll back if requested
 optionalRollback :: AppConfig -> ApiRequest -> DbHandler ()
@@ -230,50 +229,39 @@ optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} = do
    SQL.condemn
  where
    shouldCommit =
-      configDbTxAllowOverride && preferTransaction == Just Commit
+      preferTransaction == Just Commit
    shouldRollback =
-      configDbTxAllowOverride && preferTransaction == Just Rollback
+      preferTransaction == Just Rollback

-- | Runs local (transaction scoped) GUCs for every request.
+-- | Set transaction scoped settings
 setPgLocals :: AppConfig  -> KM.KeyMap JSON.Value -> BS.ByteString -> [(ByteString, ByteString)] ->
-               ApiRequest -> PgVersion -> DbHandler ()
-setPgLocals AppConfig{..} claims role roleSettings req actualPgVersion = lift $
+               ApiRequest -> Maybe Text -> DbHandler ()
+setPgLocals AppConfig{..} claims role roleSettings ApiRequest{..} tout = lift $
  SQL.statement mempty $ SQL.dynamicallyParameterized
-    ("select " <> intercalateSnippet ", " (searchPathSql : roleSql ++ roleSettingsSql ++ claimsSql ++ [methodSql, pathSql] ++ headersSql ++ cookiesSql ++ appSettingsSql))
+    -- To ensure `GRANT SET ON PARAMETER <superuser_setting> TO authenticator` works, the role settings must be set before the impersonated role.
+    -- Otherwise the GRANT SET would have to be applied to the impersonated role. See https://github.com/PostgREST/postgrest/issues/3045
+    ("select " <> intercalateSnippet ", " (searchPathSql : roleSettingsSql ++ roleSql ++ claimsSql ++ [methodSql, pathSql] ++ headersSql ++ cookiesSql ++ timezoneSql ++ timeoutSql ++ appSettingsSql))
    HD.noResult configDbPreparedStatements
  where
-    methodSql = setConfigLocal mempty ("request.method", iMethod req)
-    pathSql = setConfigLocal mempty ("request.path", iPath req)
-    headersSql = if usesLegacyGucs
-                   then setConfigLocal "request.header." <$> iHeaders req
-                   else setConfigLocalJson "request.headers" (iHeaders req)
-    cookiesSql = if usesLegacyGucs
-                   then setConfigLocal "request.cookie." <$> iCookies req
-                   else setConfigLocalJson "request.cookies" (iCookies req)
-    claimsSql = if usesLegacyGucs
-                  then setConfigLocal "request.jwt.claim." <$> [(toUtf8 $ K.toText c, toUtf8 $ unquoted v) | (c,v) <- KM.toList claims]
-                  else [setConfigLocal mempty ("request.jwt.claims", LBS.toStrict $ JSON.encode claims)]
-    roleSql = [setConfigLocal mempty ("role", role)]
-    roleSettingsSql = setConfigLocal mempty <$> roleSettings
-    appSettingsSql = setConfigLocal mempty <$> (join bimap toUtf8 <$> configAppSettings)
+    methodSql = setConfigWithConstantName ("request.method", iMethod)
+    pathSql = setConfigWithConstantName ("request.path", iPath)
+    headersSql = setConfigWithConstantNameJSON "request.headers" iHeaders
+    cookiesSql = setConfigWithConstantNameJSON "request.cookies" iCookies
+    claimsSql = [setConfigWithConstantName ("request.jwt.claims", LBS.toStrict $ JSON.encode claims)]
+    roleSql = [setConfigWithConstantName ("role", role)]
+    roleSettingsSql = setConfigWithDynamicName <$> roleSettings
+    appSettingsSql = setConfigWithDynamicName <$> (join bimap toUtf8 <$> configAppSettings)
+    timezoneSql = maybe mempty (\(PreferTimezone tz) -> [setConfigWithConstantName ("timezone", tz)]) $ preferTimezone iPreferences
+    timeoutSql = maybe mempty ((\t -> [setConfigWithConstantName ("statement_timeout", t)]) . encodeUtf8) tout
    searchPathSql =
-      let schemas = pgFmtIdentList (iSchema req : configDbExtraSearchPath) in
-      setConfigLocal mempty ("search_path", schemas)
-    usesLegacyGucs = configDbUseLegacyGucs && actualPgVersion < pgVersion140
-
-    unquoted :: JSON.Value -> Text
-    unquoted (JSON.String t) = t
-    unquoted (JSON.Number n) =
-      toS $ formatScientific Fixed (if isInteger n then Just 0 else Nothing) n
-    unquoted (JSON.Bool b) = show b
-    unquoted v = T.decodeUtf8 . LBS.toStrict $ JSON.encode v
+      let schemas = escapeIdentList (iSchema : configDbExtraSearchPath) in
+      setConfigWithConstantName ("search_path", schemas)

 -- | Runs the pre-request function.
 runPreReq :: AppConfig -> DbHandler ()
 runPreReq conf = lift $ traverse_ (SQL.statement mempty . stmt) (configDbPreRequest conf)
  where
-    stmt req = SQL.Statement
+    stmt req = SQL.dynamicallyParameterized
      ("select " <> fromQi req <> "()")
-      HE.noParams
      HD.noResult
      (configDbPreparedStatements conf)
@@ -19,7 +19,8 @@ module PostgREST.Query.QueryBuilder
 import qualified Data.ByteString.Char8           as BS
 import qualified Hasql.DynamicStatements.Snippet as SQL

-import Data.Tree (Tree (..))
+import Data.Maybe (fromJust)
+import Data.Tree  (Tree (..))

 import PostgREST.ApiRequest.Preferences   (PreferResolution (..))
 import PostgREST.Config.PgVersion         (PgVersion, pgVersion110,
@@ -27,8 +28,7 @@ import PostgREST.Config.PgVersion         (PgVersion, pgVersion110,
 import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                           Junction (..),
-                                           Relationship (..),
-                                           relIsToOne)
+                                           Relationship (..))
 import PostgREST.SchemaCache.Routine      (RoutineParam (..))

 import PostgREST.ApiRequest.Types
@@ -42,69 +42,94 @@ import PostgREST.RangeQuery        (allRange)
 import Protolude

 readPlanToQuery :: ReadPlanTree -> SQL.Snippet
-readPlanToQuery (Node ReadPlan{select,from=mainQi,fromAlias,where_=logicForest,order, range_=readRange, relToParent, relJoinConds} forest) =
+readPlanToQuery node@(Node ReadPlan{select,from=mainQi,fromAlias,where_=logicForest,order, range_=readRange, relToParent, relJoinConds, relSelect} forest) =
  "SELECT " <>
-  intercalateSnippet ", " ((pgFmtSelectItem qi <$> (if null select && null forest then defSelect else select)) ++ selects) <> " " <>
+  intercalateSnippet ", " ((pgFmtSelectItem qi <$> (if null select && null forest then defSelect else select)) ++ joinsSelects) <> " " <>
  fromFrag <> " " <>
  intercalateSnippet " " joins <> " " <>
  (if null logicForest && null relJoinConds
    then mempty
    else "WHERE " <> intercalateSnippet " AND " (map (pgFmtLogicTree qi) logicForest ++ map pgFmtJoinCondition relJoinConds)) <> " " <>
+  groupF qi select relSelect <> " " <>
  orderF qi order <> " " <>
  limitOffsetF readRange
  where
    fromFrag = fromF relToParent mainQi fromAlias
    qi = getQualifiedIdentifier relToParent mainQi fromAlias
-    defSelect = [(("*", []), Nothing, Nothing)] -- gets all the columns in case of an empty select, ignoring/obtaining these columns is done at the aggregation stage
-    (selects, joins) = foldr getSelectsJoins ([],[]) forest
+    -- gets all the columns in case of an empty select, ignoring/obtaining these columns is done at the aggregation stage
+    defSelect = [CoercibleSelectField (unknownField "*" []) Nothing Nothing Nothing Nothing]
+    joins = getJoins node
+    joinsSelects = getJoinSelects node

-getSelectsJoins :: ReadPlanTree -> ([SQL.Snippet], [SQL.Snippet]) -> ([SQL.Snippet], [SQL.Snippet])
-getSelectsJoins (Node ReadPlan{relToParent=Nothing} _) _ = ([], [])
-getSelectsJoins rr@(Node ReadPlan{select, relName, relToParent=Just rel, relAggAlias, relAlias, relJoinType, relIsSpread} forest) (selects,joins) =
+getJoinSelects :: ReadPlanTree -> [SQL.Snippet]
+getJoinSelects (Node ReadPlan{relSelect} _) =
+  mapMaybe relSelectToSnippet relSelect
+  where
+    relSelectToSnippet :: RelSelectField -> Maybe SQL.Snippet
+    relSelectToSnippet fld =
+      let aggAlias = pgFmtIdent $ rsAggAlias fld
+      in
+        case fld of
+          JsonEmbed{rsEmptyEmbed = True} ->
+            Nothing
+          JsonEmbed{rsSelName, rsEmbedMode = JsonObject} ->
+            Just $ "row_to_json(" <> aggAlias <> ".*)::jsonb AS " <> pgFmtIdent rsSelName
+          JsonEmbed{rsSelName, rsEmbedMode = JsonArray} ->
+            Just $ "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> pgFmtIdent rsSelName
+          Spread{rsSpreadSel, rsAggAlias} ->
+            Just $ intercalateSnippet ", " (pgFmtSpreadSelectItem rsAggAlias <$> rsSpreadSel)
+
+getJoins :: ReadPlanTree -> [SQL.Snippet]
+getJoins (Node _ []) = []
+getJoins (Node ReadPlan{relSelect} forest) =
+  map (\fld ->
+         let alias = rsAggAlias fld
+             matchingNode = fromJust $ find (\(Node ReadPlan{relAggAlias} _) -> alias == relAggAlias) forest
+         in getJoin fld matchingNode
+      ) relSelect
+
+getJoin :: RelSelectField -> ReadPlanTree -> SQL.Snippet
+getJoin fld node@(Node ReadPlan{relJoinType} _) =
  let
-    subquery = readPlanToQuery rr
-    aliasOrName = pgFmtIdent $ fromMaybe relName relAlias
-    aggAlias = pgFmtIdent relAggAlias
    correlatedSubquery sub al cond =
-      (if relJoinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> SQL.sql al <> " ON " <> cond
-    (sel, joi) = if relIsToOne rel
-      then
-        ( if relIsSpread
-            then SQL.sql aggAlias <> ".*"
-            else SQL.sql ("row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName)
-        , correlatedSubquery subquery aggAlias "TRUE")
-      else
-        ( SQL.sql $ "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> aliasOrName
-        , correlatedSubquery (
-            "SELECT json_agg(" <> SQL.sql aggAlias <> ") AS " <> SQL.sql aggAlias <>
-            "FROM (" <> subquery <> " ) AS " <> SQL.sql aggAlias
-          ) aggAlias $ if relJoinType == Just JTInner then SQL.sql aggAlias <> " IS NOT NULL" else "TRUE")
+      (if relJoinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> al <> " ON " <> cond
+    subquery = readPlanToQuery node
+    aggAlias = pgFmtIdent $ rsAggAlias fld
  in
-  (if null select && null forest then selects else sel:selects, joi:joins)
+    case fld of
+      JsonEmbed{rsEmbedMode = JsonObject} ->
+        correlatedSubquery subquery aggAlias "TRUE"
+      Spread{} ->
+        correlatedSubquery subquery aggAlias "TRUE"
+      JsonEmbed{rsEmbedMode = JsonArray} ->
+        let
+          subq = "SELECT json_agg(" <> aggAlias <> ")::jsonb AS " <> aggAlias <> " FROM (" <> subquery <> " ) AS " <> aggAlias
+          condition = if relJoinType == Just JTInner then aggAlias <> " IS NOT NULL" else "TRUE"
+        in correlatedSubquery subq aggAlias condition

 mutatePlanToQuery :: MutatePlan -> SQL.Snippet
 mutatePlanToQuery (Insert mainQi iCols body onConflct putConditions returnings _ applyDefaults) =
-  "INSERT INTO " <> SQL.sql (fromQi mainQi) <> SQL.sql (if null iCols then " " else "(" <> cols <> ") ") <>
+  "INSERT INTO " <> fromQi mainQi <> (if null iCols then " " else "(" <> cols <> ") ") <>
  fromJsonBodyF body iCols True False applyDefaults <>
  -- Only used for PUT
-  (if null putConditions then mempty else "WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree (QualifiedIdentifier mempty "pgrst_body") <$> putConditions)) <>
-  SQL.sql (BS.unwords [
-    maybe mempty (\(oncDo, oncCols) ->
-      if null oncCols then
-        mempty
-      else
-        " ON CONFLICT(" <> BS.intercalate ", " (pgFmtIdent <$> oncCols) <> ") " <> case oncDo of
-        IgnoreDuplicates ->
-          "DO NOTHING"
-        MergeDuplicates  ->
-          if null iCols
-             then "DO NOTHING"
-             else "DO UPDATE SET " <> BS.intercalate ", " ((pgFmtIdent . tfName) <> const " = EXCLUDED." <> (pgFmtIdent . tfName) <$> iCols)
-      ) onConflct,
+  (if null putConditions then mempty else "WHERE " <> addConfigPgrstInserted True <> " AND " <> intercalateSnippet " AND " (pgFmtLogicTree (QualifiedIdentifier mempty "pgrst_body") <$> putConditions)) <>
+  (if null putConditions && mergeDups then "WHERE " <> addConfigPgrstInserted True else mempty) <>
+  maybe mempty (\(oncDo, oncCols) ->
+    if null oncCols then
+      mempty
+    else
+      " ON CONFLICT(" <> intercalateSnippet ", " (pgFmtIdent <$> oncCols) <> ") " <> case oncDo of
+      IgnoreDuplicates ->
+        "DO NOTHING"
+      MergeDuplicates  ->
+        if null iCols
+           then "DO NOTHING"
+           else "DO UPDATE SET " <> intercalateSnippet ", " ((pgFmtIdent . cfName) <> const " = EXCLUDED." <> (pgFmtIdent . cfName) <$> iCols) <> (if null putConditions && not mergeDups then mempty else "WHERE " <> addConfigPgrstInserted False)
+    ) onConflct <> " " <>
    returningF mainQi returnings
-    ])
  where
-    cols = BS.intercalate ", " $ pgFmtIdent . tfName <$> iCols
+    cols = intercalateSnippet ", " $ pgFmtIdent . cfName <$> iCols
+    mergeDups = case onConflct of {Just (MergeDuplicates,_) -> True; _ -> False;}

 -- An update without a limit is always filtered with a WHERE
 mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings applyDefaults)
@@ -112,82 +137,83 @@ mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings a
    -- if there are no columns we cannot do UPDATE table SET {empty}, it'd be invalid syntax
    -- selecting an empty resultset from mainQi gives us the column names to prevent errors when using &select=
    -- the select has to be based on "returnings" to make computed overloaded functions not throw
-    SQL.sql $ "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"
+    "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"

  | range == allRange =
-    "UPDATE " <> mainTbl <> " SET " <> SQL.sql nonRangeCols <> " " <>
+    "UPDATE " <> mainTbl <> " SET " <> nonRangeCols <> " " <>
    fromJsonBodyF body uCols False False applyDefaults <>
    whereLogic <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings

  | otherwise =
    "WITH " <>
    "pgrst_update_body AS (" <> fromJsonBodyF body uCols True True applyDefaults <> "), " <>
    "pgrst_affected_rows AS (" <>
-      "SELECT " <> SQL.sql rangeIdF <> " FROM " <> mainTbl <>
+      "SELECT " <> rangeIdF <> " FROM " <> mainTbl <>
      whereLogic <> " " <>
      orderF mainQi ordts <> " " <>
      limitOffsetF range <>
    ") " <>
-    "UPDATE " <> mainTbl <> " SET " <> SQL.sql rangeCols <>
+    "UPDATE " <> mainTbl <> " SET " <> rangeCols <>
    "FROM pgrst_affected_rows " <>
-    "WHERE " <> SQL.sql whereRangeIdF <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    "WHERE " <> whereRangeIdF <> " " <>
+    returningF mainQi returnings

  where
    whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
-    mainTbl = SQL.sql (fromQi mainQi)
-    emptyBodyReturnedColumns = if null returnings then "NULL" else BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
-    nonRangeCols = BS.intercalate ", " (pgFmtIdent . tfName <> const " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_body") . tfName <$> uCols)
-    rangeCols = BS.intercalate ", " ((\col -> pgFmtIdent (tfName col) <> " = (SELECT " <> pgFmtIdent (tfName col) <> " FROM pgrst_update_body) ") <$> uCols)
-    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
+    mainTbl = fromQi mainQi
+    emptyBodyReturnedColumns = if null returnings then "NULL" else intercalateSnippet ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
+    nonRangeCols = intercalateSnippet ", " (pgFmtIdent . cfName <> const " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_body") . cfName <$> uCols)
+    rangeCols = intercalateSnippet ", " ((\col -> pgFmtIdent (cfName col) <> " = (SELECT " <> pgFmtIdent (cfName col) <> " FROM pgrst_update_body) ") <$> uCols)
+    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (cfName . coField <$> ordts)

 mutatePlanToQuery (Delete mainQi logicForest range ordts returnings)
  | range == allRange =
-    "DELETE FROM " <> SQL.sql (fromQi mainQi) <> " " <>
+    "DELETE FROM " <> fromQi mainQi <> " " <>
    whereLogic <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings

  | otherwise =
    "WITH " <>
    "pgrst_affected_rows AS (" <>
-      "SELECT " <> SQL.sql rangeIdF <> " FROM " <> SQL.sql (fromQi mainQi) <>
+      "SELECT " <> rangeIdF <> " FROM " <> fromQi mainQi <>
       whereLogic <> " " <>
      orderF mainQi ordts <> " " <>
      limitOffsetF range <>
    ") " <>
-    "DELETE FROM " <> SQL.sql (fromQi mainQi) <> " " <>
+    "DELETE FROM " <> fromQi mainQi <> " " <>
    "USING pgrst_affected_rows " <>
-    "WHERE " <> SQL.sql whereRangeIdF <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    "WHERE " <> whereRangeIdF <> " " <>
+    returningF mainQi returnings

  where
    whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
-    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
+    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (cfName . coField <$> ordts)

 callPlanToQuery :: CallPlan -> PgVersion -> SQL.Snippet
 callPlanToQuery (FunctionCall qi params args returnsScalar returnsSetOfScalar returnsCompositeAlias returnings) pgVer =
-  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call AS pgrst_scalar " else returnedColumns) <> " " <>
+  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call.pgrst_scalar" else returnedColumns) <> " " <>
  fromCall
  where
    fromCall = case params of
      OnePosParam prm -> "FROM " <> callIt (singleParameter args $ encodeUtf8 $ ppType prm)
      KeyParams []    -> "FROM " <> callIt mempty
-      KeyParams prms  -> fromJsonBodyF args ((\p -> TypedField (ppName p) (ppType p) Nothing) <$> prms) False True False <> ", " <>
+      KeyParams prms  -> fromJsonBodyF args ((\p -> CoercibleField (ppName p) mempty False (ppTypeMaxLength p) Nothing Nothing) <$> prms) False True False <> ", " <>
                         "LATERAL " <> callIt (fmtParams prms)

    callIt :: SQL.Snippet -> SQL.Snippet
-    callIt argument | pgVer < pgVersion130 && pgVer >= pgVersion110 && returnsCompositeAlias = "(SELECT (" <> SQL.sql (fromQi qi) <> "(" <> argument <> ")).*) pgrst_call"
-                    | otherwise                                                              = SQL.sql (fromQi qi) <> "(" <> argument <> ") pgrst_call"
+    callIt argument | pgVer < pgVersion130 && pgVer >= pgVersion110 && returnsCompositeAlias = "(SELECT (" <> fromQi qi <> "(" <> argument <> ")).*) pgrst_call"
+                    | returnsScalar || returnsSetOfScalar                                    = "(SELECT " <> fromQi qi <> "(" <> argument <> ") pgrst_scalar) pgrst_call"
+                    | otherwise                                                              = fromQi qi <> "(" <> argument <> ") pgrst_call"

    fmtParams :: [RoutineParam] -> SQL.Snippet
-    fmtParams prms = SQL.sql $ BS.intercalate ", "
+    fmtParams prms = intercalateSnippet ", "
      ((\a -> (if ppVar a then "VARIADIC " else mempty) <> pgFmtIdent (ppName a) <> " := pgrst_body." <> pgFmtIdent (ppName a)) <$> prms)

    returnedColumns :: SQL.Snippet
    returnedColumns
      | null returnings = "*"
-      | otherwise       = SQL.sql $ BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty "pgrst_call") <$> returnings)
+      | otherwise       = intercalateSnippet ", " (pgFmtColumn (QualifiedIdentifier mempty "pgrst_call") <$> returnings)

 -- | SQL query meant for COUNTing the root node of the Tree.
 -- It only takes WHERE into account and doesn't include LIMIT/OFFSET because it would reduce the COUNT.
@@ -204,7 +230,7 @@ readPlanToCountQuery (Node ReadPlan{from=mainQi, fromAlias=tblAlias, where_=logi
    then mempty
    else " WHERE " ) <>
  intercalateSnippet " AND " (
-    map (pgFmtLogicTree qi) logicForest ++
+    map (pgFmtLogicTreeCount qi) logicForest ++
    map pgFmtJoinCondition relJoinConds ++
    subQueries
  )
@@ -217,6 +243,18 @@ readPlanToCountQuery (Node ReadPlan{from=mainQi, fromAlias=tblAlias, where_=logi
      if joinType == Just JTInner
        then ("EXISTS (" <> readPlanToCountQuery readReq <> " )"):rest
        else rest
+    findNullEmbedRel fld = find (\(Node ReadPlan{relAggAlias} _) -> fld == relAggAlias) forest
+
+    -- https://github.com/PostgREST/postgrest/pull/2930#discussion_r1325293698
+    pgFmtLogicTreeCount :: QualifiedIdentifier -> CoercibleLogicTree -> SQL.Snippet
+    pgFmtLogicTreeCount qiCount (CoercibleExpr hasNot op frst) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTreeCount qiCount <$> frst) <> ")"
+      where
+        notOp =  if hasNot then "NOT" else mempty
+        opSql And = " AND "
+        opSql Or  = " OR "
+    pgFmtLogicTreeCount _ (CoercibleStmnt (CoercibleFilterNullEmbed hasNot fld)) =
+      maybe mempty (\x -> (if not hasNot then "NOT " else mempty) <> "EXISTS (" <> readPlanToCountQuery x <> ")") (findNullEmbedRel fld)
+    pgFmtLogicTreeCount qiCount (CoercibleStmnt flt) = pgFmtFilter qiCount flt

 limitedQuery :: SQL.Snippet -> Maybe Integer -> SQL.Snippet
 limitedQuery query maxRows = query <> SQL.sql (maybe mempty (\x -> " LIMIT " <> BS.pack (show x)) maxRows)
@@ -229,10 +267,12 @@ getQualifiedIdentifier rel mainQi tblAlias = case rel of

 -- FROM clause plus implicit joins
 fromF :: Maybe Relationship -> QualifiedIdentifier -> Maybe Alias -> SQL.Snippet
-fromF rel mainQi tblAlias = SQL.sql $ "FROM " <>
+fromF rel mainQi tblAlias = "FROM " <>
  (case rel of
-    Just ComputedRelationship{relFunction,relTable} -> fromQi relFunction <> "(" <> pgFmtIdent (qiName relTable) <> ")"
-    _                                               -> fromQi mainQi) <>
+    -- Due to the use of CTEs on RPC, we need to cast the parameter to the table name in case of function overloading.
+    -- See https://github.com/PostgREST/postgrest/issues/2963#issuecomment-1736557386
+    Just ComputedRelationship{relFunction,relTableAlias,relTable} -> fromQi relFunction <> "(" <> pgFmtIdent (qiName relTableAlias) <> "::" <> fromQi relTable <> ")"
+    _                                                             -> fromQi mainQi) <>
  maybe mempty (\a -> " AS " <> pgFmtIdent a) tblAlias <>
  (case rel of
    Just Relationship{relCardinality=M2M Junction{junTable=jt}} -> ", " <> fromQi jt
@@ -4,42 +4,42 @@
 {-|
 Module      : PostgREST.Query.SqlFragment
 Description : Helper functions for PostgREST.QueryBuilder.
-
-Any function that outputs a SqlFragment should be in this module.
 -}
 module PostgREST.Query.SqlFragment
  ( noLocationF
-  , SqlFragment
-  , asBinaryF
-  , asCsvF
-  , asGeoJsonF
-  , asJsonF
-  , asJsonSingleF
-  , asXmlF
+  , handlerF
  , countF
+  , groupF
  , fromQi
  , limitOffsetF
  , locationF
  , mutRangeF
  , orderF
  , pgFmtColumn
+  , pgFmtFilter
  , pgFmtIdent
-  , pgFmtIdentList
  , pgFmtJoinCondition
  , pgFmtLogicTree
  , pgFmtOrderTerm
  , pgFmtSelectItem
+  , pgFmtSpreadSelectItem
  , fromJsonBodyF
  , responseHeadersF
  , responseStatusF
+  , addConfigPgrstInserted
+  , currentSettingF
  , returningF
  , singleParameter
+  , sourceCTE
  , sourceCTEName
  , unknownEncoder
  , intercalateSnippet
  , explainF
-  , setConfigLocal
-  , setConfigLocalJson
+  , setConfigWithConstantName
+  , setConfigWithDynamicName
+  , setConfigWithConstantNameJSON
+  , escapeIdent
+  , escapeIdentList
  ) where

 import qualified Data.Aeson                      as JSON
@@ -56,44 +56,54 @@ import Control.Arrow ((***))
 import Data.Foldable                 (foldr1)
 import Text.InterpolatedString.Perl6 (qc)

-import PostgREST.ApiRequest.Types        (Alias, Cast, Field,
-                                          Filter (..),
+import PostgREST.ApiRequest.Types        (AggregateFunction (..),
+                                          Alias, Cast,
                                          FtsOperator (..),
                                          JsonOperand (..),
                                          JsonOperation (..),
                                          JsonPath,
                                          LogicOperator (..),
-                                          LogicTree (..), OpExpr (..),
+                                          OpExpr (..),
                                          OpQuantifier (..),
                                          Operation (..),
                                          OrderDirection (..),
                                          OrderNulls (..),
-                                          OrderTerm (..),
                                          QuantOperator (..),
                                          SimpleOperator (..),
                                          TrileanVal (..))
-import PostgREST.MediaType               (MTPlanFormat (..),
-                                          MTPlanOption (..))
+import PostgREST.MediaType               (MTVndPlanFormat (..),
+                                          MTVndPlanOption (..))
 import PostgREST.Plan.ReadPlan           (JoinCondition (..))
-import PostgREST.Plan.Types              (TypedField (..))
+import PostgREST.Plan.Types              (CoercibleField (..),
+                                          CoercibleFilter (..),
+                                          CoercibleLogicTree (..),
+                                          CoercibleOrderTerm (..),
+                                          CoercibleSelectField (..),
+                                          RelSelectField (..),
+                                          SpreadSelectField (..),
+                                          unknownField)
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                          rangeLimit, rangeOffset)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                          QualifiedIdentifier (..))
+import PostgREST.SchemaCache.Routine     (MediaHandler (..),
+                                          Routine (..),
+                                          funcReturnsScalar,
+                                          funcReturnsSetOfScalar,
+                                          funcReturnsSingleComposite)

-import Protolude hiding (cast)
+import Protolude hiding (Sum, cast)

-
-- | A part of a SQL query that cannot be executed independently
-type SqlFragment = ByteString
-
-noLocationF :: SqlFragment
-noLocationF = "array[]::text[]"
-
-sourceCTEName :: SqlFragment
+sourceCTEName :: Text
 sourceCTEName = "pgrst_source"

-simpleOperator :: SimpleOperator -> SqlFragment
+sourceCTE :: SQL.Snippet
+sourceCTE = "pgrst_source"
+
+noLocationF :: SQL.Snippet
+noLocationF = "array[]::text[]"
+
+simpleOperator :: SimpleOperator -> SQL.Snippet
 simpleOperator = \case
  OpNotEqual        -> "<>"
  OpContains        -> "@>"
@@ -105,7 +115,7 @@ simpleOperator = \case
  OpNotExtendsLeft  -> "&>"
  OpAdjacent        -> "-|-"

-quantOperator :: QuantOperator -> SqlFragment
+quantOperator :: QuantOperator -> SQL.Snippet
 quantOperator = \case
  OpEqual            -> "="
  OpGreaterThanEqual -> ">="
@@ -117,7 +127,7 @@ quantOperator = \case
  OpMatch            -> "~"
  OpIMatch           -> "~*"

-ftsOperator :: FtsOperator -> SqlFragment
+ftsOperator :: FtsOperator -> SQL.Snippet
 ftsOperator = \case
  FilterFts          -> "@@ to_tsquery"
  FilterFtsPlain     -> "@@ plainto_tsquery"
@@ -145,8 +155,11 @@ pgBuildArrayLiteral vals =
 "{" <> T.intercalate "," (escaped <$> vals) <> "}"

 -- TODO: refactor by following https://github.com/PostgREST/postgrest/pull/1631#issuecomment-711070833
-pgFmtIdent :: Text -> SqlFragment
-pgFmtIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
+pgFmtIdent :: Text -> SQL.Snippet
+pgFmtIdent x = SQL.sql $ escapeIdent x
+
+escapeIdent :: Text -> ByteString
+escapeIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""

 -- Only use it if the input comes from the database itself, like on `jsonb_build_object('column_from_a_table', val)..`
 pgFmtLit :: Text -> Text
@@ -164,12 +177,12 @@ trimNullChars = T.takeWhile (/= '\x0')
 -- |
 -- Format a list of identifiers and separate them by commas.
 --
-- >>> pgFmtIdentList ["schema_1", "schema_2", "SPECIAL \"@/\\#~_-"]
+-- >>> escapeIdentList ["schema_1", "schema_2", "SPECIAL \"@/\\#~_-"]
 -- "\"schema_1\", \"schema_2\", \"SPECIAL \"\"@/\\#~_-\""
-pgFmtIdentList :: [Text] -> SqlFragment
-pgFmtIdentList schemas = BS.intercalate ", " $ pgFmtIdent <$> schemas
+escapeIdentList :: [Text] -> ByteString
+escapeIdentList schemas = BS.intercalate ", " $ escapeIdent <$> schemas

-asCsvF :: SqlFragment
+asCsvF :: SQL.Snippet
 asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
  where
    asCsvHeaderF =
@@ -177,34 +190,43 @@ asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
      "  FROM (" <>
      "    SELECT json_object_keys(r)::text as k" <>
      "    FROM ( " <>
-      "      SELECT row_to_json(hh) as r from " <> sourceCTEName <> " as hh limit 1" <>
+      "      SELECT row_to_json(hh) as r from " <> sourceCTE <> " as hh limit 1" <>
      "    ) s" <>
      "  ) a" <>
      ")"
    asCsvBodyF = "coalesce(string_agg(substring(_postgrest_t::text, 2, length(_postgrest_t::text) - 2), '\n'), '')"

-asJsonSingleF :: Bool -> SqlFragment
-asJsonSingleF returnsScalar
-  | returnsScalar = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
-  | otherwise     = "coalesce(json_agg(_postgrest_t)->0, 'null')"
+addNullsToSnip :: Bool -> SQL.Snippet -> SQL.Snippet
+addNullsToSnip strip snip =
+  if strip then "json_strip_nulls(" <> snip <> ")" else  snip

-asJsonF :: Bool -> Bool -> Bool -> SqlFragment
-asJsonF returnsScalar returnsSetOfScalar returnsSingleComposite
-  | returnsSingleComposite              = "coalesce(json_agg(_postgrest_t)->0, 'null')"
-  | returnsScalar                       = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
-  | returnsSetOfScalar                  = "coalesce(json_agg(_postgrest_t.pgrst_scalar), '[]')"
-  | otherwise                           = "coalesce(json_agg(_postgrest_t), '[]')"
+asJsonSingleF :: Maybe Routine -> Bool -> SQL.Snippet
+asJsonSingleF rout strip
+  | returnsScalar = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0"  <> ", 'null')"
+  | otherwise     = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0"  <> ", 'null')"
+  where
+    returnsScalar = maybe False funcReturnsScalar rout

-asXmlF :: FieldName -> SqlFragment
-asXmlF fieldName = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
+asJsonF :: Maybe Routine -> Bool -> SQL.Snippet
+asJsonF rout strip
+  | returnsSingleComposite    = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0" <> ", 'null')"
+  | returnsScalar             = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0" <> ", 'null')"
+  | returnsSetOfScalar        = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)" <> ", '[]')"
+  | otherwise                 = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)" <> ", '[]')"
+  where
+    (returnsSingleComposite, returnsScalar, returnsSetOfScalar) = case rout of
+      Just r  -> (funcReturnsSingleComposite r, funcReturnsScalar r, funcReturnsSetOfScalar r)
+      Nothing -> (False, False, False)

-asGeoJsonF ::  SqlFragment
+asGeoJsonF ::  SQL.Snippet
 asGeoJsonF = "json_build_object('type', 'FeatureCollection', 'features', coalesce(json_agg(ST_AsGeoJSON(_postgrest_t)::json), '[]'))"

-asBinaryF :: FieldName -> SqlFragment
-asBinaryF fieldName = "coalesce(string_agg(_postgrest_t." <> pgFmtIdent fieldName <> ", ''), '')"
+customFuncF :: Maybe Routine -> QualifiedIdentifier -> QualifiedIdentifier -> SQL.Snippet
+customFuncF rout funcQi target
+  | (funcReturnsScalar <$> rout) == Just True = fromQi funcQi <> "(_postgrest_t.pgrst_scalar)"
+  | otherwise                                 = fromQi funcQi <> "(_postgrest_t::" <> fromQi target <> ")"

-locationF :: [Text] -> SqlFragment
+locationF :: [Text] -> SQL.Snippet
 locationF pKeys = [qc|(
  WITH data AS (SELECT row_to_json(_) AS row FROM {sourceCTEName} AS _ LIMIT 1)
  SELECT array_agg(json_data.key || '=' || coalesce('eq.' || json_data.value, 'is.null'))
@@ -214,34 +236,67 @@ locationF pKeys = [qc|(
  where
    fmtPKeys = T.intercalate "','" pKeys

-fromQi :: QualifiedIdentifier -> SqlFragment
+fromQi :: QualifiedIdentifier -> SQL.Snippet
 fromQi t = (if T.null s then mempty else pgFmtIdent s <> ".") <> pgFmtIdent n
  where
    n = qiName t
    s = qiSchema t

-pgFmtColumn :: QualifiedIdentifier -> Text -> SqlFragment
+pgFmtColumn :: QualifiedIdentifier -> Text -> SQL.Snippet
 pgFmtColumn table "*" = fromQi table <> ".*"
 pgFmtColumn table c   = fromQi table <> "." <> pgFmtIdent c

-pgFmtField :: QualifiedIdentifier -> Field -> SQL.Snippet
-pgFmtField table (c, []) = SQL.sql (pgFmtColumn table c)
-- Using to_jsonb instead of to_json to avoid missing operator errors when filtering:
-- "operator does not exist: json = unknown"
-pgFmtField table (c, jp) = SQL.sql ("to_jsonb(" <> pgFmtColumn table c <> ")") <> pgFmtJsonPath jp
+pgFmtCallUnary :: Text -> SQL.Snippet -> SQL.Snippet
+pgFmtCallUnary f x = SQL.sql (encodeUtf8 f) <> "(" <> x <> ")"

-pgFmtSelectItem :: QualifiedIdentifier -> (Field, Maybe Cast, Maybe Alias) -> SQL.Snippet
-pgFmtSelectItem table (f@(fName, jp), Nothing, alias) = pgFmtField table f <> SQL.sql (pgFmtAs fName jp alias)
+pgFmtField :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
+pgFmtField table CoercibleField{cfName=fn, cfJsonPath=[]}                                = pgFmtColumn table fn
+pgFmtField table CoercibleField{cfName=fn, cfToJson=doToJson, cfJsonPath=jp} | doToJson  = "to_jsonb(" <> pgFmtColumn table fn <> ")" <> pgFmtJsonPath jp
+                                                                             | otherwise = pgFmtColumn table fn <> pgFmtJsonPath jp
+
+-- Select the value of a named element from a table, applying its optional coercion mapping if any.
+pgFmtTableCoerce :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
+pgFmtTableCoerce table fld@(CoercibleField{cfTransform=(Just formatterProc)}) = pgFmtCallUnary formatterProc (pgFmtField table fld)
+pgFmtTableCoerce table f = pgFmtField table f
+
+-- | Like the previous but now we just have a name so no namespace or JSON paths.
+pgFmtCoerceNamed :: CoercibleField -> SQL.Snippet
+pgFmtCoerceNamed CoercibleField{cfName=fn, cfTransform=(Just formatterProc)} = pgFmtCallUnary formatterProc (pgFmtIdent fn) <> " AS " <> pgFmtIdent fn
+pgFmtCoerceNamed CoercibleField{cfName=fn} = pgFmtIdent fn
+
+pgFmtSelectItem :: QualifiedIdentifier -> CoercibleSelectField -> SQL.Snippet
+pgFmtSelectItem table CoercibleSelectField{csField=fld, csAggFunction=agg, csAggCast=aggCast, csCast=cast, csAlias=alias} =
+  pgFmtApplyAggregate agg aggCast (pgFmtApplyCast cast (pgFmtTableCoerce table fld)) <> pgFmtAs alias
+
+pgFmtSpreadSelectItem :: Alias -> SpreadSelectField -> SQL.Snippet
+pgFmtSpreadSelectItem aggAlias SpreadSelectField{ssSelName, ssSelAggFunction, ssSelAggCast, ssSelAlias} =
+  pgFmtApplyAggregate ssSelAggFunction ssSelAggCast fullSelName <> pgFmtAs ssSelAlias
+  where
+    fullSelName = case ssSelName of
+      "*" -> pgFmtIdent aggAlias <> ".*"
+      _   -> pgFmtIdent aggAlias <> "." <> pgFmtIdent ssSelName
+
+pgFmtApplyAggregate :: Maybe AggregateFunction -> Maybe Cast -> SQL.Snippet -> SQL.Snippet
+pgFmtApplyAggregate Nothing _ snippet = snippet
+pgFmtApplyAggregate (Just agg) aggCast snippet =
+  pgFmtApplyCast aggCast aggregatedSnippet
+  where
+    convertAggFunction :: AggregateFunction -> SQL.Snippet
+    -- Convert from e.g. Sum (the data type) to SUM
+    convertAggFunction = SQL.sql . BS.map toUpper . BS.pack . show
+    aggregatedSnippet = convertAggFunction agg <> "(" <> snippet <> ")"
+
+pgFmtApplyCast :: Maybe Cast -> SQL.Snippet -> SQL.Snippet
+pgFmtApplyCast Nothing snippet = snippet
 -- Ideally we'd quote the cast with "pgFmtIdent cast". However, that would invalidate common casts such as "int", "bigint", etc.
 -- Try doing: `select 1::"bigint"` - it'll err, using "int8" will work though. There's some parser magic that pg does that's invalidated when quoting.
 -- Not quoting should be fine, we validate the input on Parsers.
-pgFmtSelectItem table (f@(fName, jp), Just cast, alias) = "CAST (" <> pgFmtField table f <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )" <> SQL.sql (pgFmtAs fName jp alias)
+pgFmtApplyCast (Just cast) snippet = "CAST( " <> snippet <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )"

 -- TODO: At this stage there shouldn't be a Maybe since ApiRequest should ensure that an INSERT/UPDATE has a body
-fromJsonBodyF :: Maybe LBS.ByteString -> [TypedField] -> Bool -> Bool -> Bool -> SQL.Snippet
+fromJsonBodyF :: Maybe LBS.ByteString -> [CoercibleField] -> Bool -> Bool -> Bool -> SQL.Snippet
 fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
-  SQL.sql
-  (if includeSelect then "SELECT " <> parsedCols <> " " else mempty) <>
+  (if includeSelect then "SELECT " <> namedCols <> " " else mempty) <>
  "FROM (SELECT " <> jsonPlaceHolder <> " AS json_data) pgrst_payload, " <>
  -- convert a json object into a json array, this way we can use json_to_recordset for all json payloads
  -- Otherwise we'd have to use json_to_record for json objects and json_to_recordset for json arrays
@@ -250,22 +305,23 @@ fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
  (if includeDefaults
    then "LATERAL (SELECT jsonb_agg(jsonb_build_object(" <> defsJsonb <> ") || elem) AS val from jsonb_array_elements(pgrst_uniform_json.val) elem) pgrst_json_defs, "
    else mempty) <>
-  "LATERAL (SELECT * FROM " <>
+  "LATERAL (SELECT " <> parsedCols <> " FROM " <>
    (if null fields
      -- When we are inserting no columns (e.g. using default values), we can't use our ordinary `json_to_recordset`
      -- because it can't extract records with no columns (there's no valid syntax for the `AS (colName colType,...)`
      -- part). But we still need to ensure as many rows are created as there are array elements.
      then SQL.sql $ jsonArrayElementsF <> "(" <> finalBodyF <> ") _ "
-      else SQL.sql $ jsonToRecordsetF <> "(" <> finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
+      else jsonToRecordsetF <> "(" <> SQL.sql finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
    ) <>
  ") pgrst_body "
  where
-    parsedCols = BS.intercalate ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . tfName <$> fields
-    typedCols = BS.intercalate ", " $ pgFmtIdent . tfName <> const " " <> encodeUtf8 . tfIRType <$> fields
+    namedCols = intercalateSnippet ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . cfName <$> fields
+    parsedCols = intercalateSnippet ", " $ pgFmtCoerceNamed <$> fields
+    typedCols = intercalateSnippet ", " $ pgFmtIdent . cfName <> const " " <> SQL.sql . encodeUtf8 . cfIRType <$> fields
    defsJsonb = SQL.sql $ BS.intercalate "," fieldsWDefaults
    fieldsWDefaults = mapMaybe (\case
-        TypedField{tfName=nam, tfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
-        TypedField{tfDefault=Nothing} -> Nothing
+        CoercibleField{cfName=nam, cfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
+        CoercibleField{cfDefault=Nothing} -> Nothing
      ) fields
    (finalBodyF, jsonTypeofF, jsonBuildArrayF, jsonArrayElementsF, jsonToRecordsetF) =
      if includeDefaults
@@ -273,16 +329,16 @@ fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
        else ("pgrst_uniform_json.val", "json_typeof", "json_build_array", "json_array_elements", "json_to_recordset")
    jsonPlaceHolder = SQL.encoderAndParam (HE.nullable $ if includeDefaults then HE.jsonbLazyBytes else HE.jsonLazyBytes) body

-pgFmtOrderTerm :: QualifiedIdentifier -> OrderTerm -> SQL.Snippet
+pgFmtOrderTerm :: QualifiedIdentifier -> CoercibleOrderTerm -> SQL.Snippet
 pgFmtOrderTerm qi ot =
  fmtOTerm ot <> " " <>
  SQL.sql (BS.unwords [
-    maybe mempty direction $ otDirection ot,
-    maybe mempty nullOrder $ otNullOrder ot])
+    maybe mempty direction $ coDirection ot,
+    maybe mempty nullOrder $ coNullOrder ot])
  where
    fmtOTerm = \case
-      OrderTerm{otTerm}                        -> pgFmtField qi otTerm
-      OrderRelationTerm{otRelation, otRelTerm} -> pgFmtField (QualifiedIdentifier mempty otRelation) otRelTerm
+      CoercibleOrderTerm{coField=cof}                            -> pgFmtField qi cof
+      CoercibleOrderRelationTerm{coRelation, coRelTerm=(fn, jp)} -> pgFmtField (QualifiedIdentifier mempty coRelation) (unknownField fn jp)

    direction OrderAsc  = "ASC"
    direction OrderDesc = "DESC"
@@ -290,17 +346,31 @@ pgFmtOrderTerm qi ot =
    nullOrder OrderNullsFirst = "NULLS FIRST"
    nullOrder OrderNullsLast  = "NULLS LAST"

+-- | Interpret a literal in the way the planner indicated through the CoercibleField.
+pgFmtUnknownLiteralForField :: SQL.Snippet -> CoercibleField -> SQL.Snippet
+pgFmtUnknownLiteralForField value CoercibleField{cfTransform=(Just parserProc)} = pgFmtCallUnary parserProc value
+-- But when no transform is requested, we just use the literal as-is.
+pgFmtUnknownLiteralForField value _ = value

-pgFmtFilter :: QualifiedIdentifier -> Filter -> SQL.Snippet
-pgFmtFilter _ (FilterNullEmbed hasNot fld) = SQL.sql (pgFmtIdent fld) <> " IS " <> (if hasNot then "NOT" else mempty) <> " NULL"
-pgFmtFilter _ (Filter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
-pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
-   Op op val -> " " <> SQL.sql (simpleOperator op) <> " " <> unknownLiteral val
+-- | Array version of the above, used by ANY().
+pgFmtArrayLiteralForField :: [Text] -> CoercibleField -> SQL.Snippet
+-- When a transformation is requested, we need to apply the transformation to each element of the array. This could be done by just making a query with `parser(value)` for each value, but may lead to huge query lengths. Imagine `data_representations.color_from_text('...'::text)` for repeated for a hundred values. Instead we use `unnest()` to unpack a standard array literal and then apply the transformation to each element, like a map.
+-- Note the literals will be treated as text since in every case when we use ANY() the parameters are textual (coming from a query string). We want to rely on the `text->domain` parser to do the right thing.
+pgFmtArrayLiteralForField values CoercibleField{cfTransform=(Just parserProc)} = SQL.sql "(SELECT " <> pgFmtCallUnary parserProc (SQL.sql "unnest(" <> unknownLiteral (pgBuildArrayLiteral values) <> "::text[])") <> ")"
+-- When no transformation is requested, we don't need a subquery.
+pgFmtArrayLiteralForField values _ = unknownLiteral (pgBuildArrayLiteral values)

-   OpQuant op quant val -> " " <> SQL.sql (quantOperator op) <> " " <> case op of
+
+pgFmtFilter :: QualifiedIdentifier -> CoercibleFilter -> SQL.Snippet
+pgFmtFilter _ (CoercibleFilterNullEmbed hasNot fld) = pgFmtIdent fld <> " IS " <> (if not hasNot then "NOT " else mempty) <> "DISTINCT FROM NULL"
+pgFmtFilter _ (CoercibleFilter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
+pgFmtFilter table (CoercibleFilter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
+   Op op val  -> " " <> simpleOperator op <> " " <> pgFmtUnknownLiteralForField (unknownLiteral val) fld
+
+   OpQuant op quant val -> " " <> quantOperator op <> " " <> case op of
     OpLike  -> fmtQuant quant $ unknownLiteral (T.map star val)
     OpILike -> fmtQuant quant $ unknownLiteral (T.map star val)
-     _       -> fmtQuant quant $ unknownLiteral val
+     _       -> fmtQuant quant $ pgFmtUnknownLiteralForField (unknownLiteral val) fld

   -- IS cannot be prepared. `PREPARE boolplan AS SELECT * FROM projects where id IS $1` will give a syntax error.
   -- The above can be fixed by using `PREPARE boolplan AS SELECT * FROM projects where id IS NOT DISTINCT FROM $1;`
@@ -319,9 +389,9 @@ pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField
   -- + Can invalidate prepared statements: multiple parameters on an IN($1, $2, $3) will lead to using different prepared statements and not take advantage of caching.
   In vals -> " " <> case vals of
      [""] -> "= ANY('{}') "
-      _    -> "= ANY (" <> unknownLiteral (pgBuildArrayLiteral vals) <> ") "
+      _    -> "= ANY (" <> pgFmtArrayLiteralForField vals fld <> ") "

-   Fts op lang val -> " " <> SQL.sql (ftsOperator op) <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
+   Fts op lang val -> " " <> ftsOperator op <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
 where
   ftsLang = maybe mempty (\l -> unknownLiteral l <> ", ")
   notOp = if hasNot then "NOT" else mempty
@@ -333,16 +403,16 @@ pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField

 pgFmtJoinCondition :: JoinCondition -> SQL.Snippet
 pgFmtJoinCondition (JoinCondition (qi1, col1) (qi2, col2)) =
-  SQL.sql $ pgFmtColumn qi1 col1 <> " = " <> pgFmtColumn qi2 col2
+  pgFmtColumn qi1 col1 <> " = " <> pgFmtColumn qi2 col2

-pgFmtLogicTree :: QualifiedIdentifier -> LogicTree -> SQL.Snippet
-pgFmtLogicTree qi (Expr hasNot op forest) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTree qi <$> forest) <> ")"
+pgFmtLogicTree :: QualifiedIdentifier -> CoercibleLogicTree -> SQL.Snippet
+pgFmtLogicTree qi (CoercibleExpr hasNot op forest) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTree qi <$> forest) <> ")"
  where
    notOp =  if hasNot then "NOT" else mempty

    opSql And = " AND "
    opSql Or  = " OR "
-pgFmtLogicTree qi (Stmnt flt) = pgFmtFilter qi flt
+pgFmtLogicTree qi (CoercibleStmnt flt) = pgFmtFilter qi flt

 pgFmtJsonPath :: JsonPath -> SQL.Snippet
 pgFmtJsonPath = \case
@@ -353,19 +423,42 @@ pgFmtJsonPath = \case
    pgFmtJsonOperand (JKey k) = unknownLiteral k
    pgFmtJsonOperand (JIdx i) = unknownLiteral i <> "::int"

-pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SqlFragment
-pgFmtAs _ [] Nothing = mempty
-pgFmtAs fName jp Nothing = case jOp <$> lastMay jp of
-  Just (JKey key) -> " AS " <> pgFmtIdent key
-  Just (JIdx _)   -> " AS " <> pgFmtIdent (fromMaybe fName lastKey)
-    -- We get the lastKey because on:
-    -- `select=data->1->mycol->>2`, we need to show the result as [ {"mycol": ..}, {"mycol": ..} ]
-    -- `select=data->3`, we need to show the result as [ {"data": ..}, {"data": ..} ]
-    where lastKey = jVal <$> find (\case JKey{} -> True; _ -> False) (jOp <$> reverse jp)
-  Nothing -> mempty
-pgFmtAs _ _ (Just alias) = " AS " <> pgFmtIdent alias
+pgFmtAs :: Maybe Alias -> SQL.Snippet
+pgFmtAs Nothing      = mempty
+pgFmtAs (Just alias) = " AS " <> pgFmtIdent alias

-countF :: SQL.Snippet -> Bool -> (SQL.Snippet, SqlFragment)
+groupF :: QualifiedIdentifier -> [CoercibleSelectField] -> [RelSelectField] -> SQL.Snippet
+groupF qi select relSelect
+  | (noSelectsAreAggregated && noRelSelectsAreAggregated) || null groupTerms = mempty
+  | otherwise = " GROUP BY " <> intercalateSnippet ", " groupTerms
+  where
+    noSelectsAreAggregated = null $ [s | s@(CoercibleSelectField { csAggFunction = Just _ }) <- select]
+    noRelSelectsAreAggregated = all (\case Spread sels _ -> all (isNothing . ssSelAggFunction) sels; _ -> True) relSelect
+    groupTermsFromSelect = mapMaybe (pgFmtGroup qi) select
+    groupTermsFromRelSelect = mapMaybe groupTermFromRelSelectField relSelect
+    groupTerms = groupTermsFromSelect ++ groupTermsFromRelSelect
+
+groupTermFromRelSelectField :: RelSelectField -> Maybe SQL.Snippet
+groupTermFromRelSelectField (JsonEmbed { rsSelName }) =
+  Just $ pgFmtIdent rsSelName
+groupTermFromRelSelectField (Spread { rsSpreadSel, rsAggAlias }) =
+  if null groupTerms
+  then Nothing
+  else
+    Just $ intercalateSnippet ", " groupTerms
+  where
+    processField :: SpreadSelectField -> Maybe SQL.Snippet
+    processField SpreadSelectField{ssSelAggFunction = Just _} = Nothing
+    processField SpreadSelectField{ssSelName, ssSelAlias} =
+      Just $ pgFmtIdent rsAggAlias <> "." <> pgFmtIdent (fromMaybe ssSelName ssSelAlias)
+    groupTerms = mapMaybe processField rsSpreadSel
+
+pgFmtGroup :: QualifiedIdentifier -> CoercibleSelectField -> Maybe SQL.Snippet
+pgFmtGroup _  CoercibleSelectField{csAggFunction=Just _} = Nothing
+pgFmtGroup _  CoercibleSelectField{csAlias=Just alias, csAggFunction=Nothing} = Just $ pgFmtIdent alias
+pgFmtGroup qi CoercibleSelectField{csField=fld, csAlias=Nothing, csAggFunction=Nothing} = Just $ pgFmtField qi fld
+
+countF :: SQL.Snippet -> Bool -> (SQL.Snippet, SQL.Snippet)
 countF countQuery shouldCount =
  if shouldCount
    then (
@@ -375,11 +468,11 @@ countF countQuery shouldCount =
        mempty
      , "null::bigint")

-returningF :: QualifiedIdentifier -> [FieldName] -> SqlFragment
+returningF :: QualifiedIdentifier -> [FieldName] -> SQL.Snippet
 returningF qi returnings =
  if null returnings
    then "RETURNING 1" -- For mutation cases where there's no ?select, we return 1 to know how many rows were modified
-    else "RETURNING " <> BS.intercalate ", " (pgFmtColumn qi <$> returnings)
+    else "RETURNING " <> intercalateSnippet ", " (pgFmtColumn qi <$> returnings)

 limitOffsetF :: NonnegRange -> SQL.Snippet
 limitOffsetF range =
@@ -388,25 +481,30 @@ limitOffsetF range =
    limit = maybe "ALL" (\l -> unknownEncoder (BS.pack $ show l)) $ rangeLimit range
    offset = unknownEncoder (BS.pack . show $ rangeOffset range)

-responseHeadersF :: SqlFragment
+responseHeadersF :: SQL.Snippet
 responseHeadersF = currentSettingF "response.headers"

-responseStatusF :: SqlFragment
+responseStatusF :: SQL.Snippet
 responseStatusF = currentSettingF "response.status"

-currentSettingF :: SqlFragment -> SqlFragment
+addConfigPgrstInserted :: Bool -> SQL.Snippet
+addConfigPgrstInserted add =
+  let (symbol, num) =  if add then ("+", "0") else ("-", "-1") in
+  "set_config('pgrst.inserted', (coalesce(" <> currentSettingF "pgrst.inserted" <> "::int, 0) " <> symbol <> " 1)::text, true) <> '" <> num <> "'"
+
+currentSettingF :: SQL.Snippet -> SQL.Snippet
 currentSettingF setting =
  -- nullif is used because of https://gist.github.com/steve-chavez/8d7033ea5655096903f3b52f8ed09a15
  "nullif(current_setting('" <> setting <> "', true), '')"

-mutRangeF :: QualifiedIdentifier -> [FieldName] -> (SqlFragment, SqlFragment)
+mutRangeF :: QualifiedIdentifier -> [FieldName] -> (SQL.Snippet, SQL.Snippet)
 mutRangeF mainQi rangeId =
  (
-    BS.intercalate " AND " $ (\col -> pgFmtColumn mainQi col <> " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_affected_rows") col) <$> rangeId
-  , BS.intercalate ", " (pgFmtColumn mainQi <$> rangeId)
+    intercalateSnippet " AND " $ (\col -> pgFmtColumn mainQi col <> " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_affected_rows") col) <$> rangeId
+  , intercalateSnippet ", " (pgFmtColumn mainQi <$> rangeId)
  )

-orderF :: QualifiedIdentifier -> [OrderTerm] -> SQL.Snippet
+orderF :: QualifiedIdentifier -> [CoercibleOrderTerm] -> SQL.Snippet
 orderF _ []    = mempty
 orderF qi ordts = "ORDER BY " <> intercalateSnippet ", " (pgFmtOrderTerm qi <$> ordts)

@@ -421,33 +519,52 @@ intercalateSnippet :: ByteString -> [SQL.Snippet] -> SQL.Snippet
 intercalateSnippet _ [] = mempty
 intercalateSnippet frag snippets = foldr1 (\a b -> a <> SQL.sql frag <> b) snippets

-explainF :: MTPlanFormat -> [MTPlanOption] -> SQL.Snippet -> SQL.Snippet
+explainF :: MTVndPlanFormat -> [MTVndPlanOption] -> SQL.Snippet -> SQL.Snippet
 explainF fmt opts snip =
  "EXPLAIN (" <>
    SQL.sql (BS.intercalate ", " (fmtPlanFmt fmt : (fmtPlanOpt <$> opts))) <>
  ") " <> snip
  where
-    fmtPlanOpt :: MTPlanOption -> BS.ByteString
+    fmtPlanOpt :: MTVndPlanOption -> BS.ByteString
    fmtPlanOpt PlanAnalyze  = "ANALYZE"
    fmtPlanOpt PlanVerbose  = "VERBOSE"
    fmtPlanOpt PlanSettings = "SETTINGS"
    fmtPlanOpt PlanBuffers  = "BUFFERS"
    fmtPlanOpt PlanWAL      = "WAL"

-    fmtPlanFmt PlanJSON = "FORMAT JSON"
    fmtPlanFmt PlanText = "FORMAT TEXT"
+    fmtPlanFmt PlanJSON = "FORMAT JSON"

 -- | Do a pg set_config(setting, value, true) call. This is equivalent to a SET LOCAL.
-setConfigLocal :: ByteString -> (ByteString, ByteString) -> SQL.Snippet
-setConfigLocal prefix (k, v) =
-  "set_config(" <> unknownEncoder (prefix <> k) <> ", " <> unknownEncoder v <> ", true)"
+setConfigLocal :: (SQL.Snippet, ByteString) -> SQL.Snippet
+setConfigLocal (k, v) =
+  "set_config(" <> k <> ", " <> unknownEncoder v <> ", true)"
+
+-- | For when the settings are hardcoded and not parameterized
+setConfigWithConstantName :: (SQL.Snippet, ByteString) -> SQL.Snippet
+setConfigWithConstantName (k, v) = setConfigLocal ("'" <> k <> "'", v)
+
+-- | For when the settings need to be parameterized
+setConfigWithDynamicName :: (ByteString, ByteString) -> SQL.Snippet
+setConfigWithDynamicName (k, v) =
+  setConfigLocal (unknownEncoder k, v)

 -- | Starting from PostgreSQL v14, some characters are not allowed for config names (mostly affecting headers with "-").
 -- | A JSON format string is used to avoid this problem. See https://github.com/PostgREST/postgrest/issues/1857
-setConfigLocalJson :: ByteString -> [(ByteString, ByteString)] -> [SQL.Snippet]
-setConfigLocalJson prefix keyVals = [setConfigLocal mempty (prefix, gucJsonVal keyVals)]
+setConfigWithConstantNameJSON :: SQL.Snippet -> [(ByteString, ByteString)] -> [SQL.Snippet]
+setConfigWithConstantNameJSON prefix keyVals = [setConfigWithConstantName (prefix, gucJsonVal keyVals)]
  where
    gucJsonVal :: [(ByteString, ByteString)] -> ByteString
    gucJsonVal = LBS.toStrict . JSON.encode . HM.fromList . arrayByteStringToText
    arrayByteStringToText :: [(ByteString, ByteString)] -> [(Text,Text)]
    arrayByteStringToText keyVal = (T.decodeUtf8 *** T.decodeUtf8) <$> keyVal
+
+handlerF :: Maybe Routine -> QualifiedIdentifier -> MediaHandler -> SQL.Snippet
+handlerF rout target = \case
+  BuiltinAggArrayJsonStrip   -> asJsonF rout True
+  BuiltinAggSingleJson strip -> asJsonSingleF rout strip
+  BuiltinOvAggJson           -> asJsonF rout False
+  BuiltinOvAggGeoJson        -> asGeoJsonF
+  BuiltinOvAggCsv            -> asCsvF
+  CustomFunc funcQi          -> customFuncF rout funcQi target
+  NoAgg                      -> "''::text"
@@ -23,15 +23,14 @@ import qualified Hasql.DynamicStatements.Statement as SQL
 import qualified Hasql.Statement                   as SQL

 import Control.Lens ((^?))
-import Data.Maybe   (fromJust)

 import PostgREST.ApiRequest.Preferences
-import PostgREST.MediaType               (MTPlanAttrs (..),
-                                          MTPlanFormat (..),
-                                          MediaType (..),
-                                          getMediaType)
+import PostgREST.MediaType               (MTVndPlanFormat (..),
+                                          MediaType (..))
 import PostgREST.Query.SqlFragment
-import PostgREST.SchemaCache.Identifiers (FieldName)
+import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier)
+import PostgREST.SchemaCache.Routine     (MediaHandler (..), Routine,
+                                          funcReturnsSingle)

 import Protolude

@@ -51,117 +50,99 @@ data ResultSet
  -- ^ the HTTP headers to be added to the response
  , rsGucStatus  :: Maybe Text
  -- ^ the HTTP status to be added to the response
+  , rsInserted   :: Maybe Int64
+  -- ^ the number of rows inserted (Only used for upserts)
  }
  | RSPlan BS.ByteString -- ^ the plan of the query


-prepareWrite :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType ->
-                PreferRepresentation -> [Text] -> Bool -> SQL.Statement () ResultSet
-prepareWrite selectQuery mutateQuery isInsert mt rep pKeys =
+prepareWrite :: QualifiedIdentifier -> SQL.Snippet -> SQL.Snippet -> Bool -> Bool -> MediaType -> MediaHandler ->
+                Maybe PreferRepresentation -> Maybe PreferResolution -> [Text] -> Bool -> SQL.Statement () ResultSet
+prepareWrite qi selectQuery mutateQuery isInsert isPut mt handler rep resolution pKeys =
  SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
 where
+  checkUpsert snip = if isInsert && (isPut || resolution == Just MergeDuplicates) then snip else "''"
+  pgrstInsertedF = checkUpsert "nullif(current_setting('pgrst.inserted', true),'')::int"
  snippet =
-    "WITH " <> SQL.sql sourceCTEName <> " AS (" <> mutateQuery <> ") " <>
-    SQL.sql (
+    "WITH " <> sourceCTE <> " AS (" <> mutateQuery <> ") " <>
    "SELECT " <>
      "'' AS total_result_set, " <>
      "pg_catalog.count(_postgrest_t) AS page_total, " <>
      locF <> " AS header, " <>
-      bodyF <> " AS body, " <>
+      handlerF Nothing qi handler <> " AS body, " <>
      responseHeadersF <> " AS response_headers, " <>
-      responseStatusF  <> " AS response_status "
-    ) <>
+      responseStatusF  <> " AS response_status, " <>
+      pgrstInsertedF <> " AS response_inserted " <>
    "FROM (" <> selectF <> ") _postgrest_t"

  locF =
-    if isInsert && rep == HeadersOnly
-      then BS.unwords [
-        "CASE WHEN pg_catalog.count(_postgrest_t) = 1",
-          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ")",
-          "ELSE " <> noLocationF,
-        "END"]
+    if isInsert && rep == Just HeadersOnly
+      then
+        "CASE WHEN pg_catalog.count(_postgrest_t) = 1 " <>
+          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ") " <>
+          "ELSE " <> noLocationF <> " " <>
+        "END"
      else noLocationF

-  bodyF
-    | rep /= Full                       = "''"
-    | getMediaType mt == MTTextCSV      = asCsvF
-    | getMediaType mt == MTGeoJSON      = asGeoJsonF
-    | getMediaType mt == MTSingularJSON = asJsonSingleF False
-    | otherwise                         = asJsonF False False False
-
  selectF
    -- prevent using any of the column names in ?select= when no response is returned from the CTE
-    | rep /= Full = SQL.sql ("SELECT * FROM " <> sourceCTEName)
-    | otherwise   = selectQuery
+    | handler == NoAgg = "SELECT * FROM " <> sourceCTE
+    | otherwise        = selectQuery

  decodeIt :: HD.Result ResultSet
  decodeIt = case mt of
-    MTPlan{} -> planRow
-    _        -> fromMaybe (RSStandard Nothing 0 mempty mempty Nothing Nothing) <$> HD.rowMaybe (standardRow False)
+    MTVndPlan{} -> planRow
+    _           -> fromMaybe (RSStandard Nothing 0 mempty mempty Nothing Nothing Nothing) <$> HD.rowMaybe (standardRow False)

-prepareRead :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> Maybe FieldName -> Bool -> SQL.Statement () ResultSet
-prepareRead selectQuery countQuery countTotal mt binaryField =
+prepareRead :: QualifiedIdentifier -> SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> MediaHandler -> Bool -> SQL.Statement () ResultSet
+prepareRead qi selectQuery countQuery countTotal mt handler =
  SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
 where
  snippet =
-    "WITH " <>
-    SQL.sql sourceCTEName <> " AS ( " <> selectQuery <> " ) " <>
+    "WITH " <> sourceCTE <> " AS ( " <> selectQuery <> " ) " <>
    countCTEF <> " " <>
-    SQL.sql ("SELECT " <>
+    "SELECT " <>
      countResultF <> " AS total_result_set, " <>
      "pg_catalog.count(_postgrest_t) AS page_total, " <>
-      bodyF <> " AS body, " <>
+      handlerF Nothing qi handler <> " AS body, " <>
      responseHeadersF <> " AS response_headers, " <>
-      responseStatusF <> " AS response_status " <>
-    "FROM ( SELECT * FROM " <> sourceCTEName <> " ) _postgrest_t")
+      responseStatusF <> " AS response_status, " <>
+      "''" <> " AS response_inserted " <>
+    "FROM ( SELECT * FROM " <> sourceCTE <> " ) _postgrest_t"

  (countCTEF, countResultF) = countF countQuery countTotal

-  bodyF
-    | getMediaType mt == MTTextCSV                       = asCsvF
-    | getMediaType mt == MTSingularJSON                  = asJsonSingleF False
-    | getMediaType mt == MTGeoJSON                       = asGeoJsonF
-    | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
-    | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-    | otherwise                                          = asJsonF False False False
-
  decodeIt :: HD.Result ResultSet
  decodeIt = case mt of
-    MTPlan{} -> planRow
-    _        -> HD.singleRow $ standardRow True
+    MTVndPlan{} -> planRow
+    _           -> HD.singleRow $ standardRow True

-prepareCall :: Bool -> Bool -> Bool -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
-               MediaType -> Maybe FieldName -> Bool ->
+prepareCall :: QualifiedIdentifier -> Routine -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
+               MediaType -> MediaHandler -> Bool ->
               SQL.Statement () ResultSet
-prepareCall returnsScalar returnsSingleComposite returnsSetOfScalar callProcQuery selectQuery countQuery countTotal mt binaryField =
+prepareCall qi rout callProcQuery selectQuery countQuery countTotal mt handler =
  SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
  where
    snippet =
-      "WITH " <> SQL.sql sourceCTEName <> " AS (" <> callProcQuery <> ") " <>
+      "WITH " <> sourceCTE <> " AS (" <> callProcQuery <> ") " <>
      countCTEF <>
-      SQL.sql (
      "SELECT " <>
        countResultF <> " AS total_result_set, " <>
-        "pg_catalog.count(_postgrest_t) AS page_total, " <>
-        bodyF <> " AS body, " <>
+        (if funcReturnsSingle rout
+          then "1"
+          else "pg_catalog.count(_postgrest_t)") <> " AS page_total, " <>
+        handlerF (Just rout) qi handler <> " AS body, " <>
        responseHeadersF <> " AS response_headers, " <>
-        responseStatusF <> " AS response_status ") <>
+        responseStatusF <> " AS response_status, " <>
+        "''" <> " AS response_inserted " <>
      "FROM (" <> selectQuery <> ") _postgrest_t"

    (countCTEF, countResultF) = countF countQuery countTotal

-    bodyF
-     | getMediaType mt == MTSingularJSON                  = asJsonSingleF returnsScalar
-     | getMediaType mt == MTTextCSV                       = asCsvF
-     | getMediaType mt == MTGeoJSON                       = asGeoJsonF
-     | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
-     | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-     | otherwise                                          = asJsonF returnsScalar returnsSetOfScalar returnsSingleComposite
-
    decodeIt :: HD.Result ResultSet
    decodeIt = case mt of
-      MTPlan{} -> planRow
-      _        -> fromMaybe (RSStandard (Just 0) 0 mempty mempty Nothing Nothing) <$> HD.rowMaybe (standardRow True)
+      MTVndPlan{} -> planRow
+      _           -> fromMaybe (RSStandard (Just 0) 0 mempty mempty Nothing Nothing Nothing) <$> HD.rowMaybe (standardRow True)

 preparePlanRows :: SQL.Snippet -> Bool -> SQL.Statement () (Maybe Int64)
 preparePlanRows countQuery =
@@ -179,6 +160,7 @@ standardRow noLocation =
             <*> (if noLocation then pure mempty else fmap splitKeyValue <$> arrayColumn HD.bytea) <*> column HD.bytea
             <*> nullableColumn HD.bytea
             <*> nullableColumn HD.text
+             <*> nullableColumn HD.int8
  where
    splitKeyValue :: ByteString -> (ByteString, ByteString)
    splitKeyValue kv =
@@ -187,8 +169,8 @@ standardRow noLocation =

 mtSnippet :: MediaType -> SQL.Snippet -> SQL.Snippet
 mtSnippet mediaType snippet = case mediaType of
-  MTPlan (MTPlanAttrs _ fmt opts) -> explainF fmt opts snippet
-  _                               -> snippet
+  MTVndPlan _ fmt opts -> explainF fmt opts snippet
+  _                    -> snippet

 -- | We use rowList because when doing EXPLAIN (FORMAT TEXT), the result comes as many rows. FORMAT JSON comes as one.
 planRow :: HD.Result ResultSet
@@ -1,3 +1,7 @@
+{- |
+   Module      : PostgREST.Response
+   Description : Generate HTTP Response
+-}
 {-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 module PostgREST.Response
@@ -11,22 +15,18 @@ module PostgREST.Response
  , readResponse
  , singleUpsertResponse
  , updateResponse
-  , addRetryHint
-  , isServiceUnavailable
-  , optionalRollback
-  , traceHeaderMiddleware
+  , PgrstResponse(..)
  ) where

 import qualified Data.Aeson                as JSON
 import qualified Data.ByteString.Char8     as BS
 import qualified Data.ByteString.Lazy      as LBS
 import qualified Data.HashMap.Strict       as HM
-import qualified Data.List                 as L
+import           Data.Maybe                (fromJust)
 import           Data.Text.Read            (decimal)
 import qualified Network.HTTP.Types.Header as HTTP
 import qualified Network.HTTP.Types.Status as HTTP
 import qualified Network.HTTP.Types.URI    as HTTP
-import qualified Network.Wai               as Wai

 import qualified PostgREST.Error            as Error
 import qualified PostgREST.MediaType        as MediaType
@@ -36,14 +36,16 @@ import qualified PostgREST.Response.OpenAPI as OpenAPI
 import PostgREST.ApiRequest              (ApiRequest (..),
                                          InvokeMethod (..))
 import PostgREST.ApiRequest.Preferences  (PreferRepresentation (..),
-                                          PreferTransaction (..),
+                                          PreferResolution (..),
                                          Preferences (..),
-                                          shouldCount,
-                                          toAppliedHeader)
+                                          prefAppliedHeader,
+                                          shouldCount)
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
 import PostgREST.Config                  (AppConfig (..))
 import PostgREST.MediaType               (MediaType (..))
-import PostgREST.Plan                    (MutateReadPlan (..))
+import PostgREST.Plan                    (CallReadPlan (..),
+                                          MutateReadPlan (..),
+                                          WrappedReadPlan (..))
 import PostgREST.Plan.MutatePlan         (MutatePlan (..))
 import PostgREST.Query.Statements        (ResultSet (..))
 import PostgREST.Response.GucHeader      (GucHeader, unwrapGucHeader)
@@ -60,37 +62,50 @@ import qualified PostgREST.SchemaCache.Routine as Routine
 import Protolude      hiding (Handler, toS)
 import Protolude.Conv (toS)

-readResponse :: Bool -> QualifiedIdentifier -> ApiRequest -> ResultSet -> Wai.Response
-readResponse headersOnly identifier ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
-  RSStandard{..} -> do
-    let
-      (status, contentRange) = RangeQuery.rangeStatusHeader iTopLevelRange rsQueryTotal rsTableTotal
-      response = gucResponse rsGucStatus rsGucHeaders
-      headers =
-        [ contentRange
-        , ( "Content-Location"
-          , "/"
-              <> toUtf8 (qiName identifier)
-              <> if BS.null (qsCanonical iQueryParams) then mempty else "?" <> qsCanonical iQueryParams
-          )
-        ]
-        ++ contentTypeHeaders ctxApiRequest
-      rsOrErrBody = if status == HTTP.status416
-        then Error.errorPayload $ Error.ApiRequestError $ ApiRequestTypes.InvalidRange
-          $ ApiRequestTypes.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
-        else LBS.fromStrict rsBody
+data PgrstResponse = PgrstResponse {
+  pgrstStatus  :: HTTP.Status
+, pgrstHeaders :: [HTTP.Header]
+, pgrstBody    :: LBS.ByteString
+}

-    response status headers $ if headersOnly then mempty else rsOrErrBody
+readResponse :: WrappedReadPlan -> Bool -> QualifiedIdentifier -> ApiRequest -> ResultSet -> Either Error.Error PgrstResponse
+readResponse WrappedReadPlan{wrMedia} headersOnly identifier ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} resultSet =
+  case resultSet of
+    RSStandard{..} -> do
+      let
+        (status, contentRange) = RangeQuery.rangeStatusHeader iTopLevelRange rsQueryTotal rsTableTotal
+        prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing Nothing Nothing preferCount preferTransaction Nothing preferHandling preferTimezone []
+        headers =
+          [ contentRange
+          , ( "Content-Location"
+            , "/"
+                <> toUtf8 (qiName identifier)
+                <> if BS.null (qsCanonical iQueryParams) then mempty else "?" <> qsCanonical iQueryParams
+            )
+          ]
+          ++ contentTypeHeaders wrMedia ctxApiRequest
+          ++ prefHeader

-  RSPlan plan ->
-    Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
+      (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers

-createResponse :: QualifiedIdentifier -> MutateReadPlan -> ApiRequest -> ResultSet -> Wai.Response
-createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}, ..} resultSet = case resultSet of
+      let bod | status == HTTP.status416 = Error.errorPayload $ Error.ApiRequestError $ ApiRequestTypes.InvalidRange $
+                                           ApiRequestTypes.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
+              | headersOnly              = mempty
+              | otherwise                = LBS.fromStrict rsBody
+
+      Right $ PgrstResponse ovStatus ovHeaders bod
+
+    RSPlan plan ->
+      Right $ PgrstResponse HTTP.status200 (contentTypeHeaders wrMedia ctxApiRequest) $ LBS.fromStrict plan
+
+createResponse :: QualifiedIdentifier -> MutateReadPlan -> ApiRequest -> ResultSet -> Either Error.Error PgrstResponse
+createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan, mrMedia} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}, ..} resultSet = case resultSet of
  RSStandard{..} -> do
    let
      pkCols = case mrMutatePlan of { Insert{insPkCols} -> insPkCols; _ -> mempty;}
-      response = gucResponse rsGucStatus rsGucHeaders
+      prefHeader = prefAppliedHeader $
+        Preferences (if null pkCols && isNothing (qsOnConflict iQueryParams) then Nothing else preferResolution)
+        preferRepresentation Nothing preferCount preferTransaction preferMissing preferHandling preferTimezone []
      headers =
        catMaybes
          [ if null rsLocation then
@@ -104,80 +119,98 @@ createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiReques
                )
          , Just . RangeQuery.contentRangeH 1 0 $
              if shouldCount preferCount then Just rsQueryTotal else Nothing
-          , if null pkCols && isNothing (qsOnConflict iQueryParams) then
-              Nothing
-            else
-              toAppliedHeader <$> preferResolution
-          , toAppliedHeader <$> preferMissing
-          ]
+          , prefHeader ]

-    if preferRepresentation == Full then
-      response HTTP.status201 (headers ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
-    else
-      response HTTP.status201 headers mempty
+    let isInsertIfGTZero i =
+          if i <= 0 && preferResolution == Just MergeDuplicates then
+            HTTP.status200
+          else
+            HTTP.status201
+        status = maybe HTTP.status200 isInsertIfGTZero rsInserted
+        (headers', bod) = case preferRepresentation of
+          Just Full -> (headers ++ contentTypeHeaders mrMedia ctxApiRequest, LBS.fromStrict rsBody)
+          Just None -> (headers, mempty)
+          Just HeadersOnly -> (headers, mempty)
+          Nothing -> (headers, mempty)

+    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers'
+
+    Right $ PgrstResponse ovStatus ovHeaders bod
  RSPlan plan ->
-    Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
+    Right $ PgrstResponse HTTP.status200 (contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan

-updateResponse :: ApiRequest -> ResultSet -> Wai.Response
-updateResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
+updateResponse :: MutateReadPlan -> ApiRequest -> ResultSet -> Either Error.Error PgrstResponse
+updateResponse MutateReadPlan{mrMedia} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
  RSStandard{..} -> do
    let
-      response = gucResponse rsGucStatus rsGucHeaders
      contentRangeHeader =
        Just . RangeQuery.contentRangeH 0 (rsQueryTotal - 1) $
          if shouldCount preferCount then Just rsQueryTotal else Nothing
-      headers = catMaybes [contentRangeHeader, toAppliedHeader <$> preferMissing]
+      prefHeader = prefAppliedHeader $ Preferences Nothing preferRepresentation Nothing preferCount preferTransaction preferMissing preferHandling preferTimezone []
+      headers = catMaybes [contentRangeHeader, prefHeader]

-    if preferRepresentation == Full then
-        response HTTP.status200
-          (headers ++ contentTypeHeaders ctxApiRequest)
-          (LBS.fromStrict rsBody)
-      else
-        response HTTP.status204 headers mempty
+    let (status, headers', body) =
+          case preferRepresentation of
+            Just Full -> (HTTP.status200, headers ++ contentTypeHeaders mrMedia ctxApiRequest, LBS.fromStrict rsBody)
+            Just None -> (HTTP.status204, headers, mempty)
+            _         -> (HTTP.status204, headers, mempty)
+
+    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers'
+
+    Right $ PgrstResponse ovStatus ovHeaders body

  RSPlan plan ->
-    Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
+    Right $ PgrstResponse HTTP.status200 (contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan

-singleUpsertResponse :: ApiRequest -> ResultSet -> Wai.Response
-singleUpsertResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
+singleUpsertResponse :: MutateReadPlan -> ApiRequest -> ResultSet -> Either Error.Error PgrstResponse
+singleUpsertResponse MutateReadPlan{mrMedia} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
  RSStandard {..} -> do
    let
-      response = gucResponse rsGucStatus rsGucHeaders
+      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing preferRepresentation Nothing preferCount preferTransaction Nothing preferHandling preferTimezone []
+      cTHeader = contentTypeHeaders mrMedia ctxApiRequest

-    if preferRepresentation == Full then
-      response HTTP.status200 (contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
-    else
-      response HTTP.status204 [] mempty
+    let isInsertIfGTZero i = if i > 0 then HTTP.status201 else HTTP.status200
+        upsertStatus       = isInsertIfGTZero $ fromJust rsInserted
+        (status, headers, body) =
+          case preferRepresentation of
+            Just Full -> (upsertStatus, cTHeader ++ prefHeader, LBS.fromStrict rsBody)
+            Just None -> (HTTP.status204,  prefHeader, mempty)
+            _ -> (HTTP.status204, prefHeader, mempty)
+    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers
+
+    Right $ PgrstResponse ovStatus ovHeaders body

  RSPlan plan ->
-    Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
+    Right $ PgrstResponse HTTP.status200 (contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan

-deleteResponse :: ApiRequest -> ResultSet -> Wai.Response
-deleteResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
+deleteResponse :: MutateReadPlan -> ApiRequest -> ResultSet -> Either Error.Error PgrstResponse
+deleteResponse MutateReadPlan{mrMedia} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
  RSStandard {..} -> do
    let
-      response = gucResponse rsGucStatus rsGucHeaders
      contentRangeHeader =
        RangeQuery.contentRangeH 1 0 $
          if shouldCount preferCount then Just rsQueryTotal else Nothing
-      headers = [contentRangeHeader]
+      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing preferRepresentation Nothing preferCount preferTransaction Nothing preferHandling preferTimezone []
+      headers = contentRangeHeader : prefHeader

-    if preferRepresentation == Full then
-        response HTTP.status200
-          (headers ++ contentTypeHeaders ctxApiRequest)
-          (LBS.fromStrict rsBody)
-      else
-        response HTTP.status204 headers mempty
+    let (status, headers', body) =
+          case preferRepresentation of
+              Just Full -> (HTTP.status200, headers ++ contentTypeHeaders mrMedia ctxApiRequest, LBS.fromStrict rsBody)
+              Just None -> (HTTP.status204, headers, mempty)
+              _ -> (HTTP.status204, headers, mempty)
+
+    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status headers'
+
+    Right $ PgrstResponse ovStatus ovHeaders body

  RSPlan plan ->
-    Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
+    Right $ PgrstResponse HTTP.status200 (contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan

-infoIdentResponse :: QualifiedIdentifier -> SchemaCache -> Wai.Response
-infoIdentResponse identifier sCache =
+infoIdentResponse :: QualifiedIdentifier -> SchemaCache -> Either Error.Error PgrstResponse
+infoIdentResponse identifier sCache = do
  case HM.lookup identifier (dbTables sCache) of
    Just tbl -> respondInfo $ allowH tbl
-    Nothing  -> Error.errorResponseFor $ Error.ApiRequestError ApiRequestTypes.NotFound
+    Nothing  -> Left $ Error.ApiRequestError ApiRequestTypes.NotFound
  where
    allowH table =
      let hasPK = not . null $ tablePKCols table in
@@ -188,72 +221,70 @@ infoIdentResponse identifier sCache =
          ["PATCH" | tableUpdatable table] ++
          ["DELETE" | tableDeletable table]

-infoProcResponse :: Routine -> Wai.Response
+infoProcResponse :: Routine -> Either Error.Error PgrstResponse
 infoProcResponse proc | pdVolatility proc == Volatile = respondInfo "OPTIONS,POST"
                      | otherwise                     = respondInfo "OPTIONS,GET,HEAD,POST"

-infoRootResponse :: Wai.Response
+infoRootResponse :: Either Error.Error PgrstResponse
 infoRootResponse = respondInfo "OPTIONS,GET,HEAD"

-respondInfo :: ByteString -> Wai.Response
+respondInfo :: ByteString -> Either Error.Error PgrstResponse
 respondInfo allowHeader =
  let allOrigins = ("Access-Control-Allow-Origin", "*") in
-  Wai.responseLBS HTTP.status200 [allOrigins, (HTTP.hAllow, allowHeader)] mempty
+  Right $ PgrstResponse HTTP.status200 [allOrigins, (HTTP.hAllow, allowHeader)] mempty

-invokeResponse :: InvokeMethod -> Routine -> ApiRequest -> ResultSet -> Wai.Response
-invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+invokeResponse :: CallReadPlan -> InvokeMethod -> Routine -> ApiRequest -> ResultSet -> Either Error.Error PgrstResponse
+invokeResponse CallReadPlan{crMedia} invMethod proc ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} resultSet = case resultSet of
  RSStandard {..} -> do
    let
-      response = gucResponse rsGucStatus rsGucHeaders
      (status, contentRange) =
        RangeQuery.rangeStatusHeader iTopLevelRange rsQueryTotal rsTableTotal
      rsOrErrBody = if status == HTTP.status416
        then Error.errorPayload $ Error.ApiRequestError $ ApiRequestTypes.InvalidRange
          $ ApiRequestTypes.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
        else LBS.fromStrict rsBody
-      headers = [contentRange]
+      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing Nothing preferParameters preferCount preferTransaction Nothing preferHandling preferTimezone []
+      headers = contentRange : prefHeader

-    if Routine.funcReturnsVoid proc then
-        response HTTP.status204 headers mempty
-      else
-        response status
-          (headers ++ contentTypeHeaders ctxApiRequest)
-          (if invMethod == InvHead then mempty else rsOrErrBody)
+    let (status', headers', body) =
+          if Routine.funcReturnsVoid proc then
+              (HTTP.status204, headers, mempty)
+            else
+              (status,
+                headers ++ contentTypeHeaders crMedia ctxApiRequest,
+                if invMethod == InvHead then mempty else rsOrErrBody)
+
+    (ovStatus, ovHeaders) <- overrideStatusHeaders rsGucStatus rsGucHeaders status' headers'
+
+    Right $ PgrstResponse ovStatus ovHeaders body

  RSPlan plan ->
-    Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
+    Right $ PgrstResponse HTTP.status200 (contentTypeHeaders crMedia ctxApiRequest) $ LBS.fromStrict plan

-openApiResponse :: Bool -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
-openApiResponse headersOnly body conf sCache schema negotiatedByProfile =
-  Wai.responseLBS HTTP.status200
+openApiResponse :: (Text, Text) -> Bool -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Either Error.Error PgrstResponse
+openApiResponse versions headersOnly body conf sCache schema negotiatedByProfile =
+  Right $ PgrstResponse HTTP.status200
    (MediaType.toContentType MTOpenAPI : maybeToList (profileHeader schema negotiatedByProfile))
-    (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode conf sCache x y z) body)
+    (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache x y z) body)

-- | Response with headers and status overridden from GUCs.
-gucResponse
-  :: Maybe Text
-  -> Maybe BS.ByteString
-  -> HTTP.Status
-  -> [HTTP.Header]
-  -> LBS.ByteString
-  -> Wai.Response
-gucResponse rsGucStatus rsGucHeaders status headers body =
-  case (,) <$> decodeGucStatus rsGucStatus <*> decodeGucHeaders rsGucHeaders of
-    Left err -> Error.errorResponseFor err
-    Right (gucStatus, gucHeaders) ->
-      Wai.responseLBS (fromMaybe status gucStatus) (addHeadersIfNotIncluded headers (map unwrapGucHeader gucHeaders)) body
+-- Status and headers can be overridden as per https://postgrest.org/en/stable/references/transactions.html#response-headers
+overrideStatusHeaders :: Maybe Text -> Maybe BS.ByteString -> HTTP.Status -> [HTTP.Header]-> Either Error.Error (HTTP.Status, [HTTP.Header])
+overrideStatusHeaders rsGucStatus rsGucHeaders pgrstStatus pgrstHeaders = do
+  gucStatus <- decodeGucStatus rsGucStatus
+  gucHeaders <- decodeGucHeaders rsGucHeaders
+  Right (fromMaybe pgrstStatus gucStatus, addHeadersIfNotIncluded pgrstHeaders $ map unwrapGucHeader gucHeaders)

 decodeGucHeaders :: Maybe BS.ByteString -> Either Error.Error [GucHeader]
 decodeGucHeaders =
-  maybe (Right []) $ first (const Error.GucHeadersError) . JSON.eitherDecode . LBS.fromStrict
+  maybe (Right []) $ first (const . Error.ApiRequestError $ ApiRequestTypes.GucHeadersError) . JSON.eitherDecode . LBS.fromStrict

 decodeGucStatus :: Maybe Text -> Either Error.Error (Maybe HTTP.Status)
 decodeGucStatus =
-  maybe (Right Nothing) $ first (const Error.GucStatusError) . fmap (Just . toEnum . fst) . decimal
+  maybe (Right Nothing) $ first (const . Error.ApiRequestError $ ApiRequestTypes.GucStatusError) . fmap (Just . toEnum . fst) . decimal

-contentTypeHeaders :: ApiRequest -> [HTTP.Header]
-contentTypeHeaders ApiRequest{..} =
-  MediaType.toContentType iAcceptMediaType : maybeToList (profileHeader iSchema iNegotiatedByProfile)
+contentTypeHeaders :: MediaType -> ApiRequest -> [HTTP.Header]
+contentTypeHeaders mediaType ApiRequest{..} =
+  MediaType.toContentType mediaType : maybeToList (profileHeader iSchema iNegotiatedByProfile)

 profileHeader :: Schema -> Bool -> Maybe HTTP.Header
 profileHeader schema negotiatedByProfile =
@@ -262,43 +293,8 @@ profileHeader schema negotiatedByProfile =
  else
    Nothing

-addRetryHint :: Int -> Wai.Response -> Wai.Response
-addRetryHint delay response = do
-  let h = ("Retry-After", BS.pack $ show delay)
-  Wai.mapResponseHeaders (\hs -> if isServiceUnavailable response then h:hs else hs) response
-
-isServiceUnavailable :: Wai.Response -> Bool
-isServiceUnavailable response = Wai.responseStatus response == HTTP.status503
-
-optionalRollback :: AppConfig -> ApiRequest -> ExceptT Error.Error IO Wai.Response -> ExceptT Error.Error IO Wai.Response
-optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} resp = do
-  newRes <- catchError resp $ return . Error.errorResponseFor
-  return $ Wai.mapResponseHeaders preferenceApplied newRes
-  where
-    shouldCommit =
-      configDbTxAllowOverride && preferTransaction == Just Commit
-    shouldRollback =
-      configDbTxAllowOverride && preferTransaction == Just Rollback
-    preferenceApplied
-      | shouldCommit =
-          addHeadersIfNotIncluded
-            [toAppliedHeader Commit]
-      | shouldRollback =
-          addHeadersIfNotIncluded
-            [toAppliedHeader Rollback]
-      | otherwise =
-          identity
-
 -- | Add headers not already included to allow the user to override them instead of duplicating them
 addHeadersIfNotIncluded :: [HTTP.Header] -> [HTTP.Header] -> [HTTP.Header]
 addHeadersIfNotIncluded newHeaders initialHeaders =
  filter (\(nk, _) -> isNothing $ find (\(ik, _) -> ik == nk) initialHeaders) newHeaders ++
  initialHeaders
-
-traceHeaderMiddleware :: AppConfig -> Wai.Middleware
-traceHeaderMiddleware AppConfig{configServerTraceHeader} app req respond =
-  case configServerTraceHeader of
-    Nothing -> app req respond
-    Just hdr ->
-      let hdrVal = L.lookup hdr $ Wai.requestHeaders req in
-      app req (respond . Wai.mapResponseHeaders ([(hdr, fromMaybe mempty hdrVal)] ++))
@@ -12,7 +12,6 @@ import qualified Data.ByteString.Lazy  as LBS
 import qualified Data.HashMap.Strict   as HM
 import qualified Data.HashSet.InsOrd   as Set
 import qualified Data.Text             as T
-import qualified Data.Text.Encoding    as T

 import Control.Arrow              ((&&&))
 import Data.HashMap.Strict.InsOrd (InsOrdHashMap, fromList)
@@ -36,16 +35,16 @@ import PostgREST.SchemaCache.Routine      (Routine (..),
 import PostgREST.SchemaCache.Table        (Column (..), Table (..),
                                           TablesMap,
                                           tableColumnsList)
-import PostgREST.Version                  (docsVersion, prettyVersion)

 import PostgREST.MediaType

 import Protolude hiding (Proxy, get)

-encode :: AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
-encode conf sCache tables procs schemaDescription =
+encode :: (Text, Text) -> AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
+encode versions conf sCache tables procs schemaDescription =
  JSON.encode $
    postgrestSpec
+      versions
      (dbRelationships sCache)
      (concat $ HM.elems procs)
      (snd <$> HM.toList tables)
@@ -154,7 +153,7 @@ makeProcSchema pd =
  & required .~ fmap ppName (filter ppReq (pdParams pd))

 makeProcProperty :: RoutineParam -> (Text, Referenced Schema)
-makeProcProperty (RoutineParam n t _ _) = (n, Inline s)
+makeProcProperty (RoutineParam n t _ _ _) = (n, Inline s)
  where
    s = (mempty :: Schema)
          & type_ .~ toSwaggerType t
@@ -181,7 +180,7 @@ makePreferParam ts =
      _            -> []

 makeProcGetParam :: RoutineParam -> Referenced Param
-makeProcGetParam (RoutineParam n t r v) =
+makeProcGetParam (RoutineParam n t _ r v) =
  Inline $ (mempty :: Param)
    & name .~ n
    & required ?~ r
@@ -351,7 +350,7 @@ makeProcPathItem pd = ("/rpc/" ++ toS (pdName pd), pe)
      & summary .~ pSum
      & description .~ mfilter (/="") pDesc
      & tags .~ Set.fromList ["(rpc) " <> pdName pd]
-      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON]
+      & produces ?~ makeMimeList [MTApplicationJSON, MTVndSingularJSON True, MTVndSingularJSON False]
      & at 200 ?~ "OK"
    getOp = procOp
      & parameters .~ makeProcGetParams (pdParams pd)
@@ -392,12 +391,12 @@ escapeHostName "*6" = "0.0.0.0"
 escapeHostName "!6" = "0.0.0.0"
 escapeHostName h    = h

-postgrestSpec :: RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
-postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
+postgrestSpec :: (Text, Text) -> RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
+postgrestSpec (prettyVersion, docsVersion) rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
  & basePath ?~ T.unpack b
  & schemes ?~ [s']
  & info .~ ((mempty :: Info)
-      & version .~ T.decodeUtf8 prettyVersion
+      & version .~ prettyVersion
      & title .~ fromMaybe "PostgREST API" dTitle
      & description ?~ fromMaybe "This is a dynamic API generated by PostgREST" dDesc)
  & externalDocs ?~ ((mempty :: ExternalDocs)
@@ -407,8 +406,8 @@ postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
  & definitions .~ fromList (makeTableDef rels <$> ti)
  & parameters .~ fromList (makeParamDefs ti)
  & paths .~ makePathItems pds ti
-  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
-  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
+  & produces .~ makeMimeList [MTApplicationJSON, MTVndSingularJSON True, MTVndSingularJSON False, MTTextCSV]
+  & consumes .~ makeMimeList [MTApplicationJSON, MTVndSingularJSON True, MTVndSingularJSON False, MTTextCSV]
  & securityDefinitions .~ makeSecurityDefinitions securityDefName allowSecurityDef
  & security .~ [SecurityRequirement (fromList [(securityDefName, [])]) | allowSecurityDef]
    where
@@ -0,0 +1,36 @@
+module PostgREST.Response.Performance
+  ( ServerTiming (..)
+  , serverTimingHeader
+  )
+where
+import qualified Data.ByteString.Char8 as BS
+import qualified Network.HTTP.Types    as HTTP
+import           Numeric               (showFFloat)
+import           Protolude
+
+data ServerTiming =
+  ServerTiming
+    { jwt         :: Maybe Double
+    , parse       :: Maybe Double
+    , plan        :: Maybe Double
+    , transaction :: Maybe Double
+    , response    :: Maybe Double
+    }
+  deriving (Show)
+
+-- | Render the Server-Timing header from a ServerTimingData
+--
+-- >>> serverTimingHeader ServerTiming { plan=Just 0.1, transaction=Just 0.2, response=Just 0.3, jwt=Just 0.4, parse=Just 0.5}
+-- ("Server-Timing","jwt;dur=400000.0, parse;dur=500000.0, plan;dur=100000.0, transaction;dur=200000.0, response;dur=300000.0")
+serverTimingHeader :: ServerTiming -> HTTP.Header
+serverTimingHeader timing =
+  ("Server-Timing", renderTiming)
+  where
+    renderMetric metric = maybe "" (\dur -> BS.concat [metric, BS.pack $ ";dur=" <> showFFloat (Just 1) (dur * 1000000) ""])
+    renderTiming = BS.intercalate ", " $ (\(k, v) -> renderMetric k (v timing)) <$>
+      [ ("jwt", jwt)
+      , ("parse", parse)
+      , ("plan", plan)
+      , ("transaction", transaction)
+      , ("response", response)
+      ]
@@ -26,7 +26,11 @@ module PostgREST.SchemaCache
  , schemaDescription
  ) where

+import Control.Monad.Extra (whenJust)
+
+import           Data.Aeson                 ((.=))
 import qualified Data.Aeson                 as JSON
+import qualified Data.Aeson.Types           as JSON
 import qualified Data.HashMap.Strict        as HM
 import qualified Data.HashMap.Strict.InsOrd as HMI
 import qualified Data.Set                   as S
@@ -38,32 +42,57 @@ import qualified Hasql.Transaction          as SQL
 import Contravariant.Extras          (contrazip2)
 import Text.InterpolatedString.Perl6 (q)

-import PostgREST.Config.Database          (pgVersionStatement)
-import PostgREST.Config.PgVersion         (PgVersion, pgVersion100,
-                                           pgVersion110, pgVersion120)
-import PostgREST.SchemaCache.Identifiers  (AccessSet, FieldName,
-                                           QualifiedIdentifier (..),
-                                           Schema)
-import PostgREST.SchemaCache.Relationship (Cardinality (..),
-                                           Junction (..),
-                                           Relationship (..),
-                                           RelationshipsMap)
-import PostgREST.SchemaCache.Routine      (FuncVolatility (..),
-                                           PgType (..), RetType (..),
-                                           Routine (..), RoutineMap,
-                                           RoutineParam (..))
-import PostgREST.SchemaCache.Table        (Column (..), ColumnMap,
-                                           Table (..), TablesMap)
+import PostgREST.Config                      (AppConfig (..))
+import PostgREST.Config.Database             (TimezoneNames,
+                                              pgVersionStatement,
+                                              toIsolationLevel)
+import PostgREST.Config.PgVersion            (PgVersion, pgVersion100,
+                                              pgVersion110,
+                                              pgVersion120)
+import PostgREST.SchemaCache.Identifiers     (AccessSet, FieldName,
+                                              QualifiedIdentifier (..),
+                                              RelIdentifier (..),
+                                              Schema, isAnyElement)
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
+                                              Junction (..),
+                                              Relationship (..),
+                                              RelationshipsMap)
+import PostgREST.SchemaCache.Representations (DataRepresentation (..),
+                                              RepresentationsMap)
+import PostgREST.SchemaCache.Routine         (FuncVolatility (..),
+                                              MediaHandler (..),
+                                              MediaHandlerMap,
+                                              PgType (..),
+                                              RetType (..),
+                                              Routine (..),
+                                              RoutineMap,
+                                              RoutineParam (..))
+import PostgREST.SchemaCache.Table           (Column (..), ColumnMap,
+                                              Table (..), TablesMap)
+
+import qualified PostgREST.MediaType as MediaType

 import Protolude


 data SchemaCache = SchemaCache
-  { dbTables        :: TablesMap
-  , dbRelationships :: RelationshipsMap
-  , dbRoutines      :: RoutineMap
+  { dbTables          :: TablesMap
+  , dbRelationships   :: RelationshipsMap
+  , dbRoutines        :: RoutineMap
+  , dbRepresentations :: RepresentationsMap
+  , dbMediaHandlers   :: MediaHandlerMap
+  , dbTimezones       :: TimezoneNames
  }
-  deriving (Generic, JSON.ToJSON)
+
+instance JSON.ToJSON SchemaCache where
+  toJSON (SchemaCache tabs rels routs reps _ _) = JSON.object [
+      "dbTables"          .= JSON.toJSON tabs
+    , "dbRelationships"   .= JSON.toJSON rels
+    , "dbRoutines"        .= JSON.toJSON routs
+    , "dbRepresentations" .= JSON.toJSON reps
+    , "dbMediaHandlers"   .= JSON.emptyArray
+    , "dbTimezones"       .= JSON.emptyArray
+    ]

 -- | A view foreign key or primary key dependency detected on its source table
 -- Each column of the key could be referenced multiple times in the view, e.g.
@@ -103,15 +132,22 @@ data KeyDep
 -- | A SQL query that can be executed independently
 type SqlQuery = ByteString

-querySchemaCache :: [Schema] -> [Schema] -> Bool -> SQL.Transaction SchemaCache
-querySchemaCache schemas extraSearchPath prepared = do
+
+querySchemaCache :: AppConfig -> SQL.Transaction SchemaCache
+querySchemaCache AppConfig{..} = do
  SQL.sql "set local schema ''" -- This voids the search path. The following queries need this for getting the fully qualified name(schema.name) of every db object
  pgVer   <- SQL.statement mempty $ pgVersionStatement prepared
  tabs    <- SQL.statement schemas $ allTables pgVer prepared
-  keyDeps <- SQL.statement (schemas, extraSearchPath) $ allViewsKeyDependencies prepared
+  keyDeps <- SQL.statement (schemas, configDbExtraSearchPath) $ allViewsKeyDependencies prepared
  m2oRels <- SQL.statement mempty $ allM2OandO2ORels pgVer prepared
  funcs   <- SQL.statement schemas $ allFunctions pgVer prepared
  cRels   <- SQL.statement mempty $ allComputedRels prepared
+  reps    <- SQL.statement schemas $ dataRepresentations prepared
+  mHdlers <- SQL.statement schemas $ mediaHandlers pgVer prepared
+  tzones  <- SQL.statement mempty $ timezones prepared
+  _       <-
+    let sleepCall = SQL.Statement "select pg_sleep($1)" (param HE.int4) HD.noResult prepared in
+    whenJust configInternalSCSleep (`SQL.statement` sleepCall) -- only used for testing

  let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
      rels          = addInverseRels $ addM2MRels tabsWViewsPks $ addViewM2OAndO2ORels keyDeps m2oRels
@@ -120,7 +156,13 @@ querySchemaCache schemas extraSearchPath prepared = do
      dbTables = tabsWViewsPks
    , dbRelationships = getOverrideRelationshipsMap rels cRels
    , dbRoutines = funcs
+    , dbRepresentations = reps
+    , dbMediaHandlers = HM.union mHdlers initialMediaHandlers -- the custom handlers will override the initial ones
+    , dbTimezones = tzones
    }
+  where
+    schemas = toList configDbSchemas
+    prepared = configDbPreparedStatements

 -- | overrides detected relationships with the computed relationships and gets the RelationshipsMap
 getOverrideRelationshipsMap :: [Relationship] -> [Relationship] -> RelationshipsMap
@@ -146,10 +188,13 @@ getOverrideRelationshipsMap rels cRels =
 removeInternal :: [Schema] -> SchemaCache -> SchemaCache
 removeInternal schemas dbStruct =
  SchemaCache {
-      dbTables        = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
-    , dbRelationships = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
-                        HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
-    , dbRoutines      = dbRoutines dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+      dbTables          = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
+    , dbRelationships   = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
+                          HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
+    , dbRoutines        = dbRoutines dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+    , dbRepresentations = dbRepresentations dbStruct -- no need to filter, not directly exposed through the API
+    , dbMediaHandlers   = dbMediaHandlers dbStruct
+    , dbTimezones       = dbTimezones dbStruct
    }
  where
    hasInternalJunction ComputedRelationship{} = False
@@ -240,6 +285,7 @@ decodeFuncs =
                  (RoutineParam
                  <$> compositeField HD.text
                  <*> compositeField HD.text
+                  <*> compositeField HD.text
                  <*> compositeField HD.bool
                  <*> compositeField HD.bool)
              <*> (parseRetType
@@ -250,6 +296,7 @@ decodeFuncs =
                  <*> column HD.bool)
              <*> (parseVolatility <$> column HD.char)
              <*> column HD.bool
+              <*> nullableColumn (toIsolationLevel <$> HD.text)
              <*> nullableColumn HD.text

    addKey :: Routine -> (QualifiedIdentifier, Routine)
@@ -270,6 +317,42 @@ decodeFuncs =
                      | v == 's' = Stable
                      | otherwise = Volatile -- only 'v' can happen here

+decodeRepresentations :: HD.Result RepresentationsMap
+decodeRepresentations =
+  HM.fromList . map (\rep@DataRepresentation{drSourceType, drTargetType} -> ((drSourceType, drTargetType), rep)) <$> HD.rowList row
+  where
+    row = DataRepresentation
+      <$> column HD.text
+      <*> column HD.text
+      <*> column HD.text
+
+-- Selects all potential data representation transformations. To qualify the cast must be
+-- 1. to or from a domain
+-- 2. implicit
+-- For the time being it must also be to/from JSON or text, although one can imagine a future where we support special
+-- cases like CSV specific representations.
+dataRepresentations :: Bool -> SQL.Statement [Schema] RepresentationsMap
+dataRepresentations = SQL.Statement sql (arrayParam HE.text) decodeRepresentations
+  where
+    sql = [q|
+    SELECT
+      c.castsource::regtype::text,
+      c.casttarget::regtype::text,
+      c.castfunc::regproc::text
+    FROM
+      pg_catalog.pg_cast c
+    JOIN pg_catalog.pg_type src_t
+      ON c.castsource::oid = src_t.oid
+    JOIN pg_catalog.pg_type dst_t
+      ON c.casttarget::oid = dst_t.oid
+    WHERE
+      c.castcontext = 'i'
+      AND c.castmethod = 'f'
+      AND has_function_privilege(c.castfunc, 'execute')
+      AND ((src_t.typtype = 'd' AND c.casttarget IN ('json'::regtype::oid , 'text'::regtype::oid))
+       OR (dst_t.typtype = 'd' AND c.castsource IN ('json'::regtype::oid , 'text'::regtype::oid)))
+    |]
+
 allFunctions :: PgVersion -> Bool -> SQL.Statement [Schema] RoutineMap
 allFunctions pgVer = SQL.Statement sql (arrayParam HE.text) decodeFuncs
  where
@@ -312,6 +395,13 @@ funcsSqlQuery pgVer = [q|
      array_agg((
        COALESCE(name, ''), -- name
        type::regtype::text, -- type
+        CASE type
+          WHEN 'bit'::regtype THEN 'bit varying'
+          WHEN 'bit[]'::regtype THEN 'bit varying[]'
+          WHEN 'character'::regtype THEN 'character varying'
+          WHEN 'character[]'::regtype THEN 'character varying[]'
+          ELSE type::regtype::text
+        END, -- convert types that ignore the lenth and accept any value till maximum size
        idx <= (pronargs - pronargdefaults), -- is_required
        COALESCE(mode = 'v', FALSE) -- is_variadic
      ) ORDER BY idx) AS args,
@@ -341,7 +431,8 @@ funcsSqlQuery pgVer = [q|
    bt.oid <> bt.base as rettype_is_composite_alias,
    p.provolatile,
    p.provariadic > 0 as hasvariadic,
-    lower((regexp_split_to_array((regexp_split_to_array(config, '='))[2], ','))[1]) AS transaction_isolation_level
+    lower((regexp_split_to_array((regexp_split_to_array(iso_config, '='))[2], ','))[1]) AS transaction_isolation_level,
+    lower((regexp_split_to_array((regexp_split_to_array(timeout_config, '='))[2], ','))[1]) AS statement_timeout
  FROM pg_proc p
  LEFT JOIN arguments a ON a.oid = p.oid
  JOIN pg_namespace pn ON pn.oid = p.pronamespace
@@ -350,7 +441,8 @@ funcsSqlQuery pgVer = [q|
  JOIN pg_namespace tn ON tn.oid = t.typnamespace
  LEFT JOIN pg_class comp ON comp.oid = t.typrelid
  LEFT JOIN pg_description as d ON d.objoid = p.oid
-  LEFT JOIN LATERAL unnest(proconfig) config ON config like 'default_transaction_isolation%'
+  LEFT JOIN LATERAL unnest(proconfig) iso_config ON iso_config like 'default_transaction_isolation%'
+  LEFT JOIN LATERAL unnest(proconfig) timeout_config ON timeout_config like 'statement_timeout%'
  WHERE t.oid <> 'trigger'::regtype AND COALESCE(a.callable, true)
 |] <> (if pgVer >= pgVersion110 then "AND prokind = 'f'" else "AND NOT (proisagg OR proiswindow)")

@@ -516,22 +608,21 @@ tablesSqlQuery pgVer =
          c.relname::name AS table_name,
          a.attname::name AS column_name,
          d.description AS description,
-  |] <> columnDefault <>
-  [q|
+  |] <> columnDefault <> [q| AS column_default,
          not (a.attnotnull OR t.typtype = 'd' AND t.typnotnull) AS is_nullable,
+          CASE
+              WHEN t.typtype = 'd' THEN
              CASE
-                  WHEN t.typtype = 'd' THEN
-                  CASE
-                      WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
-                      ELSE format_type(a.atttypid, a.atttypmod)
-                  END
-                  ELSE
-                  CASE
-                      WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
-                      ELSE format_type(a.atttypid, a.atttypmod)
-                  END
-              END::text AS data_type,
-          t.oid AS data_type_id,
+                  WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
+                  ELSE format_type(a.atttypid, a.atttypmod)
+              END
+              ELSE
+              CASE
+                  WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
+                  ELSE format_type(a.atttypid, a.atttypmod)
+              END
+          END::text AS data_type,
+          format_type(a.atttypid, a.atttypmod)::text AS nominal_data_type,
          information_schema._pg_char_max_length(
              information_schema._pg_truetypid(a.*, t.*),
              information_schema._pg_truetypmod(a.*, t.*)
@@ -573,7 +664,7 @@ tablesSqlQuery pgVer =
          info.description,
          info.is_nullable::boolean,
          info.data_type,
-          info.data_type_id::regtype::text,
+          info.nominal_data_type,
          info.character_maximum_length,
          info.column_default,
          coalesce(enum_info.vals, '{}')) order by info.position) as columns
@@ -705,19 +796,25 @@ tablesSqlQuery pgVer =
  "ORDER BY table_schema, table_name"
  where
    relIsPartition = if pgVer >= pgVersion100 then " AND not c.relispartition " else mempty
-    columnDefault
+    columnDefault -- typbasetype and typdefaultbin handles `CREATE DOMAIN .. DEFAULT val`,  attidentity/attgenerated handles generated columns, pg_get_expr gets the default of a column
      | pgVer >= pgVersion120 = [q|
          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
            WHEN a.attidentity  = 'd' THEN format('nextval(%s)', quote_literal(seqsch.nspname || '.' || seqclass.relname))
            WHEN a.attgenerated = 's' THEN null
            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
-          END AS column_default,|]
+          END|]
      | pgVer >= pgVersion100 = [q|
          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
            WHEN a.attidentity = 'd' THEN format('nextval(%s)', quote_literal(seqsch.nspname || '.' || seqclass.relname))
            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
-          END AS column_default,|]
-      | otherwise  = "pg_get_expr(ad.adbin, ad.adrelid)::text as column_default,"
+          END|]
+      | otherwise  = [q|
+          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
+            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
+          END|]

 -- | Gets many-to-one relationships and one-to-one(O2O) relationships, which are a refinement of the many-to-one's
 allM2OandO2ORels :: PgVersion -> Bool -> SQL.Statement () [Relationship]
@@ -814,6 +911,7 @@ allComputedRels =
    (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
    (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
    (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
+    pure (QualifiedIdentifier mempty mempty) <*>
    column HD.bool <*>
    column HD.bool

@@ -1015,6 +1113,88 @@ allViewsKeyDependencies =
      having ncol = array_length(array_agg(row(col.attname, view_columns) order by pks_fks.ord), 1)
      |]

+initialMediaHandlers :: MediaHandlerMap
+initialMediaHandlers =
+  HM.insert (RelAnyElement, MediaType.MTAny            ) BuiltinOvAggJson $
+  HM.insert (RelAnyElement, MediaType.MTApplicationJSON) BuiltinOvAggJson $
+  HM.insert (RelAnyElement, MediaType.MTTextCSV        ) BuiltinOvAggCsv $
+  HM.insert (RelAnyElement, MediaType.MTGeoJSON        ) BuiltinOvAggGeoJson
+  HM.empty
+
+mediaHandlers :: PgVersion -> Bool -> SQL.Statement [Schema] MediaHandlerMap
+mediaHandlers pgVer =
+  SQL.Statement sql (arrayParam HE.text) decodeMediaHandlers
+  where
+    sql = [q|
+      with
+      all_relations as (
+        select reltype
+        from pg_class
+        where relkind in ('v','r','m','f','p')
+        union
+        select oid
+        from pg_type
+        where typname = 'anyelement'
+      ),
+      media_types as (
+          SELECT
+            t.oid,
+            lower(t.typname) as typname,
+            b.oid as base_oid,
+            b.typname AS basetypname,
+            t.typnamespace
+          FROM pg_type t
+          JOIN pg_type b ON t.typbasetype = b.oid
+          WHERE
+            t.typbasetype <> 0 and
+            (t.typname ~* '^[A-Za-z0-9.-]+/[A-Za-z0-9.\+-]+$' or t.typname = '*/*')
+      )
+      select
+        proc_schema.nspname           as handler_schema,
+        proc.proname                  as handler_name,
+        arg_schema.nspname::text      as target_schema,
+        arg_name.typname::text        as target_name,
+        media_types.typname           as media_type
+      from media_types
+        join pg_proc      proc         on proc.prorettype = media_types.oid
+        join pg_namespace proc_schema  on proc_schema.oid = proc.pronamespace
+        join pg_aggregate agg          on agg.aggfnoid = proc.oid
+        join pg_type      arg_name     on arg_name.oid = proc.proargtypes[0]
+        join pg_namespace arg_schema   on arg_schema.oid = arg_name.typnamespace
+      where
+        proc_schema.nspname = ANY($1) and
+        proc.pronargs = 1 and
+        arg_name.oid in (select reltype from all_relations)
+      union
+      select
+          typ_sch.nspname as handler_schema,
+          mtype.typname   as handler_name,
+          pro_sch.nspname as target_schema,
+          proname         as target_name,
+          mtype.typname   as media_type
+      from pg_proc proc
+        join pg_namespace pro_sch on pro_sch.oid = proc.pronamespace
+        join media_types mtype on proc.prorettype = mtype.oid
+        join pg_namespace typ_sch     on typ_sch.oid = mtype.typnamespace
+      where NOT proretset
+      |] <> (if pgVer >= pgVersion110 then " AND prokind = 'f'" else " AND NOT (proisagg OR proiswindow)")
+
+decodeMediaHandlers :: HD.Result MediaHandlerMap
+decodeMediaHandlers =
+  HM.fromList . fmap (\(x, y, z) -> ((if isAnyElement y then RelAnyElement else RelId y, z), CustomFunc x) ) <$> HD.rowList caggRow
+  where
+    caggRow = (,,)
+              <$> (QualifiedIdentifier <$> column HD.text <*> column HD.text)
+              <*> (QualifiedIdentifier <$> column HD.text <*> column HD.text)
+              <*> (MediaType.decodeMediaType . encodeUtf8 <$> column HD.text)
+
+timezones :: Bool -> SQL.Statement () TimezoneNames
+timezones = SQL.Statement sql HE.noParams decodeTimezones
+  where
+    sql = "SELECT name FROM pg_timezone_names"
+    decodeTimezones :: HD.Result TimezoneNames
+    decodeTimezones = S.fromList . map encodeUtf8 <$> HD.rowList (column HD.text)
+
 param :: HE.Value a -> HE.Params a
 param = HE.param . HE.nonNullable

@@ -3,6 +3,8 @@

 module PostgREST.SchemaCache.Identifiers
  ( QualifiedIdentifier(..)
+  , RelIdentifier(..)
+  , isAnyElement
  , Schema
  , TableName
  , FieldName
@@ -17,6 +19,9 @@ import qualified Data.Text  as T

 import Protolude

+data RelIdentifier = RelId QualifiedIdentifier | RelAnyElement
+  deriving (Eq, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
+instance Hashable RelIdentifier

 -- | Represents a pg identifier with a prepended schema name "schema.table".
 -- When qiSchema is "", the schema is defined by the pg search_path.
@@ -24,10 +29,13 @@ data QualifiedIdentifier = QualifiedIdentifier
  { qiSchema :: Schema
  , qiName   :: TableName
  }
-  deriving (Eq, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)

 instance Hashable QualifiedIdentifier

+isAnyElement :: QualifiedIdentifier -> Bool
+isAnyElement y = QualifiedIdentifier "pg_catalog" "anyelement" == y
+
 dumpQi :: QualifiedIdentifier -> Text
 dumpQi (QualifiedIdentifier s i) =
  (if T.null s then mempty else s <> ".") <> i
@@ -31,10 +31,11 @@ data Relationship = Relationship
  { relFunction     :: QualifiedIdentifier
  , relTable        :: QualifiedIdentifier
  , relForeignTable :: QualifiedIdentifier
+  , relTableAlias   :: QualifiedIdentifier
  , relToOne        :: Bool
  , relIsSelf       :: Bool
  }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)

 -- | The relationship cardinality
 -- | https://en.wikipedia.org/wiki/Cardinality_(data_modeling)
@@ -47,7 +48,7 @@ data Cardinality
  -- ^ one-to-one, this is a refinement over M2O so operating on it is pretty much the same as M2O
  | M2M Junction
  -- ^ many-to-many
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)

 type FKConstraint = Text

@@ -59,7 +60,7 @@ data Junction = Junction
  , junColsSource  :: [(FieldName, FieldName)]
  , junColsTarget  :: [(FieldName, FieldName)]
  }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)

 -- | Key based on the source table and the foreign table schema
 type RelationshipsMap = HM.HashMap (QualifiedIdentifier, Schema)  [Relationship]
@@ -0,0 +1,29 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric  #-}
+
+module PostgREST.SchemaCache.Representations
+  ( DataRepresentation(..)
+  , RepresentationsMap
+  ) where
+
+import qualified Data.Aeson          as JSON
+import qualified Data.HashMap.Strict as HM
+
+
+import Protolude
+
+-- | Data representations allow user customisation of how to present and receive data through APIs, per field.
+-- This structure is used for the library of available transforms. It answers questions like:
+-- - What function, if any, should be used to present a certain field that's been selected for API output?
+-- - How do we parse incoming data for a certain field type when inserting or updating?
+-- - And similarly, how do we parse textual data in a query string to be used as a filter?
+--
+-- Support for outputting special formats like CSV and binary data would fit into the same system.
+data DataRepresentation = DataRepresentation
+  { drSourceType :: Text
+  , drTargetType :: Text
+  , drFunction   :: Text
+  } deriving (Eq, Show, Generic, JSON.ToJSON, JSON.FromJSON)
+
+-- The representation map maps from (source type, target type) to a DR.
+type RepresentationsMap = HM.HashMap (Text, Text) DataRepresentation
@@ -14,31 +14,39 @@ module PostgREST.SchemaCache.Routine
  , funcReturnsVoid
  , funcTableName
  , funcReturnsCompositeAlias
+  , funcReturnsSingle
+  , MediaHandlerMap
+  , MediaHandler(..)
  ) where

-import qualified Data.Aeson          as JSON
-import qualified Data.HashMap.Strict as HM
+import           Data.Aeson                 ((.=))
+import qualified Data.Aeson                 as JSON
+import qualified Data.HashMap.Strict        as HM
+import qualified Hasql.Transaction.Sessions as SQL
+import qualified PostgREST.MediaType        as MediaType

 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
-                                          Schema, TableName)
+                                          RelIdentifier (..), Schema,
+                                          TableName)
+

 import Protolude

 data PgType
  = Scalar QualifiedIdentifier
  | Composite QualifiedIdentifier Bool -- True if the composite is a domain alias(used to work around a bug in pg 11 and 12, see QueryBuilder.hs)
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)

 data RetType
  = Single PgType
  | SetOf PgType
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)

 data FuncVolatility
  = Volatile
  | Stable
  | Immutable
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)

 data Routine = Function
  { pdSchema      :: Schema
@@ -48,29 +56,63 @@ data Routine = Function
  , pdReturnType  :: RetType
  , pdVolatility  :: FuncVolatility
  , pdHasVariadic :: Bool
-  , pdIsoLvl      :: Maybe Text
+  , pdIsoLvl      :: Maybe SQL.IsolationLevel
+  , pdTimeout     :: Maybe Text
  }
-  deriving (Eq, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Generic)
+-- need to define JSON manually bc SQL.IsolationLevel doesn't have a JSON instance(and we can't define one for that type without getting a compiler error)
+instance JSON.ToJSON Routine where
+  toJSON (Function sch nam desc params ret vol hasVar _ tout) = JSON.object
+    [
+      "pdSchema"      .= sch
+    , "pdName"        .= nam
+    , "pdDescription" .= desc
+    , "pdParams"      .= JSON.toJSON params
+    , "pdReturnType"  .= JSON.toJSON ret
+    , "pdVolatility"  .= JSON.toJSON vol
+    , "pdHasVariadic" .= JSON.toJSON hasVar
+    , "pdTimeout"     .= tout
+    ]

 data RoutineParam = RoutineParam
-  { ppName :: Text
-  , ppType :: Text
-  , ppReq  :: Bool
-  , ppVar  :: Bool
+  { ppName          :: Text
+  , ppType          :: Text
+  , ppTypeMaxLength :: Text
+  , ppReq           :: Bool
+  , ppVar           :: Bool
  }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)

 -- Order by least number of params in the case of overloaded functions
 instance Ord Routine where
-  Function schema1 name1 des1 prms1 rt1 vol1 hasVar1 iso1 `compare` Function schema2 name2 des2 prms2 rt2 vol2 hasVar2 iso2
+  Function schema1 name1 des1 prms1 rt1 vol1 hasVar1 iso1 tout1 `compare` Function schema2 name2 des2 prms2 rt2 vol2 hasVar2 iso2 tout2
    | schema1 == schema2 && name1 == name2 && length prms1 < length prms2  = LT
    | schema2 == schema2 && name1 == name2 && length prms1 > length prms2  = GT
-    | otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1, iso1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2, iso2)
+    | otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1, iso1, tout1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2, iso2, tout2)

 -- | A map of all procs, all of which can be overloaded(one entry will have more than one Routine).
 -- | It uses a HashMap for a faster lookup.
 type RoutineMap = HM.HashMap QualifiedIdentifier [Routine]

+-- | A media handler can be an aggregate over a composite type or a function over a scalar
+data MediaHandler
+   -- non overridable builtins
+   = BuiltinAggSingleJson Bool
+   | BuiltinAggArrayJsonStrip
+   -- these builtins are overridable
+   | BuiltinOvAggJson
+   | BuiltinOvAggGeoJson
+   | BuiltinOvAggCsv
+   -- custom
+   | CustomFunc QualifiedIdentifier
+   | NoAgg
+   deriving (Eq, Show)
+
+funcReturnsSingle :: Routine -> Bool
+funcReturnsSingle proc = case proc of
+  Function{pdReturnType = Single _} -> True
+  _                                 -> False
+
 funcReturnsScalar :: Routine -> Bool
 funcReturnsScalar proc = case proc of
  Function{pdReturnType = Single (Scalar{})} -> True
@@ -102,3 +144,5 @@ funcTableName proc = case pdReturnType proc of
  SetOf  (Composite qi _) -> Just $ qiName qi
  Single (Composite qi _) -> Just $ qiName qi
  _                       -> Nothing
+
+type MediaHandlerMap = HM.HashMap (RelIdentifier, MediaType.MediaType) MediaHandler
@@ -1,58 +1,53 @@
+{-# LANGUAGE CPP #-}
+
 module PostgREST.Unix
-  ( runAppWithSocket
-  , installSignalHandlers
+  ( installSignalHandlers
+  , createAndBindDomainSocket
  ) where

-import qualified Network.Socket           as Socket
-import qualified Network.Wai.Handler.Warp as Warp
-import qualified System.Posix.Signals     as Signals
+#ifndef mingw32_HOST_OS
+import qualified System.Posix.Signals as Signals
+#endif
+import System.Posix.Types       (FileMode)
+import System.PosixCompat.Files (setFileMode)

-import Network.Wai        (Application)
-import System.Directory   (removeFile)
-import System.IO.Error    (isDoesNotExistError)
-import System.Posix.Files (setFileMode)
-import System.Posix.Types (FileMode)
-
-import qualified PostgREST.AppState as AppState
-import qualified PostgREST.Workers  as Workers
-
-import Protolude
-
-
-- | Run the PostgREST application with user defined socket.
-runAppWithSocket :: Warp.Settings -> Application -> FileMode -> FilePath -> IO ()
-runAppWithSocket settings app socketFileMode socketFilePath =
-  bracket createAndBindSocket Socket.close $ \socket -> do
-    Socket.listen socket Socket.maxListenQueue
-    Warp.runSettingsSocket settings socket app
-  where
-    createAndBindSocket = do
-      deleteSocketFileIfExist socketFilePath
-      sock <- Socket.socket Socket.AF_UNIX Socket.Stream Socket.defaultProtocol
-      Socket.bind sock $ Socket.SockAddrUnix socketFilePath
-      setFileMode socketFilePath socketFileMode
-      return sock
-
-    deleteSocketFileIfExist path =
-      removeFile path `catch` handleDoesNotExist
-
-    handleDoesNotExist e
-      | isDoesNotExistError e = return ()
-      | otherwise = throwIO e
+import           Data.String      (String)
+import qualified Network.Socket   as NS
+import           Protolude
+import           System.Directory (removeFile)
+import           System.IO.Error  (isDoesNotExistError)

 -- | Set signal handlers, only for systems with signals
-installSignalHandlers :: AppState.AppState -> IO ()
-installSignalHandlers appState = do
-  let interrupt = throwTo (AppState.getMainThreadId appState) UserInterrupt
+installSignalHandlers :: ThreadId -> IO () -> IO () -> IO ()
+#ifndef mingw32_HOST_OS
+installSignalHandlers tid usr1 usr2 = do
+  let interrupt = throwTo tid UserInterrupt
  install Signals.sigINT interrupt
  install Signals.sigTERM interrupt
-
-  -- The SIGUSR1 signal updates the internal 'SchemaCache' by running
-  -- 'connectionWorker' exactly as before.
-  install Signals.sigUSR1 $ Workers.connectionWorker appState
-
-  -- Re-read the config on SIGUSR2
-  install Signals.sigUSR2 $ Workers.reReadConfig False appState
+  install Signals.sigUSR1 usr1
+  install Signals.sigUSR2 usr2
  where
    install signal handler =
      void $ Signals.installHandler signal (Signals.Catch handler) Nothing
+#else
+installSignalHandlers _ _ _ = pass
+#endif
+
+-- | Create a unix domain socket and bind it to the given path.
+-- | The socket file will be deleted if it already exists.
+createAndBindDomainSocket :: String -> FileMode -> IO NS.Socket
+createAndBindDomainSocket path mode = do
+  unless NS.isUnixDomainSocketAvailable $
+    panic "Cannot run with unix socket on non-unix platforms. Consider deleting the `server-unix-socket` config entry in order to continue."
+  deleteSocketFileIfExist path
+  sock <- NS.socket NS.AF_UNIX NS.Stream NS.defaultProtocol
+  NS.bind sock $ NS.SockAddrUnix path
+  NS.listen sock (max 2048 NS.maxListenQueue)
+  setFileMode path mode
+  return sock
+  where
+    deleteSocketFileIfExist path' =
+      removeFile path' `catch` handleDoesNotExist
+    handleDoesNotExist e
+      | isDoesNotExistError e = return ()
+      | otherwise = throwIO e
@@ -1,343 +0,0 @@
-{-# LANGUAGE LambdaCase      #-}
-{-# LANGUAGE NamedFieldPuns  #-}
-{-# LANGUAGE RecordWildCards #-}
-
-module PostgREST.Workers
-  ( connectionWorker
-  , reReadConfig
-  , runListener
-  , runAdmin
-  ) where
-
-import qualified Data.ByteString            as BS
-import qualified Data.Text                  as T
-import qualified Hasql.Notifications        as SQL
-import qualified Hasql.Session              as SQL
-import qualified Hasql.Transaction.Sessions as SQL
-import qualified Network.HTTP.Types.Status  as HTTP
-import qualified Network.Wai                as Wai
-import qualified Network.Wai.Handler.Warp   as Warp
-
-import Control.Retry    (RetryStatus, capDelay, exponentialBackoff,
-                         retrying, rsPreviousDelay)
-import Hasql.Connection (acquire)
-
-import Network.Socket
-import Network.Socket.ByteString
-
-import PostgREST.AppState         (AppState)
-import PostgREST.Config           (AppConfig (..), readAppConfig)
-import PostgREST.Config.Database  (queryDbSettings, queryPgVersion,
-                                   queryRoleSettings)
-import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
-import PostgREST.Error            (checkIsFatal)
-import PostgREST.SchemaCache      (querySchemaCache)
-
-import qualified PostgREST.AppState as AppState
-
-import Protolude
-
-
-- | Current database connection status data ConnectionStatus
-data ConnectionStatus
-  = NotConnected
-  | Connected PgVersion
-  | FatalConnectionError Text
-  deriving (Eq)
-
-- | Schema cache status
-data SCacheStatus
-  = SCLoaded
-  | SCOnRetry
-  | SCFatalFail
-
-- | The purpose of this worker is to obtain a healthy connection to pg and an
-- up-to-date schema cache(SchemaCache).  This method is meant to be called
-- multiple times by the same thread, but does nothing if the previous
-- invocation has not terminated. In all cases this method does not halt the
-- calling thread, the work is performed in a separate thread.
--
-- Background thread that does the following :
--  1. Tries to connect to pg server and will keep trying until success.
--  2. Checks if the pg version is supported and if it's not it kills the main
--     program.
--  3. Obtains the sCache. If this fails, it goes back to 1.
-connectionWorker :: AppState -> IO ()
-connectionWorker appState = do
-  runExclusively (AppState.getWorkerSem appState) work
-  -- Prevents multiple workers to be running at the same time. Could happen on
-  -- too many SIGUSR1s.
-  where
-    runExclusively mvar action = mask_ $ do
-      success <- tryPutMVar mvar ()
-      when success $ do
-        void $ forkIO $ action `finally` takeMVar mvar
-    work = do
-      AppConfig{..} <- AppState.getConfig appState
-      AppState.logWithZTime appState "Attempting to connect to the database..."
-      connected <- establishConnection appState
-      case connected of
-        FatalConnectionError reason ->
-          -- Fatal error when connecting
-          AppState.logWithZTime appState reason >> killThread (AppState.getMainThreadId appState)
-        NotConnected ->
-          -- Unreachable because establishConnection will keep trying to connect
-          return ()
-        Connected actualPgVersion -> do
-          -- Procede with initialization
-          AppState.putPgVersion appState actualPgVersion
-          when configDbChannelEnabled $
-            AppState.signalListener appState
-          AppState.logWithZTime appState "Connection successful"
-          -- this could be fail because the connection drops, but the
-          -- loadSchemaCache will pick the error and retry again
-          when configDbConfig $ reReadConfig False appState
-          scStatus <- loadSchemaCache appState
-          case scStatus of
-            SCLoaded ->
-              -- do nothing and proceed if the load was successful
-              return ()
-            SCOnRetry ->
-              -- retry reloading the schema cache
-              work
-            SCFatalFail ->
-              -- die if our schema cache query has an error
-              killThread $ AppState.getMainThreadId appState
-
-- | Repeatedly flush the pool, and check if a connection from the
-- pool allows access to the PostgreSQL database.
--
-- Releasing the pool is key for rapid recovery. Otherwise, the pool
-- timeout would have to be reached for new healthy connections to be acquired.
-- Which might not happen if the server is busy with requests. No idle
-- connection, no pool timeout.
--
-- The connection tries are capped, but if the connection times out no error is
-- thrown, just 'False' is returned.
-establishConnection :: AppState -> IO ConnectionStatus
-establishConnection appState =
-  retrying retrySettings shouldRetry $
-    const $ AppState.flushPool appState >> getConnectionStatus
-  where
-    retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
-    delayMicroseconds = 32000000 -- 32 seconds
-    backoffMicroseconds = 1000000 -- 1 second
-
-    getConnectionStatus :: IO ConnectionStatus
-    getConnectionStatus = do
-      pgVersion <- AppState.usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
-      case pgVersion of
-        Left e -> do
-          AppState.logPgrstError appState e
-          case checkIsFatal e of
-            Just reason ->
-              return $ FatalConnectionError reason
-            Nothing ->
-              return NotConnected
-        Right version ->
-          if version < minimumPgVersion then
-            return . FatalConnectionError $
-              "Cannot run in this PostgreSQL version, PostgREST needs at least "
-              <> pgvName minimumPgVersion
-          else
-            return . Connected  $ version
-
-    shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
-    shouldRetry rs isConnSucc = do
-      let
-        delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
-        itShould = NotConnected == isConnSucc
-      when itShould . AppState.logWithZTime appState $
-        "Attempting to reconnect to the database in "
-        <> (show delay::Text)
-        <> " seconds..."
-      when itShould $ AppState.putRetryNextIn appState delay
-      return itShould
-
-- | Load the SchemaCache by using a connection from the pool.
-loadSchemaCache :: AppState -> IO SCacheStatus
-loadSchemaCache appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  result <-
-    let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
-    AppState.usePool appState . transaction SQL.ReadCommitted SQL.Read $
-      querySchemaCache (toList configDbSchemas) configDbExtraSearchPath configDbPreparedStatements
-  case result of
-    Left e -> do
-      case checkIsFatal e of
-        Just hint -> do
-          AppState.logWithZTime appState "A fatal error ocurred when loading the schema cache"
-          AppState.logPgrstError appState e
-          AppState.logWithZTime appState hint
-          return SCFatalFail
-        Nothing -> do
-          AppState.putSchemaCache appState Nothing
-          AppState.logWithZTime appState "An error ocurred when loading the schema cache"
-          AppState.logPgrstError appState e
-          return SCOnRetry
-
-    Right sCache -> do
-      AppState.putSchemaCache appState (Just sCache)
-      AppState.logWithZTime appState "Schema cache loaded"
-      return SCLoaded
-
-runListener :: AppConfig -> AppState -> IO ()
-runListener AppConfig{configDbChannelEnabled} appState =
-  when configDbChannelEnabled $ listener appState
-
-- | Starts a dedicated pg connection to LISTEN for notifications.  When a
-- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
-- cache.  It uses the connectionWorker in case the LISTEN connection dies.
-listener :: AppState -> IO ()
-listener appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  let dbChannel = toS configDbChannel
-
-  -- The listener has to wait for a signal from the connectionWorker.
-  -- This is because when the connection to the db is lost, the listener also
-  -- tries to recover the connection, but not with the same pace as the connectionWorker.
-  -- Not waiting makes stderr quickly fill with connection retries messages from the listener.
-  AppState.waitListener appState
-
-  -- forkFinally allows to detect if the thread dies
-  void . flip forkFinally (handleFinally dbChannel) $ do
-    dbOrError <- acquire $ toUtf8 configDbUri
-    case dbOrError of
-      Right db -> do
-        AppState.logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
-        AppState.putIsListenerOn appState True
-        SQL.listen db $ SQL.toPgIdentifier dbChannel
-        SQL.waitForNotifications handleNotification db
-      _ ->
-        die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
-  where
-    handleFinally dbChannel _ = do
-      -- if the thread dies, we try to recover
-      AppState.logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
-      AppState.putIsListenerOn appState False
-      -- assume the pool connection was also lost, call the connection worker
-      connectionWorker appState
-      -- retry the listener
-      listener appState
-
-    handleNotification _ msg
-      | BS.null msg            = cacheReloader
-      | msg == "reload schema" = cacheReloader
-      | msg == "reload config" = reReadConfig False appState
-      | otherwise              = pure () -- Do nothing if anything else than an empty message is sent
-
-    cacheReloader =
-      -- reloads the schema cache + restarts pool connections
-      -- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
-      connectionWorker appState
-
-- | Re-reads the config plus config options from the db
-reReadConfig :: Bool -> AppState -> IO ()
-reReadConfig startingUp appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  dbSettings <-
-    if configDbConfig then do
-      qDbSettings <- AppState.usePool appState $ queryDbSettings configDbPreparedStatements
-      case qDbSettings of
-        Left e -> do
-          AppState.logWithZTime appState
-            "An error ocurred when trying to query database settings for the config parameters"
-          case checkIsFatal e of
-            Just hint -> do
-              AppState.logPgrstError appState e
-              AppState.logWithZTime appState hint
-              killThread (AppState.getMainThreadId appState)
-            Nothing -> do
-              AppState.logPgrstError appState e
-          pure mempty
-        Right x -> pure x
-    else
-      pure mempty
-  roleSettings <-
-    if configDbConfig then do
-      rSettings <- AppState.usePool appState $ queryRoleSettings configDbPreparedStatements
-      case rSettings of
-        Left e -> do
-          AppState.logWithZTime appState "An error ocurred when trying to query the role settings"
-          AppState.logPgrstError appState e
-          pure mempty
-        Right x -> pure x
-    else
-      pure mempty
-  readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings >>= \case
-    Left err   ->
-      if startingUp then
-        panic err -- die on invalid config if the program is starting up
-      else
-        AppState.logWithZTime appState $ "Failed reloading config: " <> err
-    Right newConf -> do
-      AppState.putConfig appState newConf
-      if startingUp then
-        pass
-      else
-        AppState.logWithZTime appState "Config reloaded"
-
-runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
-runAdmin conf@AppConfig{configAdminServerPort} appState settings =
-  whenJust configAdminServerPort $ \adminPort -> do
-    AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
-    void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
-  where
-    whenJust :: Applicative m => Maybe a -> (a -> m ()) -> m ()
-    whenJust mg f = maybe (pure ()) f mg
-    adminApp = admin appState conf
-
-- | PostgREST admin application
-admin :: AppState.AppState -> AppConfig -> Wai.Application
-admin appState appConfig req respond  = do
-  isMainAppReachable  <- any isRight <$> reachMainApp appConfig
-  isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
-  isConnectionUp      <-
-    if configDbChannelEnabled appConfig
-      then AppState.getIsListenerOn appState
-      else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
-
-  case Wai.pathInfo req of
-    ["ready"] ->
-      respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
-    ["live"] ->
-      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
-    _ ->
-      respond $ Wai.responseLBS HTTP.status404 [] mempty
-
-- Try to connect to the main app socket
-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
-- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
-- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
-reachMainApp :: AppConfig -> IO [Either IOException ()]
-reachMainApp AppConfig{..} =
-  case configServerUnixSocket of
-    Just path ->  do
-      sock <- socket AF_UNIX Stream 0
-      (:[]) <$> try (do
-        connect sock $ SockAddrUnix path
-        withSocketsDo $ bracket (pure sock) close sendEmpty)
-    Nothing -> do
-      let
-        host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
-             | otherwise                                             = Just configServerHost
-        filterAddrs xs =
-          case configServerHost of
-              "*4" -> ipv4Addrs xs ++ ipv6Addrs xs
-              "!4" -> ipv4Addrs xs
-              "*6" -> ipv6Addrs xs ++ ipv4Addrs xs
-              "!6" -> ipv6Addrs xs
-              _    -> xs
-        ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
-        ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
-
-      addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
-      tryAddr `traverse` filterAddrs addrs
-  where
-    sendEmpty sock = void $ send sock mempty
-    tryAddr :: AddrInfo -> IO (Either IOException ())
-    tryAddr addr = do
-      sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
-      try $ do
-        connect sock $ addrAddress addr
-        withSocketsDo $ bracket (pure sock) close sendEmpty
@@ -12,5 +12,5 @@ nix:
 extra-deps:
  - git: https://github.com/PostgREST/postgresql-libpq.git
    commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
-  - hasql-notifications-0.2.0.4
-  - hasql-pool-0.9
+  - hasql-notifications-0.2.0.6
+  - hasql-pool-0.10
@@ -16,19 +16,19 @@ packages:
    commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
    git: https://github.com/PostgREST/postgresql-libpq.git
 - completed:
-    hackage: hasql-notifications-0.2.0.4@sha256:9a09fa9b97feadd9492c8bd8bc6b9cffe0513510102f08374b0c45ecd479ed67,2028
+    hackage: hasql-notifications-0.2.0.6@sha256:16d783f5cd1660fad924fd3769380889de5804e057f09b304dcdc3a3ff11eb3c,2028
    pantry-tree:
-      sha256: 56f9e240728e7a65711dde45fa2e2075b914e32cd370424aaa4572392378a60e
+      sha256: 2319743501bb3c0bef801014ce61308b8666cef86ae5a97a0a283c0c1ec12d4f
      size: 452
  original:
-    hackage: hasql-notifications-0.2.0.4
+    hackage: hasql-notifications-0.2.0.6
 - completed:
-    hackage: hasql-pool-0.9@sha256:db7a37f6b3a922c37adc3c7ced47a7c10786d1f171e47a735a6e812a587ba44c,2111
+    hackage: hasql-pool-0.10@sha256:912197a328acb85505f98bb9700d61f366b87659ca45126c5c2d636687b801c3,2112
    pantry-tree:
-      sha256: 49b1181d28c6f5317e794671c2dae155754b834bdcfa30f7e5dbad28e4cf0249
+      sha256: b655c540a49764a8d16b62941137e295b936b96edc0785eb9250972f0f92dc47
      size: 346
  original:
-    hackage: hasql-pool-0.9
+    hackage: hasql-pool-0.10
 snapshots:
 - completed:
    sha256: 4905c93319aa94aa53da8f41d614d7bacdbfe6c63a8c6132d32e6e62f24a9af4
@@ -11,9 +11,15 @@ main =
    [ "-XOverloadedStrings"
    , "-XNoImplicitPrelude"
    , "-XStandaloneDeriving"
+    , "-XDuplicateRecordFields"
    , "-isrc"
    , "src/PostgREST/Query/SqlFragment.hs"
    , "src/PostgREST/ApiRequest/Preferences.hs"
    , "src/PostgREST/ApiRequest/QueryParams.hs"
+    , "src/PostgREST/Response/Performance.hs"
    , "src/PostgREST/Error.hs"
+    , "src/PostgREST/MediaType.hs"
+    , "src/PostgREST/Config.hs"
+    , "src/PostgREST/Plan.hs"
+    , "src/PostgREST/Response.hs"
    ]
@@ -1,4 +1,5 @@
 db-schema = "provided_through_alias"
+db-pool-timeout = 5
 max-rows = 1000
 pre-request = "check_alias"
 role-claim-key = ".aliased"
@@ -1,3 +1,4 @@
+db-aggregates-enabled = false
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
@@ -7,26 +8,30 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 5
+db-pool-automatic-recovery = true
 db-pre-request = "check_alias"
 db-prepared-statements = true
 db-root-spec = "open_alias"
 db-schemas = "provided_through_alias"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
-db-use-legacy-gucs = true
 jwt-aud = ""
 jwt-role-claim-key = ".\"aliased\""
 jwt-secret = ""
 jwt-secret-is-base64 = true
+jwt-cache-max-lifetime = 0
 log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
+server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000
 server-trace-header = ""
+server-timing-enabled = false
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""
@@ -1,3 +1,4 @@
+db-aggregates-enabled = false
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
@@ -7,26 +8,30 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
+db-pool-automatic-recovery = true
 db-pre-request = ""
 db-prepared-statements = false
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
-db-use-legacy-gucs = true
 jwt-aud = ""
 jwt-role-claim-key = ".\"role\""
 jwt-secret = ""
 jwt-secret-is-base64 = true
+jwt-cache-max-lifetime = 0
 log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
+server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000
 server-trace-header = ""
+server-timing-enabled = false
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""
@@ -1,3 +1,4 @@
+db-aggregates-enabled = false
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
@@ -7,26 +8,30 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
+db-pool-automatic-recovery = true
 db-pre-request = ""
 db-prepared-statements = false
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
-db-use-legacy-gucs = true
 jwt-aud = ""
 jwt-role-claim-key = ".\"role\""
 jwt-secret = ""
 jwt-secret-is-base64 = true
+jwt-cache-max-lifetime = 0
 log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
+server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000
 server-trace-header = ""
+server-timing-enabled = false
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""
@@ -1,3 +1,4 @@
+db-aggregates-enabled = false
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
@@ -7,26 +8,30 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
+db-pool-automatic-recovery = true
 db-pre-request = ""
 db-prepared-statements = true
 db-root-spec = ""
 db-schemas = "public"
 db-config = false
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
-db-use-legacy-gucs = true
 jwt-aud = ""
 jwt-role-claim-key = ".\"role\""
 jwt-secret = ""
 jwt-secret-is-base64 = false
+jwt-cache-max-lifetime = 0
 log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
+server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000
 server-trace-header = ""
+server-timing-enabled = false
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""
@@ -1,4 +1,5 @@
-db-anon-role = "other"
+db-aggregates-enabled = false
+db-anon-role = "pre_config_role"
 db-channel = "postgrest"
 db-channel-enabled = false
 db-extra-search-path = "public,extensions,other"
@@ -7,26 +8,30 @@ db-plan-enabled = true
 db-pool = 1
 db-pool-acquisition-timeout = 30
 db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
+db-pool-automatic-recovery = false
 db-pre-request = "test.other_custom_headers"
 db-prepared-statements = false
 db-root-spec = "other_root"
 db-schemas = "test,other_tenant1,other_tenant2"
 db-config = true
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "postgresql://"
-db-use-legacy-gucs = false
 jwt-aud = "https://otherexample.org"
-jwt-role-claim-key = ".\"other\".\"role\""
+jwt-role-claim-key = ".\"other\".\"pre_config_role\""
 jwt-secret = "ODERREALLYREALLYREALLYREALLYVERYSAFE"
 jwt-secret-is-base64 = true
+jwt-cache-max-lifetime = 86400
 log-level = "info"
 openapi-mode = "disabled"
 openapi-security-active = false
 openapi-server-proxy-uri = "https://otherexample.org/api"
-raw-media-types = "application/vnd.pgrst.other-db-config"
+server-cors-allowed-origins = "http://example.com"
 server-host = "0.0.0.0"
 server-port = 80
 server-trace-header = "traceparent"
+server-timing-enabled = true
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001
@@ -1,3 +1,4 @@
+db-aggregates-enabled = false
 db-anon-role = "anonymous"
 db-channel = "postgrest"
 db-channel-enabled = false
@@ -7,26 +8,30 @@ db-plan-enabled = true
 db-pool = 1
 db-pool-acquisition-timeout = 30
 db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
+db-pool-automatic-recovery = false
 db-pre-request = "test.custom_headers"
 db-prepared-statements = false
 db-root-spec = "root"
 db-schemas = "test,tenant1,tenant2"
 db-config = true
+db-pre-config = "postgrest.preconf"
 db-tx-end = "commit-allow-override"
 db-uri = "postgresql://"
-db-use-legacy-gucs = false
 jwt-aud = "https://example.org"
 jwt-role-claim-key = ".\"a\".\"role\""
 jwt-secret = "OVERRIDE=REALLY=REALLY=REALLY=REALLY=VERY=SAFE"
 jwt-secret-is-base64 = false
+jwt-cache-max-lifetime = 86400
 log-level = "info"
 openapi-mode = "ignore-privileges"
 openapi-security-active = true
 openapi-server-proxy-uri = "https://example.org/api"
-raw-media-types = "application/vnd.pgrst.db-config"
+server-cors-allowed-origins = "http://example.com"
 server-host = "0.0.0.0"
 server-port = 80
 server-trace-header = "CF-Ray"
+server-timing-enabled = true
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001
@@ -1,3 +1,4 @@
+db-aggregates-enabled = true
 db-anon-role = "root"
 db-channel = "postgrest"
 db-channel-enabled = false
@@ -7,26 +8,30 @@ db-plan-enabled = true
 db-pool = 1
 db-pool-acquisition-timeout = 30
 db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
+db-pool-automatic-recovery = false
 db-pre-request = "please_run_fast"
 db-prepared-statements = false
 db-root-spec = "openapi_v3"
 db-schemas = "multi,tenant,setup"
 db-config = false
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "tmp_db"
-db-use-legacy-gucs = false
 jwt-aud = "https://postgrest.org"
 jwt-role-claim-key = ".\"user\"[0].\"real-role\""
 jwt-secret = "c2VjdXJpdHl0aHJvdWdob2JzY3VyaXR5"
 jwt-secret-is-base64 = true
+jwt-cache-max-lifetime = 86400
 log-level = "info"
 openapi-mode = "ignore-privileges"
 openapi-security-active = true
 openapi-server-proxy-uri = "https://postgrest.org"
-raw-media-types = "application/vnd.pgrst.config"
+server-cors-allowed-origins = "http://example.com"
 server-host = "0.0.0.0"
 server-port = 80
 server-trace-header = "X-Request-Id"
+server-timing-enabled = true
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001
@@ -1,3 +1,4 @@
+db-aggregates-enabled = false
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
@@ -7,26 +8,30 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
+db-pool-automatic-recovery = true
 db-pre-request = ""
 db-prepared-statements = true
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
-db-use-legacy-gucs = true
 jwt-aud = ""
 jwt-role-claim-key = ".\"role\""
 jwt-secret = ""
 jwt-secret-is-base64 = false
+jwt-cache-max-lifetime = 0
 log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
+server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000
 server-trace-header = ""
+server-timing-enabled = false
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""
@@ -1,5 +1,6 @@
 PGRST_APP_SETTINGS_test2: test
 PGRST_APP_SETTINGS_test: test
+PGRST_DB_AGGREGATES_ENABLED: true
 PGRST_DB_ANON_ROLE: root
 PGRST_DB_CHANNEL: postgrest
 PGRST_DB_CHANNEL_ENABLED: false
@@ -9,27 +10,31 @@ PGRST_DB_PLAN_ENABLED: true
 PGRST_DB_POOL: 1
 PGRST_DB_POOL_ACQUISITION_TIMEOUT: 30
 PGRST_DB_POOL_MAX_LIFETIME: 3600
+PGRST_DB_POOL_MAX_IDLETIME: 60
+PGRST_DB_POOL_AUTOMATIC_RECOVERY: false
 PGRST_DB_PREPARED_STATEMENTS: false
 PGRST_DB_PRE_REQUEST: please_run_fast
 PGRST_DB_ROOT_SPEC: openapi_v3
 PGRST_DB_SCHEMAS: multi,   tenant,setup
 PGRST_DB_CONFIG: false
+PGRST_DB_PRE_CONFIG: "postgrest.pre_config"
 PGRST_DB_TX_END: rollback-allow-override
 PGRST_DB_URI: tmp_db
-PGRST_DB_EMBED_DEFAULT_JOIN: inner
 PGRST_DB_USE_LEGACY_GUCS: false
 PGRST_JWT_AUD: 'https://postgrest.org'
 PGRST_JWT_ROLE_CLAIM_KEY: '.user[0]."real-role"'
 PGRST_JWT_SECRET: c2VjdXJpdHl0aHJvdWdob2JzY3VyaXR5
 PGRST_JWT_SECRET_IS_BASE64: true
+PGRST_JWT_CACHE_MAX_LIFETIME: 86400
 PGRST_LOG_LEVEL: info
 PGRST_OPENAPI_MODE: 'ignore-privileges'
 PGRST_OPENAPI_SECURITY_ACTIVE: true
 PGRST_OPENAPI_SERVER_PROXY_URI: 'https://postgrest.org'
-PGRST_RAW_MEDIA_TYPES: application/vnd.pgrst.config
+PGRST_SERVER_CORS_ALLOWED_ORIGINS: "http://example.com"
 PGRST_SERVER_HOST: 0.0.0.0
 PGRST_SERVER_PORT: 80
 PGRST_SERVER_TRACE_HEADER: X-Request-Id
+PGRST_SERVER_TIMING_ENABLED: true
 PGRST_SERVER_UNIX_SOCKET: /tmp/pgrst_io_test.sock
 PGRST_SERVER_UNIX_SOCKET_MODE: 777
 PGRST_ADMIN_SERVER_PORT: 3001
@@ -1,3 +1,4 @@
+db-aggregates-enabled = true
 db-anon-role = "root"
 db-channel = "postgrest"
 db-channel-enabled = false
@@ -7,26 +8,30 @@ db-plan-enabled = true
 db-pool = 1
 db-pool-acquisition-timeout = 30
 db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
+db-pool-automatic-recovery = false
 db-pre-request = "please_run_fast"
 db-prepared-statements = false
 db-root-spec = "openapi_v3"
 db-schemas = "multi,   tenant,setup"
 db-config = false
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "tmp_db"
-db-use-legacy-gucs = false
 jwt-aud = "https://postgrest.org"
 jwt-role-claim-key = ".user[0].\"real-role\""
 jwt-secret = "c2VjdXJpdHl0aHJvdWdob2JzY3VyaXR5"
 jwt-secret-is-base64 = true
+jwt-cache-max-lifetime = 86400
 log-level = "info"
 openapi-mode = "ignore-privileges"
 openapi-security-active = true
 openapi-server-proxy-uri = "https://postgrest.org"
-raw-media-types = "application/vnd.pgrst.config"
+server-cors-allowed-origins = "http://example.com"
 server-host = "0.0.0.0"
 server-port = 80
 server-trace-header = "X-Request-Id"
+server-timing-enabled = true
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001
@@ -8,6 +8,3 @@ db-channel-enabled = 13

 # expects integer or string
 db-max-rows = true
-
-# expects string
-raw-media-types = true
@@ -3,12 +3,13 @@ CREATE ROLE db_config_authenticator LOGIN NOINHERIT;
 -- reloadable config options
 ALTER ROLE db_config_authenticator SET pgrst.jwt_aud = 'https://example.org';
 ALTER ROLE db_config_authenticator SET pgrst.openapi_server_proxy_uri = 'https://example.org/api';
-ALTER ROLE db_config_authenticator SET pgrst.raw_media_types = 'application/vnd.pgrst.db-config';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_secret = 'REALLY=REALLY=REALLY=REALLY=VERY=SAFE';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_secret_is_base64 = 'false';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_role_claim_key = '."a"."role"';
+ALTER ROLE db_config_authenticator SET pgrst.db_aggregates_enabled = 'false';
 ALTER ROLE db_config_authenticator SET pgrst.db_anon_role = 'anonymous';
 ALTER ROLE db_config_authenticator SET pgrst.db_tx_end = 'commit-allow-override';
+ALTER ROLE db_config_authenticator SET pgrst.db_pre_config = 'postgrest.preconf';
 ALTER ROLE db_config_authenticator SET pgrst.db_schemas = 'test, tenant1, tenant2';
 ALTER ROLE db_config_authenticator SET pgrst.db_root_spec = 'root';
 ALTER ROLE db_config_authenticator SET pgrst.db_plan_enabled = 'true';
@@ -17,7 +18,9 @@ ALTER ROLE db_config_authenticator SET pgrst.db_pre_request = 'test.custom_heade
 ALTER ROLE db_config_authenticator SET pgrst.db_max_rows = '1000';
 ALTER ROLE db_config_authenticator SET pgrst.db_extra_search_path = 'public, extensions';
 ALTER ROLE db_config_authenticator SET pgrst.not_existing = 'should be ignored';
+ALTER ROLE db_config_authenticator SET pgrst.server_cors_allowed_origins = 'http://example.com';
 ALTER ROLE db_config_authenticator SET pgrst.server_trace_header = 'CF-Ray';
+ALTER ROLE db_config_authenticator SET pgrst.server_timing_enabled = 'true';

 -- override with database specific setting
 ALTER ROLE db_config_authenticator IN DATABASE :DBNAME SET pgrst.jwt_secret = 'OVERRIDE=REALLY=REALLY=REALLY=REALLY=VERY=SAFE';
@@ -40,18 +43,18 @@ ALTER ROLE db_config_authenticator SET pgrst.db_channel_enabled = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_channel = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_pool = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_pool_timeout = 'ignored';
-ALTER ROLE db_config_authenticator SET pgrst.db_config = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_acquisition_timeout = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_lifetime = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_idletime = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_config = 'true';

 -- other authenticator reloadable config options
 CREATE ROLE other_authenticator LOGIN NOINHERIT;
 ALTER ROLE other_authenticator SET pgrst.jwt_aud = 'https://otherexample.org';
 ALTER ROLE other_authenticator SET pgrst.openapi_server_proxy_uri = 'https://otherexample.org/api';
-ALTER ROLE other_authenticator SET pgrst.raw_media_types = 'application/vnd.pgrst.other-db-config';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret = 'ODERREALLYREALLYREALLYREALLYVERYSAFE';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret_is_base64 = 'true';
-ALTER ROLE other_authenticator SET pgrst.jwt_role_claim_key = '."other"."role"';
-ALTER ROLE other_authenticator SET pgrst.db_anon_role = 'other';
-ALTER ROLE other_authenticator SET pgrst.db_tx_end = 'rollback-allow-override';
+ALTER ROLE other_authenticator SET pgrst.db_aggregates_enabled = 'false';
 ALTER ROLE other_authenticator SET pgrst.db_schemas = 'test, other_tenant1, other_tenant2';
 ALTER ROLE other_authenticator SET pgrst.db_root_spec = 'other_root';
 ALTER ROLE other_authenticator SET pgrst.db_plan_enabled = 'true';
@@ -61,7 +64,35 @@ ALTER ROLE other_authenticator SET pgrst.db_max_rows = '100';
 ALTER ROLE other_authenticator SET pgrst.db_extra_search_path = 'public, extensions, other';
 ALTER ROLE other_authenticator SET pgrst.openapi_mode = 'disabled';
 ALTER ROLE other_authenticator SET pgrst.openapi_security_active = 'false';
+ALTER ROLE other_authenticator SET pgrst.server_cors_allowed_origins = 'http://example.com';
 ALTER ROLE other_authenticator SET pgrst.server_trace_header = 'traceparent';
+ALTER ROLE other_authenticator SET pgrst.db_pre_config = 'postgrest.pre_config';
+ALTER ROLE other_authenticator SET pgrst.server_timing_enabled = 'true';
+
+create schema postgrest;
+grant usage on schema postgrest to db_config_authenticator;
+grant usage on schema postgrest to other_authenticator;
+
+-- pre-config hook
+create or replace function postgrest.pre_config()
+returns void as $$
+begin
+  if current_user = 'other_authenticator' then
+    perform
+      set_config('pgrst.jwt_role_claim_key', '."other"."pre_config_role"', true)
+    , set_config('pgrst.db_anon_role', 'pre_config_role', true)
+    , set_config('pgrst.db_schemas', 'will be overriden with the above ALTER ROLE.. db_schemas', true)
+    , set_config('pgrst.db_tx_end', 'rollback-allow-override', true);
+  else
+    null;
+  end if;
+end $$ language plpgsql;
+
+create or replace function postgrest.preconf()
+returns void as $$
+begin
+  null;
+end $$ language plpgsql;

 -- authenticator used for tests that manipulate statement timeout
 CREATE ROLE timeout_authenticator LOGIN NOINHERIT;
@@ -1,6 +1,7 @@
-\ir big_schema.sql
+-- \ir big_schema.sql big schema test currently skipped, see test_io.py
 \ir db_config.sql

+set check_function_bodies = false; -- to allow conditionals based on the pg version
 set search_path to public;

 CREATE ROLE postgrest_test_anonymous;
@@ -14,7 +15,21 @@ alter role postgrest_test_serializable set default_transaction_isolation = 'seri
 CREATE ROLE postgrest_test_repeatable_read;
 alter role postgrest_test_repeatable_read set default_transaction_isolation = 'REPEATABLE READ';

-GRANT postgrest_test_anonymous, postgrest_test_author, postgrest_test_serializable, postgrest_test_repeatable_read TO :PGUSER;
+CREATE ROLE postgrest_test_w_superuser_settings;
+alter role postgrest_test_w_superuser_settings set log_min_duration_statement = 1;
+alter role postgrest_test_w_superuser_settings set log_min_messages = 'fatal';
+
+DO $do$BEGIN
+  IF (SELECT current_setting('server_version_num')::INT >= 150000) THEN
+    ALTER ROLE postgrest_test_w_superuser_settings SET log_min_duration_sample = 12345;
+    GRANT SET ON PARAMETER log_min_duration_sample to postgrest_test_authenticator;
+  END IF;
+END$do$;
+
+GRANT
+  postgrest_test_anonymous, postgrest_test_author,
+  postgrest_test_serializable, postgrest_test_repeatable_read,
+  postgrest_test_w_superuser_settings TO :PGUSER;

 CREATE SCHEMA v1;
 GRANT USAGE ON SCHEMA v1 TO postgrest_test_anonymous;
@@ -23,7 +38,7 @@ CREATE TABLE authors_only ();
 GRANT SELECT ON authors_only TO postgrest_test_author;

 CREATE TABLE projects AS SELECT FROM generate_series(1,5);
-GRANT SELECT ON projects TO postgrest_test_anonymous;
+GRANT SELECT ON projects TO postgrest_test_anonymous, postgrest_test_w_superuser_settings;

 create function get_guc_value(name text) returns text as $$
  select nullif(current_setting(name), '')::text;
@@ -90,7 +105,7 @@ create or replace function sleep(seconds double precision) returns void as $$
 $$ language sql;

 create or replace function hello() returns text as $$
-  select 'hello';
+  select 'hello'::text;
 $$ language sql;

 create table cats(id uuid primary key, name text);
@@ -142,3 +157,44 @@ returns text as $$
  select current_setting('transaction_isolation', true);
 $$
 language sql set default_transaction_isolation = 'REPEATABLE READ';
+
+create or replace function create_function() returns void as $_$
+  drop function if exists mult_them(int, int);
+  create or replace function mult_them(a int, b int) returns int as $$
+    select a*b;
+  $$ language sql;
+  notify pgrst, 'reload schema';
+$_$ language sql security definer;
+
+create or replace function migrate_function() returns void as $_$
+  drop function if exists mult_them(int, int);
+  create or replace function mult_them(c int, d int) returns int as $$
+    select c*d;
+  $$ language sql;
+  notify pgrst, 'reload schema';
+$_$ language sql security definer;
+
+create or replace function get_pgrst_version() returns text
+  language sql
+as $$
+select application_name
+from pg_stat_activity
+where application_name ilike 'postgrest%'
+limit 1;
+$$;
+
+create function terminate_pgrst() returns setof record as $$
+select pg_terminate_backend(pid) from pg_stat_activity where application_name iLIKE '%postgrest%';
+$$ language sql security definer;
+
+create or replace function one_sec_timeout() returns void as $$
+  select pg_sleep(3);
+$$ language sql set statement_timeout = '1s';
+
+create or replace function four_sec_timeout() returns void as $$
+  select pg_sleep(3);
+$$ language sql set statement_timeout = '4s';
+
+create function get_postgres_version() returns int as $$
+  select current_setting('server_version_num')::int;
+$$ language sql;
@@ -4,6 +4,10 @@ cli:
    args: ['--help']
  - name: help short
    args: ['-h']
+  - name: version long
+    args: ['--version']
+  - name: version short
+    args: ['-v']
  - name: example long
    args: ['--example']
  - name: example short
@@ -115,11 +119,6 @@ cli:
    use_defaultenv: true
    env:
      PGRST_DB_TX_END: rollback
-  - name: raw-media-types list
-    expect: 'raw-media-types = "image/png,image/jpeg"'
-    use_defaultenv: true
-    env:
-      PGRST_RAW_MEDIA_TYPES: ' image/png ,  image/jpeg '

 roleclaims:
  - key: '.postgrest.a_role'
@@ -17,6 +17,21 @@ import requests_unixsocket
 from config import *


+def sleep_until_postgrest_scache_reload():
+    "Sleep until schema cache reload"
+    time.sleep(0.3)
+
+
+def sleep_until_postgrest_config_reload():
+    "Sleep until config reload"
+    time.sleep(0.2)
+
+
+def sleep_until_postgrest_full_reload():
+    "Sleep until schema cache plus config reload"
+    time.sleep(0.3)
+
+
 class PostgrestTimedOut(Exception):
    "Connecting to PostgREST endpoint timed out."

@@ -64,6 +79,7 @@ def run(
    host=None,
    wait_for_readiness=True,
    no_pool_connection_available=False,
+    no_startup_stdout=True,
 ):
    "Run PostgREST and yield an endpoint that is ready for connections."

@@ -104,7 +120,8 @@ def run(
            if wait_for_readiness:
                wait_until_ready(adminurl + "/ready")

-            process.stdout.read()
+            if no_startup_stdout:
+                process.stdout.read()

            if no_pool_connection_available:
                sleep_pool_connection(baseurl, 10)
@@ -1,6 +1,6 @@
 "Unit tests for Input/Ouput of PostgREST seen as a black box."

-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from operator import attrgetter
 import os
 import re
@@ -169,7 +169,7 @@ def test_app_settings_flush_pool(defaultenv):

        # SIGUSR1 causes the postgres connection pool to be flushed
        postgrest.process.send_signal(signal.SIGUSR1)
-        time.sleep(0.1)
+        sleep_until_postgrest_scache_reload()

        uri = "/rpc/get_guc_value?name=app.settings.external_api_secret"
        response = postgrest.session.get(uri)
@@ -197,6 +197,13 @@ def test_flush_pool_no_interrupt(defaultenv):
        t.join()


+def test_random_port_bound(defaultenv):
+    "PostgREST should bind to a random port when PGRST_SERVER_PORT is 0."
+
+    with run(env=defaultenv, port="0") as postgrest:
+        assert True  # liveness check is done by run(), so we just need to check that it doesn't fail
+
+
 def test_app_settings_reload(tmp_path, defaultenv):
    "App settings should be reloaded from file when PostgREST is sent SIGUSR2."
    config = (CONFIGSDIR / "sigusr2-settings.config").read_text()
@@ -213,7 +220,7 @@ def test_app_settings_reload(tmp_path, defaultenv):
        # reload
        postgrest.process.send_signal(signal.SIGUSR2)

-        time.sleep(0.1)
+        sleep_until_postgrest_config_reload()

        response = postgrest.session.get(uri)
        assert response.text == '"Jane"'
@@ -237,7 +244,7 @@ def test_jwt_secret_reload(tmp_path, defaultenv):
        # reload config
        postgrest.process.send_signal(signal.SIGUSR2)

-        time.sleep(0.1)
+        sleep_until_postgrest_config_reload()

        response = postgrest.session.get("/authors_only", headers=headers)
        assert response.status_code == 200
@@ -267,14 +274,14 @@ def test_jwt_secret_external_file_reload(tmp_path, defaultenv):

        # SIGUSR1 doesn't reload external files, at least when db-config=false
        postgrest.process.send_signal(signal.SIGUSR1)
-        time.sleep(0.1)
+        sleep_until_postgrest_scache_reload()

        response = postgrest.session.get("/authors_only", headers=headers)
        assert response.status_code == 401

        # reload config and external file with SIGUSR2
        postgrest.process.send_signal(signal.SIGUSR2)
-        time.sleep(0.1)
+        sleep_until_postgrest_config_reload()

        response = postgrest.session.get("/authors_only", headers=headers)
        assert response.status_code == 200
@@ -285,7 +292,7 @@ def test_jwt_secret_external_file_reload(tmp_path, defaultenv):
        # reload config and external file with NOTIFY
        response = postgrest.session.post("/rpc/reload_pgrst_config")
        assert response.status_code == 204
-        time.sleep(0.1)
+        sleep_until_postgrest_config_reload()

        response = postgrest.session.get("/authors_only", headers=headers)
        assert response.status_code == 401
@@ -308,13 +315,11 @@ def test_db_schema_reload(tmp_path, defaultenv):

        # reload config
        postgrest.process.send_signal(signal.SIGUSR2)
+        sleep_until_postgrest_config_reload()

        # reload schema cache to verify that the config reload actually happened
        postgrest.process.send_signal(signal.SIGUSR1)
-
-        # takes max 1 second to load the internal cache(big_schema.sql included now)
-        # TODO this could go back to time.sleep(0.1) if the big_schema is put in another test suite
-        time.sleep(1)
+        sleep_until_postgrest_scache_reload()

        response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
        assert response.text == '"\\"v1\\", \\"public\\""'
@@ -334,7 +339,7 @@ def test_db_schema_notify_reload(defaultenv):
            "/rpc/change_db_schema_and_full_reload", data={"schemas": "v1"}
        )

-        time.sleep(0.2)
+        sleep_until_postgrest_full_reload()

        response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
        assert response.text == '"\\"v1\\", \\"public\\""'
@@ -362,7 +367,7 @@ def test_max_rows_reload(defaultenv):
        # reload config
        postgrest.process.send_signal(signal.SIGUSR2)

-        time.sleep(0.1)
+        sleep_until_postgrest_config_reload()

        response = postgrest.session.head("/projects")
        assert response.status_code == 200
@@ -392,7 +397,7 @@ def test_max_rows_notify_reload(defaultenv):
            "/rpc/change_max_rows_config", data={"val": 1, "notify": True}
        )

-        time.sleep(0.1)
+        sleep_until_postgrest_config_reload()

        response = postgrest.session.head("/projects")
        assert response.status_code == 200
@@ -503,7 +508,7 @@ def test_change_statement_timeout(defaultenv, metapostgrest):

        # trigger schema refresh
        postgrest.process.send_signal(signal.SIGUSR1)
-        time.sleep(0.1)
+        sleep_until_postgrest_scache_reload()

        response = postgrest.session.get("/rpc/sleep?seconds=1")
        assert response.status_code == 500
@@ -514,7 +519,7 @@ def test_change_statement_timeout(defaultenv, metapostgrest):

        # trigger role setting refresh
        postgrest.process.send_signal(signal.SIGUSR1)
-        time.sleep(0.1)
+        sleep_until_postgrest_scache_reload()

        response = postgrest.session.get("/rpc/sleep?seconds=1")
        assert response.status_code == 204
@@ -550,13 +555,15 @@ def test_pool_size(defaultenv, metapostgrest):
        assert delta > 1 and delta < 1.5


-def test_pool_acquisition_timeout(defaultenv, metapostgrest):
-    "Verify that PGRST_DB_POOL_ACQUISITON_TIMEOUT times out when the pool is empty"
+@pytest.mark.parametrize("level", ["crit", "error", "warn", "info"])
+def test_pool_acquisition_timeout(level, defaultenv, metapostgrest):
+    "Verify that PGRST_DB_POOL_ACQUISITION_TIMEOUT times out when the pool is empty"

    env = {
        **defaultenv,
        "PGRST_DB_POOL": "1",
        "PGRST_DB_POOL_ACQUISITION_TIMEOUT": "1",  # 1 second
+        "PGRST_LOG_LEVEL": level,
    }

    with run(env=env, no_pool_connection_available=True) as postgrest:
@@ -567,8 +574,12 @@ def test_pool_acquisition_timeout(defaultenv, metapostgrest):

        # ensure the message appears on the logs as well
        output = sorted(postgrest.read_stdout(nlines=2))
-        assert " 504 " in output[0]
-        assert "Timed out acquiring connection from connection pool." in output[1]
+
+        if level == "crit":
+            assert len(output) == 0
+        else:
+            assert " 504 " in output[0]
+            assert "Timed out acquiring connection from connection pool." in output[1]


 def test_change_statement_timeout_held_connection(defaultenv, metapostgrest):
@@ -665,7 +676,7 @@ def test_admin_ready_includes_schema_cache_state(defaultenv, metapostgrest):

        # force a reconnection so the new role setting is picked up
        postgrest.process.send_signal(signal.SIGUSR1)
-        time.sleep(0.1)
+        sleep_until_postgrest_scache_reload()

        response = postgrest.admin.get("/ready")
        assert response.status_code == 503
@@ -832,7 +843,7 @@ def test_notify_reloading_catalog_cache(defaultenv):
        # change it to a bigint
        response = postgrest.session.post("/rpc/drop_change_cats")
        assert response.status_code == 204
-        time.sleep(0.1)
+        sleep_until_postgrest_scache_reload()

        # next request should succeed with a bigint value
        response = postgrest.session.get("/cats?id=eq.1")
@@ -860,7 +871,7 @@ def test_role_settings(defaultenv):

        response = postgrest.session.get("/rpc/reload_pgrst_config")
        assert response.status_code == 204
-        time.sleep(0.1)
+        sleep_until_postgrest_config_reload()

        response = postgrest.session.get("/rpc/get_guc_value?name=statement_timeout")
        assert response.text == '"5s"'
@@ -946,6 +957,44 @@ def test_isolation_level(defaultenv):
        assert response.text == '"serializable"'


+def test_schema_cache_reloading(defaultenv):
+    "schema cache should reload successfully"
+
+    # If DB_POOL=1, then the second request(/rpc/migrate_function) will just wait(PGRST_DB_POOL_ACQUISITION_TIMEOUT=10) for the schema cache reload to finish.
+    # This is bc the only pool connection will be busy with the PGRST_INTERNAL_SCHEMA_CACHE_SLEEP(does a pg_sleep)
+    # So this must be tested with a DB_POOL size of at least 2. That way the second request will pick the other pool connection and proceed.
+
+    env = {
+        **defaultenv,
+        "PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "1",
+        "PGRST_DB_CHANNEL_ENABLED": "true",
+        "PGRST_DB_POOL": "2",
+    }
+
+    internal_sleep = int(env["PGRST_INTERNAL_SCHEMA_CACHE_SLEEP"])
+
+    with run(env=env, wait_for_readiness=False) as postgrest:
+        time.sleep(2 * internal_sleep + 0.1)  # wait for readiness manually
+
+        response = postgrest.session.post("/rpc/create_function")
+        assert response.status_code == 204
+
+        time.sleep(
+            internal_sleep / 2
+        )  # wait to be inside the schema cache reload process
+
+        response = postgrest.session.post("/rpc/migrate_function")
+        assert response.status_code == 204
+
+        time.sleep(
+            2 * internal_sleep
+        )  # wait enough time to ensure the schema cache state remains
+
+        response = postgrest.session.get("/rpc/mult_them?c=3&d=4")
+        assert response.text == "12"
+        assert response.status_code == 200
+
+
 # TODO: This test fails now because of https://github.com/PostgREST/postgrest/pull/2122
 # The stack size of 1K(-with-rtsopts=-K1K) is not enough and this fails with "stack overflow"
 # A stack size of 200K seems to be enough for succeess
@@ -962,3 +1011,367 @@ def test_openapi_in_big_schema(defaultenv):
    with run(env=env) as postgrest:
        response = postgrest.session.get("/")
        assert response.status_code == 200
+
+
+@pytest.mark.parametrize("dburi_type", ["no_params", "no_params_qmark", "with_params"])
+def test_get_pgrst_version_with_uri_connection_string(dburi_type, dburi, defaultenv):
+    "The fallback_application_name should be added to the db-uri if it has a URI format"
+    defaultenv_without_libpq = {
+        key: value
+        for key, value in defaultenv.items()
+        if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
+    }
+
+    env = {
+        "no_params": {**defaultenv, "PGRST_DB_URI": "postgresql://"},
+        "no_params_qmark": {**defaultenv, "PGRST_DB_URI": "postgresql://?"},
+        "with_params": {**defaultenv_without_libpq, "PGRST_DB_URI": dburi.decode()},
+    }
+
+    with run(env=env[dburi_type]) as postgrest:
+        response = postgrest.session.post("/rpc/get_pgrst_version")
+        version = '"%s"' % response.headers["Server"].replace(
+            "postgrest/", "PostgREST "
+        )
+        assert response.text == version
+
+
+def test_get_pgrst_version_with_keyval_connection_string(defaultenv):
+    "The fallback_application_name should be added to the db-uri if it has a keyword/value format"
+    uri = f'dbname={defaultenv["PGDATABASE"]} host={defaultenv["PGHOST"]} user={defaultenv["PGUSER"]}'
+    defaultenv_without_libpq = {
+        key: value
+        for key, value in defaultenv.items()
+        if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
+    }
+    env = {**defaultenv_without_libpq, "PGRST_DB_URI": uri}
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.post("/rpc/get_pgrst_version")
+        version = '"%s"' % response.headers["Server"].replace(
+            "postgrest/", "PostgREST "
+        )
+        assert response.text == version
+
+
+def test_log_postgrest_version(defaultenv):
+    "Should show the PostgREST version in the logs"
+
+    with run(env=defaultenv, no_startup_stdout=False) as postgrest:
+        version = postgrest.session.head("/").headers["Server"].split("/")[1]
+
+        assert (
+            "Starting PostgREST %s..." % version
+            in postgrest.process.stdout.readline().decode()
+        )
+
+
+def test_succeed_w_role_having_superuser_settings(defaultenv):
+    "Should succeed when having superuser settings on the impersonated role"
+
+    env = {**defaultenv, "PGRST_DB_CONFIG": "true", "PGRST_JWT_SECRET": SECRET}
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        headers = jwtauthheader({"role": "postgrest_test_w_superuser_settings"}, SECRET)
+        response = postgrest.session.get("/projects", headers=headers)
+        print(response.text)
+        assert response.status_code == 200
+
+
+def test_get_granted_superuser_setting(defaultenv):
+    "Should succeed when the impersonated role has granted superuser settings"
+
+    env = {**defaultenv, "PGRST_DB_CONFIG": "true", "PGRST_JWT_SECRET": SECRET}
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        response_ver = postgrest.session.get("/rpc/get_postgres_version")
+        pg_ver = eval(response_ver.text)
+        if pg_ver >= 150000:
+            headers = jwtauthheader(
+                {"role": "postgrest_test_w_superuser_settings"}, SECRET
+            )
+            response = postgrest.session.get(
+                "/rpc/get_guc_value?name=log_min_duration_sample", headers=headers
+            )
+            assert response.text == '"12345ms"'
+
+
+def test_fail_with_invalid_dbname_and_automatic_recovery_disabled(defaultenv):
+    "Should fail without retries when automatic recovery is disabled and dbname is invalid"
+    dbname = "INVALID"
+    uri = f'postgresql://?dbname={dbname}&host={defaultenv["PGHOST"]}&user={defaultenv["PGUSER"]}'
+    env = {
+        **defaultenv,
+        "PGRST_DB_URI": uri,
+        "PGRST_DB_POOL_AUTOMATIC_RECOVERY": "false",
+    }
+
+    with run(env=env, wait_for_readiness=False) as postgrest:
+        exitCode = wait_until_exit(postgrest)
+        assert exitCode == 1
+
+
+def test_fail_with_automatic_recovery_disabled_and_terminated_using_query(defaultenv):
+    "Should fail without retries when automatic recovery is disabled and pg_terminate_backend(pid) is called"
+
+    env = {
+        **defaultenv,
+        "PGRST_DB_POOL_AUTOMATIC_RECOVERY": "false",
+    }
+
+    with run(env=env) as postgrest:
+        os.system(
+            f'psql -d {defaultenv["PGDATABASE"]} -U {defaultenv["PGUSER"]} -h {defaultenv["PGHOST"]} --set ON_ERROR_STOP=1 -a -c "SELECT terminate_pgrst()"'
+        )
+
+        exitCode = wait_until_exit(postgrest)
+        assert exitCode == 1
+
+
+def test_server_timing_jwt_should_decrease_on_subsequent_requests(defaultenv):
+    "assert that server-timing duration for JWT should decrease on subsequent requests"
+
+    env = {
+        **defaultenv,
+        "PGRST_SERVER_TIMING_ENABLED": "true",
+        "PGRST_JWT_CACHE_MAX_LIFETIME": "86400",
+        "PGRST_JWT_SECRET": "@/dev/stdin",
+        "PGRST_DB_CONFIG": "false",
+    }
+
+    headers = jwtauthheader(
+        {
+            "role": "postgrest_test_author",
+            "exp": int(
+                (datetime.now(timezone.utc) + timedelta(minutes=30)).timestamp()
+            ),
+        },
+        SECRET,
+    )
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        first_timings = postgrest.session.get("/authors_only", headers=headers).headers[
+            "Server-Timing"
+        ]
+        second_timings = postgrest.session.get(
+            "/authors_only", headers=headers
+        ).headers["Server-Timing"]
+
+        first_dur = parse_server_timings_header(first_timings)["jwt"]
+        second_dur = parse_server_timings_header(second_timings)["jwt"]
+
+        # their difference should be atleast 300, implying
+        # that JWT Caching is working as expected
+        assert (first_dur - second_dur) > 300.0
+
+
+# just added to complete code coverage
+def test_jwt_caching_works_with_db_plan_disabled(defaultenv):
+    "assert that JWT caching words even when Server-Timing header is not returned"
+
+    env = {
+        **defaultenv,
+        "PGRST_SERVER_TIMING_ENABLED": "true",
+        "PGRST_JWT_CACHE_MAX_LIFETIME": "86400",
+        "PGRST_JWT_SECRET": "@/dev/stdin",
+        "PGRST_DB_CONFIG": "false",
+    }
+
+    headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        first_request = postgrest.session.get("/authors_only", headers=headers)
+        second_request = postgrest.session.get("/authors_only", headers=headers)
+
+        # in this case we don't get server-timing in response headers
+        # so we can't compare durations, we just check if request succeeds
+        assert first_request.status_code == 200 and second_request.status_code == 200
+
+
+def test_server_timing_jwt_should_not_decrease_when_caching_disabled(defaultenv):
+    "assert than jwt duration should not decrease when disabled"
+
+    env = {
+        **defaultenv,
+        "PGRST_SERVER_TIMING_ENABLED": "true",
+        "PGRST_JWT_CACHE_MAX_LIFETIME": "0",  # cache disabled
+        "PGRST_JWT_SECRET": "@/dev/stdin",
+        "PGRST_DB_CONFIG": "false",
+    }
+
+    headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        warmup_req = postgrest.session.get("/authors_only", headers=headers)
+        first_timings = postgrest.session.get("/authors_only", headers=headers).headers[
+            "Server-Timing"
+        ]
+        second_timings = postgrest.session.get(
+            "/authors_only", headers=headers
+        ).headers["Server-Timing"]
+
+        first_dur = parse_server_timings_header(first_timings)["jwt"]
+        second_dur = parse_server_timings_header(second_timings)["jwt"]
+
+        # their difference should be less than 150
+        # implying that token is not cached
+        assert (first_dur - second_dur) < 150.0
+
+
+def test_jwt_cache_with_no_exp_claim(defaultenv):
+    "assert than jwt duration should decrease"
+
+    env = {
+        **defaultenv,
+        "PGRST_SERVER_TIMING_ENABLED": "true",
+        "PGRST_JWT_CACHE_MAX_LIFETIME": "86400",
+        "PGRST_JWT_SECRET": "@/dev/stdin",
+        "PGRST_DB_CONFIG": "false",
+    }
+
+    headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)  # no exp
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        first_timings = postgrest.session.get("/authors_only", headers=headers).headers[
+            "Server-Timing"
+        ]
+        second_timings = postgrest.session.get(
+            "/authors_only", headers=headers
+        ).headers["Server-Timing"]
+
+        first_dur = parse_server_timings_header(first_timings)["jwt"]
+        second_dur = parse_server_timings_header(second_timings)["jwt"]
+
+        # their difference should be atleast 300, implying
+        # that JWT Caching is working as expected
+        assert (first_dur - second_dur) > 300.0
+
+
+def test_preflight_request_with_cors_allowed_origin_config(defaultenv):
+    "OPTIONS preflight request should return Access-Control-Allow-Origin equal to origin"
+
+    env = {
+        **defaultenv,
+        "PGRST_SERVER_CORS_ALLOWED_ORIGINS": "http://example.com, http://example2.com",
+    }
+
+    headers = {
+        "Accept": "*/*",
+        "Origin": "http://example.com",
+        "Access-Control-Request-Method": "POST",
+        "Access-Control-Request-Headers": "Content-Type",
+    }
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.options("/items", headers=headers)
+        assert (
+            response.headers["Access-Control-Allow-Origin"] == "http://example.com"
+            and response.headers["Access-Control-Allow-Credentials"] == "true"
+        )
+
+
+def test_preflight_request_with_empty_cors_allowed_origin_config(defaultenv):
+    "OPTIONS preflight request should allow all origins when config is present but empty"
+
+    env = {
+        **defaultenv,
+        "PGRST_SERVER_CORS_ALLOWED_ORIGINS": "",
+    }
+
+    headers = {
+        "Accept": "*/*",
+        "Origin": "http://anyorigin.com",
+        "Access-Control-Request-Method": "POST",
+        "Access-Control-Request-Headers": "Content-Type",
+    }
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.options("/items", headers=headers)
+        assert response.headers["Access-Control-Allow-Origin"] == "*"
+        assert "POST" in response.headers["Access-Control-Allow-Methods"]
+
+
+def test_no_preflight_request_with_CORS_config_should_return_header(defaultenv):
+    "GET no preflight request should return Access-Control-Allow-Origin equal to origin"
+
+    env = {
+        **defaultenv,
+        "PGRST_SERVER_CORS_ALLOWED_ORIGINS": "http://example.com, http://example2.com",
+    }
+
+    headers = {
+        "Accept": "*/*",
+        "Origin": "http://example.com",
+    }
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.get("/items", headers=headers)
+        assert response.headers["Access-Control-Allow-Origin"] == "http://example.com"
+
+
+def test_no_preflight_request_with_CORS_config_should_not_return_header(defaultenv):
+    "GET no preflight request should not return Access-Control-Allow-Origin"
+
+    env = {
+        **defaultenv,
+        "PGRST_SERVER_CORS_ALLOWED_ORIGINS": "http://example.com, http://example2.com",
+    }
+
+    headers = {
+        "Accept": "*/*",
+        "Origin": "http://invalid.com",
+    }
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.get("/items", headers=headers)
+        assert "Access-Control-Allow-Origin" not in response.headers
+
+
+def test_fail_with_3_sec_statement_and_1_sec_statement_timeout(defaultenv):
+    "statement that takes three seconds to execute should fail with one second timeout"
+
+    with run(env=defaultenv) as postgrest:
+        response = postgrest.session.post("/rpc/one_sec_timeout")
+
+        assert response.status_code == 500
+        assert (
+            response.text
+            == '{"code":"57014","details":null,"hint":null,"message":"canceling statement due to statement timeout"}'
+        )
+
+
+def test_passes_with_3_sec_statement_and_4_sec_statement_timeout(defaultenv):
+    "statement that takes three seconds to execute should succeed with four second timeout"
+
+    with run(env=defaultenv) as postgrest:
+        response = postgrest.session.post("/rpc/four_sec_timeout")
+
+        assert response.status_code == 204
+
+
+@pytest.mark.parametrize("level", ["crit", "error", "warn", "info"])
+def test_db_error_logging_to_stderr(level, defaultenv, metapostgrest):
+    "verify that DB errors are logged to stderr"
+
+    role = "timeout_authenticator"
+    set_statement_timeout(metapostgrest, role, 500)
+
+    env = {
+        **defaultenv,
+        "PGUSER": role,
+        "PGRST_DB_ANON_ROLE": role,
+        "PGRST_LOG_LEVEL": level,
+    }
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.get("/rpc/sleep?seconds=1")
+        assert response.status_code == 500
+
+        # ensure the message appears on the logs
+        output = sorted(postgrest.read_stdout(nlines=2))
+
+        if level == "crit":
+            assert len(output) == 0
+        else:
+            assert " 500 " in output[0]
+            assert "canceling statement due to statement timeout" in output[1]
@@ -40,3 +40,20 @@ def authheader(token):
 def jwtauthheader(claim, secret):
    "Authorization header with signed JWT."
    return authheader(jwt.encode(claim, secret))
+
+
+def parse_server_timings_header(header):
+    """Parse the Server-Timing header into a dict of metric names to values.
+
+    The header is a comma-separated list of metrics, each of which has a name
+    and a duration. The duration may be followed by a semicolon and a list of
+    parameters, but we ignore those.
+
+    See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Server-Timing
+    """
+    timings = {}
+    for timing in header.split(","):
+        name, duration_text, *_ = timing.split(";")
+        _, duration = duration_text.split("=")
+        timings[name] = float(duration)
+    return timings
@@ -102,7 +102,7 @@ postJsonArrayTest(){

 echo "Running memory usage tests.."

-jsonKeyTest "1M" "POST" "/rpc/leak?columns=blob" "16M"
+jsonKeyTest "1M" "POST" "/rpc/leak?columns=blob" "24M"
 jsonKeyTest "1M" "POST" "/leak?columns=blob" "16M"
 jsonKeyTest "1M" "PATCH" "/leak?id=eq.1&columns=blob" "16M"

@@ -114,8 +114,8 @@ jsonKeyTest "50M" "POST" "/rpc/leak?columns=blob" "172M"
 jsonKeyTest "50M" "POST" "/leak?columns=blob" "172M"
 jsonKeyTest "50M" "PATCH" "/leak?id=eq.1&columns=blob" "172M"

-postJsonArrayTest "1000" "/perf_articles?columns=id,body" "14M"
-postJsonArrayTest "10000" "/perf_articles?columns=id,body" "14M"
+postJsonArrayTest "1000" "/perf_articles?columns=id,body" "15M"
+postJsonArrayTest "10000" "/perf_articles?columns=id,body" "15M"
 postJsonArrayTest "100000" "/perf_articles?columns=id,body" "24M"

 trap - int term exit
@@ -20,8 +20,7 @@ spec =
          ""
        `shouldRespondWith`
          ""
-          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "http://example.com"
-                           , "Access-Control-Allow-Credentials" <:> "true"
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*"
                           , "Access-Control-Allow-Methods" <:> "GET, POST, PATCH, PUT, DELETE, OPTIONS, HEAD"
                           , "Access-Control-Allow-Headers" <:> "Authorization, Foo, Bar, Accept, Accept-Language, Content-Language"
                           , "Access-Control-Max-Age" <:> "86400" ]
@@ -40,7 +39,7 @@ spec =
                             \Date, Location, Server, Transfer-Encoding, Range-Unit"]
          }

-    it "allows INFO body through even with CORS request headers present to postflight request" $
+    it "allows INFO body through even with CORS request headers present to postflight request" $ do
      request methodOptions "/items"
          [ ("Host", "localhost:3000")
          , ("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0")
@@ -54,3 +53,17 @@ spec =
        `shouldRespondWith`
          ""
          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
+
+      request methodOptions "/items"
+          [ ("Accept", "application/json") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
+
+      request methodOptions "/shops"
+          [ ("Accept", "application/geo+json") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
@@ -1,68 +0,0 @@
-module Feature.LegacyGucsSpec where
-
-import Network.Wai (Application)
-
-import Network.HTTP.Types
-import Test.Hspec          hiding (pendingWith)
-import Test.Hspec.Wai
-import Test.Hspec.Wai.JSON
-
-import Protolude  hiding (get)
-import SpecHelper
-
-spec :: SpecWith ((), Application)
-spec =
-  describe "remote procedure call with legacy gucs disabled" $ do
-    it "custom header is set" $
-      request methodPost "/rpc/get_guc_value" [("Custom-Header", "test")]
-        [json| { "prefix": "request.headers", "name": "custom-header" } |]
-          `shouldRespondWith`
-          [json|"test"|]
-          { matchStatus  = 200
-          , matchHeaders = [ matchContentTypeJson ]
-          }
-
-    it "standard header is set" $
-      request methodPost "/rpc/get_guc_value" [("Origin", "http://example.com")]
-        [json| { "prefix": "request.headers", "name": "origin" } |]
-          `shouldRespondWith`
-          [json|"http://example.com"|]
-          { matchStatus  = 200
-          , matchHeaders = [ matchContentTypeJson ]
-          }
-
-    it "current role is available as GUC claim" $
-      request methodPost "/rpc/get_guc_value" []
-        [json| { "prefix": "request.jwt.claims", "name": "role" } |]
-          `shouldRespondWith`
-          [json|"postgrest_test_anonymous"|]
-          { matchStatus  = 200
-          , matchHeaders = [ matchContentTypeJson ]
-          }
-
-    it "single cookie ends up as claims" $
-      request methodPost "/rpc/get_guc_value" [("Cookie","acookie=cookievalue")]
-        [json| {"prefix": "request.cookies", "name":"acookie"} |]
-          `shouldRespondWith`
-          [json|"cookievalue"|]
-          { matchStatus = 200
-          , matchHeaders = []
-          }
-
-    it "multiple cookies ends up as claims" $
-      request methodPost "/rpc/get_guc_value" [("Cookie","acookie=cookievalue;secondcookie=anothervalue")]
-        [json| {"prefix": "request.cookies", "name":"secondcookie"} |]
-          `shouldRespondWith`
-          [json|"anothervalue"|]
-          { matchStatus = 200
-          , matchHeaders = []
-          }
-
-    it "gets the Authorization value" $
-      request methodPost "/rpc/get_guc_value" [authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIn0.Xod-F15qsGL0WhdOCr2j3DdKuTw9QJERVgoFD3vGaWA"]
-        [json| {"prefix": "request.headers", "name":"authorization"} |]
-          `shouldRespondWith`
-          [json|"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIn0.Xod-F15qsGL0WhdOCr2j3DdKuTw9QJERVgoFD3vGaWA"|]
-          { matchStatus = 200
-          , matchHeaders = []
-          }
@@ -35,6 +35,11 @@ spec actualPgVersion = describe "OpenAPI" $ do
            (acceptHdrs "application/openapi+json") ""
      `shouldRespondWith` 415

+  it "should respond to openapi request with unsupported media type with 415" $
+    request methodGet "/"
+            (acceptHdrs "text/csv") ""
+      `shouldRespondWith` 415
+
  it "includes postgrest.org current version api docs" $ do
    r <- simpleBody <$> get "/"

@@ -3,7 +3,7 @@ module Feature.OpenApi.RootSpec where
 import Network.HTTP.Types
 import Network.Wai        (Application)

-import Test.Hspec
+import Test.Hspec          hiding (pendingWith)
 import Test.Hspec.Wai
 import Test.Hspec.Wai.JSON

@@ -12,7 +12,7 @@ import Protolude hiding (get)
 spec :: SpecWith ((), Application)
 spec =
  describe "root spec function" $ do
-    it "accepts application/openapi+json" $
+    it "accepts application/openapi+json" $ do
      request methodGet "/"
        [("Accept","application/openapi+json")] "" `shouldRespondWith`
        [json|{
@@ -20,3 +20,12 @@ spec =
           "info": {"title": "PostgREST API", "description": "This is a dynamic API generated by PostgREST"}
          }|]
        { matchHeaders = ["Content-Type" <:> "application/openapi+json; charset=utf-8"] }
+
+    it "accepts application/json" $ do
+      request methodGet "/"
+        [("Accept","application/json")] "" `shouldRespondWith`
+        [json|{
+           "swagger": "2.0",
+           "info": {"title": "PostgREST API", "description": "This is a dynamic API generated by PostgREST"}
+          }|]
+        { matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"] }
@@ -0,0 +1,168 @@
+module Feature.Query.AggregateFunctionsSpec where
+
+import Network.Wai (Application)
+
+import Test.Hspec          hiding (pendingWith)
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import Protolude  hiding (get)
+import SpecHelper
+
+allowed :: SpecWith ((), Application)
+allowed =
+  describe "aggregate functions" $ do
+    context "performing a count without specifying a field" $ do
+      it "returns the count of all rows when no other fields are selected" $
+        get "/entities?select=count()" `shouldRespondWith`
+          [json|[{ "count": 4 }]|] { matchHeaders = [matchContentTypeJson] }
+      it "allows you to specify an alias for the count" $
+        get "/entities?select=cnt:count()" `shouldRespondWith`
+          [json|[{ "cnt": 4 }]|] { matchHeaders = [matchContentTypeJson] }
+      it "allows you to cast the result of the count" $
+        get "/entities?select=count()::text" `shouldRespondWith`
+          [json|[{ "count": "4" }]|] { matchHeaders = [matchContentTypeJson] }
+      it "returns the count grouped by all provided fields when other fields are selected" $
+        get "/projects?select=c:count(),client_id&order=client_id.desc" `shouldRespondWith`
+          [json|[{ "c": 1, "client_id": null }, { "c": 2, "client_id": 2 }, { "c": 2, "client_id": 1}]|] { matchHeaders = [matchContentTypeJson] }
+
+    context "performing a count by using it as a column (backwards compat)" $ do
+      it "returns the count of all rows when no other fields are selected" $
+        get "/entities?select=count" `shouldRespondWith`
+          [json|[{ "count": 4 }]|] { matchHeaders = [matchContentTypeJson] }
+      it "returns the embedded count of another resource" $
+        get "/clients?select=name,projects(count)'" `shouldRespondWith`
+          [json|[{"name":"Microsoft","projects":[{"count": 2}]}, {"name":"Apple","projects":[{"count": 2}]}]|] { matchHeaders = [matchContentTypeJson] }
+
+    context "performing an aggregation on one or more fields" $ do
+      it "supports sum()" $
+        get "/project_invoices?select=invoice_total.sum()" `shouldRespondWith`
+          [json|[{"sum":8800}]|] { matchHeaders = [matchContentTypeJson] }
+      it "supports avg()" $
+        get "/project_invoices?select=invoice_total.avg()" `shouldRespondWith`
+          [json|[{"avg":1100.0000000000000000}]|] { matchHeaders = [matchContentTypeJson] }
+      it "supports min()" $
+        get "/project_invoices?select=invoice_total.min()" `shouldRespondWith`
+          [json|[{ "min": 100 }]|] { matchHeaders = [matchContentTypeJson] }
+      it "supports max()" $
+        get "/project_invoices?select=invoice_total.max()" `shouldRespondWith`
+          [json|[{ "max": 4000 }]|] { matchHeaders = [matchContentTypeJson] }
+      it "supports count()" $
+        get "/project_invoices?select=invoice_total.count()" `shouldRespondWith`
+          [json|[{ "count": 8 }]|] { matchHeaders = [matchContentTypeJson] }
+      it "groups by any fields selected that do not have an aggregate applied" $
+        get "/project_invoices?select=invoice_total.sum(),invoice_total.max(),invoice_total.min(),project_id&order=project_id.desc" `shouldRespondWith`
+          [json|[
+            {"sum":4100,"max":4000,"min":100,"project_id":4},
+            {"sum":3200,"max":2000,"min":1200,"project_id":3},
+            {"sum":1200,"max":700,"min":500,"project_id":2},
+            {"sum":300,"max":200,"min":100,"project_id":1} ]|]
+          { matchHeaders = [matchContentTypeJson] }
+      it "supports the use of aliases on fields that will be used in the group by" $
+        get "/project_invoices?select=invoice_total.sum(),invoice_total.max(),invoice_total.min(),pid:project_id&order=project_id.desc" `shouldRespondWith`
+          [json|[
+            {"sum":4100,"max":4000,"min":100,"pid":4},
+            {"sum":3200,"max":2000,"min":1200,"pid":3},
+            {"sum":1200,"max":700,"min":500,"pid":2},
+            {"sum":300,"max":200,"min":100,"pid":1}]|]
+          { matchHeaders = [matchContentTypeJson] }
+      it "allows you to specify an alias for the aggregate" $
+        get "/project_invoices?select=total_charged:invoice_total.sum(),project_id&order=project_id.desc" `shouldRespondWith`
+          [json|[
+             {"total_charged":4100,"project_id":4},
+             {"total_charged":3200,"project_id":3},
+             {"total_charged":1200,"project_id":2},
+             {"total_charged":300,"project_id":1}]|] { matchHeaders = [matchContentTypeJson] }
+      it "allows you to cast the result of the aggregate" $
+        get "/project_invoices?select=total_charged:invoice_total.sum()::text,project_id&order=project_id.desc" `shouldRespondWith`
+          [json|[
+             {"total_charged":"4100","project_id":4},
+             {"total_charged":"3200","project_id":3},
+             {"total_charged":"1200","project_id":2},
+             {"total_charged":"300","project_id":1}]|] { matchHeaders = [matchContentTypeJson] }
+      it "allows you to cast the input argument of the aggregate" $
+        get "/trash_details?select=jsonb_col->>key::integer.sum()" `shouldRespondWith`
+          [json|[{"sum": 24}]|] { matchHeaders = [matchContentTypeJson] }
+      it "allows the combination of an alias, a before cast, and an after cast" $
+        get "/trash_details?select=s:jsonb_col->>key::integer.sum()::text" `shouldRespondWith`
+          [json|[{"s": "24"}]|] { matchHeaders = [matchContentTypeJson] }
+      it "supports use of aggregates on RPC functions that return table values" $
+        get "/rpc/getallprojects?select=id.max()" `shouldRespondWith`
+          [json|[{"max": 5}]|] { matchHeaders = [matchContentTypeJson] }
+      it "allows the use of an JSON-embedded relationship column as part of the group by" $
+        get "/project_invoices?select=project_id,total:invoice_total.sum(),projects(name)&order=project_id" `shouldRespondWith`
+          [json|[
+            {"project_id": 1, "total": 300,  "projects": {"name": "Windows 7"}},
+            {"project_id": 2, "total": 1200, "projects": {"name": "Windows 10"}},
+            {"project_id": 3, "total": 3200, "projects": {"name": "IOS"}},
+            {"project_id": 4, "total": 4100, "projects": {"name": "OSX"}}]|] { matchHeaders = [matchContentTypeJson] }
+    context "performing aggregations that involve JSON-embedded relationships" $ do
+      it "supports sum()" $
+        get "/projects?select=name,project_invoices(invoice_total.sum())" `shouldRespondWith`
+          [json|[
+            {"name":"Windows 7","project_invoices":[{"sum": 300}]},
+            {"name":"Windows 10","project_invoices":[{"sum": 1200}]},
+            {"name":"IOS","project_invoices":[{"sum": 3200}]},
+            {"name":"OSX","project_invoices":[{"sum": 4100}]},
+            {"name":"Orphan","project_invoices":[{"sum": null}]}]|]
+          { matchHeaders = [matchContentTypeJson] }
+      it "supports max()" $
+        get "/projects?select=name,project_invoices(invoice_total.max())" `shouldRespondWith`
+          [json|[{"name":"Windows 7","project_invoices":[{"max": 200}]},
+            {"name":"Windows 10","project_invoices":[{"max": 700}]},
+            {"name":"IOS","project_invoices":[{"max": 2000}]},
+            {"name":"OSX","project_invoices":[{"max": 4000}]},
+            {"name":"Orphan","project_invoices":[{"max": null}]}]|]
+          { matchHeaders = [matchContentTypeJson] }
+      it "supports avg()" $
+        get "/projects?select=name,project_invoices(invoice_total.avg())" `shouldRespondWith`
+          [json|[{"name":"Windows 7","project_invoices":[{"avg": 150.0000000000000000}]},
+            {"name":"Windows 10","project_invoices":[{"avg": 600.0000000000000000}]},
+            {"name":"IOS","project_invoices":[{"avg": 1600.0000000000000000}]},
+            {"name":"OSX","project_invoices":[{"avg": 2050.0000000000000000}]},
+            {"name":"Orphan","project_invoices":[{"avg": null}]}]|]
+          { matchHeaders = [matchContentTypeJson] }
+      it "supports min()" $
+        get "/projects?select=name,project_invoices(invoice_total.min())" `shouldRespondWith`
+          [json|[{"name":"Windows 7","project_invoices":[{"min": 100}]},
+            {"name":"Windows 10","project_invoices":[{"min": 500}]},
+            {"name":"IOS","project_invoices":[{"min": 1200}]},
+            {"name":"OSX","project_invoices":[{"min": 100}]},
+            {"name":"Orphan","project_invoices":[{"min": null}]}]|]
+          { matchHeaders = [matchContentTypeJson] }
+      it "supports all at once" $
+        get "/projects?select=name,project_invoices(invoice_total.max(),invoice_total.min(),invoice_total.avg(),invoice_total.sum(),invoice_total.count())" `shouldRespondWith`
+          [json|[
+            {"name":"Windows 7","project_invoices":[{"avg": 150.0000000000000000, "max": 200, "min": 100, "sum": 300, "count": 2}]},
+            {"name":"Windows 10","project_invoices":[{"avg": 600.0000000000000000, "max": 700, "min": 500, "sum": 1200, "count": 2}]},
+            {"name":"IOS","project_invoices":[{"avg": 1600.0000000000000000, "max": 2000, "min": 1200, "sum": 3200, "count": 2}]},
+            {"name":"OSX","project_invoices":[{"avg": 2050.0000000000000000, "max": 4000, "min": 100, "sum": 4100, "count": 2}]},
+            {"name":"Orphan","project_invoices":[{"avg": null, "max": null, "min": null, "sum": null, "count": 0}]}]|]
+          { matchHeaders = [matchContentTypeJson] }
+
+    context "performing aggregations on spreaded fields from an embedded resource" $ do
+      it "supports the use of aggregates on spreaded fields" $ do
+        get "/budget_expenses?select=total_expenses:expense_amount.sum(),...budget_categories(budget_owner,total_budget:budget_amount.sum())&order=budget_categories(budget_owner)" `shouldRespondWith`
+          [json|[
+            {"total_expenses": 600.52,"budget_owner": "Brian Smith",  "total_budget": 2000.42},
+            {"total_expenses": 100.22, "budget_owner": "Jane Clarkson","total_budget": 7000.41},
+            {"total_expenses": 900.27, "budget_owner": "Sally Hughes", "total_budget": 500.23}]|]
+          { matchHeaders = [matchContentTypeJson] }
+      it "supports the use of aggregates on spreaded fields when only aggregates are supplied" $ do
+        get "/budget_expenses?select=...budget_categories(total_budget:budget_amount.sum())" `shouldRespondWith`
+          [json|[{"total_budget": 9501.06}]|]
+          { matchHeaders = [matchContentTypeJson] }
+
+disallowed :: SpecWith ((), Application)
+disallowed =
+  describe "attempting to use an aggregate when aggregate functions are disallowed" $ do
+    it "prevents the use of aggregates" $
+      get "/project_invoices?select=invoice_total.sum()" `shouldRespondWith`
+        [json|{
+          "hint":null,
+          "details":null,
+          "code":"PGRST123",
+          "message":"Use of aggregate functions is not allowed"
+        }|]
+        { matchStatus = 400
+        , matchHeaders = [matchContentTypeJson] }
@@ -104,6 +104,47 @@ spec = describe "computed relationships" $ do
        [json|[ {"name":"Final Fantasy I","designer":{"name":"Hironobu Sakaguchi"}} ]|]
        { matchStatus  = 200 }

+  it "applies data representations to response" $ do
+    -- A smoke test for data reps in the presence of computed relations.
+
+    -- The data rep here title cases the designer name before presentation. So here the lowercase version will be saved,
+    -- but the title case version returned. Pulling in a computed relation should not confuse this.
+    request methodPatch "/designers?select=name,videogames:computed_videogames(name)&id=eq.1"
+      [("Prefer", "return=representation"), ("Prefer", "tx=commit")]
+      [json| {"name": "sidney k. meier"} |]
+      `shouldRespondWith`
+      [json|[{"name":"Sidney K. Meier","videogames":[{"name":"Civilization I"}, {"name":"Civilization II"}]}]|]
+      { matchStatus = 200 }
+
+    -- Verify it was saved the way we requested (there's no text data rep for this column, so if we select with the wrong casing, it should fail.)
+    get "/designers?select=id&name=eq.Sidney%20K.%20Meier"
+      `shouldRespondWith`
+      [json|[]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
+    -- But with the right casing it works.
+    get "/designers?select=id,name&name=eq.sidney%20k.%20meier"
+      `shouldRespondWith`
+      [json|[{"id": 1, "name":"Sidney K. Meier"}]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
+
+    -- Most importantly, if you read it back even via a computed relation, the data rep should be applied.
+    get "/videogames?select=name,designer:computed_designers(*)&id=eq.1"
+      `shouldRespondWith`
+      [json|[
+        {"name":"Civilization I","designer":{"id": 1, "name":"Sidney K. Meier"}}
+      ]|] { matchHeaders = [matchContentTypeJson] }
+
+    -- reset the test fixture
+    request methodPatch "/designers?id=eq.1"
+      [("Prefer", "tx=commit")]
+      [json| {"name": "Sid Meier"} |]
+      `shouldRespondWith` 204
+    -- need to poke the second one too to prevent inherent ordering from changing
+    request methodPatch "/designers?id=eq.2"
+      [("Prefer", "tx=commit")]
+      [json| {"name": "Hironobu Sakaguchi"} |]
+      `shouldRespondWith` 204
+
  it "works with self joins" $
    get "/web_content?select=name,child_web_content(name),parent_web_content(name)&id=in.(0,1)"
    `shouldRespondWith`
@@ -151,3 +192,29 @@ spec = describe "computed relationships" $ do
          {"name":"Windows 10","computed_clients":{"name":"Microsoft"}}
        ]}
      ]|] { matchHeaders = [matchContentTypeJson] }
+
+  -- https://github.com/PostgREST/postgrest/issues/2963
+  context "can be defined using overloaded functions" $ do
+    it "tables" $ do
+      get "/items?select=*,computed_rel_overload(*)&limit=1"
+        `shouldRespondWith`
+        [json|
+          [{"id":1,"computed_rel_overload":[{"id":1}]}]
+        |] { matchHeaders = [matchContentTypeJson] }
+      get "/items2?select=*,computed_rel_overload(*)&limit=1"
+        `shouldRespondWith`
+        [json|
+          [{"id":1,"computed_rel_overload":[{"id":1},{"id":2}]}]
+        |] { matchHeaders = [matchContentTypeJson] }
+
+    it "rpc" $ do
+      get "/rpc/search?id=1&select=*,computed_rel_overload(*)"
+        `shouldRespondWith`
+        [json|
+          [{"id":1,"computed_rel_overload":[{"id":1}]}]
+        |] { matchHeaders = [matchContentTypeJson] }
+      get "/rpc/search2?id=1&select=*,computed_rel_overload(*)"
+        `shouldRespondWith`
+        [json|
+          [{"id":1,"computed_rel_overload":[{"id":1},{"id":2}]}]
+        |] { matchHeaders = [matchContentTypeJson] }
@@ -0,0 +1,302 @@
+module Feature.Query.CustomMediaSpec where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Network.Wai.Test    (SResponse (simpleBody, simpleHeaders, simpleStatus))
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+import Text.Heredoc        (str)
+
+import Protolude  hiding (get)
+import SpecHelper
+
+spec :: SpecWith ((), Application)
+spec = describe "custom media types" $ do
+  context "for tables with aggregate" $ do
+    it "can query if there's an aggregate defined for the table" $ do
+      r <- request methodGet "/lines" (acceptHdrs "application/vnd.twkb") ""
+      liftIO $ do
+        simpleBody r `shouldBe` readFixtureFile "lines.twkb"
+        simpleHeaders r `shouldContain` [("Content-Type", "application/vnd.twkb")]
+
+    it "can query by id if there's an aggregate defined for the table" $ do
+      r <- request methodGet "/lines?id=eq.1" (acceptHdrs "application/vnd.twkb") ""
+      liftIO $ do
+        simpleBody r `shouldBe` readFixtureFile "1.twkb"
+        simpleHeaders r `shouldContain` [("Content-Type", "application/vnd.twkb")]
+
+    it "will fail if there's no aggregate defined for the table" $ do
+      request methodGet "/lines" (acceptHdrs "text/plain") ""
+        `shouldRespondWith`
+        [json| {"code":"PGRST107","details":null,"hint":null,"message":"None of these media types are available: text/plain"} |]
+        { matchStatus  = 415
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "can get raw xml output with Accept: text/xml if there's an aggregate defined" $ do
+      request methodGet "/xmltest" (acceptHdrs "text/xml") ""
+        `shouldRespondWith`
+        "<myxml>foo</myxml>bar<foobar><baz/></foobar>"
+        { matchStatus = 200
+        , matchHeaders = ["Content-Type" <:> "text/xml; charset=utf-8"]
+        }
+
+  -- TODO SOH (start of heading) is being added to results
+  context "for tables with anyelement aggregate" $ do
+    it "will use the application/vnd.geo2+json media type for any table" $
+      request methodGet "/lines" (acceptHdrs "application/vnd.geo2+json") ""
+        `shouldRespondWith`
+        "\SOH{\"type\": \"FeatureCollection\", \"hello\": \"world\"}"
+        { matchStatus  = 200
+        , matchHeaders = ["Content-Type" <:> "application/vnd.geo2+json"]
+        }
+
+    it "will use the more specific application/vnd.geo2 handler for this table" $ do
+      request methodGet "/shop_bles" (acceptHdrs "application/vnd.geo2+json") ""
+        `shouldRespondWith`
+        "\SOH\"anyelement overridden\""
+        { matchStatus  = 200
+        , matchHeaders = ["Content-Type" <:> "application/vnd.geo2+json"]
+        }
+
+      request methodGet "/rpc/get_shop_bles" (acceptHdrs "application/vnd.geo2+json") ""
+        `shouldRespondWith`
+        "\SOH\"anyelement overridden\""
+        { matchStatus  = 200
+        , matchHeaders = ["Content-Type" <:> "application/vnd.geo2+json"]
+        }
+
+  context "Proc that returns scalar" $ do
+    it "can get raw output with Accept: text/html" $ do
+      request methodGet "/rpc/welcome.html" (acceptHdrs "text/html") ""
+        `shouldRespondWith`
+        [str|
+            |<html>
+            |  <head>
+            |    <title>PostgREST</title>
+            |  </head>
+            |  <body>
+            |    <h1>Welcome to PostgREST</h1>
+            |  </body>
+            |</html>
+            |]
+        { matchStatus = 200
+        , matchHeaders = ["Content-Type" <:> "text/html"]
+        }
+
+    it "can get raw output with Accept: text/plain" $ do
+      request methodGet "/rpc/welcome" (acceptHdrs "text/plain") ""
+        `shouldRespondWith` "Welcome to PostgREST"
+        { matchStatus = 200
+        , matchHeaders = ["Content-Type" <:> "text/plain; charset=utf-8"]
+        }
+
+    it "can get raw xml output with Accept: text/xml" $ do
+      request methodGet "/rpc/return_scalar_xml" (acceptHdrs "text/xml") ""
+        `shouldRespondWith`
+        "<my-xml-tag/>"
+        { matchStatus = 200
+        , matchHeaders = ["Content-Type" <:> "text/xml; charset=utf-8"]
+        }
+
+    it "can get raw xml output with Accept: text/xml" $ do
+      request methodGet "/rpc/welcome.xml" (acceptHdrs "text/xml") ""
+        `shouldRespondWith`
+        "<html>\n  <head>\n    <title>PostgREST</title>\n  </head>\n  <body>\n    <h1>Welcome to PostgREST</h1>\n  </body>\n</html>"
+        { matchStatus = 200
+        , matchHeaders = ["Content-Type" <:> "text/xml; charset=utf-8"]
+        }
+
+    it "should fail with function returning text and Accept: text/xml" $ do
+      request methodGet "/rpc/welcome" (acceptHdrs "text/xml") ""
+        `shouldRespondWith`
+        [json|
+          {"code":"PGRST107","details":null,"hint":null,"message":"None of these media types are available: text/xml"}
+        |]
+        { matchStatus = 415
+        , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+        }
+
+  context "Proc that returns scalar based on a table" $ do
+    it "can get an image with Accept: image/png" $ do
+      r <- request methodGet "/rpc/ret_image" (acceptHdrs "image/png") ""
+      liftIO $ do
+        simpleBody r `shouldBe` readFixtureFile "A.png"
+        simpleHeaders r `shouldContain` [("Content-Type", "image/png")]
+
+  context "Proc that returns set of scalars and Accept: text/plain" $
+    it "will err because only scalars work with media type domains" $ do
+      request methodGet "/rpc/welcome_twice"
+          (acceptHdrs "text/plain")
+          ""
+        `shouldRespondWith`
+          [json|{"code":"PGRST107","details":null,"hint":null,"message":"None of these media types are available: text/plain"}|]
+          { matchStatus = 415
+          , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+          }
+
+  context "Proc that returns rows and accepts custom media type" $ do
+    it "works if it has an aggregate defined" $ do
+      r <- request methodGet "/rpc/get_lines" [("Accept", "application/vnd.twkb")] ""
+      liftIO $ do
+        simpleBody r `shouldBe` readFixtureFile "lines.twkb"
+        simpleHeaders r `shouldContain` [("Content-Type", "application/vnd.twkb")]
+
+    it "fails if doesn't have an aggregate defined" $ do
+      request methodGet "/rpc/get_lines"
+          (acceptHdrs "application/octet-stream") ""
+        `shouldRespondWith`
+          [json| {"code":"PGRST107","details":null,"hint":null,"message":"None of these media types are available: application/octet-stream"} |]
+          { matchStatus = 415 }
+
+    -- TODO SOH (start of heading) is being added to results
+    it "works if there's an anyelement aggregate defined" $ do
+      request methodGet "/rpc/get_lines" (acceptHdrs "application/vnd.geo2+json") ""
+        `shouldRespondWith`
+        "\SOH{\"type\": \"FeatureCollection\", \"hello\": \"world\"}"
+        { matchStatus  = 200
+        , matchHeaders = ["Content-Type" <:> "application/vnd.geo2+json"]
+        }
+
+  context "overriding" $ do
+    it "will override the application/json handler for a single table" $
+      request methodGet "/ov_json" (acceptHdrs "application/json") ""
+        `shouldRespondWith`
+        [json| {"overridden": "true"} |]
+        { matchStatus  = 200
+        , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+        }
+
+    -- TODO SOH (start of heading) is being added to results
+    it "will override the application/geo+json handler for a single table" $
+      request methodGet "/lines?id=eq.1" (acceptHdrs "application/geo+json") ""
+        `shouldRespondWith`
+        "\SOH{\"crs\": {\"type\": \"name\", \"properties\": {\"name\": \"EPSG:4326\"}}, \"type\": \"FeatureCollection\", \"features\": [{\"type\": \"Feature\", \"geometry\": {\"type\": \"LineString\", \"coordinates\": [[1, 1], [5, 5]]}, \"properties\": {\"id\": 1, \"name\": \"line-1\"}}]}"
+        { matchStatus  = 200
+        , matchHeaders = ["Content-Type" <:> "application/geo+json; charset=utf-8"]
+        }
+
+    it "will not override vendored media types like application/vnd.pgrst.object" $
+      request methodGet "/projects?id=eq.1" (acceptHdrs "application/vnd.pgrst.object") ""
+        `shouldRespondWith`
+        [json|{"id":1,"name":"Windows 7","client_id":1}|]
+        { matchStatus  = 200
+        , matchHeaders = ["Content-Type" <:> "application/vnd.pgrst.object+json; charset=utf-8"]
+        }
+
+  context "matches requested media type correctly" $ do
+    -- https://github.com/PostgREST/postgrest/issues/1462
+    it "will match image/png according to q values" $ do
+      r1 <- request methodGet "/rpc/ret_image" (acceptHdrs "image/png, */*") ""
+      liftIO $ do
+        simpleBody r1 `shouldBe` readFixtureFile "A.png"
+        simpleHeaders r1 `shouldContain` [("Content-Type", "image/png")]
+
+      r2 <- request methodGet "/rpc/ret_image" (acceptHdrs "text/html,application/xhtml+xml,application/xml;q=0.9,image/png,*/*;q=0.8") ""
+      liftIO $ do
+        simpleBody r2 `shouldBe` readFixtureFile "A.png"
+        simpleHeaders r2 `shouldContain` [("Content-Type", "image/png")]
+
+    -- https://github.com/PostgREST/postgrest/issues/2170
+    it "will match json in presence of text/plain" $ do
+      r <- request methodGet "/projects?id=eq.1" (acceptHdrs "text/plain, application/json") ""
+      liftIO $ do
+        simpleStatus r `shouldBe` status200
+        simpleHeaders r `shouldContain` [("Content-Type", "application/json; charset=utf-8")]
+
+    -- https://github.com/PostgREST/postgrest/issues/1102
+    it "will match a custom text/tab-separated-values" $ do
+      request methodGet "/projects?id=in.(1,2)" (acceptHdrs "text/tab-separated-values") ""
+        `shouldRespondWith`
+        "id\tname\tclient_id\n1\tWindows 7\t1\n2\tWindows 10\t1\n"
+        { matchStatus  = 200
+        , matchHeaders = ["Content-Type" <:> "text/tab-separated-values"]
+        }
+
+    -- https://github.com/PostgREST/postgrest/issues/1371#issuecomment-519248984
+    it "will match a custom text/csv with BOM" $ do
+      r <- request methodGet "/lines" (acceptHdrs "text/csv") ""
+      liftIO $ do
+        simpleBody r `shouldBe` readFixtureFile "lines.csv"
+        simpleHeaders r `shouldContain` [("Content-Type", "text/csv; charset=utf-8")]
+        simpleHeaders r `shouldContain` [("Content-Disposition", "attachment; filename=\"lines.csv\"")]
+
+  context "any media type" $ do
+    context "on functions" $ do
+      -- TODO not correct, it should return the generic "application/octet-stream"
+      it "returns application/json for */* if not explicitly set" $ do
+        request methodGet "/rpc/ret_any_mt" (acceptHdrs "*/*") ""
+          `shouldRespondWith` "any"
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }
+
+      it "accepts any media type and sets it as a header" $ do
+        request methodGet "/rpc/ret_any_mt" (acceptHdrs "app/bingo") ""
+          `shouldRespondWith` "any"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "app/bingo"]
+          }
+
+        request methodGet "/rpc/ret_any_mt" (acceptHdrs "text/bango") ""
+          `shouldRespondWith` "any"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "text/bango"]
+          }
+
+        request methodGet "/rpc/ret_any_mt" (acceptHdrs "image/boingo") ""
+          `shouldRespondWith` "any"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "image/boingo"]
+          }
+
+      it "returns custom media type for */* if explicitly set" $ do
+        request methodGet "/rpc/ret_some_mt" (acceptHdrs "*/*") ""
+          `shouldRespondWith` "groucho"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "app/groucho"]
+          }
+
+      it "accepts some media types if there's conditional logic" $ do
+        request methodGet "/rpc/ret_some_mt" (acceptHdrs "app/chico") ""
+          `shouldRespondWith` "chico"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "app/chico"]
+          }
+
+        request methodGet "/rpc/ret_some_mt" (acceptHdrs "app/harpo") ""
+          `shouldRespondWith` "harpo"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "app/harpo"]
+          }
+
+        request methodGet "/rpc/ret_some_mt" (acceptHdrs "text/csv") ""
+          `shouldRespondWith` 415
+
+    context "on tables" $ do
+      -- TODO not correct, it should return the generic "application/octet-stream"
+      it "returns application/json for */* if not explicitly set" $ do
+        request methodGet "/some_numbers?val=eq.1" (acceptHdrs "*/*") ""
+          `shouldRespondWith` "anything\n1"
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }
+
+      it "accepts any media type and sets it as a header" $ do
+        request methodGet "/some_numbers?val=eq.2" (acceptHdrs "magic/number") ""
+          `shouldRespondWith` "magic\n2"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "magic/number"]
+          }
+        request methodGet "/some_numbers?val=eq.3" (acceptHdrs "crazy/bingo") ""
+          `shouldRespondWith` "crazy\n3"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "crazy/bingo"]
+          }
+        request methodGet "/some_numbers?val=eq.4" (acceptHdrs "unknown/unknown") ""
+          `shouldRespondWith` "anything\n4"
+          { matchStatus  = 200
+          , matchHeaders = ["Content-Type" <:> "unknown/unknown"]
+          }
@@ -37,7 +37,8 @@ spec =
        request methodDelete "/items?id=eq.2" [("Prefer", "return=representation"), ("Prefer", "count=exact")] ""
          `shouldRespondWith` [json|[{"id":2}]|]
          { matchStatus  = 200
-          , matchHeaders = ["Content-Range" <:> "*/1"]
+          , matchHeaders = ["Content-Range" <:> "*/1"
+                           , "Preference-Applied" <:> "return=representation, count=exact"]
          }

      it "ignores ?select= when return not set or return=minimal" $ do
@@ -57,7 +58,8 @@ spec =
            ""
            { matchStatus  = 204
            , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=minimal"]
            }

      it "returns the deleted item and shapes the response" $
@@ -137,7 +139,8 @@ spec =
          `shouldRespondWith`
            ""
            { matchStatus = 204
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal" ]
            }

      it "suceeds deleting the row with no explicit select by default" $
@@ -1,30 +0,0 @@
-module Feature.Query.HtmlRawOutputSpec where
-
-import Network.Wai (Application)
-
-import Network.HTTP.Types
-import Test.Hspec         hiding (pendingWith)
-import Test.Hspec.Wai
-import Text.Heredoc
-
-import Protolude  hiding (get)
-import SpecHelper (acceptHdrs)
-
-spec :: SpecWith ((), Application)
-spec = describe "When raw-media-types is set to \"text/html\"" $
-  it "can get raw output with Accept: text/html" $
-    request methodGet "/rpc/welcome.html" (acceptHdrs "text/html") ""
-      `shouldRespondWith`
-      [str|
-          |<html>
-          |  <head>
-          |    <title>PostgREST</title>
-          |  </head>
-          |  <body>
-          |    <h1>Welcome to PostgREST</h1>
-          |  </body>
-          |</html>
-          |]
-      { matchStatus = 200
-      , matchHeaders = ["Content-Type" <:> "text/html"]
-      }
@@ -43,7 +43,8 @@ spec actualPgVersion = do
          , "enum": "foo"
          }] |] `shouldRespondWith` [json|[{"integer":14,"varchar":"testing!"}]|]
          { matchStatus  = 201
-          , matchHeaders = [matchContentTypeJson]
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "return=representation"]
          }

      it "ignores &select when return not set or using return=minimal" $ do
@@ -69,7 +70,8 @@ spec actualPgVersion = do
          `shouldRespondWith`
            ""
            { matchStatus  = 201
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal"]
            }

    context "non uniform json array" $ do
@@ -99,7 +101,8 @@ spec actualPgVersion = do
          { matchStatus  = 201
          , matchHeaders = [ matchContentTypeJson
                           , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/1" ]
+                           , "Content-Range" <:> "*/1"
+                           , "Preference-Applied" <:> "return=representation, count=exact"]
          }

      it "can rename and cast the selected columns" $
@@ -110,7 +113,8 @@ spec actualPgVersion = do
          { matchStatus  = 201
          , matchHeaders = [ matchContentTypeJson
                           , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/*" ]
+                           , "Content-Range" <:> "*/*"
+                           , "Preference-Applied" <:> "return=representation"]
          }

      it "should not throw and return location header when selecting without PK" $
@@ -120,7 +124,8 @@ spec actualPgVersion = do
          { matchStatus  = 201
          , matchHeaders = [ matchContentTypeJson
                           , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/*" ]
+                           , "Content-Range" <:> "*/*"
+                           , "Preference-Applied" <:> "return=representation"]
          }

    context "requesting headers only representation" $ do
@@ -133,7 +138,8 @@ spec actualPgVersion = do
            { matchStatus  = 201
            , matchHeaders = [ matchHeaderAbsent hContentType
                             , "Location" <:> "/projects?id=eq.11"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
            }

      when (actualPgVersion >= pgVersion110) $
@@ -146,7 +152,8 @@ spec actualPgVersion = do
              { matchStatus  = 201
              , matchHeaders = [ matchHeaderAbsent hContentType
                               , "Location" <:> "/car_models?name=eq.Enzo&year=eq.2021"
-                               , "Content-Range" <:> "*/*" ]
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
              }

    context "requesting no representation" $
@@ -193,7 +200,8 @@ spec actualPgVersion = do
              ""
              { matchStatus  = 201
              , matchHeaders = [ matchHeaderAbsent hContentType
-                               , "Location" <:> "/auto_incrementing_pk?id=eq.2" ]
+                               , "Location" <:> "/auto_incrementing_pk?id=eq.2"
+                               , "Preference-Applied" <:> "return=headers-only"]
              }

      context "into a table with simple pk" $
@@ -227,7 +235,8 @@ spec actualPgVersion = do
            `shouldRespondWith`
              [json| [{ "a":"bar", "b":"baz" }] |]
              { matchStatus  = 201
-              , matchHeaders = [matchHeaderAbsent hLocation]
+              , matchHeaders = [matchHeaderAbsent hLocation
+                               , "Preference-Applied" <:> "return=representation"]
              }

        it "returns empty array when no items inserted, and return=rep" $ do
@@ -391,6 +400,22 @@ spec actualPgVersion = do
          `shouldRespondWith` [json|[{ id: 20 }]|]
            { matchStatus  = 201 }

+    -- https://github.com/PostgREST/postgrest/issues/2861
+    context "bit and char columns with length" $ do
+      it "should insert to a bit column with length" $
+        request methodPost "/bitchar_with_length?select=bit"
+            [("Prefer", "return=representation")]
+            [json|{"bit": "10101"}|]
+          `shouldRespondWith` [json|[{ "bit": "10101" }]|]
+            { matchStatus  = 201 }
+
+      it "should insert to a char column with length" $
+        request methodPost "/bitchar_with_length?select=char"
+            [("Prefer", "return=representation")]
+            [json|{"char": "abcde"}|]
+          `shouldRespondWith` [json|[{ "char": "abcde" }]|]
+            { matchStatus  = 201 }
+
    context "POST with ?columns parameter" $ do
      it "ignores json keys not included in ?columns" $ do
        request methodPost "/articles?columns=id,body" [("Prefer", "return=representation")]
@@ -469,7 +494,7 @@ spec actualPgVersion = do
                  {"id": 6, "name": "Sechs", "field-with_sep": 6, "settings":null,"arr_data":[1,2,3]}
                ]|]
                { matchStatus  = 201
-                , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+                , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
                }

        it "inserts view default values(field-with_sep) when json keys are undefined" $
@@ -484,7 +509,7 @@ spec actualPgVersion = do
                {"id": 8, "name": "Default", "field-with_sep": 1, "settings":null,"arr_data":null}
              ]|]
              { matchStatus  = 201
-              , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
              }

        it "doesn't insert json duplicate keys(since it uses jsonb)" $
@@ -493,7 +518,7 @@ spec actualPgVersion = do
            `shouldRespondWith`
              [json| [ { "data": { "a": 2 }, "id": 3 } ] |]
              { matchStatus  = 201
-              , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
              }

        when (actualPgVersion >= pgVersion100) $
@@ -503,7 +528,7 @@ spec actualPgVersion = do
              `shouldRespondWith`
                [json| [{"data":{"foo": "bar"},"slug":"foo"}] |] -- id 1 was inserted here, we don't get it for idempotence in the tests
                { matchStatus  = 201
-                , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+                , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
                }

        when (actualPgVersion >= pgVersion120) $
@@ -529,6 +554,15 @@ spec actualPgVersion = do
                  }|])
                { matchStatus  = 400 }

+        it "inserts a default on a DOMAIN with default" $
+          request methodPost "/evil_friends?columns=id,name" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+              [json| { "name": "Lu" } |]
+            `shouldRespondWith`
+              [json| [{"id": 666, "name": "Lu"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+              }
+
    it "inserts json that has duplicate keys" $ do
      request methodPost "/tbl_w_json" [("Prefer", "return=representation")]
          [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
@@ -694,7 +728,8 @@ spec actualPgVersion = do
        `shouldRespondWith`
          ""
          { matchStatus = 201
-          , matchHeaders = [matchHeaderAbsent hContentType]
+          , matchHeaders = [matchHeaderAbsent hContentType
+                           , "Preference-Applied" <:> "return=minimal"]
          }

  describe "Inserting into VIEWs" $ do
@@ -717,7 +752,8 @@ spec actualPgVersion = do
            { matchStatus  = 201
            , matchHeaders = [ matchHeaderAbsent hContentType
                             , "Location" <:> "/with_multiple_pks?pk1=eq.1&pk2=eq.2"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
            }

    context "requesting header only representation" $ do
@@ -729,7 +765,8 @@ spec actualPgVersion = do
            { matchStatus  = 201
            , matchHeaders = [ matchHeaderAbsent hContentType
                             , "Location" <:> "/compound_pk_view?k1=eq.1&k2=eq.test"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
            }

      it "should not throw and return location header when a PK is null" $
@@ -740,5 +777,120 @@ spec actualPgVersion = do
            { matchStatus  = 201
            , matchHeaders = [ matchHeaderAbsent hContentType
                             , "Location" <:> "/test_null_pk_competitors_sponsors?id=eq.1&sponsor_id=is.null"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
            }
+
+
+  -- Data representations for payload parsing requires Postgres 10 or above.
+  when (actualPgVersion >= pgVersion100) $ do
+    describe "Data representations" $ do
+      context "on regular table" $ do
+        it "parses values in POST body" $
+          -- we don't check that the parsing is correct here, just that it's happening. If it doesn't happen we'll get a
+          -- an "invalid input syntax for type integer:" error.
+          request methodPost "/datarep_todos" [("Prefer", "return=headers-only")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 201
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Location" <:> "/datarep_todos?id=eq.5"
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
+              }
+
+        it "parses values in POST body and formats individually selected values in return=representation" $
+          request methodPost "/datarep_todos?select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "label_color": "#001100"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "parses values in POST body and formats values in return=representation" $
+          request methodPost "/datarep_todos" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00", "icon_image": "3q2+7w", "created_at":-15, "budget": "-100000000000000.13"} |]
+            `shouldRespondWith`
+            [json| [{"id":5,"name": "party", "label_color": "#001100", "due_at":"2018-01-03T11:00:00Z", "icon_image": "3q2+7w==", "created_at":-15, "budget": "-100000000000000.13"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+      context "with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPost "/datarep_todos?columns=id,label_color&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "invalid but should be ignored"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "name":null, "label_color": "#001100", "due_at": "2018-01-01T00:00:00Z"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "fails without parsing anything if at least one specified column doesn't exist" $
+          request methodPost "/datarep_todos?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }
+
+      context "on updatable view" $ do
+        it "parses values in POST body" $
+          -- we don't check that the parsing is correct here, just that it's happening. If it doesn't happen we'll get a
+          -- an "invalid input syntax for type integer:" error.
+          request methodPost "/datarep_todos_computed" [("Prefer", "return=headers-only")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 201
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Location" <:> "/datarep_todos_computed?id=eq.5"
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
+              }
+
+        it "parses values in POST body and formats individually selected values in return=representation" $
+          request methodPost "/datarep_todos_computed?select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "label_color": "#001100"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "parses values in POST body and formats values in return=representation" $
+          request methodPost "/datarep_todos_computed" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5,"name": "party", "label_color": "#001100", "due_at":"2018-01-03T11:00:00Z", "dark_color":"#000880"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+      context "on updatable views with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPost "/datarep_todos_computed?columns=id,label_color&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "invalid but should be ignored"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "name":null, "label_color": "#001100", "due_at": "2018-01-01T00:00:00Z"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "fails without parsing anything if at least one specified column doesn't exist" $
+          request methodPost "/datarep_todos_computed?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos_computed' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }
@@ -103,7 +103,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do

      it "can get array of objects" $ do
        get "/json_arr?select=data->0->>a&id=in.(5,6)" `shouldRespondWith`
-          [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
+          [json|[{"a":"A"}, {"a":"[1,2,3]"}]|]
          { matchHeaders = [matchContentTypeJson] }
        get "/json_arr?select=data->0->a->>2&id=in.(5,6)" `shouldRespondWith`
          [json| [{"a":null}, {"a":"3"}] |]
@@ -275,7 +275,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
        [json| [{"data":8}, {"data":7}] |]
        { matchHeaders = [matchContentTypeJson] }
      get "/json_arr?select=data->-2->>a&id=in.(5,6)" `shouldRespondWith`
-        [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
+        [json| [{"a":"A"}, {"a":"[1,2,3]"}] |]
        { matchHeaders = [matchContentTypeJson] }

    it "can filter with negative indexes" $ do
@@ -300,3 +300,11 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         "code": "PGRST100",
         "hint": null} |]
      { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
+
+  it "works when an RPC returns a dynamic TABLE with a composite type" $
+    get "/rpc/returns_complex?select=val->r&val->i=gt.0.5&order=val->>i.desc" `shouldRespondWith`
+      [json|[
+        {"r":0.3},
+        {"r":0.2}
+      ]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
@@ -225,13 +225,11 @@ spec =

      it "succeeds on PUT on the v2 schema" $
        request methodPut "/children?id=eq.111" [("Content-Profile", "v2"), ("Prefer", "return=representation")]
-          [json| [ { "id": 111, "name": "child v2-111", "parent_id": null } ]|]
+          [json|[{"id": 111, "name": "child v2-111", "parent_id": null}]|]
          `shouldRespondWith`
-          [json|[{ "id": 111, "name": "child v2-111", "parent_id": null }]|]
-          {
-            matchStatus = 200
-          , matchHeaders = [matchContentTypeJson, "Content-Profile" <:> "v2"]
-          }
+          [json|[{"id": 111, "name": "child v2-111", "parent_id": null}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchContentTypeJson, "Content-Profile" <:> "v2"]}

    context "OpenAPI output" $ do
      it "succeeds in reading table definition from default schema v1 if no schema is selected via header" $ do
@@ -0,0 +1,126 @@
+module Feature.Query.NullsStripSpec where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import Protolude  hiding (get)
+import SpecHelper
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "Stripping null values from JSON response" $ do
+    let arrayStrip = ("Accept", "application/vnd.pgrst.array+json;nulls=stripped")
+    let singularStrip = ("Accept", "application/vnd.pgrst.object+json;nulls=stripped")
+
+    context "strip nulls from response" $ do
+      it "strips nulls when Accept: application/vnd.pgrst.array+json;nulls=stripped" $ do
+        request methodGet  "/organizations?select=*"
+          [arrayStrip]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"name":"Referee Org","manager_id":1},{"id":2,"name":"Auditor Org","manager_id":2},{"id":3,"name":"Acme","referee":1,"auditor":2,"manager_id":3},{"id":4,"name":"Umbrella","referee":1,"auditor":2,"manager_id":4},{"id":5,"name":"Cyberdyne","referee":3,"auditor":4,"manager_id":5},{"id":6,"name":"Oscorp","referee":3,"auditor":4,"manager_id":6}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPost  "/organizations?select=*"
+          [arrayStrip,("Prefer","return=representation")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7,"name":"John","manager_id":6}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPatch  "/organizations?id=eq.3&select=*"
+          [arrayStrip, ("Prefer","return=representation")]
+          [json|{"name":"John","referee":null}|]
+          `shouldRespondWith`
+          [json|[{"id":3,"name":"John","auditor":2,"manager_id":3}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.array;nulls=stripped" $
+        request methodGet  "/organizations?select=*"
+          [("Accept","application/vnd.pgrst.array;nulls=stripped")]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"name":"Referee Org","manager_id":1},{"id":2,"name":"Auditor Org","manager_id":2},{"id":3,"name":"Acme","referee":1,"auditor":2,"manager_id":3},{"id":4,"name":"Umbrella","referee":1,"auditor":2,"manager_id":4},{"id":5,"name":"Cyberdyne","referee":3,"auditor":4,"manager_id":5},{"id":6,"name":"Oscorp","referee":3,"auditor":4,"manager_id":6}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.object+json;nulls=stripped" $
+        request methodGet  "/organizations?limit=1"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"id":1,"name":"Referee Org","manager_id":1}|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTSingularStrip]
+          }
+
+      it "throws error when Accept: application/vnd.pgrst.object+json;nulls=stripped and result not singular" $
+        request methodGet  "/organizations?select=*"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"details":"The result contains 6 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+          { matchStatus  = 406
+          , matchHeaders = [matchContentTypeSingular]
+          }
+
+    context "strip nulls from response even if explicitly selected" $ do
+      it "strips nulls when Accept: application/vnd.pgrst.array+json;nulls=stripped" $ do
+        request methodGet  "/organizations?select=id,referee,auditor"
+          [arrayStrip]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1},{"id":2},{"id":3,"referee":1,"auditor":2},{"id":4,"referee":1,"auditor":2},{"id":5,"referee":3,"auditor":4},{"id":6,"referee":3,"auditor":4}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPost  "/organizations?select=id,referee,auditor"
+          [arrayStrip,("Prefer","return=representation")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPatch  "/organizations?id=eq.3&select=id,name,referee,auditor"
+          [arrayStrip, ("Prefer","return=representation")]
+          [json|{"name":"John","referee":null}|]
+          `shouldRespondWith`
+          [json|[{"id":3,"name":"John","auditor":2}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.object+json;nulls=stripped" $
+        request methodGet  "/organizations?select=id,referee,auditor&limit=1"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"id":1}|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTSingularStrip]
+          }
+
+    context "doesn't strip nulls" $ do
+      it "doesn't strips nulls when Accept: application/vnd.pgrst.array+json" $
+        request methodGet  "/organizations?select=id,referee,auditor"
+          [("Accept", "application/vnd.pgrst.array+json")]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"referee":null,"auditor":null},{"id":2,"referee":null,"auditor":null},{"id":3,"referee":1,"auditor":2},{"id":4,"referee":1,"auditor":2},{"id":5,"referee":3,"auditor":4},{"id":6,"referee":3,"auditor":4}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }
@@ -8,7 +8,6 @@ import Network.Wai.Test (SResponse (..))

 import           Data.Aeson.Lens
 import           Data.Aeson.QQ
-import qualified Data.ByteString      as BS
 import qualified Data.ByteString.Lazy as LBS
 import qualified Data.Text            as T
 import           Network.HTTP.Types
@@ -33,7 +32,7 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
        totalCost `shouldBe`
          if actualPgVersion > pgVersion120
@@ -49,7 +48,7 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
        totalCost `shouldBe`
          if actualPgVersion > pgVersion120
@@ -65,8 +64,8 @@ spec actualPgVersion = do
              resHeaders = simpleHeaders r

          liftIO $ do
-            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=buffers; charset=utf-8")
-            resBody `shouldSatisfy` (\t -> T.isInfixOf "Shared Hit Blocks" (decodeUtf8 $ BS.toStrict t))
+            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=buffers; charset=utf-8")
+            resBody `shouldSatisfy` (\t -> T.isInfixOf "Shared Hit Blocks" (decodeUtf8 $ LBS.toStrict t))
        else do
          -- analyze is required for buffers on pg < 13
          r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=analyze|buffers") ""
@@ -75,7 +74,7 @@ spec actualPgVersion = do
              resHeaders = simpleHeaders r

          liftIO $ do
-            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=analyze|buffers; charset=utf-8")
+            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze|buffers; charset=utf-8")
            blocks `shouldBe` Just [aesonQQ| 1.0 |]

    when (actualPgVersion >= pgVersion120) $
@@ -86,7 +85,7 @@ spec actualPgVersion = do
            resHeaders = simpleHeaders r

        liftIO $ do
-          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=settings; charset=utf-8")
+          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=settings; charset=utf-8")
          searchPath `shouldBe`
            Just [aesonQQ|
              {
@@ -102,7 +101,7 @@ spec actualPgVersion = do
            resHeaders = simpleHeaders r

        liftIO $ do
-          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=analyze|wal; charset=utf-8")
+          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze|wal; charset=utf-8")
          walRecords `shouldBe` Just [aesonQQ|0|]

    it "outputs columns info when using the verbose option" $ do
@@ -112,7 +111,7 @@ spec actualPgVersion = do
          resHeaders = simpleHeaders r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=verbose; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=verbose; charset=utf-8")
        cols `shouldBe` Just [aesonQQ| ["projects.id", "projects.name", "projects.client_id"] |]

    it "outputs the plan for application/json " $ do
@@ -151,7 +150,7 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
        totalCost `shouldBe` 3.27

@@ -164,7 +163,7 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
        totalCost `shouldBe` 12.45

@@ -177,7 +176,7 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
        totalCost `shouldBe` 15.68

@@ -191,9 +190,65 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe` 1.29
+        totalCost `shouldBe` 3.55
+
+    it "outputs the total cost for 2 upserts" $ do
+      r <- request methodPost "/tiobe_pls"
+            [("Prefer","resolution=merge-duplicates"), ("Accept","application/vnd.pgrst.plan+json")]
+            [json| [ { "name": "Python", "rank": 19 }, { "name": "Go", "rank": 20} ]|]
+
+      let totalCost  = planCost r
+          resStatus  = simpleStatus r
+          resHeaders = simpleHeaders r
+
+      liftIO $ do
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
+        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
+        totalCost `shouldBe` 5.53
+
+    it "outputs the total cost for an upsert with 10 rows" $ do
+      r <- request methodPost "/tiobe_pls"
+            [("Prefer","resolution=merge-duplicates"), ("Accept","application/vnd.pgrst.plan+json")]
+            (getInsertDataForTiobePlsTable 10)
+
+      let totalCost  = planCost r
+          resStatus  = simpleStatus r
+          resHeaders = simpleHeaders r
+
+      liftIO $ do
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
+        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
+        totalCost `shouldBe` 5.53
+
+    it "outputs the total cost for an upsert with 100 rows" $ do
+      r <- request methodPost "/tiobe_pls"
+            [("Prefer","resolution=merge-duplicates"), ("Accept","application/vnd.pgrst.plan+json")]
+            (getInsertDataForTiobePlsTable 100)
+
+      let totalCost  = planCost r
+          resStatus  = simpleStatus r
+          resHeaders = simpleHeaders r
+
+      liftIO $ do
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
+        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
+        totalCost `shouldBe` 5.53
+
+    it "outputs the total cost for an upsert with 1000 rows" $ do
+      r <- request methodPost "/tiobe_pls"
+            [("Prefer","resolution=merge-duplicates"), ("Accept","application/vnd.pgrst.plan+json")]
+            (getInsertDataForTiobePlsTable 1000)
+
+      let totalCost  = planCost r
+          resStatus  = simpleStatus r
+          resHeaders = simpleHeaders r
+
+      liftIO $ do
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
+        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
+        totalCost `shouldBe` 5.53

    it "outputs the plan for application/vnd.pgrst.object" $ do
      r <- request methodDelete "/projects?id=eq.6"
@@ -216,21 +271,10 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
        totalCost `shouldBe` 68.56

-    it "outputs the plan for text/xml" $ do
-      r <- request methodGet "/rpc/return_scalar_xml"
-        (acceptHdrs "application/vnd.pgrst.plan+json; for=\"text/xml\"; options=verbose") ""
-
-      let aggCol  = simpleBody r ^? nth 0 . key "Plan" . key "Output" . nth 2
-          resHeaders = simpleHeaders r
-
-      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"text/xml\"; options=verbose; charset=utf-8")
-        aggCol `shouldBe` Just [aesonQQ| "COALESCE(xmlagg(return_scalar_xml.pgrst_scalar), ''::xml)" |]
-
  describe "text format" $ do
    it "outputs the total cost for a function call" $ do
      r <- request methodGet "/projects?id=in.(1,2,3)"
@@ -241,7 +285,7 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
        resBody `shouldSatisfy` (\t -> LBS.take 9 t == "Aggregate")

@@ -254,7 +298,7 @@ spec actualPgVersion = do
          resStatus  = simpleStatus r

      liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; for=\"application/json\"; charset=utf-8")
        resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
        resBody `shouldSatisfy` (\t -> LBS.take 9 t == "Aggregate")

@@ -304,12 +348,12 @@ spec actualPgVersion = do
        r1 <- request methodGet "/users?select=*,tasks!inner(*)&tasks.id=eq.1"
               [planHdr] ""

-        liftIO $ planCost r1 `shouldSatisfy` (< 20876.14)
+        liftIO $ planCost r1 `shouldSatisfy` (< 20888.83)

        r2 <- request methodGet "/users?select=*,tasks(*)&tasks.id=eq.1&tasks=not.is.null"
               [planHdr] ""

-        liftIO $ planCost r2 `shouldSatisfy` (< 20876.14)
+        liftIO $ planCost r2 `shouldSatisfy` (< 20888.83)

  describe "function call costs" $ do
    it "should not exceed cost when calling setof composite proc" $ do
@@ -328,7 +372,7 @@ spec actualPgVersion = do
      r <- request methodGet "/rpc/add_them?a=3&b=4"
             [planHdr] ""

-      liftIO $ planCost r `shouldSatisfy` (< 1.18)
+      liftIO $ planCost r `shouldSatisfy` (< 0.11)

    context "function inlining" $ do
      it "should inline a zero argument function(the function won't appear in the plan tree)" $ do
@@ -338,7 +382,7 @@ spec actualPgVersion = do
        let resBody = simpleBody r

        liftIO $ do
-          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getallusers" (decodeUtf8 $ BS.toStrict t))
+          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getallusers" (decodeUtf8 $ LBS.toStrict t))

      it "should inline a function with arguments(the function won't appear in the plan tree)" $ do
        r <- request methodGet "/rpc/getitemrange?min=10&max=15"
@@ -347,7 +391,72 @@ spec actualPgVersion = do
        let resBody = simpleBody r

        liftIO $ do
-          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getitemrange" (decodeUtf8 $ BS.toStrict t))
+          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getitemrange" (decodeUtf8 $ LBS.toStrict t))
+
+    context "index usage" $ do
+      it "should use an index for a json arrow operator filter" $ do
+        r <- request methodGet "/bets?data_json->>contractId=eq.1"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index Cond" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for a jsonb arrow operator filter" $ do
+        r <- request methodGet "/bets?data_jsonb->>contractId=eq.1"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for ordering on a json arrow operator" $ do
+        r <- request methodGet "/bets?order=data_json->>contractId"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for ordering on a jsonb arrow operator" $ do
+        r <- request methodGet "/bets?order=data_jsonb->>contractId"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
+
+  describe "custom media types" $ do
+    it "outputs the plan for a scalar function text/xml" $ do
+      r <- request methodGet "/rpc/return_scalar_xml"
+        (acceptHdrs "application/vnd.pgrst.plan+json; for=\"text/xml\"; options=verbose") ""
+
+      let aggCol  = simpleBody r ^? nth 0 . key "Plan" . key "Output" . nth 2
+          resHeaders = simpleHeaders r
+
+      liftIO $ do
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"text/xml\"; options=verbose; charset=utf-8")
+        aggCol `shouldBe` Just [aesonQQ| "return_scalar_xml.pgrst_scalar" |]
+
+    it "outputs the plan for an aggregate application/vnd.twkb" $ do
+      r <- request methodGet "/lines"
+        (acceptHdrs "application/vnd.pgrst.plan+json; for=\"application/vnd.twkb\"; options=verbose") ""
+
+      let aggCol  = simpleBody r ^? nth 0 . key "Plan" . key "Output" . nth 2
+          resHeaders = simpleHeaders r
+
+      liftIO $ do
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/vnd.twkb\"; options=verbose; charset=utf-8")
+        aggCol `shouldBe`
+          (
+            if actualPgVersion >= pgVersion120
+              then Just [aesonQQ| "twkb_agg(ROW(lines.id, lines.name, lines.geom)::lines)" |]
+              else Just [aesonQQ| "twkb_agg(ROW(pgrst_source.id, pgrst_source.name, pgrst_source.geom)::lines)" |]
+          )

 disabledSpec :: SpecWith ((), Application)
 disabledSpec =
@@ -0,0 +1,121 @@
+module Feature.Query.PreferencesSpec where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import Protolude  hiding (get)
+import SpecHelper
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "test prefer headers and preference-applied headers" $ do
+
+    context "check behaviour of Prefer: handling=strict" $ do
+      it "throws error when handling=strict and invalid prefs are given" $
+        request methodGet  "/items" [("Prefer", "handling=strict, anything")] ""
+          `shouldRespondWith`
+          [json|{"details":"Invalid preferences: anything","message":"Invalid preferences given with handling=strict","code":"PGRST122","hint":null}|]
+          { matchStatus = 400 }
+
+      it "throw error when handling=strict and invalid prefs are given with multiples in separate prefers" $
+        request methodGet  "/items" [("Prefer", "handling=strict"),("Prefer","something, else")] ""
+          `shouldRespondWith`
+          [json|{"details":"Invalid preferences: something, else","message":"Invalid preferences given with handling=strict","code":"PGRST122","hint":null}|]
+          { matchStatus = 400 }
+
+      it "throws error with post request" $
+        request methodPost  "/organizations?select=*"
+          [("Prefer","return=representation, handling=strict, anything")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|{"details":"Invalid preferences: anything","message":"Invalid preferences given with handling=strict","code":"PGRST122","hint":null}|]
+          { matchStatus = 400 }
+
+      it "throws error with rpc" $
+        request methodPost "/rpc/overloaded_unnamed_param"
+          [("Content-Type", "application/json"), ("Prefer", "handling=strict, anything")]
+          [json|{}|]
+          `shouldRespondWith`
+          [json|{"details":"Invalid preferences: anything","message":"Invalid preferences given with handling=strict","code":"PGRST122","hint":null}|]
+          { matchStatus = 400 }
+
+    context "check behaviour of Prefer: handling=lenient" $ do
+      it "does not throw error when handling=lenient and invalid prefs" $
+        request methodGet  "/items" [("Prefer", "handling=lenient, anything")] ""
+          `shouldRespondWith` 200
+
+      it "does not throw error when handling=lenient and invalid prefs in multiples prefers" $
+        request methodGet  "/items" [("Prefer", "handling=lenient"), ("Prefer", "anything")] ""
+          `shouldRespondWith` 200
+
+      it "does not throw error with post request" $
+        request methodPost  "/organizations?select=*"
+          [("Prefer","return=representation, handling=lenient, anything")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}]|]
+          { matchStatus  = 201
+          , matchHeaders = [ matchContentTypeJson ]
+          }
+
+      it "does not throw error with rpc" $
+        request methodPost "/rpc/overloaded_unnamed_param"
+          [("Content-Type", "application/json"), ("Prefer", "handling=lenient, anything")]
+          [json|{}|]
+          `shouldRespondWith`
+          [json| 1 |]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+           }
+
+    context "test Prefer: timezone=America/Los_Angeles" $ do
+      it "should change timezone with handling=strict" $
+        request methodGet "/timestamps"
+          [("Prefer", "handling=strict, timezone=America/Los_Angeles")]
+          ""
+          `shouldRespondWith`
+          [json|[{"t":"2023-10-18T05:37:59.611-07:00"}, {"t":"2023-10-18T07:37:59.611-07:00"}, {"t":"2023-10-18T09:37:59.611-07:00"}]|]
+          { matchStatus = 200
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "handling=strict, timezone=America/Los_Angeles"]}
+
+      it "should change timezone without handling=strict" $
+        request methodGet "/timestamps"
+          [("Prefer", "timezone=America/Los_Angeles")]
+          ""
+          `shouldRespondWith`
+          [json|[{"t":"2023-10-18T05:37:59.611-07:00"}, {"t":"2023-10-18T07:37:59.611-07:00"}, {"t":"2023-10-18T09:37:59.611-07:00"}]|]
+          { matchStatus = 200
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "timezone=America/Los_Angeles"] }
+
+    context "test Prefer: timezone=Invalid/Timezone" $ do
+      it "should throw error with handling=strict" $
+        request methodGet "/timestamps"
+          [("Prefer", "handling=strict, timezone=Invalid/Timezone")]
+          ""
+          `shouldRespondWith`
+          [json|{"code":"PGRST122","details":"Invalid preferences: timezone=Invalid/Timezone","hint":null,"message":"Invalid preferences given with handling=strict"}|]
+          { matchStatus = 400 }
+
+      it "should return with default timezone without handling or with handling=lenient" $ do
+        request methodGet "/timestamps"
+          [("Prefer", "timezone=Invalid/Timezone")]
+          ""
+          `shouldRespondWith`
+          [json|[{"t":"2023-10-18T12:37:59.611+00:00"}, {"t":"2023-10-18T14:37:59.611+00:00"}, {"t":"2023-10-18T16:37:59.611+00:00"}]|]
+          { matchStatus = 200
+          , matchHeaders = [matchContentTypeJson]}
+
+        request methodGet "/timestamps"
+          [("Prefer", "handling=lenient, timezone=Invalid/Timezone")]
+          ""
+          `shouldRespondWith`
+          [json|[{"t":"2023-10-18T12:37:59.611+00:00"}, {"t":"2023-10-18T14:37:59.611+00:00"}, {"t":"2023-10-18T16:37:59.611+00:00"}]|]
+          { matchStatus = 200
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "handling=lenient"]}
@@ -1042,56 +1042,6 @@ spec actualPgVersion = do
        [json|[{"a$num$":100}]|]
        { matchHeaders = [matchContentTypeJson] }

-  context "binary output" $ do
-    it "can query if a single column is selected" $
-      request methodGet "/images_base64?select=img&name=eq.A.png" (acceptHdrs "application/octet-stream") ""
-        `shouldRespondWith` "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEUAAAD/AAAb/40iAAAAP0lEQVQI12NgwAbYG2AE/wEYwQMiZB4ACQkQYZEAIgqAhAGIKLCAEQ8kgMT/P1CCEUwc4IMSzA3sUIIdCHECAGSQEkeOTUyCAAAAAElFTkSuQmCC"
-        { matchStatus = 200
-        , matchHeaders = ["Content-Type" <:> "application/octet-stream"]
-        }
-
-    it "can get raw output with Accept: text/plain" $
-      request methodGet "/projects?select=name&id=eq.1" (acceptHdrs "text/plain") ""
-        `shouldRespondWith` "Windows 7"
-        { matchStatus = 200
-        , matchHeaders = ["Content-Type" <:> "text/plain; charset=utf-8"]
-        }
-
-    it "can get raw xml output with Accept: text/xml" $
-      request methodGet "/xmltest?select=xml" (acceptHdrs "text/xml") ""
-        `shouldRespondWith`
-        "<myxml>foo</myxml>bar<foobar><baz/></foobar>"
-        { matchStatus = 200
-        , matchHeaders = ["Content-Type" <:> "text/xml; charset=utf-8"]
-        }
-
-    it "fails if a single column is not selected" $ do
-      request methodGet "/images?select=img,name&name=eq.A.png" (acceptHdrs "application/octet-stream") ""
-        `shouldRespondWith`
-          [json| {"message":"application/octet-stream requested but more than one column was selected","code":"PGRST113","details":null,"hint":null} |]
-          { matchStatus = 406 }
-
-      request methodGet "/images?select=*&name=eq.A.png"
-          (acceptHdrs "application/octet-stream")
-          ""
-        `shouldRespondWith`
-          [json| {"message":"application/octet-stream requested but more than one column was selected","code":"PGRST113","details":null,"hint":null} |]
-          { matchStatus = 406 }
-
-      request methodGet "/images?name=eq.A.png"
-          (acceptHdrs "application/octet-stream")
-          ""
-        `shouldRespondWith`
-          [json| {"message":"application/octet-stream requested but more than one column was selected","code":"PGRST113","details":null,"hint":null} |]
-          { matchStatus = 406 }
-
-    it "concatenates results if more than one row is returned" $
-      request methodGet "/images_base64?select=img&name=in.(A.png,B.png)" (acceptHdrs "application/octet-stream") ""
-        `shouldRespondWith` "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEUAAAD/AAAb/40iAAAAP0lEQVQI12NgwAbYG2AE/wEYwQMiZB4ACQkQYZEAIgqAhAGIKLCAEQ8kgMT/P1CCEUwc4IMSzA3sUIIdCHECAGSQEkeOTUyCAAAAAElFTkSuQmCCiVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEX///8AAP94wDzzAAAAL0lEQVQIW2NgwAb+HwARH0DEDyDxwAZEyGAhLODqHmBRzAcn5GAS///A1IF14AAA5/Adbiiz/0gAAAAASUVORK5CYII="
-        { matchStatus = 200
-        , matchHeaders = ["Content-Type" <:> "application/octet-stream"]
-        }
-
  describe "values with quotes in IN and NOT IN" $ do
    it "succeeds when only quoted values are present" $ do
      get "/w_or_wo_comma_names?name=in.(\"Hebdon, John\")" `shouldRespondWith`
@@ -1328,3 +1278,132 @@ spec actualPgVersion = do
      get "/articles?body=imatch(any).{stop,thing}&select=id" `shouldRespondWith`
        [json|[{"id":1}, {"id":2}]|]
        { matchHeaders = [matchContentTypeJson] }
+
+  describe "Data representations for customisable value formatting and parsing" $ do
+    it "formats a single column" $
+      get "/datarep_todos?select=id,label_color&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"label_color":"#000000"},{"id":2,"label_color":"#000100"},{"id":3,"label_color":"#01E240"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats two columns with different formatters" $
+      get "/datarep_todos?select=id,label_color,due_at&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"},{"id":2,"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"},{"id":3,"label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "fails in some reasonable way when selecting fields that don't exist" $
+      get "/datarep_todos?select=id,label_color,banana" `shouldRespondWith`
+        [json| {"code":"42703","details":null,"hint":null,"message":"column datarep_todos.banana does not exist"} |]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+    it "formats columns in views including computed columns" $
+      get "/datarep_todos_computed?select=id,label_color,dark_color" `shouldRespondWith`
+        [json| [
+          {"id":1, "label_color":"#000000", "dark_color":"#000000"},
+          {"id":2, "label_color":"#000100", "dark_color":"#000080"},
+          {"id":3, "label_color":"#01E240", "dark_color":"#00F120"},
+          {"id":4, "label_color":"", "dark_color":""}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats and allows rename" $
+      get "/datarep_todos?select=id,clr:label_color&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"clr":"#000000"},{"id":2,"clr":"#000100"},{"id":3,"clr":"#01E240"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats, renames and allows manual casting on top" $
+      get "/datarep_todos?select=id,clr:label_color::text&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"clr":"\"#000000\""},{"id":2,"clr":"\"#000100\""},{"id":3,"clr":"\"#01E240\""}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats nulls" $
+      -- due_at is formatted as NULL but label_color NULLs become empty strings-- it's up to the formatting function.
+      get "/datarep_todos?select=id,label_color,due_at&id=gt.2&id=lt.5" `shouldRespondWith`
+        [json| [{"id":3,"label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z"},{"id":4,"label_color":"","due_at":null}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats star select" $
+      get "/datarep_todos?select=*&id=lt.4" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+           {"id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+           {"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats implicit star select" $
+      get "/datarep_todos?id=lt.4" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+ {"id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+ {"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats star and explicit mix" $
+      get "/datarep_todos?select=due_at,*&id=lt.4" `shouldRespondWith`
+        [json| [
+          {"due_at":"2018-01-02T00:00:00Z","id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+           {"due_at":"2018-01-03T00:00:00Z","id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+           {"due_at":"2018-01-01T14:12:34.123456Z","id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats through join" $
+      get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)" `shouldRespondWith`
+        [json| [{"id":1,"name":"school related","first_item":{"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"}},{"id":2,"name":"do these first","first_item":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats through join with star select" $
+      get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"school related","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}},
+          {"id":2,"name":"do these first","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "uses text parser on value for filter given through query parameters" $
+      get "/datarep_todos?select=id,due_at&label_color=eq.000100" `shouldRespondWith`
+        [json| [{"id":2,"due_at":"2018-01-03T00:00:00Z"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "in the absense of text parser, does not try to use the JSON parser for query parameters" $
+      get "/datarep_todos?select=id,due_at&due_at=eq.Z" `shouldRespondWith`
+        -- we prove the parser is not used because it'd replace the Z with `+00:00` and a different error message.
+        [json| {"code":"22007","details":null,"hint":null,"message":"invalid input syntax for type timestamp with time zone: \"Z\""} |]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+    -- Before PG 11, this will fail because we need arrays of domain type values. The docs should explain data reps are
+    -- not supported in this case.
+    when (actualPgVersion >= pgVersion110) $ do
+      it "uses text parser for filter with 'IN' predicates" $
+        get "/datarep_todos?select=id,due_at&label_color=in.(000100,01E240)" `shouldRespondWith`
+          [json| [
+            {"id":2, "due_at": "2018-01-03T00:00:00Z"},
+            {"id":3, "due_at": "2018-01-01T14:12:34.123456Z"}
+          ] |]
+          { matchHeaders = [matchContentTypeJson] }
+      it "uses text parser for filter with 'NOT IN' predicates" $
+        get "/datarep_todos?select=id,due_at&label_color=not.in.(000000,01E240)" `shouldRespondWith`
+          [json| [
+            {"id":2, "due_at": "2018-01-03T00:00:00Z"}
+          ] |]
+          { matchHeaders = [matchContentTypeJson] }
+    it "uses text parser on value for filter across relations" $
+      get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100" `shouldRespondWith`
+        [json| [{"id":1,"name":"school related","datarep_todos":null},{"id":2,"name":"do these first","datarep_todos":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    -- This is not supported by data reps (would be hard to make it work with high performance). So the test just
+    -- verifies we don't panic or add inappropriate SQL to the filters.
+    it "fails safely on user trying to use ilike operator on data reps column" $
+      get "/datarep_todos?select=id,name&label_color=ilike.#*100" `shouldRespondWith` (
+        if actualPgVersion >= pgVersion110 then
+        [json|
+          {"code":"42883","details":null,"hint":"No operator matches the given name and argument types. You might need to add explicit type casts.","message":"operator does not exist: public.color ~~* unknown"}
+        |]
+        else
+        [json|
+          {"code":"42883","details":null,"hint":"No operator matches the given name and argument type(s). You might need to add explicit type casts.","message":"operator does not exist: public.color ~~* unknown"}
+        |])
+        { matchStatus  = 404
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+  context "searching for an empty string" $ do
+    it "works with an empty eq filter" $
+      get "/empty_string?string=eq.&select=id,string" `shouldRespondWith`
+        [json|
+          [{"id":1,"string":""}]
+        |]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
@@ -145,6 +145,14 @@ spec = do
        it "returns whole range with status 200" $
          get "/items" `shouldRespondWith` 200

+      context "count with an empty body" $ do
+        it "returns empty body with Content-Range */0" $
+          request methodGet "/items?id=eq.0"
+            [("Prefer", "count=exact")] ""
+            `shouldRespondWith`
+              [json|[]|]
+              { matchHeaders = ["Content-Range" <:> "*/0"] }
+
      context "when I don't want the count" $ do
        it "returns range Content-Range with /*" $
          request methodGet "/menagerie"
@@ -211,11 +219,24 @@ spec = do
                             , "Content-Range" <:> "2-4/*" ]
            }

-      it "succeeds if offset equals 0 as a no-op" $
-        get "/items?select=id&offset=0&order=id"
-          `shouldRespondWith`
-            [json|[{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13},{"id":14},{"id":15}]|]
-            { matchHeaders = ["Content-Range" <:> "0-14/*"] }
+      context "succeeds if offset equals 0 as a no-op" $ do
+        it  "no items" $ do
+          get "/items?offset=0&id=eq.0"
+            `shouldRespondWith`
+              [json|[]|]
+              { matchHeaders = ["Content-Range" <:> "*/*"] }
+
+          request methodGet "/items?offset=0&id=eq.0"
+            [("Prefer", "count=exact")] ""
+            `shouldRespondWith`
+              [json|[]|]
+              { matchHeaders = ["Content-Range" <:> "*/0"] }
+
+        it  "one or more items" $
+          get "/items?select=id&offset=0&order=id"
+            `shouldRespondWith`
+              [json|[{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13},{"id":14},{"id":15}]|]
+              { matchHeaders = ["Content-Range" <:> "0-14/*"] }

      it "succeeds if offset is negative as a no-op" $
        get "/items?select=id&offset=-4&order=id"
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`index-state: hackage.haskell.org 2023-10-13T13:54:33Z`