bump version to 11.2.0

This allows using doctests on the Response.hs module

.github/actions/setup-nix/action.yaml

@@ -11,7 +11,9 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: cachix/install-nix-action@v18
+    - uses: cachix/install-nix-action@v22
+      with:
+        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12
       with:
         name: postgrest

.github/dependabot.yml

@@ -1,6 +1,11 @@
 version: 2
 updates:
-- package-ecosystem: github-actions
+  - package-ecosystem: github-actions
-  directory: /
+    directory: /
-  schedule:
+    schedule:
-    interval: weekly
+      interval: weekly
+
+  - package-ecosystem: github-actions
+    directory: /.github/actions/setup-nix
+    schedule:
+      interval: weekly

.github/get_cirrusci_freebsd

@@ -7,12 +7,13 @@ set -euo pipefail
 # https://docs.github.com/en/rest/reference/checks#list-check-suites-for-a-git-reference
 
 cirrus_artifact_name=bin
+gh_auth_header="Authorization: Bearer $GITHUB_TOKEN"
 gh_accept_header="Accept: application/vnd.github.v3+json"
 
 get_gh_check_runs_url() {
   gh_checks_list_url="https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$GITHUB_COMMIT/check-suites"
   >&2 echo "Getting list of check-suites from $gh_checks_list_url ..."
-  curl --fail -H "$gh_accept_header"  "$gh_checks_list_url" \
+  curl --fail -H "$gh_auth_header" -H "$gh_accept_header"  "$gh_checks_list_url" \
     | jq -r '.check_suites[] | select(.app.slug == "cirrus-ci") | .check_runs_url'
 }
 
@@ -21,7 +22,7 @@ wait_for_cirrusci() {
   >&2 echo "Waiting to CirrusCI run to complete (two hours maximum)..."
   for _ in $(seq 1 120); do
     echo "Checking for CirrusCI task status at $gh_check_runs_url ..."
-    status=$(curl --fail "$gh_check_runs_url" | jq -r '.check_runs[] | .status')
+    status=$(curl --fail -H "$gh_auth_header" "$gh_check_runs_url" | jq -r '.check_runs[] | .status')
     if [ "$status" == "completed" ]; then
       break
     else
@@ -37,7 +38,7 @@ wait_for_cirrusci() {
 get_cirrus_taskid() {
   gh_check_runs_url="$(get_gh_check_runs_url)"
   >&2 echo "Getting the CirrusCI task id from $gh_check_runs_url ..."
-  curl --fail -H "$gh_accept_header" "$gh_check_runs_url" \
+  curl --fail -H "$gh_auth_header" -H "$gh_accept_header" "$gh_check_runs_url" \
     | jq -r '.check_runs[] | .external_id'
 }
 

.github/scripts/arm/build.sh

@@ -4,11 +4,15 @@
 
 [ -z "$1" ] && { echo "Missing 1st argument: PostgREST github commit SHA"; exit 1; }
 [ -z "$2" ] && { echo "Missing 2nd argument: Build environment directory name"; exit 1; }
+[ -z "$3" ] && { echo "Missing 3rd argument: GHC version"; exit 1; }
 
 PGRST_GITHUB_COMMIT="$1"
 SCRIPT_DIR="$2"
 
 DOCKER_BUILD_DIR="$SCRIPT_DIR/docker-env"
+# latest is a shortcut documented on https://www.haskell.org/ghcup/guide/#tags-and-shortcuts
+CABAL_VERSION="latest"
+GHC_VERSION="$3"
 
 install_packages() {
   sudo apt-get update -y
@@ -26,13 +30,14 @@ install_ghcup() {
 
 install_cabal() {
   ghcup upgrade
-  ghcup install cabal 3.6.0.0
+  ghcup install cabal $CABAL_VERSION
-  ghcup set cabal 3.6.0.0
+  ghcup set cabal $CABAL_VERSION
 }
 
 install_ghc() {
-  ghcup install ghc 8.10.7
+  ghcup upgrade
-  ghcup set ghc 8.10.7
+  ghcup install ghc $GHC_VERSION
+  ghcup set ghc $GHC_VERSION
 }
 
 install_packages
@@ -41,8 +46,8 @@ install_packages
 [ -f ~/.ghcup/env ] && source ~/.ghcup/env
 
 ghcup --version || install_ghcup
-cabal --version || install_cabal
+ghcup set cabal $CABAL_VERSION || install_cabal
-ghc --version || install_ghc
+ghcup set ghc $GHC_VERSION || install_ghc
 
 cd ~/$SCRIPT_DIR
 

.github/scripts/arm/docker-env/Dockerfile

@@ -13,4 +13,6 @@ EXPOSE 3000
 
 USER 1000
 
-CMD postgrest
+# Use the array form to avoid running the command using bash, which does not handle `SIGTERM` properly. 
+# See https://docs.docker.com/compose/faq/#why-do-my-services-take-10-seconds-to-recreate-or-stop 
+CMD ["postgrest"]

.github/workflows/cachix.yaml

@@ -0,0 +1,78 @@
+name: Cachix
+
+# This workflow serves to
+#   - keep cachix up to date with the main branch
+#   - incrementally update cachix for large dependency
+#     updates, e.g. after running postgrest-nixpkgs-upgrade,
+#     which can cause the main CI workflow to time out
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - rel-*
+    tags:
+      - v*
+
+jobs:
+  Seed-Cachix:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: Linux
+            runs-on: ubuntu-latest
+          - os: MacOS
+            runs-on: macos-latest
+    name: Seed ${{ matrix.os }}
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+        with:
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: Install cachix tooling
+        run: |
+          nix-env -f default.nix -iA devTools.pushCachix.bin
+          postgrest-push-cachix
+
+      - name: Seed dynamic postgrest build
+        run: |
+          nix-build -A postgrestPackage
+          postgrest-push-cachix
+
+      - name: Seed style tools
+        run: |
+          nix-build -A style
+          postgrest-push-cachix
+
+      - name: Seed test tools
+        run: |
+          nix-build -A tests
+          postgrest-push-cachix
+
+      - name: Seed static toolchain
+        if: matrix.os == 'Linux'
+        run: |
+          nix-build -A packagesStatic.haskellPackages.hello
+          postgrest-push-cachix
+
+      - name: Seed static postgresql build (for libpq)
+        if: matrix.os == 'Linux'
+        run: |
+          nix-build -A packagesStatic.pkgs.postgresql
+          postgrest-push-cachix
+
+      - name: Seed static postgrest build
+        if: matrix.os == 'Linux'
+        run: |
+          nix-build -A postgrestStatic
+          postgrest-push-cachix
+
+      - name: Build and push everything to Cachix
+        run: |
+          nix-build
+          postgrest-push-cachix

.github/workflows/ci.yaml

@@ -46,7 +46,7 @@ jobs:
       - name: Run coverage (IO tests and Spec tests against PostgreSQL 15)
         run: postgrest-coverage
       - name: Upload coverage to codecov
-        uses: codecov/codecov-action@v3.1.1
+        uses: codecov/codecov-action@v3.1.4
         with:
           files: ./coverage/codecov.json
 
@@ -84,11 +84,7 @@ jobs:
 
       - name: Run IO tests
         if: always()
-        run: postgrest-with-postgresql-${{ matrix.pgVersion }} -f test/io/fixtures.sql postgrest-test-io
+        run: postgrest-with-postgresql-${{ matrix.pgVersion }} -f test/io/fixtures.sql postgrest-test-io -vv
-
-      - name: Run query cost tests
-        if: always()
-        run: postgrest-with-postgresql-${{ matrix.pgVersion }} postgrest-test-querycost
 
 
   Test-Memory-Nix:
@@ -112,7 +108,6 @@ jobs:
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
-          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
           tools: tests
 
       - name: Build static executable
@@ -135,12 +130,18 @@ jobs:
           path: postgrest-docker.tar.gz
           if-no-files-found: error
 
-      - name: Build and push everything to Cachix (main branch only)
+
-        if: ${{ github.ref == 'refs/heads/main' }}
+  Build-Macos-Nix:
+    name: Build MacOS (Nix)
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+
+      - name: Build everything
         run: |
           nix-build
-          nix-env -f default.nix -iA devTools
-          postgrest-push-cachix
 
 
   Build-Stack:
@@ -202,7 +203,8 @@ jobs:
       - name: Get FreeBSD executable from CirrusCI
         env:
           # GITHUB_SHA does weird things for pull request, so we roll our own:
-          GITHUB_COMMIT: ${{github.event.pull_request.head.sha || github.sha}}
+          GITHUB_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: .github/get_cirrusci_freebsd
       - name: Save executable as artifact
         uses: actions/upload-artifact@v3
@@ -214,12 +216,15 @@ jobs:
   Build-Cabal:
     strategy:
       matrix:
-        ghc: ['8.10.7', '9.2.4']
+        ghc: ['9.0.2', '9.2.4']
       fail-fast: false
     name: Build Linux (Cabal, GHC ${{ matrix.ghc }})
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Workaround runner image issue
+        # https://github.com/actions/runner-images/issues/7061
+        run: sudo chown -R "$USER" /usr/local/.ghcup
       - name: ghcup
         run: |
           ghcup install ghc ${{ matrix.ghc }}
@@ -242,18 +247,23 @@ jobs:
         run: cabal build --enable-tests --enable-benchmarks all
 
   Build-Cabal-Arm:
-    name: Build aarch64 (Cabal)
+    strategy:
+      matrix:
+        ghc: ['9.2.4']
+      fail-fast: false
+    name: Build aarch64 (Cabal, GHC ${{ matrix.ghc }})
     if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/rel-') }}
     runs-on: ubuntu-latest
     outputs:
       remotepath: ${{ steps.Remote-Dir.outputs.remotepath }}
     env:
       GITHUB_COMMIT: ${{ github.sha }}
+      GHC_VERSION: ${{ matrix.ghc }}
     steps:
       - uses: actions/checkout@v3
       - id: Remote-Dir
         name: Unique directory name for the remote build
-        run: echo "::set-output name=remotepath::postgrest-build-$(uuidgen)"
+        run: echo "remotepath=postgrest-build-$(uuidgen)" >> "$GITHUB_OUTPUT"
       - name: Copy script files to the remote server
         uses: appleboy/scp-action@master
         with:
@@ -275,8 +285,8 @@ jobs:
           fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
           command_timeout: 120m
           script_stop: true
-          envs: GITHUB_COMMIT,REMOTE_DIR
+          envs: GITHUB_COMMIT,REMOTE_DIR,GHC_VERSION
-          script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR"
+          script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR" "GHC_VERSION"
       - name: Download binaries from remote server
         uses: nicklasfrahm/scp-action@main
         with:
@@ -290,7 +300,7 @@ jobs:
       - name: Extract downloaded binaries
         run: tar -xvf result.tar.xz && rm result.tar.xz
       - name: Save aarch64 executable as artifact
-        uses: actions/upload-artifact@v2.3.1
+        uses: actions/upload-artifact@v3
         with:
           name: postgrest-ubuntu-aarch64
           path: result/postgrest
@@ -326,14 +336,14 @@ jobs:
             exit 1
           else
             echo "Version to be released is $cabal_version"
-            echo "::set-output name=version::$cabal_version"
+            echo "version=$cabal_version" >> "$GITHUB_OUTPUT"
           fi
 
           if [[ "$cabal_version" != *.*.*.* ]]; then
             echo "Version is for a full release (version does not have four components)"
           else
             echo "Version is for a pre-release (version has four components, e.g., 1.1.1.1)"
-            echo "::set-output name=isprerelease::1"
+            echo "isprerelease=1" >> "$GITHUB_OUTPUT"
           fi
       - name: Identify changes from CHANGELOG.md
         run: |
@@ -424,7 +434,6 @@ jobs:
     name: Release on Docker Hub
     runs-on: ubuntu-latest
     needs:
-      - Build-Cabal-Arm
       - Prepare-Release
     env:
       GITHUB_COMMIT: ${{ github.sha }}
@@ -459,18 +468,6 @@ jobs:
           else
             echo "Skipping pushing to 'latest' tag for v$VERSION pre-release..."
           fi
-      - name: Publish images for ARM builds on Docker Hub
-        uses: appleboy/ssh-action@master
-        env:
-          REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
-        with:
-          host: ${{ secrets.SSH_ARM_HOST }}
-          username: ubuntu
-          key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
-          fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
-          script_stop: true
-          envs: GITHUB_COMMIT,DOCKER_REPO,DOCKER_USER,DOCKER_PASS,REMOTE_DIR,VERSION,ISPRERELEASE
-          script: bash ~/$REMOTE_DIR/docker-publish.sh "$GITHUB_COMMIT" "$DOCKER_REPO" "$DOCKER_USER" "$DOCKER_PASS" "$REMOTE_DIR" "$VERSION" "$ISPRERELEASE"
 # TODO: Enable dockerhub description update again, once a solution for the permission problem is found:
 # https://github.com/docker/hub-feedback/issues/1927
 #      - name: Update descriptions on Docker Hub
@@ -484,17 +481,49 @@ jobs:
 #            echo "Skipping updating description for pre-release..."
 #          fi
 
+  Release-Docker-Arm:
+    name: Release Arm Builds on Docker Hub
+    runs-on: ubuntu-latest
+    needs:
+      - Build-Cabal-Arm
+      - Prepare-Release
+      - Release-Docker
+    env:
+      GITHUB_COMMIT: ${{ github.sha }}
+      DOCKER_REPO: postgrest
+      DOCKER_USER: stevechavez
+      DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+      VERSION: ${{ needs.Prepare-Release.outputs.version }}
+      ISPRERELEASE: ${{ needs.Prepare-Release.outputs.isprerelease }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Publish images for ARM builds on Docker Hub
+        uses: appleboy/ssh-action@master
+        env:
+          REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
+        with:
+          host: ${{ secrets.SSH_ARM_HOST }}
+          username: ubuntu
+          key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
+          fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
+          script_stop: true
+          envs: GITHUB_COMMIT,DOCKER_REPO,DOCKER_USER,DOCKER_PASS,REMOTE_DIR,VERSION,ISPRERELEASE
+          script: bash ~/$REMOTE_DIR/docker-publish.sh "$GITHUB_COMMIT" "$DOCKER_REPO" "$DOCKER_USER" "$DOCKER_PASS" "$REMOTE_DIR" "$VERSION" "$ISPRERELEASE"
+
   Clean-Arm-Server:
     name: Remove copied files from server
     needs:
       - Build-Cabal-Arm
-      - Release-Docker
+      - Release-Docker-Arm
-    if: ${{ always() && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/rel-')) }}
+    if: success() ||
+      needs.Build-Cabal-Arm.result == 'failure' ||
+      needs.Build-Cabal-Arm.result == 'cancelled' ||
+      (needs.Build-Cabal-Arm.result == 'success' && !startsWith(github.ref, 'refs/tags/v'))
     runs-on: ubuntu-latest
     env:
       REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
     steps:
-      - uses: actions/checkout@v2.4.0
+      - uses: actions/checkout@v3
       - name: Remove uploaded files from server
         uses: appleboy/ssh-action@master
         with:

.github/workflows/report.yaml

@@ -22,7 +22,7 @@ jobs:
           name: loadtest.md
           path: artifacts
       - name: Upload to GitHub Checks
-        uses: LouisBrunner/checks-action@v1.5.0
+        uses: LouisBrunner/checks-action@v1.6.2
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sha: ${{ github.event.workflow_run.head_sha }}

ARCHITECTURE.md

@@ -55,12 +55,6 @@ It builds the OpenAPI response using the schema cache.
 
 This module provides functions to deal with JWT authorization.
 
-### Workers.hs
-
-This spawns threads which are used to execute concurrent jobs.
-
-Jobs include connection recovery, a listener for the PostgreSQL LISTEN command, and an admin server.
-
 ### SchemaCache.hs
 
 This queries the PostgreSQL system catalogs and caches the metadata into a SchemaCache type,
@@ -68,3 +62,7 @@ This queries the PostgreSQL system catalogs and caches the metadata into a Schem
 ### AppState.hs
 
 The state of the App which is kept across requests.
+
+This spawns threads which are used to execute concurrent jobs.
+
+Jobs include connection recover and a listener for the PostgreSQL LISTEN command.

BACKERS.md

@@ -46,12 +46,14 @@ PostgREST ongoing development is only possible thanks to our Sponsors and Backer
 
 ## Lead Backers
 
+- [Roboflow](https://github.com/roboflow)
 - Evans Fernandes
 - [Jan Sommer](https://github.com/nerfpops)
 - [Franz Gusenbauer](https://www.igutech.at/)
 
 ## Backers
 
+- Zac Miller
 - Tsingson Qin
 - Michel Pelletier
 - Jay Hannah

CHANGELOG.md

@@ -5,12 +5,166 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+## [11.2.0] - 2023-08-10
+
+### Added
+
+ - #2523, Data representations - @aljungberg
+   + Allows for flexible API output formatting and input parsing on a per-column type basis using regular SQL functions configured in the database
+   + Enables greater flexibility in the form and shape of your APIs, both for output and input, making PostgREST a more versatile general-purpose API server
+   + Examples include base64 encode/decode your binary data (like a `bytea` column containing an image), choose whether to present a timestamp column as seconds since the Unix epoch or as an ISO 8601 string, or represent fixed precision decimals as strings, not doubles, to preserve precision
+   + ...and accept the same in `POST/PUT/PATCH` by configuring the reverse transformation(s)
+   + Other use-cases include custom representation of enums, arrays, nested objects, CSS hex colour strings, gzip compressed fields, metric to imperial conversions, and much more
+   + Works when using the `select` parameter to select only a subset of columns, embedding through complex joins, renaming fields, with views and computed columns
+   + Works when filtering on a formatted column without extra indexes by parsing to the canonical representation
+   + Works for data `RETURNING` operations, such as requesting the full body in a POST/PUT/PATCH with `Prefer: return=representation`
+   + Works for batch updates and inserts
+   + Completely optional, define the functions in the database and they will be used automatically everywhere
+   + Data representations preserve the ability to write to the original column and require no extra storage or complex triggers (compared to using `GENERATED ALWAYS` columns)
+   + Note: data representations require Postgres 10 (Postgres 11 if using `IN` predicates); data representations are not implemented for RPC
+ - #2647, Allow to verify the PostgREST version in SQL: `select distinct application_name from pg_stat_activity`. - @laurenceisla
+ - #2856, Add the `--version` CLI option that prints the version information - @laurenceisla
+ - #1655, Improve `details` field of the singular error response - @taimoorzaeem
+ - #740, Add `Preference-Applied` in response for `Prefer: return=representation/headers-only/minimal` - @taimoorzaeem
+ - #1601, Add optional `nulls=stripped` parameter for mediatypes `application/vnd.pgrst.array+json` and `application/vnd.pgrst.object+json` - @taimoorzaeem
+
+### Fixed
+
+ - #2821, Fix OPTIONS not accepting all available media types - @steve-chavez
+ - #2834, Fix compilation on Ubuntu by being compatible with GHC 9.0.2 - @steve-chavez
+ - #2840, Fix `Prefer: missing=default` with DOMAIN default values - @steve-chavez
+ - #2849, Fix HEAD unnecessarily executing aggregates - @steve-chavez
+ - #2594, Fix unused index on jsonb/jsonb arrow filter and order (``/bets?data->>contractId=eq.1`` and ``/bets?order=data->>contractId``) - @steve-chavez
+ - #2861, Fix character and bit columns with fixed length not inserting/updating properly - @laurenceisla
+   + Fixes the error "value too long for type character(1)" when the char length of the column was bigger than one.
+ - #2862, Fix null filtering on embedded resource when using a column name equal to the relation name - @steve-chavez
+ - #1586, Fix function parameters of type character and bit not ignoring length - @laurenceisla
+   + Fixes the error "value too long for type character(1)" when the char length of the parameter was bigger than one.
+ - #2881, Fix error when a function returns `RECORD` or `SET OF RECORD` - @laurenceisla
+ - #2896, Fix applying superuser settings for impersonated role - @steve-chavez
+
+### Deprecated
+
+ - #2863, Deprecate resource embedding target disambiguation - @steve-chavez
+   + The `/table?select=*,other!fk(*)` must be used to disambiguate
+   + The server aids in choosing the `!fk` by sending a `hint` on the error whenever an ambiguous request happens.
+
+## [11.1.0] - 2023-06-07
+
+### Added
+
+ - #2786, Limit idle postgresql connection lifetime - @robx
+   + New option `db-pool-max-idletime` (default 30s).
+   + This is equivalent to the old option `db-pool-timeout` of PostgREST 10.0.0.
+   + A config alias for `db-pool-timeout` is included.
+ - #2703, Add pre-config function - @steve-chavez
+    + New config option `db-pre-config`(empty by default)
+    + Allows using the in-database configuration without SUPERUSER
+ - #2781, When `db-channel-enabled` is false, start automatic connection recovery on a new request when pool connections are closed with `pg_terminate_backend` - @steve-chavez
+    + Mitigates the lack of LISTEN/NOTIFY for schema cache reloading on read replicas.
+
+### Fixed
+
+ - #2791, Fix dropping schema cache reload notifications  - @steve-chavez
+ - #2801, Stop retrying connection when "no password supplied" - @steve-chavez
+
+## [11.0.1] - 2023-04-27
+
+### Fixed
+
+ - #2762, Fixes "permission denied for schema" error during schema cache load - @steve-chavez
+ - #2756, Fix bad error message on generated columns when using `Prefer: missing=default` - @steve-chavez
+ - #1139, Allow a 30 second skew for JWT validation - @steve-chavez
+   + It used to be 1 second, which was too strict
+
+## [11.0.0] - 2023-04-16
+
+### Added
+
+ - #1414, Add related orders - @steve-chavez
+   + On a many-to-one or one-to-one relationship, you can order a parent by a child column `/projects?select=*,clients(*)&order=clients(name).desc.nullsfirst`
+ - #1233, #1907, #2566, Allow spreading embedded resources - @steve-chavez
+   + On a many-to-one or one-to-one relationship, you can unnest a json object with `/projects?select=*,...clients(client_name:name)`
+   + Allows including the join table columns when resource embedding
+   + Allows disambiguating a recursive m2m embed
+   + Allows disambiguating an embed that has a many-to-many relationship using two foreign keys on a junction
+ - #2340, Allow embedding without selecting any column - @steve-chavez
+ - #2563, Allow `is.null` or `not.is.null` on an embedded resource - @steve-chavez
+   + Offers a more flexible replacement for `!inner`, e.g. `/projects?select=*,clients(*)&clients=not.is.null`
+   + Allows doing an anti join, e.g. `/projects?select=*,clients(*)&clients=is.null`
+   + Allows using or across related tables conditions
+ - #1100, Customizable OpenAPI title - @AnthonyFisi
+ - #2506, Add `server-trace-header` for tracing HTTP requests.  - @steve-chavez
+   + When the client sends the request header specified in the config it will be included in the response headers.
+ - #2694, Make `db-root-spec` stable. - @steve-chavez
+   + This can be used to override the OpenAPI spec with a custom database function
+ - #1567, On bulk inserts, missing values can get the column DEFAULT by using the `Prefer: missing=default` header - @steve-chavez
+ - #2501, Allow filtering by`IS DISTINCT FROM` using the `isdistinct` operator, e.g. `/people?alias=isdistinct.foo`
+ - #1569, Allow `any/all` modifiers on the `eq,like,ilike,gt,gte,lt,lte,match,imatch` operators, e.g. `/tbl?id=eq(any).{1,2,3}` - @steve-chavez
+   - This converts the input into an array type
+ - #2561, Configurable role settings - @steve-chavez
+   - Database roles that are members of the connection role get their settings applied, e.g. doing
+     `ALTER ROLE anon SET statement_timeout TO '5s'` will result in that `statement_timeout` getting applied for that role.
+   - Works when switching roles when a JWT is sent
+   - Settings can be reloaded with `NOTIFY pgrst, 'reload config'`.
+ - #2468, Configurable transaction isolation level with `default_transaction_isolation` - @steve-chavez
+   - Can be set per function `create function .. set default_transaction_isolation = 'repeatable read'`
+   - Or per role `alter role .. set default_transaction_isolation = 'serializable'`
+
+### Fixed
+
+ - #2651, Add the missing `get` path item for RPCs to the OpenAPI output - @laurenceisla
+ - #2648, Fix inaccurate error codes with new ones - @laurenceisla
+   + `PGRST204`: Column is not found
+   + `PGRST003`: Timed out when acquiring connection to db
+ - #1652, Fix function call with arguments not inlining - @steve-chavez
+ - #2705, Fix bug when using the `Range` header on `PATCH/DELETE` - @laurenceisla
+   + Fix the`"message": "syntax error at or near \"RETURNING\""` error
+   + Fix doing a limited update/delete when an `order` query parameter was present
+ - #2742, Fix db settings and pg version queries not getting prepared  - @steve-chavez
+ - #2618, Fix `PATCH` requests not recognizing embedded filters and using the top-level resource instead - @steve-chavez
+
+### Changed
+
+ - #2705, The `Range` header is now only considered on `GET` requests and is ignored for any other method - @laurenceisla
+   + Other methods should use the `limit/offset` query parameters for sub-ranges
+   + `PUT` requests no longer return an error when this header is present (using `limit/offset` still triggers the error)
+ - #2733, Remove bulk RPC call with the `Prefer: params=multiple-objects` header. A function with a JSON array or object parameter should be used instead.
+
+## [10.2.0] - 2023-04-12
+
+### Added
+
+ - #2663, Limit maximal postgresql connection lifetime - @robx
+   + New option `db-pool-max-lifetime` (default 30m)
+   + `db-pool-acquisition-timeout` is no longer optional and defaults to 10s
+   + Fixes postgresql resource leak with long-lived connections (#2638)
+
+### Fixed
+
+ - #2667, Fix `db-pool-acquisition-timeout` not logging to stderr when the timeout is reached - @steve-chavez
+
+## [10.1.2] - 2023-02-01
+
+### Fixed
+
+ - #2565, Fix bad M2M embedding on RPC - @steve-chavez
+ - #2575, Replace misleading error message when no function is found with a hint containing functions/parameters names suggestions - @laurenceisla
+ - #2582, Move explanation about "single parameters" from the `message` to the `details` in the error output - @laurenceisla
+ - #2569, Replace misleading error message when no relationship is found with a hint containing parent/child names suggestions - @laurenceisla
+ - #1405, Add the required OpenAPI items object when the parameter is an array - @laurenceisla
+ - #2592, Add upsert headers for POST requests to the OpenAPI output - @laurenceisla
+ - #2623, Fix FK pointing to VIEW instead of TABLE in OpenAPI output - @laurenceisla
+ - #2622, Consider any PostgreSQL authentication failure as fatal and exit immediately - @michivi
+ - #2620, Fix `NOTIFY pgrst` not reloading the db connections catalog cache - @steve-chavez
+
 ## [10.1.1] - 2022-11-08
 
 ### Fixed
 
  - #2548, Fix regression when embedding views with partial references to multi column FKs - @wolfgangwalther
  - #2558, Fix regression when requesting limit=0 and `db-max-row` is set - @laurenceisla
+ - #2542, Return a clear error without hitting the database when trying to update or insert an unknown column with `?columns` - @aljungberg
 
 ## [10.1.0] - 2022-10-28
 

cabal.project.non-nix

@@ -17,4 +17,4 @@ packages: .
 source-repository-package
   type: git
   location: https://github.com/PostgREST/postgresql-libpq.git
-  tag: 33ff97db570b5b432255f5f24a68db51453f6eb8
+  tag: 890a0a16cf57dd401420fdc6c7d576fb696003bc

default.nix

@@ -41,6 +41,7 @@ let
       allOverlays.postgresql-legacy
       allOverlays.postgresql-future
       (allOverlays.haskell-packages { inherit compiler; })
+      allOverlays.slocat
     ];
 
   # Evaluated expression of the Nixpkgs repository.
@@ -65,11 +66,17 @@ let
   postgrest =
     pkgs.haskell.packages."${compiler}".callCabal2nix name src { };
 
-  # Function that derives a fully static Haskell package based on
+  # Functionality that derives a fully static Haskell package based on
   # nh2/static-haskell-nix
   staticHaskellPackage =
     import nix/static-haskell-package.nix { inherit nixpkgs system compiler patches allOverlays; };
 
+  # Static executable.
+  postgrestStatic =
+    lib.justStaticExecutables (lib.dontCheck (staticHaskellPackage name src).package);
+
+  packagesStatic = (staticHaskellPackage name src).survey;
+
   # Options passed to cabal in dev tools and tests
   devCabalOptions =
     "-f dev --test-show-detail=direct";
@@ -94,10 +101,6 @@ rec {
   postgrestPackage =
     lib.dontCheck postgrest;
 
-  # Static executable.
-  postgrestStatic =
-    lib.justStaticExecutables (lib.dontCheck (staticHaskellPackage name src));
-
   # Profiled dynamic executable.
   postgrestProfiled =
     lib.enableExecutableProfiling (
@@ -119,14 +122,13 @@ rec {
   cabalTools =
     pkgs.callPackage nix/tools/cabalTools.nix { inherit devCabalOptions postgrest; };
 
+  withTools =
+    pkgs.callPackage nix/tools/withTools.nix { inherit cabalTools devCabalOptions postgresqlVersions postgrest; };
+
   # Development tools.
   devTools =
     pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie withTools; };
 
-  # Docker images and loading script.
-  docker =
-    pkgs.callPackage nix/tools/docker { postgrest = postgrestStatic; };
-
   # Load testing tools.
   loadtest =
     pkgs.callPackage nix/tools/loadtest.nix { inherit withTools; };
@@ -155,7 +157,12 @@ rec {
       inherit (pkgs.haskell.packages."${compiler}") hpc-codecov;
       inherit (pkgs.haskell.packages."${compiler}") weeder;
     };
+} // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux rec {
+  # Static executable.
+  inherit postgrestStatic;
+  inherit packagesStatic;
 
-  withTools =
+  # Docker images and loading script.
-    pkgs.callPackage nix/tools/withTools.nix { inherit devCabalOptions postgresqlVersions postgrest; };
+  docker =
+    pkgs.callPackage nix/tools/docker { postgrest = postgrestStatic; };
 }

nix/README.md

@@ -5,24 +5,14 @@ for developing, testing and building PostgREST.
 
 ## Getting started with Nix
 
-You'll need to [get Nix](https://nixos.org/download.html). The installer will
+You'll need to [get Nix](https://nixos.org/download.html). Follow the recommended installation for your operating system from the official download website.
-create your Nix store in the `/nix/` directory, where all build artifacts and
-their dependencies will be stored. It will also link the Nix executables like
-`nix-env`, `nix-build` and `nix-shell` into your PATH. Nix will manage all
-other PostgREST dependencies from here on out. To clean up older build
-artifacts from the `/nix/store`, you can run `nix-collect-garbage`.
-
-If you are on a system that does not support nix, for example Windows, you can
-run the nix development environment in a docker container. Inside the `nix/`
-directory run `docker-compose run --rm nix` to start the docker container. This
-will set up the binary cache and launch `nix-shell` automatically.
 
 ## Building PostgREST
 
 To build PostgREST from your local checkout of the repository, run:
 
 ```bash
-nix-build --attr postgrestPackage
+$ nix-build --attr postgrestPackage
 
 ```
 
@@ -39,10 +29,10 @@ We recommend that you use the PostgREST binary cache on
 
 ```bash
 # Install cachix:
-nix-env -iA cachix -f https://cachix.org/api/v1/install
+$ nix-env -iA cachix -f https://cachix.org/api/v1/install
 
 # Set cachix up to use the PostgREST binary cache:
-cachix use postgrest
+$ cachix use postgrest
 
 ```
 
@@ -56,7 +46,7 @@ following command will put you into a new shell that has GHC and Cabal on the
 PATH:
 
 ```bash
-nix-shell
+$ nix-shell
 
 ```
 
@@ -146,10 +136,10 @@ Note: Once inside nix-shell, the utilities work from any directory inside
 the PostgREST repo. Paths are resolved relative to the repo root:
 
 ```bash
-$ cd src
+[nix-shell]$ cd src
 # Even though the current directory is ./src, the config path must still start
 # from the repo root:
-$ postgrest-run test/io/configs/simple.conf
+[nix-shell]$ postgrest-run test/io/configs/simple.conf
 ```
 
 ## Testing
@@ -177,21 +167,21 @@ run with `postgrest-test-io`. The test runner under the hood is
 
 ```bash
 # Filter the tests to run by name, including all that contain 'config':
-postgrest-test-io -k config
+[nix-shell]$ postgrest-test-io -k config
 
 # Run tests in parallel using xdist, specifying the number of processes:
-postgrest-test-io -n auto
+[nix-shell]$ postgrest-test-io -n auto
-postgrest-test-io -n 8
+[nix-shell]$ postgrest-test-io -n 8
 ```
 
 The memory tests check that we don't surpass a memory threshold for big request bodies.
 
 ```bash
 # Build the dependencies needed for the memory test
-nix-shell --arg memory true
+$ nix-shell --arg memory true
 
 # Run the memory test
-postgrest-test-memory
+[nix-shell]$ postgrest-test-memory
 ```
 
 The loadtests ensure that performance doesn't drop on a change. Underlyingly they use
@@ -199,25 +189,25 @@ The loadtests ensure that performance doesn't drop on a change. Underlyingly the
 
 ```bash
 # Run the loadtests on the latest commit(HEAD)
-postgrest-loadtest
+[nix-shell]$ postgrest-loadtest
 
 # You can loadtest comparing to a different branch
-postgrest-loadtest-against master
+[nix-shell]$ postgrest-loadtest-against master
+
+# You can simulate latency client/postgrest and postgrest/database
+[nix-shell]$ PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest
+
+# You can build postgrest directly with cabal for faster iteration
+[nix-shell]$ PGRST_BUILD_CABAL=1 postgrest-loadtest
 
 # Produce a markdown report to be used on CI
-postgrest-loadtest-report
+[nix-shell]$ postgrest-loadtest-report
-```
-
-Our query cost tests ensure that our generated queries don't surpass a threshold EXPLAIN cost.
-
-```bash
-postgrest-test-querycost
 ```
 
 doctests for some of our modules are also available:
 
 ```bash
-postgrest-test-doctest
+[nix-shell]$ postgrest-test-doctest
 ```
 
 ## Code coverage
@@ -226,11 +216,11 @@ Code coverage is available under the `postgrest-coverage` command. This will pro
 
 ```bash
 # Will run all the tests and produce a coverage dir
-postgrest-coverage
+[nix-shell]$ postgrest-coverage
 
 # Visualize the output
-cd coverage
+[nix-shell]$ cd coverage
-python -mSimpleHTTPServer 8080
+[nix-shell]$ python -mSimpleHTTPServer 8080
 ```
 
 ## Linting and styling code

nix/nixpkgs-version.nix

@@ -1,6 +1,6 @@
 # Pinned version of Nixpkgs, generated with postgrest-nixpkgs-upgrade.
 {
-  date = "2022-10-28";
+  date = "2023-03-25";
-  rev = "f44ba1be526c8da9e79a5759feca2365204003f6";
+  rev = "dbf5322e93bcc6cfc52268367a8ad21c09d76fea";
-  tarballHash = "0npbwsdjw88py5w2pjflwh94wgi4jmnmls0k1n7q8m6h94w1y1ps";
+  tarballHash = "0lwk4v9dkvd28xpqch0b0jrac4xl9lwm6snrnzx8k5lby72kmkng";
 }

nix/overlays/default.nix

@@ -7,4 +7,5 @@
   postgresql-default = import ./postgresql-default.nix;
   postgresql-legacy = import ./postgresql-legacy.nix;
   postgresql-future = import ./postgresql-future.nix;
+  slocat = import ./slocat.nix;
 }

nix/overlays/haskell-packages.nix

@@ -29,33 +29,33 @@ let
       # To fill in the sha256:
       #   update-nix-fetchgit nix/overlays/haskell-packages.nix
 
-      hashtables = lib.dontCheck prev.hashtables_1_3_1;
-      hasql = lib.dontCheck prev.hasql_1_6_1_4;
-      hasql-dynamic-statements = lib.dontCheck prev.hasql-dynamic-statements_0_3_1_2;
-      hasql-pool = lib.dontCheck
-        (prev.callHackageDirect
-          {
-            pkg = "hasql-pool";
-            ver = "0.8.0.6";
-            sha256 = "sha256-2u/cwPk8XfXffaDRzGeyzhL+9k2+2T4b8bGOZwz8AX0=";
-          }
-          { });
-      hasql-transaction = lib.dontCheck prev.hasql-transaction_1_0_1_2;
-      isomorphism-class = lib.unmarkBroken prev.isomorphism-class;
-      lens = lib.dontCheck prev.lens_5_2;
-      postgresql-binary = lib.dontCheck prev.postgresql-binary_0_13_1;
-      text-builder = lib.dontCheck prev.text-builder_0_6_7;
-      text-builder-dev = lib.dontCheck prev.text-builder-dev_0_3_3;
-
       postgresql-libpq = lib.dontCheck
         (prev.callCabal2nix "postgresql-libpq"
           (super.fetchFromGitHub {
             owner = "PostgREST";
             repo = "postgresql-libpq";
-            rev = "cef92cb4c07b56568dffdbf4b719258b82183119"; # master
+            rev = "890a0a16cf57dd401420fdc6c7d576fb696003bc"; # master
-            sha256 = "0r59klrz47qcnd22s47h612mlz3jbg40wwalfj3f6djwg0cdyr85";
+            sha256 = "1wmyhldk0k14y8whp1p4akrkqxf5snh8qsbm7fv5f7kz95nyffd0";
           })
           { });
+
+      hasql-notifications = lib.dontCheck
+        (prev.callHackageDirect
+          {
+            pkg = "hasql-notifications";
+            ver = "0.2.0.5";
+            sha256 = "sha256-KV36zs/RTgJh/oBBPZaNiQshusmakWnjYTcdoNa4JLA=";
+          }
+          { });
+
+      hasql-pool = lib.dontCheck
+        (prev.callHackageDirect
+          {
+            pkg = "hasql-pool";
+            ver = "0.10";
+            sha256 = "sha256-kHzoqtNV9BFWnn1h560JRqMooQRwxokVKgDRBexamNI=";
+          }
+          { });
     } // extraOverrides final prev;
 in
 {

nix/overlays/slocat.nix

@@ -0,0 +1,13 @@
+final: prev:
+{
+  slocat = prev.buildGoModule {
+    name = "slocat";
+    src = prev.fetchFromGitHub {
+      owner = "robx";
+      repo = "slocat";
+      rev = "52e7512c6029fd00483e41ccce260a3b4b9b3b64";
+      sha256 = "sha256-qn6luuh5wqREu3s8RfuMCP5PKdS2WdwPrujRYTpfzQ8=";
+    };
+    vendorSha256 = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo=";
+  };
+}

nix/static-haskell-package.nix

@@ -59,4 +59,7 @@ let
   survey =
     import "${patched-static-haskell-nix}/survey" { inherit normalPkgs compiler defaultCabalPackageVersionComingWithGhc; };
 in
-survey.haskellPackages."${name}"
+{
+  inherit survey;
+  package = survey.haskellPackages."${name}";
+}

nix/tools/cabalTools.nix

@@ -37,12 +37,22 @@ let
     checkedShellScript
       {
         name = "postgrest-run";
-        docs = "Run PostgREST after buidling it interactively with cabal-install";
+        docs = "Run PostgREST after building it interactively with cabal-install";
-        args = [ "ARG_LEFTOVERS([PostgREST arguments])" ];
+        args =
+          [
+            "ARG_USE_ENV([PGRST_DB_ANON_ROLE], [postgrest_test_anonymous], [PostgREST anonymous role])"
+            "ARG_USE_ENV([PGRST_DB_POOL], [1], [PostgREST pool size])"
+            "ARG_USE_ENV([PGRST_DB_POOL_ACQUISITION_TIMEOUT], [1], [PostgREST pool size])"
+            "ARG_LEFTOVERS([PostgREST arguments])"
+          ];
         inRootDir = true;
         withEnv = postgrest.env;
       }
       ''
+        export PGRST_DB_ANON_ROLE
+        export PGRST_DB_POOL
+        export PGRST_DB_POOL_ACQUISITION_TIMEOUT
+
         exec ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
           postgrest "''${_arg_leftovers[@]}"
       '';

nix/tools/devTools.nix

@@ -77,7 +77,6 @@ let
       }
       ''
         ${tests}/bin/postgrest-test-spec
-        ${tests}/bin/postgrest-test-querycost
         ${tests}/bin/postgrest-test-doctests
         ${tests}/bin/postgrest-test-io
         ${style}/bin/postgrest-lint
@@ -165,6 +164,7 @@ let
           # The following unsets all GIT_ variables.
           unset "''${!GIT_@}"
 
+          # shellcheck disable=SC2317
           function restore () {
             ref="$(git stash list --format=format:%gD --grep "$1" -n1)"
             # this will avoid merge conflicts when applying the stash
@@ -304,4 +304,5 @@ buildToolbox
     hsieGraphModules
     hsieGraphSymbols
   ];
+  extra = { inherit pushCachix; };
 }

nix/tools/loadtest.nix

@@ -56,11 +56,14 @@ let
         export PGRST_LOG_LEVEL="crit"
 
         mkdir -p "$(dirname "$_arg_output")"
+        abs_output="$(realpath "$_arg_output")"
 
         # shellcheck disable=SC2145
         ${withTools.withPg} --fixtures "$_arg_testdir"/fixtures.sql \
+        ${withTools.withSlowPg} \
         ${withTools.withPgrst} \
-        sh -c "cd \"$_arg_testdir\" && ${runner} -targets targets.http -output \"$_arg_output\" \"''${_arg_leftovers[@]}\""
+        ${withTools.withSlowPgrst} \
+        sh -c "cd \"$_arg_testdir\" && ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
         ${vegeta}/bin/vegeta report -type=text "$_arg_output"
       '';
 

nix/tools/release/default.nix

@@ -72,14 +72,15 @@ let
         today_date="$(date '+%Y%m%d')"
         today_date_for_changelog="$(date '+%Y-%m-%d')"
         bump_pre="$major.$minor.$patch.$today_date"
+        bump_pre_minor="$major.$((minor+1)).0.$today_date"
         bump_patch="$major.$minor.$((patch+1))"
         bump_minor="$major.$((minor+1)).0"
         bump_major="$((major+1)).0.0"
 
         PS3="Please select the new version: "
-        select new_version in "$bump_pre" "$bump_patch" "$bump_minor" "$bump_major"; do
+        select new_version in "$bump_pre" "$bump_pre_minor" "$bump_patch" "$bump_minor" "$bump_major"; do
           case "$REPLY" in
-            1|2|3|4)
+            1|2|3|4|5)
               echo "Selected $new_version"
               break
               ;;
@@ -95,7 +96,7 @@ let
         echo "Committing ..."
         git add postgrest.cabal > /dev/null
 
-        if [[ "$new_version" != "$bump_pre" ]]; then
+        if [[ "$new_version" != "$bump_pre" && "$new_version" != "$bump_pre_minor" ]]; then
           echo "Updating CHANGELOG.md ..."
           sed -i -E "s/Unreleased/&\n\n## [$new_version] - $today_date_for_changelog/" CHANGELOG.md > /dev/null
           git add CHANGELOG.md > /dev/null
@@ -106,7 +107,7 @@ let
         echo "Tagging ..."
         git tag "v$new_version" > /dev/null
 
-        trap "Couldn't find remote. Please push manually ..." ERR
+        trap "echo Remote not found. Please push manually ..." ERR
         remote="$(git remote -v | grep PostgREST/postgrest | grep push | cut -f1)"
         trap "" ERR
 

nix/tools/tests.nix

@@ -32,18 +32,6 @@ let
           test:spec -- "''${_arg_leftovers[@]}"
       '';
 
-  testQuerycost =
-    checkedShellScript
-      {
-        name = "postgrest-test-querycost";
-        docs = "Run the Haskell test suite for query costs";
-        inRootDir = true;
-        withEnv = postgrest.env;
-      }
-      ''
-        ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:querycost
-      '';
-
   testDoctests =
     checkedShellScript
       {
@@ -140,7 +128,7 @@ let
           rm -rf coverage/*
 
           # build once before running all the tests
-          ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest lib:postgrest test:spec test:querycost
+          ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest lib:postgrest test:spec
 
           (
             trap 'echo Found dead code: Check file list above.' ERR ;
@@ -155,14 +143,11 @@ let
           HPCTIXFILE="$tmpdir"/spec.tix \
             ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec
 
-          HPCTIXFILE="$tmpdir"/querycost.tix \
-            ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:querycost
-
           # Note: No coverage for doctests, as doctests leverage GHCi and GHCi does not support hpc
 
           # collect all the tix files
           ${ghc}/bin/hpc sum  --union --exclude=Paths_postgrest --output="$tmpdir"/tests.tix \
-            "$tmpdir"/io*.tix "$tmpdir"/spec.tix "$tmpdir"/querycost.tix
+            "$tmpdir"/io*.tix "$tmpdir"/spec.tix
 
           # prepare the overlay
           ${ghc}/bin/hpc overlay --output="$tmpdir"/overlay.tix test/coverage.overlay
@@ -234,7 +219,6 @@ buildToolbox
   tools =
     [
       testSpec
-      testQuerycost
       testDoctests
       testSpecIdempotence
       testIO

nix/tools/withTools.nix

@@ -1,6 +1,7 @@
 { bash-completion
 , buildToolbox
 , cabal-install
+, cabalTools
 , checkedShellScript
 , curl
 , devCabalOptions
@@ -8,15 +9,20 @@
 , lib
 , postgresqlVersions
 , postgrest
+, slocat
 , writeText
 }:
 let
   withTmpDb =
     { name, postgresql }:
+    let
+      commandName = "postgrest-with-${name}";
+      superuserRole = "postgres";
+    in
     checkedShellScript
       {
-        name = "postgrest-with-${name}";
+        name = commandName;
-        docs = "Run the given command in a temporary database with ${name}";
+        docs = "Run the given command in a temporary database with ${name}. If you wish to mutate the database, login with the '${superuserRole}' role.";
         args =
           [
             "ARG_OPTIONAL_SINGLE([fixtures], [f], [SQL file to load fixtures from], [test/spec/fixtures/load.sql])"
@@ -25,6 +31,8 @@ let
             "ARG_USE_ENV([PGUSER], [postgrest_test_authenticator], [Authenticator PG role])"
             "ARG_USE_ENV([PGDATABASE], [postgres], [PG database name])"
             "ARG_USE_ENV([PGRST_DB_SCHEMAS], [test], [Schema to expose])"
+            "ARG_USE_ENV([PGTZ], [utc], [Timezone to use])"
+            "ARG_USE_ENV([PGOPTIONS], [-c search_path=public,test], [PG options to use])"
           ];
         positionalCompletion = "_command";
         inRootDir = true;
@@ -53,18 +61,26 @@ let
         export PGUSER
         export PGDATABASE
         export PGRST_DB_SCHEMAS
+        export PGTZ
+        export PGOPTIONS
+
+        HBA_FILE="$tmpdir/pg_hba.conf"
+        echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
+        echo "local $PGDATABASE all trust" >> "$HBA_FILE"
 
         log "Initializing database cluster..."
         # We try to make the database cluster as independent as possible from the host
         # by specifying the timezone, locale and encoding.
-        PGTZ=UTC initdb --no-locale --encoding=UTF8 --nosync -U "$PGUSER" --auth=trust \
+        # initdb -U creates a superuser(man initdb)
+        PGTZ=UTC initdb --no-locale --encoding=UTF8 --nosync -U "${superuserRole}" --auth=trust \
           >> "$setuplog"
 
         log "Starting the database cluster..."
         # Instead of listening on a local port, we will listen on a unix domain socket.
-        pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -k $PGHOST -c log_statement=\"all\"" \
+        pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
           >> "$setuplog"
 
+        # shellcheck disable=SC2317
         stop () {
           log "Stopping the database cluster..."
           pg_ctl stop -m i >> "$setuplog"
@@ -72,10 +88,17 @@ let
         }
         trap stop EXIT
 
-        log "Loading fixtures..."
+        log "Creating a minimally privileged $PGUSER connection role..."
-        psql -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
+        createuser "$PGUSER" -U "${superuserRole}" --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login
+
+        log "Loading fixtures under the ${superuserRole} role..."
+        psql -U "${superuserRole}" -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
 
         log "Done. Running command..."
+
+        echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U ${superuserRole}"
+        echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log"
+
         ("$_arg_command" "''${_arg_leftovers[@]}")
       '';
 
@@ -125,6 +148,81 @@ let
 
   withPg = builtins.head withPgVersions;
 
+  withSlowPg =
+    checkedShellScript
+      {
+        name = "postgrest-with-slow-pg";
+        docs = "Run the given command with simulated high latency postgresql";
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
+            "ARG_LEFTOVERS([command arguments])"
+            "ARG_USE_ENV([PGHOST], [], [PG host (socket name)])"
+            "ARG_USE_ENV([PGDELAY], [0ms], [extra PG latency (duration)])"
+          ];
+        positionalCompletion = "_command";
+        inRootDir = true;
+        redirectTixFiles = false;
+        withTmpDir = true;
+      }
+      ''
+        delay="''${PGDELAY:-0ms}"
+        echo "delaying data to/from postgres by $delay"
+
+        REALPGHOST="$PGHOST"
+        export PGHOST="$tmpdir/socket"
+        mkdir -p "$PGHOST"
+
+        ${slocat}/bin/slocat -delay "$delay" -src "$PGHOST/.s.PGSQL.5432" -dst "$REALPGHOST/.s.PGSQL.5432" &
+        SLOCAT_PID=$!
+        # shellcheck disable=SC2317
+        stop_slocat() {
+          kill "$SLOCAT_PID" || true
+          wait "$SLOCAT_PID" || true
+        }
+        trap stop_slocat EXIT
+        sleep 1 # should wait for socket file to appear instead
+
+        ("$_arg_command" "''${_arg_leftovers[@]}")
+      '';
+
+  withSlowPgrst =
+    checkedShellScript
+      {
+        name = "postgrest-with-slow-postgrest";
+        docs = "Run the given command with simulated high latency postgrest";
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
+            "ARG_LEFTOVERS([command arguments])"
+            "ARG_USE_ENV([PGRST_SERVER_UNIX_SOCKET], [], [PostgREST host (socket name)])"
+            "ARG_USE_ENV([PGRST_DELAY], [0ms], [extra PostgREST latency (duration)])"
+          ];
+        positionalCompletion = "_command";
+        inRootDir = true;
+        redirectTixFiles = false;
+        withTmpDir = true;
+      }
+      ''
+        delay="''${PGRST_DELAY:-0ms}"
+        echo "delaying data to/from PostgREST by $delay"
+
+        REAL_PGRST_SERVER_UNIX_SOCKET="$PGRST_SERVER_UNIX_SOCKET"
+        export PGRST_SERVER_UNIX_SOCKET="$tmpdir/postgrest.socket"
+
+        ${slocat}/bin/slocat -delay "$delay" -src "$PGRST_SERVER_UNIX_SOCKET" -dst "$REAL_PGRST_SERVER_UNIX_SOCKET" &
+        SLOCAT_PID=$!
+        # shellcheck disable=SC2317
+        stop_slocat() {
+          kill "$SLOCAT_PID" || true
+          wait "$SLOCAT_PID" || true
+        }
+        trap stop_slocat EXIT
+        sleep 1 # should wait for socket file to appear instead
+
+        ("$_arg_command" "''${_arg_leftovers[@]}")
+      '';
+
   withGit =
     let
       name = "postgrest-with-git";
@@ -245,17 +343,25 @@ let
         export PGRST_SERVER_UNIX_SOCKET="$tmpdir"/postgrest.socket
 
         rm -f result
-        echo -n "Building postgrest... "
+        if [ -z "''${PGRST_BUILD_CABAL:-}" ]; then
-        nix-build -A postgrestPackage > "$tmpdir"/build.log 2>&1 || {
+          echo -n "Building postgrest (nix)... "
-          echo "failed, output:"
+          nix-build -A postgrestPackage > "$tmpdir"/build.log 2>&1 || {
-          cat "$tmpdir"/build.log
+            echo "failed, output:"
-          exit 1
+            cat "$tmpdir"/build.log
-        }
+            exit 1
+          }
+          PGRST_CMD=./result/bin/postgrest
+        else
+          echo -n "Building postgrest (cabal)... "
+          postgrest-build
+          PGRST_CMD=postgrest-run
+        fi
         echo "done."
 
         echo -n "Starting postgrest... "
-        ./result/bin/postgrest ${legacyConfig} > "$tmpdir"/run.log 2>&1 &
+        $PGRST_CMD ${legacyConfig} > "$tmpdir"/run.log 2>&1 &
         pid=$!
+        # shellcheck disable=SC2317
         cleanup() {
           kill "$pid" || true
         }
@@ -275,7 +381,7 @@ in
 buildToolbox
 {
   name = "postgrest-with";
-  tools = [ withPgAll withGit withPgrst ] ++ withPgVersions;
+  tools = [ withPgAll withGit withPgrst withSlowPg withSlowPgrst ] ++ withPgVersions;
   # make withTools available for other nix files
-  extra = { inherit withGit withPg withPgAll withPgrst; };
+  extra = { inherit withGit withPg withPgAll withPgrst withSlowPg withSlowPgrst; };
 }

postgrest.cabal

@@ -1,5 +1,5 @@
 name:               postgrest
-version:            10.1.1
+version:            11.2.0
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                     for tables, views, and functions, supporting all HTTP methods that security
@@ -34,7 +34,8 @@ library
   default-extensions: OverloadedStrings
                       NoImplicitPrelude
   hs-source-dirs:     src
-  exposed-modules:    PostgREST.App
+  exposed-modules:    PostgREST.Admin
+                      PostgREST.App
                       PostgREST.AppState
                       PostgREST.Auth
                       PostgREST.CLI
@@ -46,8 +47,9 @@ library
                       PostgREST.Cors
                       PostgREST.SchemaCache
                       PostgREST.SchemaCache.Identifiers
-                      PostgREST.SchemaCache.Proc
+                      PostgREST.SchemaCache.Routine
                       PostgREST.SchemaCache.Relationship
+                      PostgREST.SchemaCache.Representations
                       PostgREST.SchemaCache.Table
                       PostgREST.Error
                       PostgREST.Logger
@@ -60,6 +62,7 @@ library
                       PostgREST.Plan.CallPlan
                       PostgREST.Plan.MutatePlan
                       PostgREST.Plan.ReadPlan
+                      PostgREST.Plan.Types
                       PostgREST.RangeQuery
                       PostgREST.ApiRequest
                       PostgREST.ApiRequest.Preferences
@@ -69,10 +72,9 @@ library
                       PostgREST.Response.OpenAPI
                       PostgREST.Response.GucHeader
                       PostgREST.Version
-                      PostgREST.Workers
   other-modules:      Paths_postgrest
   build-depends:      base                      >= 4.9 && < 4.17
-                    , HTTP                      >= 4000.3.7 && < 4000.4
+                    , HTTP                      >= 4000.3.7 && < 4000.5
                     , Ranged-sets               >= 0.3 && < 0.5
                     , aeson                     >= 2.0.3 && < 2.2
                     , auto-update               >= 0.1.4 && < 0.2
@@ -85,11 +87,13 @@ library
                     , contravariant-extras      >= 0.3.3 && < 0.4
                     , cookie                    >= 0.4.2 && < 0.5
                     , either                    >= 4.4.1 && < 5.1
+                    , extra                     >= 1.7.0 && < 2.0
+                    , fuzzyset                  >= 0.2.3
                     , gitrev                    >= 1.2 && < 1.4
                     , hasql                     >= 1.6.1.1 && < 1.7
                     , hasql-dynamic-statements  >= 0.3.1 && < 0.4
                     , hasql-notifications       >= 0.1 && < 0.3
-                    , hasql-pool                >= 0.8.0.6 && < 0.9
+                    , hasql-pool                >= 0.10 && < 0.11
                     , hasql-transaction         >= 1.0.1 && < 1.1
                     , heredoc                   >= 0.2 && < 0.3
                     , http-types                >= 0.12.2 && < 0.13
@@ -97,11 +101,11 @@ library
                     , interpolatedstring-perl6  >= 1 && < 1.1
                     , jose                      >= 0.8.5.1 && < 0.11
                     , lens                      >= 4.14 && < 5.3
-                    , lens-aeson                >= 1.0.1 && < 1.2
+                    , lens-aeson                >= 1.0.1 && < 1.3
                     , mtl                       >= 2.2.2 && < 2.3
                     , network                   >= 2.6 && < 3.2
                     , network-uri               >= 2.6.1 && < 2.8
-                    , optparse-applicative      >= 0.13 && < 0.17
+                    , optparse-applicative      >= 0.13 && < 0.18
                     , parsec                    >= 3.1.11 && < 3.2
                     , protolude                 >= 0.3.1 && < 0.4
                     , regex-tdfa                >= 1.2.2 && < 1.4
@@ -185,6 +189,8 @@ test-suite spec
                       Feature.CorsSpec
                       Feature.ExtraSearchPathSpec
                       Feature.LegacyGucsSpec
+                      Feature.NoSuperuserSpec
+                      Feature.ObservabilitySpec
                       Feature.OpenApi.DisabledOpenApiSpec
                       Feature.OpenApi.IgnorePrivOpenApiSpec
                       Feature.OpenApi.OpenApiSpec
@@ -209,15 +215,17 @@ test-suite spec
                       Feature.Query.QuerySpec
                       Feature.Query.RangeSpec
                       Feature.Query.RawOutputTypesSpec
+                      Feature.Query.RelatedQueriesSpec
                       Feature.Query.RpcSpec
                       Feature.Query.SingularSpec
+                      Feature.Query.NullsStrip
+                      Feature.Query.SpreadQueriesSpec
                       Feature.Query.UnicodeSpec
                       Feature.Query.UpdateSpec
                       Feature.Query.UpsertSpec
                       Feature.RollbackSpec
                       Feature.RpcPreRequestGucsSpec
                       SpecHelper
-                      TestTypes
   build-depends:      base              >= 4.9 && < 4.17
                     , aeson             >= 2.0.3 && < 2.2
                     , aeson-qq          >= 0.8.1 && < 0.9
@@ -227,69 +235,32 @@ test-suite spec
                     , bytestring        >= 0.10.8 && < 0.12
                     , case-insensitive  >= 1.2 && < 1.3
                     , containers        >= 0.5.7 && < 0.7
-                    , hasql-pool        >= 0.8.0.2 && < 0.9
+                    , hasql-pool        >= 0.10 && < 0.11
                     , hasql-transaction >= 1.0.1 && < 1.1
                     , heredoc           >= 0.2 && < 0.3
-                    , hspec             >= 2.3 && < 2.9
+                    , hspec             >= 2.3 && < 2.10
                     , hspec-wai         >= 0.10 && < 0.12
                     , hspec-wai-json    >= 0.10 && < 0.12
                     , http-types        >= 0.12.3 && < 0.13
                     , lens              >= 4.14 && < 5.3
-                    , lens-aeson        >= 1.0.1 && < 1.2
+                    , lens-aeson        >= 1.0.1 && < 1.3
                     , monad-control     >= 1.0.1 && < 1.1
                     , postgrest
                     , process           >= 1.4.2 && < 1.7
                     , protolude         >= 0.3.1 && < 0.4
                     , regex-tdfa        >= 1.2.2 && < 1.4
+                    , scientific        >= 0.3.4 && < 0.4
                     , text              >= 1.2.2 && < 1.3
                     , transformers-base >= 0.4.4 && < 0.5
                     , wai               >= 3.2.1 && < 3.3
                     , wai-extra         >= 3.0.19 && < 3.2
-  ghc-options:        -O0 -Werror -Wall -fwarn-identities
+  ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
                       -fno-warn-missing-signatures
                       -fwrite-ide-info
                       -- https://github.com/PostgREST/postgrest/issues/387
                       -with-rtsopts=-K33K
 
-test-suite querycost
-  type:               exitcode-stdio-1.0
-  default-language:   Haskell2010
-  default-extensions: OverloadedStrings
-                      QuasiQuotes
-                      NoImplicitPrelude
-  hs-source-dirs:     test/spec
-  main-is:            QueryCost.hs
-  other-modules:      SpecHelper
-  build-depends:      base              >= 4.9 && < 4.17
-                    , aeson             >= 2.0.3 && < 2.2
-                    , base64-bytestring >= 1 && < 1.3
-                    , bytestring        >= 0.10.8 && < 0.12
-                    , case-insensitive  >= 1.2 && < 1.3
-                    , containers        >= 0.5.7 && < 0.7
-                    , contravariant     >= 1.4 && < 1.6
-                    , hasql             >= 1.6 && < 1.7
-                    , hasql-dynamic-statements >= 0.3.1 && < 0.4
-                    , hasql-pool        >= 0.8.0.2 && < 0.9
-                    , hasql-transaction >= 1.0.1 && < 1.1
-                    , heredoc           >= 0.2 && < 0.3
-                    , hspec             >= 2.3 && < 2.9
-                    , hspec-wai         >= 0.10 && < 0.12
-                    , hspec-wai-json    >= 0.10 && < 0.12
-                    , http-types        >= 0.12.3 && < 0.13
-                    , lens              >= 4.14 && < 5.3
-                    , lens-aeson        >= 1.0.1 && < 1.2
-                    , postgrest
-                    , process           >= 1.4.2 && < 1.7
-                    , protolude         >= 0.3.1 && < 0.4
-                    , regex-tdfa        >= 1.2.2 && < 1.4
-                    , wai-extra         >= 3.0.19 && < 3.2
-  ghc-options:        -O0 -Werror -Wall -fwarn-identities
-                      -fno-spec-constr -optP-Wno-nonportable-include-path
-                      -fwrite-ide-info
-                      -- https://github.com/PostgREST/postgrest/issues/387
-                      -with-rtsopts=-K1K
-
 test-suite doctests
   type:               exitcode-stdio-1.0
   default-language:   Haskell2010

src/PostgREST/Admin.hs

@@ -0,0 +1,87 @@
+{-# LANGUAGE NamedFieldPuns  #-}
+{-# LANGUAGE RecordWildCards #-}
+
+module PostgREST.Admin
+  ( runAdmin
+  ) where
+
+import qualified Data.Text                 as T
+import qualified Hasql.Session             as SQL
+import qualified Network.HTTP.Types.Status as HTTP
+import qualified Network.Wai               as Wai
+import qualified Network.Wai.Handler.Warp  as Warp
+
+import Control.Monad.Extra (whenJust)
+
+import Network.Socket
+import Network.Socket.ByteString
+
+import PostgREST.AppState (AppState)
+import PostgREST.Config   (AppConfig (..))
+
+import qualified PostgREST.AppState as AppState
+
+import Protolude
+
+runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
+runAdmin conf@AppConfig{configAdminServerPort} appState settings =
+  whenJust configAdminServerPort $ \adminPort -> do
+    AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
+    void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
+  where
+    adminApp = admin appState conf
+
+-- | PostgREST admin application
+admin :: AppState.AppState -> AppConfig -> Wai.Application
+admin appState appConfig req respond  = do
+  isMainAppReachable  <- any isRight <$> reachMainApp appConfig
+  isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
+  isConnectionUp      <-
+    if configDbChannelEnabled appConfig
+      then AppState.getIsListenerOn appState
+      else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
+
+  case Wai.pathInfo req of
+    ["ready"] ->
+      respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
+    ["live"] ->
+      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
+    _ ->
+      respond $ Wai.responseLBS HTTP.status404 [] mempty
+
+-- Try to connect to the main app socket
+-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
+-- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
+-- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
+reachMainApp :: AppConfig -> IO [Either IOException ()]
+reachMainApp AppConfig{..} =
+  case configServerUnixSocket of
+    Just path ->  do
+      sock <- socket AF_UNIX Stream 0
+      (:[]) <$> try (do
+        connect sock $ SockAddrUnix path
+        withSocketsDo $ bracket (pure sock) close sendEmpty)
+    Nothing -> do
+      let
+        host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
+             | otherwise                                             = Just configServerHost
+        filterAddrs xs =
+          case configServerHost of
+              "*4" -> ipv4Addrs xs ++ ipv6Addrs xs
+              "!4" -> ipv4Addrs xs
+              "*6" -> ipv6Addrs xs ++ ipv4Addrs xs
+              "!6" -> ipv6Addrs xs
+              _    -> xs
+        ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
+        ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
+
+      addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
+      tryAddr `traverse` filterAddrs addrs
+  where
+    sendEmpty sock = void $ send sock mempty
+    tryAddr :: AddrInfo -> IO (Either IOException ())
+    tryAddr addr = do
+      sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
+      try $ do
+        connect sock $ addrAddress addr
+        withSocketsDo $ bracket (pure sock) close sendEmpty

src/PostgREST/ApiRequest.hs

@@ -3,6 +3,7 @@ Module      : PostgREST.Request.ApiRequest
 Description : PostgREST functions to translate HTTP request to a domain type called ApiRequest.
 -}
 {-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE MultiWayIf      #-}
 {-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 
@@ -32,43 +33,33 @@ import qualified Data.Set              as S
 import qualified Data.Text.Encoding    as T
 import qualified Data.Vector           as V
 
+import Data.Either.Combinators (mapBoth)
+
 import Control.Arrow             ((***))
 import Data.Aeson.Types          (emptyArray, emptyObject)
 import Data.List                 (lookup, union)
 import Data.Ranged.Ranges        (emptyRange, rangeIntersection,
                                   rangeIsEmpty)
-import Data.Tree                 (Tree (..))
 import Network.HTTP.Types.Header (RequestHeaders, hCookie)
 import Network.HTTP.Types.URI    (parseSimpleQuery)
 import Network.Wai               (Request (..))
 import Network.Wai.Parse         (parseHttpAccept)
 import Web.Cookie                (parseCookies)
 
-import PostgREST.ApiRequest.Preferences  (PreferCount (..),
-                                          PreferParameters (..),
-                                          PreferRepresentation (..),
-                                          PreferResolution (..),
-                                          PreferTransaction (..))
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
 import PostgREST.ApiRequest.Types        (ApiRequestError (..),
-                                          RangeError (..),
+                                          RangeError (..))
-                                          SelectItem (..))
 import PostgREST.Config                  (AppConfig (..),
                                           OpenAPIMode (..))
-import PostgREST.MediaType               (MTPlanAttrs (..),
+import PostgREST.MediaType               (MTPlanFormat (..),
-                                          MTPlanFormat (..),
                                           MediaType (..))
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           convertToLimitZeroRange,
                                           hasLimitZero,
                                           rangeRequested)
-import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier (..),
                                           Schema)
-import PostgREST.SchemaCache.Proc        (ProcDescription (..),
-                                          ProcParam (..), ProcsMap,
-                                          procReturnsScalar)
 
 import qualified PostgREST.ApiRequest.Preferences as Preferences
 import qualified PostgREST.ApiRequest.QueryParams as QueryParams
@@ -90,6 +81,7 @@ data Payload
       -- ^ Keys of the object or if it's an array these keys are guaranteed to
       -- be the same across all its objects
       }
+  | ProcessedUrlEncoded { payArray  :: [(Text, Text)], payKeys :: S.Set Text }
   | RawJSON { payRaw  :: LBS.ByteString }
   | RawPay  { payRaw  :: LBS.ByteString }
 
@@ -114,41 +106,9 @@ data PathInfo
       }
 -- | The target db object of a user action
 data Target = TargetIdent QualifiedIdentifier
-            | TargetProc{tProc :: ProcDescription, tpIsRootSpec :: Bool}
+            | TargetProc{tProc :: QualifiedIdentifier, tpIsRootSpec :: Bool}
             | TargetDefaultSpec{tdsSchema :: Schema} -- The default spec offered at root "/"
 
--- | RPC query param value `/rpc/func?v=<value>`, used for VARIADIC functions on form-urlencoded POST and GETs
--- | It can be fixed `?v=1` or repeated `?v=1&v=2&v=3.
-data RpcParamValue = Fixed Text | Variadic [Text]
-instance JSON.ToJSON RpcParamValue where
-  toJSON (Fixed    v) = JSON.toJSON v
-  toJSON (Variadic v) = JSON.toJSON v
-
-toRpcParamValue :: ProcDescription -> (Text, Text) -> (Text, RpcParamValue)
-toRpcParamValue proc (k, v) | prmIsVariadic k = (k, Variadic [v])
-                            | otherwise       = (k, Fixed v)
-  where
-    prmIsVariadic prm = isJust $ find (\ProcParam{ppName, ppVar} -> ppName == prm && ppVar) $ pdParams proc
-
--- | Convert rpc params `/rpc/func?a=val1&b=val2` to json `{"a": "val1", "b": "val2"}
-jsonRpcParams :: ProcDescription -> [(Text, Text)] -> Payload
-jsonRpcParams proc prms =
-  if not $ pdHasVariadic proc then -- if proc has no variadic param, save steps and directly convert to json
-    ProcessedJSON (JSON.encode $ HM.fromList $ second JSON.toJSON <$> prms) (S.fromList $ fst <$> prms)
-  else
-    let paramsMap = HM.fromListWith mergeParams $ toRpcParamValue proc <$> prms in
-    ProcessedJSON (JSON.encode paramsMap) (S.fromList $ HM.keys paramsMap)
-  where
-    mergeParams :: RpcParamValue -> RpcParamValue -> RpcParamValue
-    mergeParams (Variadic a) (Variadic b) = Variadic $ b ++ a
-    mergeParams v _                       = v -- repeated params for non-variadic parameters are not merged
-
-targetToJsonRpcParams :: Maybe Target -> [(Text, Text)] -> Maybe Payload
-targetToJsonRpcParams target params =
-  case target of
-    Just TargetProc{tProc} -> Just $ jsonRpcParams tProc params
-    _                      -> Nothing
-
 {-|
   Describes what the user wants to do. This data type is a
   translation of the raw elements of an HTTP request into domain
@@ -157,37 +117,60 @@ targetToJsonRpcParams target params =
   if it is an action we are able to perform.
 -}
 data ApiRequest = ApiRequest {
-    iAction               :: Action                           -- ^ Similar but not identical to HTTP method, e.g. Create/Invoke both POST
+    iAction              :: Action                           -- ^ Similar but not identical to HTTP method, e.g. Create/Invoke both POST
-  , iRange                :: HM.HashMap Text NonnegRange      -- ^ Requested range of rows within response
+  , iRange               :: HM.HashMap Text NonnegRange      -- ^ Requested range of rows within response
-  , iTopLevelRange        :: NonnegRange                      -- ^ Requested range of rows from the top level
+  , iTopLevelRange       :: NonnegRange                      -- ^ Requested range of rows from the top level
-  , iTarget               :: Target                           -- ^ The target, be it calling a proc or accessing a table
+  , iTarget              :: Target                           -- ^ The target, be it calling a proc or accessing a table
-  , iPayload              :: Maybe Payload                    -- ^ Data sent by client and used for mutation actions
+  , iPayload             :: Maybe Payload                    -- ^ Data sent by client and used for mutation actions
-  , iPreferRepresentation :: PreferRepresentation             -- ^ If client wants created items echoed back
+  , iPreferences         :: Preferences.Preferences          -- ^ Prefer header values
-  , iPreferParameters     :: Maybe PreferParameters           -- ^ How to pass parameters to a stored procedure
+  , iQueryParams         :: QueryParams.QueryParams
-  , iPreferCount          :: Maybe PreferCount                -- ^ Whether the client wants a result count
+  , iColumns             :: S.Set FieldName                  -- ^ parsed colums from &columns parameter and payload
-  , iPreferResolution     :: Maybe PreferResolution           -- ^ Whether the client wants to UPSERT or ignore records on PK conflict
+  , iHeaders             :: [(ByteString, ByteString)]       -- ^ HTTP request headers
-  , iPreferTransaction    :: Maybe PreferTransaction          -- ^ Whether the clients wants to commit or rollback the transaction
+  , iCookies             :: [(ByteString, ByteString)]       -- ^ Request Cookies
-  , iQueryParams          :: QueryParams.QueryParams
+  , iPath                :: ByteString                       -- ^ Raw request path
-  , iColumns              :: S.Set FieldName                  -- ^ parsed colums from &columns parameter and payload
+  , iMethod              :: ByteString                       -- ^ Raw request method
-  , iHeaders              :: [(ByteString, ByteString)]       -- ^ HTTP request headers
+  , iSchema              :: Schema                           -- ^ The request schema. Can vary depending on profile headers.
-  , iCookies              :: [(ByteString, ByteString)]       -- ^ Request Cookies
+  , iNegotiatedByProfile :: Bool                             -- ^ If schema was was chosen according to the profile spec https://www.w3.org/TR/dx-prof-conneg/
-  , iPath                 :: ByteString                       -- ^ Raw request path
+  , iAcceptMediaType     :: MediaType                        -- ^ The media type in the Accept header
-  , iMethod               :: ByteString                       -- ^ Raw request method
+  , iContentMediaType    :: MediaType                        -- ^ The media type in the Content-Type header
-  , iSchema               :: Schema                           -- ^ The request schema. Can vary depending on profile headers.
-  , iNegotiatedByProfile  :: Bool                             -- ^ If schema was was chosen according to the profile spec https://www.w3.org/TR/dx-prof-conneg/
-  , iAcceptMediaType      :: MediaType                        -- ^ The media type in the Accept header
-  , iBinaryField          :: Maybe FieldName                  -- ^ field used for raw output
   }
 
 -- | Examines HTTP request and translates it into user intent.
-userApiRequest :: AppConfig -> SchemaCache -> Request -> RequestBody -> Either ApiRequestError ApiRequest
+userApiRequest :: AppConfig -> Request -> RequestBody -> Either ApiRequestError ApiRequest
-userApiRequest conf sCache req reqBody = do
+userApiRequest conf req reqBody = do
-  qPrms <- first QueryParamError $ QueryParams.parse $ rawQueryString req
+  pInfo@PathInfo{..} <- getPathInfo conf $ pathInfo req
-  pInfo <- getPathInfo conf $ pathInfo req
+  act <- getAction pInfo method
-  act <- getAction pInfo $ requestMethod req
+  qPrms <- first QueryParamError $ QueryParams.parse (pathIsProc && act `elem` [ActionInvoke InvGet, ActionInvoke InvHead]) $ rawQueryString req
-  mediaTypes <- getMediaTypes conf (requestHeaders req) act pInfo
+  (acceptMediaType, contentMediaType) <- getMediaTypes conf hdrs act pInfo
-  negotiatedSchema <- getSchema conf (requestHeaders req) (requestMethod req)
+  (schema, negotiatedByProfile) <- getSchema conf hdrs method
-  apiRequest conf sCache req reqBody qPrms pInfo act mediaTypes negotiatedSchema
+  (topLevelRange, ranges) <- getRanges method qPrms hdrs
+  (payload, columns) <- getPayload reqBody contentMediaType qPrms act pInfo
+  return $ ApiRequest {
+    iAction = act
+  , iTarget = if | pathIsProc    -> TargetProc (QualifiedIdentifier schema pathName) pathIsRootSpec
+                 | pathIsDefSpec -> TargetDefaultSpec schema
+                 | otherwise     -> TargetIdent $ QualifiedIdentifier schema pathName
+  , iRange = ranges
+  , iTopLevelRange = topLevelRange
+  , iPayload = payload
+  , iPreferences = Preferences.fromHeaders hdrs
+  , iQueryParams = qPrms
+  , iColumns = columns
+  , iHeaders = iHdrs
+  , iCookies = iCkies
+  , iPath = rawPathInfo req
+  , iMethod = method
+  , iSchema = schema
+  , iNegotiatedByProfile = negotiatedByProfile
+  , iAcceptMediaType = acceptMediaType
+  , iContentMediaType = contentMediaType
+  }
+  where
+    method = requestMethod req
+    hdrs = requestHeaders req
+    lookupHeader    = flip lookup hdrs
+    iHdrs = [ (CI.foldedCase k, v) | (k,v) <- hdrs, k /= hCookie]
+    iCkies = maybe [] parseCookies $ lookupHeader "Cookie"
 
 getPathInfo :: AppConfig -> [Text] -> Either ApiRequestError PathInfo
 getPathInfo AppConfig{configOpenApiMode, configDbRootSpec} path =
@@ -223,7 +206,7 @@ getAction PathInfo{pathIsProc, pathIsDefSpec} method =
 
 getMediaTypes :: AppConfig -> RequestHeaders -> Action -> PathInfo -> Either ApiRequestError (MediaType, MediaType)
 getMediaTypes conf hdrs action path = do
-   acceptMediaType <- findAcceptMediaType conf action path accepts
+   acceptMediaType <- negotiateContent conf action path accepts
    pure (acceptMediaType, contentMediaType)
   where
     accepts = maybe [MTAny] (map MediaType.decodeMediaType . parseHttpAccept) $ lookupHeader "accept"
@@ -249,127 +232,71 @@ getSchema AppConfig{configDbSchemas} hdrs method = do
     acceptProfile = T.decodeUtf8 <$> lookupHeader "Accept-Profile"
     lookupHeader    = flip lookup hdrs
 
-apiRequest :: AppConfig -> SchemaCache -> Request -> RequestBody -> QueryParams.QueryParams -> PathInfo -> Action -> (MediaType, MediaType) -> (Schema, Bool) -> Either ApiRequestError ApiRequest
+getRanges :: ByteString -> QueryParams -> RequestHeaders -> Either ApiRequestError (NonnegRange, HM.HashMap Text NonnegRange)
-apiRequest conf sCache req reqBody queryparams@QueryParams{..} PathInfo{pathName, pathIsProc, pathIsRootSpec, pathIsDefSpec} action (acceptMediaType, contentMediaType) (schema, negotiatedByProfile)
+getRanges method QueryParams{qsOrder,qsRanges} hdrs
   | isInvalidRange = Left $ InvalidRange (if rangeIsEmpty headerRange then LowerGTUpper else NegativeLimit)
-  | shouldParsePayload && isLeft payload = either (Left . InvalidBody) witness payload
-  | not expectParams && not (L.null qsParams) = Left $ ParseRequestError "Unexpected param or filter missing operator" ("Failed to parse " <> show qsParams)
   | method `elem` ["PATCH", "DELETE"] && not (null qsRanges) && null qsOrder = Left LimitNoOrderError
-  | method == "PUT" && topLevelRange /= allRange = Left PutRangeNotAllowedError
+  | method == "PUT" && topLevelRange /= allRange = Left PutLimitNotAllowedError
-  | otherwise = do
+  | otherwise = Right (topLevelRange, ranges)
-     checkedTarget <- target
+  where
-     bField <- binaryField conf acceptMediaType checkedTarget queryparams
+    -- According to the RFC (https://www.rfc-editor.org/rfc/rfc9110.html#name-range),
-     return ApiRequest {
+    -- the Range header must be ignored for all methods other than GET
-      iAction = action
+    headerRange = if method == "GET" then rangeRequested hdrs else allRange
-      , iTarget = checkedTarget
+    limitRange = fromMaybe allRange (HM.lookup "limit" qsRanges)
-      , iRange = ranges
+    headerAndLimitRange = rangeIntersection headerRange limitRange
-      , iTopLevelRange = topLevelRange
+    -- Bypass all the ranges and send only the limit zero range (0 <= x <= -1) if
-      , iPayload = relevantPayload
+    -- limit=0 is present in the query params (not allowed for the Range header)
-      , iPreferRepresentation = fromMaybe None preferRepresentation
+    ranges = HM.insert "limit" (convertToLimitZeroRange limitRange headerAndLimitRange) qsRanges
-      , iPreferParameters = preferParameters
+    -- The only emptyRange allowed is the limit zero range
-      , iPreferCount = preferCount
+    isInvalidRange = topLevelRange == emptyRange && not (hasLimitZero limitRange)
-      , iPreferResolution = preferResolution
+    topLevelRange = fromMaybe allRange $ HM.lookup "limit" ranges -- if no limit is specified, get all the request rows
-      , iPreferTransaction = preferTransaction
-      , iQueryParams = queryparams
-      , iColumns = payloadColumns
-      , iHeaders = [ (CI.foldedCase k, v) | (k,v) <- hdrs, k /= hCookie]
-      , iCookies = maybe [] parseCookies $ lookupHeader "Cookie"
-      , iPath = rawPathInfo req
-      , iMethod = method
-      , iSchema = schema
-      , iNegotiatedByProfile = negotiatedByProfile
-      , iAcceptMediaType = acceptMediaType
-      , iBinaryField = bField
-      }
- where
-  expectParams = pathIsProc && method /= "POST"
 
-  columns = case action of
+getPayload :: RequestBody -> MediaType -> QueryParams.QueryParams -> Action -> PathInfo -> Either ApiRequestError (Maybe Payload, S.Set FieldName)
-    ActionMutate MutationCreate -> qsColumns
+getPayload reqBody contentMediaType QueryParams{qsColumns} action PathInfo{pathIsProc}= do
-    ActionMutate MutationUpdate -> qsColumns
+  checkedPayload <- if shouldParsePayload then payload else Right Nothing
-    ActionInvoke InvPost        -> qsColumns
+  let cols = case (checkedPayload, columns) of
-    _                           -> Nothing
+        (Just ProcessedJSON{payKeys}, _)       -> payKeys
+        (Just ProcessedUrlEncoded{payKeys}, _) -> payKeys
+        (Just RawJSON{}, Just cls)             -> cls
+        _                                      -> S.empty
+  return (checkedPayload, cols)
+  where
+    payload :: Either ApiRequestError (Maybe Payload)
+    payload = mapBoth InvalidBody Just $ case (contentMediaType, pathIsProc) of
+      (MTApplicationJSON, _) ->
+        if isJust columns
+          then Right $ RawJSON reqBody
+          else note "All object keys must match" . payloadAttributes reqBody
+                 =<< if LBS.null reqBody && pathIsProc
+                       then Right emptyObject
+                       else first BS.pack $ JSON.eitherDecode reqBody
+      (MTTextCSV, _) -> do
+        json <- csvToJson <$> first BS.pack (CSV.decodeByName reqBody)
+        note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json
+      (MTUrlEncoded, isProc) -> do
+        let params = (T.decodeUtf8 *** T.decodeUtf8) <$> parseSimpleQuery (LBS.toStrict reqBody)
+        if isProc
+          then Right $ ProcessedUrlEncoded params (S.fromList $ fst <$> params)
+          else
+            let paramsMap = HM.fromList $ (identity *** JSON.String) <$> params in
+            Right $ ProcessedJSON (JSON.encode paramsMap) $ S.fromList (HM.keys paramsMap)
+      (MTTextPlain, True) -> Right $ RawPay reqBody
+      (MTTextXML, True) -> Right $ RawPay reqBody
+      (MTOctetStream, True) -> Right $ RawPay reqBody
+      (ct, _) -> Left $ "Content-Type not acceptable: " <> MediaType.toMime ct
 
-  payloadColumns =
+    shouldParsePayload = case (action, contentMediaType) of
-    case (contentMediaType, action) of
+      (ActionMutate MutationCreate, _)       -> True
-      (_, ActionInvoke InvGet)  -> S.fromList $ fst <$> qsParams
+      (ActionInvoke InvPost, _)              -> True
-      (_, ActionInvoke InvHead) -> S.fromList $ fst <$> qsParams
+      (ActionMutate MutationSingleUpsert, _) -> True
-      (MTUrlEncoded, _)         -> S.fromList $ map (T.decodeUtf8 . fst) $ parseSimpleQuery $ LBS.toStrict reqBody
+      (ActionMutate MutationUpdate, _)       -> True
-      _ -> case (relevantPayload, columns) of
+      _                                      -> False
-        (Just ProcessedJSON{payKeys}, _) -> payKeys
-        (Just RawJSON{}, Just cls)       -> cls
-        _                                -> S.empty
-  payload :: Either ByteString Payload
-  payload = case (contentMediaType, pathIsProc) of
-    (MTApplicationJSON, _) ->
-      if isJust columns
-        then Right $ RawJSON reqBody
-        else note "All object keys must match" . payloadAttributes reqBody
-               =<< if LBS.null reqBody && pathIsProc
-                     then Right emptyObject
-                     else first BS.pack $ JSON.eitherDecode reqBody
-    (MTTextCSV, _) -> do
-      json <- csvToJson <$> first BS.pack (CSV.decodeByName reqBody)
-      note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json
-    (MTUrlEncoded, _) ->
-      let paramsMap = HM.fromList $ (T.decodeUtf8 *** JSON.String . T.decodeUtf8) <$> parseSimpleQuery (LBS.toStrict reqBody) in
-      Right $ ProcessedJSON (JSON.encode paramsMap) $ S.fromList (HM.keys paramsMap)
-    (MTTextPlain, True) -> Right $ RawPay reqBody
-    (MTTextXML, True) -> Right $ RawPay reqBody
-    (MTOctetStream, True) -> Right $ RawPay reqBody
-    (ct, _) -> Left $ "Content-Type not acceptable: " <> MediaType.toMime ct
-  topLevelRange = fromMaybe allRange $ HM.lookup "limit" ranges -- if no limit is specified, get all the request rows
 
-  target
+    columns = case action of
-    | pathIsProc    = (`TargetProc` pathIsRootSpec) <$> callFindProc schema pathName
+      ActionMutate MutationCreate -> qsColumns
-    | pathIsDefSpec = Right $ TargetDefaultSpec schema
+      ActionMutate MutationUpdate -> qsColumns
-    | otherwise     = Right $ TargetIdent $ QualifiedIdentifier schema pathName
+      ActionInvoke InvPost        -> qsColumns
-    where
+      _                           -> Nothing
-      callFindProc procSch procNam = findProc
-        (QualifiedIdentifier procSch procNam) payloadColumns (preferParameters == Just SingleObject) (dbProcs sCache)
-        contentMediaType (action == ActionInvoke InvPost)
-
-  shouldParsePayload = case (action, contentMediaType) of
-    (ActionMutate MutationCreate, _)       -> True
-    (ActionInvoke InvPost, MTUrlEncoded)   -> False
-    (ActionInvoke InvPost, _)              -> True
-    (ActionMutate MutationSingleUpsert, _) -> True
-    (ActionMutate MutationUpdate, _)       -> True
-    _                                      -> False
-  relevantPayload = case (contentMediaType, action) of
-    -- Though ActionInvoke GET/HEAD doesn't really have a payload, we use the payload variable as a way
-    -- to store the query string arguments to the function.
-    (_, ActionInvoke InvGet)             -> targetToJsonRpcParams (rightToMaybe target) qsParams
-    (_, ActionInvoke InvHead)            -> targetToJsonRpcParams (rightToMaybe target) qsParams
-    (MTUrlEncoded, ActionInvoke InvPost) -> targetToJsonRpcParams (rightToMaybe target) $ (T.decodeUtf8 *** T.decodeUtf8) <$> parseSimpleQuery (LBS.toStrict reqBody)
-    _ | shouldParsePayload               -> rightToMaybe payload
-      | otherwise                        -> Nothing
-  method          = requestMethod req
-  hdrs            = requestHeaders req
-  lookupHeader    = flip lookup hdrs
-  Preferences.Preferences{..} = Preferences.fromHeaders hdrs
-  headerRange = rangeRequested hdrs
-  limitRange = fromMaybe allRange (HM.lookup "limit" qsRanges)
-  headerAndLimitRange = rangeIntersection headerRange limitRange
-
-  -- Bypass all the ranges and send only the limit zero range (0 <= x <= -1) if
-  -- limit=0 is present in the query params (not allowed for the Range header)
-  ranges = HM.insert "limit" (convertToLimitZeroRange limitRange headerAndLimitRange) qsRanges
-  -- The only emptyRange allowed is the limit zero range
-  isInvalidRange = topLevelRange == emptyRange && not (hasLimitZero limitRange)
-
-{-|
-  Find the best match from a list of media types accepted by the
-  client in order of decreasing preference and a list of types
-  producible by the server.  If there is no match but the client
-  accepts */* then return the top server pick.
--}
-mutuallyAgreeable :: [MediaType] -> [MediaType] -> Maybe MediaType
-mutuallyAgreeable sProduces cAccepts =
-  let exact = listToMaybe $ L.intersect cAccepts sProduces in
-  if isNothing exact && MTAny `elem` cAccepts
-     then listToMaybe sProduces
-     else exact
 
 type CsvData = V.Vector (M.Map Text LBS.ByteString)
 
@@ -420,115 +347,33 @@ payloadAttributes raw json =
   where
     emptyPJArray = ProcessedJSON (JSON.encode emptyArray) S.empty
 
-findAcceptMediaType :: AppConfig -> Action -> PathInfo -> [MediaType] -> Either ApiRequestError MediaType
-findAcceptMediaType conf action path accepts =
-  case mutuallyAgreeable (requestMediaTypes conf action path) accepts of
-    Just ct ->
-      Right ct
-    Nothing ->
-      Left . MediaTypeError $ map MediaType.toMime accepts
 
-requestMediaTypes :: AppConfig -> Action -> PathInfo -> [MediaType]
+-- | Do content negotiation. i.e. choose a media type based on the intersection of accepted/produced media types.
-requestMediaTypes conf action path =
+negotiateContent :: AppConfig -> Action -> PathInfo -> [MediaType] -> Either ApiRequestError MediaType
+negotiateContent conf action path accepts =
+  case firstAcceptedPick of
+    Just MTAny -> Right MTApplicationJSON -- by default(for */*) we respond with json
+    Just mt    -> Right mt
+    Nothing    -> Left . MediaTypeError $ map MediaType.toMime accepts
+  where
+    -- if there are multiple accepted media types, pick the first
+    firstAcceptedPick = listToMaybe $ L.intersect accepts $ producedMediaTypes conf action path
+
+producedMediaTypes :: AppConfig -> Action -> PathInfo -> [MediaType]
+producedMediaTypes conf action path =
   case action of
     ActionRead _    -> defaultMediaTypes ++ rawMediaTypes
     ActionInvoke _  -> invokeMediaTypes
-    ActionInspect _ -> [MTOpenAPI, MTApplicationJSON]
+    ActionInfo      -> defaultMediaTypes
-    ActionInfo      -> [MTTextCSV]
+    ActionMutate _  -> defaultMediaTypes
-    _               -> defaultMediaTypes
+    ActionInspect _ -> inspectMediaTypes
   where
+    inspectMediaTypes = [MTOpenAPI, MTApplicationJSON, MTArrayJSONStrip, MTAny]
     invokeMediaTypes =
       defaultMediaTypes
         ++ rawMediaTypes
         ++ [MTOpenAPI | pathIsRootSpec path]
     defaultMediaTypes =
-      [MTApplicationJSON, MTSingularJSON, MTGeoJSON, MTTextCSV] ++
+      [MTApplicationJSON, MTArrayJSONStrip, MTSingularJSON True, MTSingularJSON False, MTGeoJSON, MTTextCSV] ++
-      [MTPlan $ MTPlanAttrs Nothing PlanJSON mempty | configDbPlanEnabled conf]
+      [MTPlan MTApplicationJSON PlanText mempty | configDbPlanEnabled conf] ++ [MTAny]
     rawMediaTypes = configRawMediaTypes conf `union` [MTOctetStream, MTTextPlain, MTTextXML]
-
-{-|
-  Search a pg proc by matching name and arguments keys to parameters. Since a function can be overloaded,
-  the name is not enough to find it. An overloaded function can have a different volatility or even a different return type.
--}
-findProc :: QualifiedIdentifier -> S.Set Text -> Bool -> ProcsMap -> MediaType -> Bool -> Either ApiRequestError ProcDescription
-findProc qi argumentsKeys paramsAsSingleObject allProcs contentMediaType isInvPost =
-  case matchProc of
-    ([], [])     -> Left $ NoRpc (qiSchema qi) (qiName qi) (S.toList argumentsKeys) paramsAsSingleObject contentMediaType isInvPost
-    -- If there are no functions with named arguments, fallback to the single unnamed argument function
-    ([], [proc]) -> Right proc
-    ([], procs)  -> Left $ AmbiguousRpc (toList procs)
-    -- Matches the functions with named arguments
-    ([proc], _)  -> Right proc
-    (procs, _)   -> Left $ AmbiguousRpc (toList procs)
-  where
-    matchProc = overloadedProcPartition $ HM.lookupDefault mempty qi allProcs -- first find the proc by name
-    -- The partition obtained has the form (overloadedProcs,fallbackProcs)
-    -- where fallbackProcs are functions with a single unnamed parameter
-    overloadedProcPartition = foldr select ([],[])
-    select proc ~(ts,fs)
-      | matchesParams proc         = (proc:ts,fs)
-      | hasSingleUnnamedParam proc = (ts,proc:fs)
-      | otherwise                  = (ts,fs)
-    -- If the function is called with post and has a single unnamed parameter
-    -- it can be called depending on content type and the parameter type
-    hasSingleUnnamedParam ProcDescription{pdParams=[ProcParam{ppType}]} = isInvPost && case (contentMediaType, ppType) of
-      (MTApplicationJSON, "json")  -> True
-      (MTApplicationJSON, "jsonb") -> True
-      (MTTextPlain, "text")        -> True
-      (MTTextXML, "xml")           -> True
-      (MTOctetStream, "bytea")     -> True
-      _                            -> False
-    hasSingleUnnamedParam _ = False
-    matchesParams proc =
-      let
-        params = pdParams proc
-        firstType = (ppType <$> headMay params)
-      in
-      -- exceptional case for Prefer: params=single-object
-      if paramsAsSingleObject
-        then length params == 1 && (firstType == Just "json" || firstType == Just "jsonb")
-      -- If the function has no parameters, the arguments keys must be empty as well
-      else if null params
-        then null argumentsKeys && not (isInvPost && contentMediaType `elem` [MTOctetStream, MTTextPlain, MTTextXML])
-      -- A function has optional and required parameters. Optional parameters have a default value and
-      -- don't require arguments for the function to be executed, required parameters must have an argument present.
-      else case L.partition ppReq params of
-      -- If the function only has required parameters, the arguments keys must match those parameters
-        (reqParams, [])        -> argumentsKeys == S.fromList (ppName <$> reqParams)
-      -- If the function only has optional parameters, the arguments keys can match none or any of them(a subset)
-        ([], optParams)        -> argumentsKeys `S.isSubsetOf` S.fromList (ppName <$> optParams)
-      -- If the function has required and optional parameters, the arguments keys have to match the required parameters
-      -- and can match any or none of the default parameters.
-        (reqParams, optParams) -> argumentsKeys `S.difference` S.fromList (ppName <$> optParams) == S.fromList (ppName <$> reqParams)
-
--- | If raw(binary) output is requested, check that MediaType is one of the
--- admitted rawMediaTypes and that`?select=...` contains only one field other
--- than `*`
-binaryField :: AppConfig -> MediaType -> Target -> QueryParams -> Either ApiRequestError (Maybe FieldName)
-binaryField AppConfig{configRawMediaTypes} acceptMediaType target QueryParams{qsSelect}
-  | returnsScalar target && isRawMediaType =
-      Right $ Just "pgrst_scalar"
-  | isRawMediaType =
-      let
-        fieldName = fstFieldName qsSelect
-      in
-      case fieldName of
-        Just fld -> Right $ Just fld
-        Nothing  -> Left $ BinaryFieldError acceptMediaType
-  | otherwise =
-      Right Nothing
-  where
-    isRawMediaType = acceptMediaType `elem` configRawMediaTypes `union` [MTOctetStream, MTTextPlain, MTTextXML] || isRawPlan acceptMediaType
-    isRawPlan mt = case mt of
-      MTPlan (MTPlanAttrs (Just MTOctetStream) _ _) -> True
-      MTPlan (MTPlanAttrs (Just MTTextPlain) _ _)   -> True
-      MTPlan (MTPlanAttrs (Just MTTextXML) _ _)     -> True
-      _                                             -> False
-    returnsScalar :: Target -> Bool
-    returnsScalar (TargetProc proc _) = procReturnsScalar proc
-    returnsScalar _                   = False
-
-    fstFieldName :: [Tree SelectItem] -> Maybe FieldName
-    fstFieldName [Node SelectField{selField=("*", _)} []] = Nothing
-    fstFieldName [Node SelectField{selField=(fld, _)} []] = Just fld
-    fstFieldName _                                        = Nothing

src/PostgREST/ApiRequest/Preferences.hs

@@ -9,6 +9,7 @@
 module PostgREST.ApiRequest.Preferences
   ( Preferences(..)
   , PreferCount(..)
+  , PreferMissing(..)
   , PreferParameters(..)
   , PreferRepresentation(..)
   , PreferResolution(..)
@@ -33,6 +34,7 @@ import Protolude
 -- >>> deriving instance Show PreferParameters
 -- >>> deriving instance Show PreferCount
 -- >>> deriving instance Show PreferTransaction
+-- >>> deriving instance Show PreferMissing
 -- >>> deriving instance Show Preferences
 
 -- | Preferences recognized by the application.
@@ -43,6 +45,7 @@ data Preferences
     , preferParameters     :: Maybe PreferParameters
     , preferCount          :: Maybe PreferCount
     , preferTransaction    :: Maybe PreferTransaction
+    , preferMissing        :: Maybe PreferMissing
     }
 
 -- |
@@ -57,17 +60,19 @@ data Preferences
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
+--     , preferMissing = Nothing
 --     }
 --
 -- Multiple headers can also be used:
 --
--- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact")]
+-- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
 --     , preferRepresentation = Nothing
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
+--     , preferMissing = Just ApplyNulls
 --     }
 --
 -- If a preference is set more than once, only the first is used:
@@ -92,23 +97,25 @@ data Preferences
 --
 -- Preferences can be separated by arbitrary amounts of space, lower-case header is also recognized:
 --
--- >>> pPrint $ fromHeaders [("prefer", "count=exact,    tx=commit   ,return=minimal")]
+-- >>> pPrint $ fromHeaders [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default")]
 -- Preferences
 --     { preferResolution = Nothing
---     , preferRepresentation = Just None
+--     , preferRepresentation = Just Full
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Just Commit
+--     , preferMissing = Just ApplyDefaults
 --     }
 --
 fromHeaders :: [HTTP.Header] -> Preferences
 fromHeaders headers =
   Preferences
-    { preferResolution = parsePrefs [MergeDuplicates, IgnoreDuplicates]
+    { preferResolution     = parsePrefs [MergeDuplicates, IgnoreDuplicates]
     , preferRepresentation = parsePrefs [Full, None, HeadersOnly]
-    , preferParameters = parsePrefs [SingleObject, MultipleObjects]
+    , preferParameters     = parsePrefs [SingleObject]
-    , preferCount = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
+    , preferCount          = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
-    , preferTransaction = parsePrefs [Commit, Rollback]
+    , preferTransaction    = parsePrefs [Commit, Rollback]
+    , preferMissing        = parsePrefs [ApplyDefaults, ApplyNulls]
     }
   where
     prefHeaders = filter ((==) HTTP.hPrefer . fst) headers
@@ -161,6 +168,8 @@ data PreferRepresentation
   | None        -- ^ Return nothing from the mutated data.
   deriving Eq
 
+instance ToAppliedHeader PreferRepresentation
+
 instance ToHeaderValue PreferRepresentation where
   toHeaderValue Full        = "return=representation"
   toHeaderValue None        = "return=minimal"
@@ -169,13 +178,10 @@ instance ToHeaderValue PreferRepresentation where
 -- | How to pass parameters to stored procedures.
 data PreferParameters
   = SingleObject    -- ^ Pass all parameters as a single json object to a stored procedure.
-  | MultipleObjects -- ^ Pass an array of json objects as params to a stored procedure.
   deriving Eq
 
--- TODO: Deprecate params=multiple-objects in next major version
 instance ToHeaderValue PreferParameters where
   toHeaderValue SingleObject    = "params=single-object"
-  toHeaderValue MultipleObjects = "params=multiple-objects"
 
 -- | How to determine the count of (expected) results
 data PreferCount
@@ -204,3 +210,17 @@ instance ToHeaderValue PreferTransaction where
   toHeaderValue Rollback = "tx=rollback"
 
 instance ToAppliedHeader PreferTransaction
+
+-- |
+-- How to handle the insertion/update when the keys specified in ?columns are not present
+-- in the json body.
+data PreferMissing
+  = ApplyDefaults  -- ^ Use the default column value for missing values.
+  | ApplyNulls     -- ^ Use the null value for missing values.
+  deriving Eq
+
+instance ToHeaderValue PreferMissing where
+  toHeaderValue ApplyDefaults = "missing=default"
+  toHeaderValue ApplyNulls    = "missing=null"
+
+instance ToAppliedHeader PreferMissing

src/PostgREST/ApiRequest/QueryParams.hs

@@ -30,14 +30,13 @@ import Data.Ranged.Ranges            (Range (..))
 import Data.Tree                     (Tree (..))
 import Text.Parsec.Error             (errorMessages,
                                       showErrorMessages)
-import Text.Parsec.Prim              (parserFail)
 import Text.ParserCombinators.Parsec (GenParser, ParseError, Parser,
                                       anyChar, between, char, digit,
                                       eof, errorPos, letter,
                                       lookAhead, many1, noneOf,
                                       notFollowedBy, oneOf,
-                                      optionMaybe, sepBy1, string,
+                                      optionMaybe, sepBy, sepBy1,
-                                      try, (<?>))
+                                      string, try, (<?>))
 
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           rangeGeq, rangeLimit,
@@ -46,35 +45,21 @@ import PostgREST.SchemaCache.Identifiers (FieldName)
 
 import PostgREST.ApiRequest.Types (EmbedParam (..), EmbedPath, Field,
                                    Filter (..), FtsOperator (..),
-                                   JoinType (..), JsonOperand (..),
+                                   Hint, JoinType (..),
+                                   JsonOperand (..),
                                    JsonOperation (..), JsonPath,
                                    ListVal, LogicOperator (..),
                                    LogicTree (..), OpExpr (..),
-                                   Operation (..),
+                                   OpQuantifier (..), Operation (..),
                                    OrderDirection (..),
                                    OrderNulls (..), OrderTerm (..),
-                                   QPError (..), SelectItem (..),
+                                   QPError (..), QuantOperator (..),
+                                   SelectItem (..),
                                    SimpleOperator (..), SingleVal,
                                    TrileanVal (..))
 
 import Protolude hiding (try)
 
-
--- $setup
--- Setup for doctests
--- >>> import Text.Pretty.Simple (pPrint)
--- >>> deriving instance Show QPError
--- >>> deriving instance Show TrileanVal
--- >>> deriving instance Show FtsOperator
--- >>> deriving instance Show SimpleOperator
--- >>> deriving instance Show Operation
--- >>> deriving instance Show OpExpr
--- >>> deriving instance Show JsonOperand
--- >>> deriving instance Show JsonOperation
--- >>> deriving instance Show Filter
--- >>> deriving instance Show JoinType
--- >>> deriving instance Show SelectItem
-
 data QueryParams =
   QueryParams
     { qsCanonical      :: ByteString
@@ -108,39 +93,45 @@ data QueryParams =
 --
 -- The canonical representation of the query string has parameters sorted alphabetically:
 --
--- >>> qsCanonical <$> parse "a=1&c=3&b=2&d"
+-- >>> qsCanonical <$> parse True "a=1&c=3&b=2&d"
 -- Right "a=1&b=2&c=3&d="
 --
 -- 'select' is a reserved parameter that selects the fields to be returned:
 --
--- >>> qsSelect <$> parse "select=name,location"
+-- >>> qsSelect <$> parse False "select=name,location"
 -- Right [Node {rootLabel = SelectField {selField = ("name",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectField {selField = ("location",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]
 --
 -- Filters are parameters whose value contains an operator, separated by a '.' from its value:
 --
--- >>> qsFilters <$> parse "a.b=eq.0"
+-- >>> qsFilters <$> parse False "a.b=eq.0"
--- Right [(["a"],Filter {field = ("b",[]), opExpr = OpExpr False (Op OpEqual "0")})]
+-- Right [(["a"],Filter {field = ("b",[]), opExpr = OpExpr False (OpQuant OpEqual Nothing "0")})]
 --
 -- If the operator specified in a filter does not exist, parsing the query string fails:
 --
--- >>> qsFilters <$> parse "a.b=noop.0"
+-- >>> qsFilters <$> parse False "a.b=noop.0"
--- Left (QPError "\"failed to parse filter (noop.0)\" (line 1, column 6)" "unknown single value operator noop")
+-- Left (QPError "\"failed to parse filter (noop.0)\" (line 1, column 1)" "unexpected \"o\" expecting \"not\" or operator (eq, gt, ...)")
-parse :: ByteString -> Either QPError QueryParams
+parse :: Bool -> ByteString -> Either QPError QueryParams
-parse qs =
+parse isRpcGet qs = do
-  QueryParams
+  rOrd                      <- pRequestOrder `traverse` order
-    canonical
+  rLogic                    <- pRequestLogicTree `traverse` logic
-    params
+  rCols                     <- pRequestColumns columns
-    ranges
+  rSel                      <- pRequestSelect select
-    <$> pRequestOrder `traverse` order
+  (rFlts, params)           <- L.partition hasOp <$> pRequestFilter isRpcGet `traverse` filters
-    <*> pRequestLogicTree `traverse` logic
+  (rFltsRoot, rFltsNotRoot) <- pure $ L.partition hasRootFilter rFlts
-    <*> pRequestColumns columns
+  rOnConflict               <- pRequestOnConflict `traverse` onConflict
-    <*> pRequestSelect select
+
-    <*> pRequestFilter `traverse` filters
+  let rFltsFields           = S.fromList (fst <$> filters)
-    <*> (fmap snd <$> (pRequestFilter `traverse` filtersRoot))
+      params'               = mapMaybe (\case {(_, Filter (fld, _) (NoOpExpr v)) -> Just (fld,v); _ -> Nothing}) params
-    <*> pRequestFilter `traverse` filtersNotRoot
+      rFltsRoot'            = snd <$> rFltsRoot
-    <*> pure (S.fromList (fst <$> filters))
+
-    <*> sequenceA (pRequestOnConflict <$> onConflict)
+  return $ QueryParams canonical params' ranges rOrd rLogic rCols rSel rFlts rFltsRoot' rFltsNotRoot rFltsFields rOnConflict
   where
+    hasRootFilter, hasOp :: (EmbedPath, Filter) -> Bool
+    hasRootFilter ([], _) = True
+    hasRootFilter _       = False
+    hasOp (_, Filter (_, _) (NoOpExpr _)) = False
+    hasOp _                               = True
+
     logic = filter (endingIn ["and", "or"] . fst) nonemptyParams
     select = fromMaybe "*" $ lookupParam "select"
     onConflict = lookupParam "on_conflict"
@@ -167,32 +158,11 @@ parse qs =
     endingIn xx key = lastWord `elem` xx
       where lastWord = L.last $ T.split (== '.') key
 
-    (filters, params) = L.partition isParam filtersAndParams
+    filters = filter (isFilter . fst) nonemptyParams
-    isParam (k, v) = isEmbedPath k || hasOperator v || hasFtsOperator v
+    isFilter k = not (endingIn reservedEmbeddable k) && notElem k reserved
-
-    filtersAndParams = filter (isFilterOrParam . fst)  nonemptyParams
-    isFilterOrParam k = not (endingIn reservedEmbeddable k) && notElem k reserved
     reserved = ["select", "columns", "on_conflict"]
     reservedEmbeddable = ["order", "limit", "offset", "and", "or"]
 
-    (filtersNotRoot, filtersRoot) = L.partition isNotRoot filters
-    isNotRoot = flip T.isInfixOf "." . fst
-
-    -- TODO: These checks are redundant to the parsers, should use parsers to differentiate params
-    hasOperator val =
-      case T.splitOn "." val of
-        "not" : _ : _ -> True
-        "is" : _      -> True
-        "in" : _      -> True
-        x : _         -> isJust (operator x) || isJust (ftsOperator x)
-        _             -> False
-
-    hasFtsOperator val =
-      case T.splitOn "(" val of
-        x : _ : _ -> isJust $ ftsOperator x
-        _         -> False
-
-    isEmbedPath = T.isInfixOf "."
     replaceLast x s = T.intercalate "." $ L.init (T.split (=='.') s) <> [x]
 
     ranges :: HM.HashMap Text (Range Integer)
@@ -209,39 +179,31 @@ parse qs =
         offsetParams =
           HM.fromList [(k, maybe allRange rangeGeq (readMaybe v)) | (k,v) <- offsets]
 
-operator :: Text -> Maybe SimpleOperator
+simpleOperator :: Parser SimpleOperator
-operator = \case
+simpleOperator =
-  "eq"     -> Just OpEqual
+  try (string "neq" $> OpNotEqual) <|>
-  "gte"    -> Just OpGreaterThanEqual
+  try (string "cs" $> OpContains) <|>
-  "gt"     -> Just OpGreaterThan
+  try (string "cd" $> OpContained) <|>
-  "lte"    -> Just OpLessThanEqual
+  try (string "ov" $> OpOverlap) <|>
-  "lt"     -> Just OpLessThan
+  try (string "sl" $> OpStrictlyLeft) <|>
-  "neq"    -> Just OpNotEqual
+  try (string "sr" $> OpStrictlyRight) <|>
-  "like"   -> Just OpLike
+  try (string "nxr" $> OpNotExtendsRight) <|>
-  "ilike"  -> Just OpILike
+  try (string "nxl" $> OpNotExtendsLeft) <|>
-  "cs"     -> Just OpContains
+  try (string "adj" $> OpAdjacent) <?>
-  "cd"     -> Just OpContained
+  "unknown single value operator"
-  "ov"     -> Just OpOverlap
-  "sl"     -> Just OpStrictlyLeft
-  "sr"     -> Just OpStrictlyRight
-  "nxr"    -> Just OpNotExtendsRight
-  "nxl"    -> Just OpNotExtendsLeft
-  "adj"    -> Just OpAdjacent
-  "match"  -> Just OpMatch
-  "imatch" -> Just OpIMatch
-  _        -> Nothing
-
-ftsOperator :: Text -> Maybe FtsOperator
-ftsOperator = \case
-  "fts"   -> Just FilterFts
-  "plfts" -> Just FilterFtsPlain
-  "phfts" -> Just FilterFtsPhrase
-  "wfts"  -> Just FilterFtsWebsearch
-  _       -> Nothing
-
-
--- PARSERS
 
+quantOperator :: Parser QuantOperator
+quantOperator =
+  try (string "eq" $> OpEqual) <|>
+  try (string "gte" $> OpGreaterThanEqual) <|>
+  try (string "gt" $> OpGreaterThan) <|>
+  try (string "lte" $> OpLessThanEqual) <|>
+  try (string "lt" $> OpLessThan) <|>
+  try (string "like" $> OpLike) <|>
+  try (string "ilike" $> OpILike) <|>
+  try (string "match" $> OpMatch) <|>
+  try (string "imatch" $> OpIMatch) <?>
+  "unknown single value operator"
 
 pRequestSelect :: Text -> Either QPError [Tree SelectItem]
 pRequestSelect selStr =
@@ -251,11 +213,25 @@ pRequestOnConflict :: Text -> Either QPError [FieldName]
 pRequestOnConflict oncStr =
   mapError $ P.parse pColumns ("failed to parse on_conflict parameter (" <> toS oncStr <> ")") (toS oncStr)
 
-pRequestFilter :: (Text, Text) -> Either QPError (EmbedPath, Filter)
+-- |
-pRequestFilter (k, v) = mapError $ (,) <$> path <*> (Filter <$> fld <*> oper)
+-- Parse `id=eq.1`(id, eq.1) into (EmbedPath, Filter)
+--
+-- >>> pRequestFilter False ("id", "eq.1")
+-- Right ([],Filter {field = ("id",[]), opExpr = OpExpr False (OpQuant OpEqual Nothing "1")})
+--
+-- >>> pRequestFilter False ("id", "val")
+-- Left (QPError "\"failed to parse filter (val)\" (line 1, column 1)" "unexpected \"v\" expecting \"not\" or operator (eq, gt, ...)")
+--
+-- >>> pRequestFilter True ("id", "val")
+-- Right ([],Filter {field = ("id",[]), opExpr = NoOpExpr "val"})
+pRequestFilter :: Bool -> (Text, Text) -> Either QPError (EmbedPath, Filter)
+pRequestFilter isRpcGet (k, v) = mapError $ (,) <$> path <*> (Filter <$> fld <*> oper)
   where
     treePath = P.parse pTreePath ("failed to parse tree path (" ++ toS k ++ ")") $ toS k
-    oper = P.parse (pOpExpr pSingleVal) ("failed to parse filter (" ++ toS v ++ ")") $ toS v
+    oper = P.parse parseFlt ("failed to parse filter (" ++ toS v ++ ")") $ toS v
+    parseFlt = if isRpcGet
+      then pOpExpr pSingleVal <|> pure (NoOpExpr v)
+      else pOpExpr pSingleVal
     path = fst <$> treePath
     fld = snd <$> treePath
 
@@ -314,20 +290,28 @@ pTreePath = do
 -- >>> P.parse pFieldForest "" "*,client(*,nested(*))"
 -- Right [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectRelation {selRelation = "client", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectRelation {selRelation = "nested", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]}]}]
 --
+-- >>> P.parse pFieldForest "" "*,...client(*),other(*)"
+-- Right [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SpreadRelation {selRelation = "client", selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]},Node {rootLabel = SelectRelation {selRelation = "other", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]}]
+--
+-- >>> P.parse pFieldForest "" ""
+-- Right []
+--
 -- >>> P.parse pFieldForest "" "id,clients(name[])"
 -- Left (line 1, column 16):
 -- unexpected '['
--- expecting letter, digit, "-", "!", "(", "->>", "->", "::", ")", "," or end of input
+-- expecting letter, digit, "-", "->>", "->", "::", ")", "," or end of input
+--
+-- >>> P.parse pFieldForest "" "data->>-78xy"
+-- Left (line 1, column 11):
+-- unexpected 'x'
+-- expecting digit, "->", "::", ".", "," or end of input
 pFieldForest :: Parser [Tree SelectItem]
-pFieldForest = pFieldTree `sepBy1` lexeme (char ',')
+pFieldForest = pFieldTree `sepBy` lexeme (char ',')
   where
-    pFieldTree :: Parser (Tree SelectItem)
+    pFieldTree =  Node <$> try pSpreadRelationSelect <*> between (char '(') (char ')') pFieldForest <|>
-    pFieldTree =  try (Node <$> pRelationSelect <*> between (char '(') (char ')') pFieldForest) <|>
+                  Node <$> try pRelationSelect       <*> between (char '(') (char ')') pFieldForest <|>
                   Node <$> pFieldSelect <*> pure []
 
-pStar :: Parser Text
-pStar = string "*" $> "*"
-
 -- |
 -- Parse field names
 --
@@ -393,6 +377,23 @@ pFieldName =
 --
 -- >>> P.parse pJsonPath "" "->0.desc"
 -- Right [JArrow {jOp = JIdx {jVal = "+0"}}]
+--
+-- Fails on badly formed negatives
+--
+-- >>> P.parse pJsonPath "" "->>-78xy"
+-- Left (line 1, column 7):
+-- unexpected 'x'
+-- expecting digit, "->", "::", ".", "," or end of input
+--
+-- >>> P.parse pJsonPath "" "->>--34"
+-- Left (line 1, column 5):
+-- unexpected "-"
+-- expecting digit
+--
+-- >>> P.parse pJsonPath "" "->>-xy-4"
+-- Left (line 1, column 5):
+-- unexpected "x"
+-- expecting digit
 pJsonPath :: Parser JsonPath
 pJsonPath = many pJsonOperation
   where
@@ -448,27 +449,12 @@ aliasSeparator = char ':' >> notFollowedBy (char ':')
 -- Left (line 1, column 6):
 -- unexpected '>'
 pRelationSelect :: Parser SelectItem
-pRelationSelect = lexeme $ try ( do
+pRelationSelect = lexeme $ do
     alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
     name <- pFieldName
-    prm1 <- optionMaybe pEmbedParam
+    (hint, jType) <- pEmbedParams
-    prm2 <- optionMaybe pEmbedParam
     try (void $ lookAhead (string "("))
-    return $ SelectRelation name alias (embedParamHint prm1 <|> embedParamHint prm2) (embedParamJoin prm1 <|> embedParamJoin prm2)
+    return $ SelectRelation name alias hint jType
-  )
-  where
-    pEmbedParam :: Parser EmbedParam
-    pEmbedParam =
-      char '!' *> (
-        try (string "left"  $> EPJoinType JTLeft)  <|>
-        try (string "inner" $> EPJoinType JTInner) <|>
-        try (EPHint <$> pFieldName))
-    embedParamHint prm = case prm of
-      Just (EPHint hint) -> Just hint
-      _                  -> Nothing
-    embedParamJoin prm = case prm of
-      Just (EPJoinType jt) -> Just jt
-      _                    -> Nothing
 
 -- |
 -- Parse regular fields in select
@@ -506,43 +492,123 @@ pRelationSelect = lexeme $ try ( do
 -- unexpected end of input
 -- expecting letter or digit
 pFieldSelect :: Parser SelectItem
-pFieldSelect = lexeme $
+pFieldSelect = lexeme $ try (do
-  try (
-    do
-      alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
-      fld <- pField
-      cast' <- optionMaybe (string "::" *> pIdentifier)
-      pEnd
-      return $ SelectField fld (toS <$> cast') alias
-  )
-  <|> do
     s <- pStar
     pEnd
-    return $ SelectField (s, []) Nothing Nothing
+    return $ SelectField (s, []) Nothing Nothing)
+  <|> do
+    alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
+    fld <- pField
+    cast' <- optionMaybe (string "::" *> pIdentifier)
+    pEnd
+    return $ SelectField fld (toS <$> cast') alias
   where
     pEnd = try (void $ lookAhead (string ")")) <|>
            try (void $ lookAhead (string ",")) <|>
            try eof
+    pStar = string "*" $> "*"
+
+
+-- |
+-- Parse spread relations in select
+--
+-- >>> P.parse pSpreadRelationSelect "" "...rel(*)"
+-- Right (SpreadRelation {selRelation = "rel", selHint = Nothing, selJoinType = Nothing})
+--
+-- >>> P.parse pSpreadRelationSelect "" "...rel!hint!inner(*)"
+-- Right (SpreadRelation {selRelation = "rel", selHint = Just "hint", selJoinType = Just JTInner})
+--
+-- >>> P.parse pSpreadRelationSelect "" "rel(*)"
+-- Left (line 1, column 1):
+-- unexpected "r"
+-- expecting "..."
+--
+-- >>> P.parse pSpreadRelationSelect "" "alias:...rel(*)"
+-- Left (line 1, column 1):
+-- unexpected "a"
+-- expecting "..."
+--
+-- >>> P.parse pSpreadRelationSelect "" "...rel->jsonpath(*)"
+-- Left (line 1, column 9):
+-- unexpected '>'
+pSpreadRelationSelect :: Parser SelectItem
+pSpreadRelationSelect = lexeme $ do
+    name <- string "..." >> pFieldName
+    (hint, jType) <- pEmbedParams
+    try (void $ lookAhead (string "("))
+    return $ SpreadRelation name hint jType
+
+pEmbedParams :: Parser (Maybe Hint, Maybe JoinType)
+pEmbedParams = do
+  prm1 <- optionMaybe pEmbedParam
+  prm2 <- optionMaybe pEmbedParam
+  return (embedParamHint prm1 <|> embedParamHint prm2, embedParamJoin prm1 <|> embedParamJoin prm2)
+  where
+    pEmbedParam :: Parser EmbedParam
+    pEmbedParam =
+      char '!' *> (
+        try (string "left"  $> EPJoinType JTLeft)  <|>
+        try (string "inner" $> EPJoinType JTInner) <|>
+        try (EPHint <$> pFieldName))
+    embedParamHint prm = case prm of
+      Just (EPHint hint) -> Just hint
+      _                  -> Nothing
+    embedParamJoin prm = case prm of
+      Just (EPJoinType jt) -> Just jt
+      _                    -> Nothing
 
 -- |
 -- Parse operator expression used in horizontal filtering
 --
 -- >>> P.parse (pOpExpr pSingleVal) "" "fts().value"
--- Left (line 1, column 7):
+-- Left (line 1, column 5):
--- unknown single value operator fts()
+-- unexpected ")"
+-- expecting operator (eq, gt, ...)
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "eq(any).value"
+-- Right (OpExpr False (OpQuant OpEqual (Just QuantAny) "value"))
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "eq(all).value"
+-- Right (OpExpr False (OpQuant OpEqual (Just QuantAll) "value"))
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "not.eq(all).value"
+-- Right (OpExpr True (OpQuant OpEqual (Just QuantAll) "value"))
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "eq().value"
+-- Left (line 1, column 4):
+-- unexpected ")"
+-- expecting operator (eq, gt, ...)
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "is().value"
+-- Left (line 1, column 3):
+-- unexpected "("
+-- expecting operator (eq, gt, ...)
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "in().value"
+-- Left (line 1, column 3):
+-- unexpected "("
+-- expecting operator (eq, gt, ...)
 pOpExpr :: Parser SingleVal -> Parser OpExpr
-pOpExpr pSVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)) <|> OpExpr False <$> pOperation
+pOpExpr pSVal = do
+  boolExpr <- try (string "not" *> pDelimiter $> True) <|> pure False
+  OpExpr boolExpr <$> pOperation
   where
     pOperation :: Parser Operation
-    pOperation = pIn <|> pIs <|> try pFts <|> pOp <?> "operator (eq, gt, ...)"
+    pOperation = pIn <|> pIs <|> pIsDist <|> try pFts <|> try pSimpleOp <|> try pQuantOp <?> "operator (eq, gt, ...)"
 
     pIn = In <$> (try (string "in" *> pDelimiter) *> pListVal)
     pIs = Is <$> (try (string "is" *> pDelimiter) *> pTriVal)
 
-    pOp = do
+    pIsDist = IsDistinctFrom <$> (try (string "isdistinct" *> pDelimiter) *> pSVal)
-      opStr <- try (P.manyTill anyChar (try pDelimiter))
+
-      op <- parseMaybe ("unknown single value operator " <> opStr) . operator $ toS opStr
+    pSimpleOp = do
-      Op op <$> pSVal
+      op <- simpleOperator
+      pDelimiter *> (Op op <$> pSVal)
+
+    pQuantOp = do
+      op <- quantOperator
+      quant <- optionMaybe $ try (between (char '(') (char ')') (try (string "any" $> QuantAny) <|> string "all" $> QuantAll))
+      pDelimiter *> (OpQuant op quant <$> pSVal)
 
     pTriVal = try (ciString "null"    $> TriNull)
           <|> try (ciString "unknown" $> TriUnknown)
@@ -551,15 +617,14 @@ pOpExpr pSVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)
           <?> "null or trilean value (unknown, true, false)"
 
     pFts = do
-      opStr <- try (P.many (noneOf ".("))
+      op <-  try (string "fts"   $> FilterFts)
-      op <- parseMaybe ("unknown fts operator " <> opStr) . ftsOperator $ toS opStr
+         <|> try (string "plfts" $> FilterFtsPlain)
+         <|> try (string "phfts" $> FilterFtsPhrase)
+         <|> try (string "wfts"  $> FilterFtsWebsearch)
+
       lang <- optionMaybe $ try (between (char '(') (char ')') pIdentifier)
       pDelimiter >> Fts op (toS <$> lang) <$> pSVal
 
-    parseMaybe :: [Char] -> Maybe a -> Parser a
-    parseMaybe err Nothing = parserFail err
-    parseMaybe _ (Just x)  = pure x
-
     -- case insensitive char and string
     ciChar :: Char -> GenParser Char state Char
     ciChar c = char c <|> char (toUpper c)
@@ -583,24 +648,119 @@ pQuotedValue = toS <$> (char '"' *> many pCharsOrSlashed <* char '"')
 pDelimiter :: Parser Char
 pDelimiter = char '.' <?> "delimiter (.)"
 
+-- |
+-- Parses the elements in the order query parameter
+--
+-- >>> P.parse pOrder "" "name.desc.nullsfirst"
+-- Right [OrderTerm {otTerm = ("name",[]), otDirection = Just OrderDesc, otNullOrder = Just OrderNullsFirst}]
+--
+-- >>> P.parse pOrder "" "json_col->key.asc.nullslast"
+-- Right [OrderTerm {otTerm = ("json_col",[JArrow {jOp = JKey {jVal = "key"}}]), otDirection = Just OrderAsc, otNullOrder = Just OrderNullsLast}]
+--
+-- >>> P.parse pOrder "" "clients(json_col->key).desc.nullsfirst"
+-- Right [OrderRelationTerm {otRelation = "clients", otRelTerm = ("json_col",[JArrow {jOp = JKey {jVal = "key"}}]), otDirection = Just OrderDesc, otNullOrder = Just OrderNullsFirst}]
+--
+-- >>> P.parse pOrder "" "clients(name,id)"
+-- Left (line 1, column 8):
+-- unexpected '('
+-- expecting letter, digit, "-", "->>", "->", delimiter (.), "," or end of input
+--
+-- >>> P.parse pOrder "" "name,clients(name),id"
+-- Right [OrderTerm {otTerm = ("name",[]), otDirection = Nothing, otNullOrder = Nothing},OrderRelationTerm {otRelation = "clients", otRelTerm = ("name",[]), otDirection = Nothing, otNullOrder = Nothing},OrderTerm {otTerm = ("id",[]), otDirection = Nothing, otNullOrder = Nothing}]
+--
+-- >>> P.parse pOrder "" "id.ac"
+-- Left (line 1, column 4):
+-- unexpected "c"
+-- expecting "asc", "desc", "nullsfirst" or "nullslast"
+--
+-- >>> P.parse pOrder "" "id.descc"
+-- Left (line 1, column 8):
+-- unexpected 'c'
+-- expecting delimiter (.), "," or end of input
+--
+-- >>> P.parse pOrder "" "id.nulsfist"
+-- Left (line 1, column 4):
+-- unexpected "n"
+-- expecting "asc", "desc", "nullsfirst" or "nullslast"
+--
+-- >>> P.parse pOrder "" "id.nullslasttt"
+-- Left (line 1, column 13):
+-- unexpected 't'
+-- expecting "," or end of input
+--
+-- >>> P.parse pOrder "" "id.smth34"
+-- Left (line 1, column 4):
+-- unexpected "s"
+-- expecting "asc", "desc", "nullsfirst" or "nullslast"
+--
+-- >>> P.parse pOrder "" "id.asc.nlsfst"
+-- Left (line 1, column 8):
+-- unexpected "l"
+-- expecting "nullsfirst" or "nullslast"
+--
+-- >>> P.parse pOrder "" "id.asc.nullslasttt"
+-- Left (line 1, column 17):
+-- unexpected 't'
+-- expecting "," or end of input
+--
+-- >>> P.parse pOrder "" "id.asc.smth34"
+-- Left (line 1, column 8):
+-- unexpected "s"
+-- expecting "nullsfirst" or "nullslast"
 pOrder :: Parser [OrderTerm]
-pOrder = lexeme pOrderTerm `sepBy1` char ','
+pOrder = lexeme (try pOrderRelationTerm <|> pOrderTerm) `sepBy1` char ','
-
-pOrderTerm :: Parser OrderTerm
-pOrderTerm = do
-  fld <- pField
-  dir <- optionMaybe $
-         try (pDelimiter *> string "asc" $> OrderAsc) <|>
-         try (pDelimiter *> string "desc" $> OrderDesc)
-  nls <- optionMaybe pNulls <* pEnd <|>
-         pEnd $> Nothing
-  return $ OrderTerm fld dir nls
   where
+    pOrderTerm = do
+      fld <- pField
+      dir <- optionMaybe pOrdDir
+      nls <- optionMaybe pNulls <* pEnd <|>
+             pEnd $> Nothing
+      return $ OrderTerm fld dir nls
+
+    pOrderRelationTerm = do
+      nam <- pFieldName
+      fld <- between (char '(') (char ')') pField
+      dir <- optionMaybe pOrdDir
+      nls <- optionMaybe pNulls <* pEnd <|> pEnd $> Nothing
+      return $ OrderRelationTerm nam fld dir nls
+
+    pNulls :: Parser OrderNulls
     pNulls = try (pDelimiter *> string "nullsfirst" $> OrderNullsFirst) <|>
              try (pDelimiter *> string "nullslast"  $> OrderNullsLast)
-    pEnd = try (void $ lookAhead (char ',')) <|>
-           try eof
 
+    pOrdDir :: Parser OrderDirection
+    pOrdDir = try (pDelimiter *> string "asc" $> OrderAsc) <|>
+              try (pDelimiter *> string "desc" $> OrderDesc)
+
+    pEnd = try (void $ lookAhead (char ',')) <|> try eof
+
+-- |
+-- Parses the elements inside or/and
+--
+-- >>> P.parse pLogicTree "" "or()"
+-- Left (line 1, column 4):
+-- unexpected ")"
+-- expecting field name (* or [a..z0..9_$]), negation operator (not) or logic operator (and, or)
+--
+-- >>> P.parse pLogicTree "" "or(id.in.1,2,id.eq.3)"
+-- Left (line 1, column 10):
+-- unexpected "1"
+-- expecting "("
+--
+-- >>> P.parse pLogicTree "" "or)("
+-- Left (line 1, column 3):
+-- unexpected ")"
+-- expecting "("
+--
+-- >>> P.parse pLogicTree "" "and(ord(id.eq.1,id.eq.1),id.eq.2)"
+-- Left (line 1, column 7):
+-- unexpected "d"
+-- expecting "("
+--
+-- >>> P.parse pLogicTree "" "or(id.eq.1,not.xor(id.eq.2,id.eq.3))"
+-- Left (line 1, column 16):
+-- unexpected "x"
+-- expecting logic operator (and, or)
 pLogicTree :: Parser LogicTree
 pLogicTree = Stmnt <$> try pLogicFilter
              <|> Expr <$> pNot <*> pLogicOp <*> (lexeme (char '(') *> pLogicTree `sepBy1` lexeme (char ',') <* lexeme (char ')'))

src/PostgREST/ApiRequest/Types.hs

@@ -19,6 +19,7 @@ module PostgREST.ApiRequest.Types
   , NodeName
   , OpExpr(..)
   , Operation (..)
+  , OpQuantifier(..)
   , OrderDirection(..)
   , OrderNulls(..)
   , OrderTerm(..)
@@ -27,35 +28,45 @@ module PostgREST.ApiRequest.Types
   , SingleVal
   , TrileanVal(..)
   , SimpleOperator(..)
+  , QuantOperator(..)
   , FtsOperator(..)
   , SelectItem(..)
   ) where
 
 import PostgREST.MediaType                (MediaType (..))
-import PostgREST.SchemaCache.Identifiers  (FieldName)
+import PostgREST.SchemaCache.Identifiers  (FieldName,
-import PostgREST.SchemaCache.Proc         (ProcDescription (..))
+                                           QualifiedIdentifier)
-import PostgREST.SchemaCache.Relationship (Relationship)
+import PostgREST.SchemaCache.Relationship (Relationship,
+                                           RelationshipsMap)
+import PostgREST.SchemaCache.Routine      (Routine (..))
 
 import Protolude
 
--- | The select value in `/tbl?select=alias:field::cast`
+-- | The value in `/tbl?select=alias:field::cast`
 data SelectItem
   = SelectField
     { selField :: Field
     , selCast  :: Maybe Cast
     , selAlias :: Maybe Alias
     }
+-- | The value in `/tbl?select=alias:another_tbl(*)`
   | SelectRelation
     { selRelation :: FieldName
     , selAlias    :: Maybe Alias
     , selHint     :: Maybe Hint
     , selJoinType :: Maybe JoinType
     }
-  deriving (Eq)
+-- | The value in `/tbl?select=...another_tbl(*)`
+  | SpreadRelation
+    { selRelation :: FieldName
+    , selHint     :: Maybe Hint
+    , selJoinType :: Maybe JoinType
+    }
+  deriving (Eq, Show)
 
 data ApiRequestError
   = AmbiguousRelBetween Text Text [Relationship]
-  | AmbiguousRpc [ProcDescription]
+  | AmbiguousRpc [Routine]
   | BinaryFieldError MediaType
   | MediaTypeError [ByteString]
   | InvalidBody ByteString
@@ -64,40 +75,53 @@ data ApiRequestError
   | InvalidRpcMethod ByteString
   | LimitNoOrderError
   | NotFound
-  | NoRelBetween Text Text Text
+  | NoRelBetween Text Text (Maybe Text) Text RelationshipsMap
-  | NoRpc Text Text [Text] Bool MediaType Bool
+  | NoRpc Text Text [Text] Bool MediaType Bool [QualifiedIdentifier] [Routine]
   | NotEmbedded Text
-  | ParseRequestError Text Text
+  | PutLimitNotAllowedError
-  | PutRangeNotAllowedError
   | QueryParamError QPError
+  | RelatedOrderNotToOne Text Text
+  | SpreadNotToOne Text Text
+  | UnacceptableFilter Text
   | UnacceptableSchema [Text]
   | UnsupportedMethod ByteString
+  | ColumnNotFound Text Text
+  deriving Show
 
 data QPError = QPError Text Text
+  deriving Show
 data RangeError
   = NegativeLimit
   | LowerGTUpper
   | OutOfBounds Text Text
+  deriving Show
 
 type NodeName = Text
 type Depth = Integer
 
-data OrderTerm = OrderTerm
+data OrderTerm
-  { otTerm      :: Field
+  = OrderTerm
-  , otDirection :: Maybe OrderDirection
+    { otTerm      :: Field
-  , otNullOrder :: Maybe OrderNulls
+    , otDirection :: Maybe OrderDirection
-  }
+    , otNullOrder :: Maybe OrderNulls
-  deriving (Eq)
+    }
+  | OrderRelationTerm
+    { otRelation  :: FieldName
+    , otRelTerm   :: Field
+    , otDirection :: Maybe OrderDirection
+    , otNullOrder :: Maybe OrderNulls
+    }
+  deriving (Eq, Show)
 
 data OrderDirection
   = OrderAsc
   | OrderDesc
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data OrderNulls
   = OrderNullsFirst
   | OrderNullsLast
-  deriving (Eq)
+  deriving (Eq, Show)
 
 type Field = (FieldName, JsonPath)
 type Cast = Text
@@ -114,7 +138,7 @@ data EmbedParam
 data JoinType
   = JTInner
   | JTLeft
-  deriving Eq
+  deriving (Eq, Show)
 
 -- | Path of the embedded levels, e.g "clients.projects.name=eq.." gives Path
 -- ["clients", "projects"]
@@ -128,7 +152,7 @@ type JsonPath = [JsonOperation]
 data JsonOperation
   = JArrow { jOp :: JsonOperand }
   | J2Arrow { jOp :: JsonOperand }
-  deriving (Eq)
+  deriving (Eq, Show, Ord)
 
 -- | Represents the key(`->'key'`) or index(`->'1`::int`), the index is Text
 -- because we reuse our escaping functons and let pg do the casting with
@@ -136,7 +160,7 @@ data JsonOperation
 data JsonOperand
   = JKey { jVal :: Text }
   | JIdx { jVal :: Text }
-  deriving (Eq)
+  deriving (Eq, Show, Ord)
 
 -- | Boolean logic expression tree e.g. "and(name.eq.N,or(id.eq.1,id.eq.2))" is:
 --
@@ -148,29 +172,36 @@ data JsonOperand
 data LogicTree
   = Expr Bool LogicOperator [LogicTree]
   | Stmnt Filter
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data LogicOperator
   = And
   | Or
-  deriving Eq
+  deriving (Eq, Show)
 
-data Filter = Filter
+data Filter
+  = Filter
   { field  :: Field
   , opExpr :: OpExpr
   }
-  deriving (Eq)
+  deriving (Eq, Show)
 
-data OpExpr =
+data OpExpr
-  OpExpr Bool Operation
+  = OpExpr Bool Operation
-  deriving (Eq)
+  | NoOpExpr Text
+  deriving (Eq, Show)
+
+data OpQuantifier = QuantAny | QuantAll
+  deriving (Eq, Show)
 
 data Operation
   = Op SimpleOperator SingleVal
+  | OpQuant QuantOperator (Maybe OpQuantifier) SingleVal
   | In ListVal
   | Is TrileanVal
+  | IsDistinctFrom SingleVal
   | Fts FtsOperator (Maybe Language) SingleVal
-  deriving (Eq)
+  deriving (Eq, Show)
 
 type Language = Text
 
@@ -186,17 +217,23 @@ data TrileanVal
   | TriFalse
   | TriNull
   | TriUnknown
-  deriving Eq
+  deriving (Eq, Show)
 
-data SimpleOperator
+-- Operators that are quantifiable, i.e. they can be used with the any/all modifiers
+data QuantOperator
   = OpEqual
   | OpGreaterThanEqual
   | OpGreaterThan
   | OpLessThanEqual
   | OpLessThan
-  | OpNotEqual
   | OpLike
   | OpILike
+  | OpMatch
+  | OpIMatch
+  deriving (Eq, Show)
+
+data SimpleOperator
+  = OpNotEqual
   | OpContains
   | OpContained
   | OpOverlap
@@ -205,14 +242,13 @@ data SimpleOperator
   | OpNotExtendsRight
   | OpNotExtendsLeft
   | OpAdjacent
-  | OpMatch
+  deriving (Eq, Show)
-  | OpIMatch
-  deriving Eq
 
+--
 -- | Operators for full text search operators
 data FtsOperator
   = FilterFts
   | FilterFtsPlain
   | FilterFtsPhrase
   | FilterFtsWebsearch
-  deriving Eq
+  deriving (Eq, Show)

src/PostgREST/App.hs

@@ -26,10 +26,13 @@ import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
                                  setServerName)
 import System.Posix.Types       (FileMode)
 
+import qualified Data.HashMap.Strict        as HM
+import qualified Data.Text.Encoding         as T
 import qualified Hasql.Transaction.Sessions as SQL
 import qualified Network.Wai                as Wai
 import qualified Network.Wai.Handler.Warp   as Warp
 
+import qualified PostgREST.Admin            as Admin
 import qualified PostgREST.ApiRequest       as ApiRequest
 import qualified PostgREST.ApiRequest.Types as ApiRequestTypes
 import qualified PostgREST.AppState         as AppState
@@ -40,18 +43,18 @@ import qualified PostgREST.Logger           as Logger
 import qualified PostgREST.Plan             as Plan
 import qualified PostgREST.Query            as Query
 import qualified PostgREST.Response         as Response
-import qualified PostgREST.Workers          as Workers
 
-import PostgREST.ApiRequest       (Action (..), ApiRequest (..),
+import PostgREST.ApiRequest          (Action (..), ApiRequest (..),
-                                   Mutation (..), Target (..))
+                                      Mutation (..), Target (..))
-import PostgREST.AppState         (AppState)
+import PostgREST.AppState            (AppState)
-import PostgREST.Auth             (AuthResult (..))
+import PostgREST.Auth                (AuthResult (..))
-import PostgREST.Config           (AppConfig (..), LogLevel (..))
+import PostgREST.Config              (AppConfig (..))
-import PostgREST.Config.PgVersion (PgVersion (..))
+import PostgREST.Config.PgVersion    (PgVersion (..))
-import PostgREST.Error            (Error)
+import PostgREST.Error               (Error)
-import PostgREST.Query            (DbHandler)
+import PostgREST.Query               (DbHandler)
-import PostgREST.SchemaCache      (SchemaCache (..))
+import PostgREST.SchemaCache         (SchemaCache (..))
-import PostgREST.Version          (prettyVersion)
+import PostgREST.SchemaCache.Routine (Routine (..))
+import PostgREST.Version             (docsVersion, prettyVersion)
 
 import Protolude hiding (Handler)
 
@@ -64,14 +67,14 @@ type SocketRunner = Warp.Settings -> Wai.Application -> FileMode -> FilePath ->
 run :: SignalHandlerInstaller -> Maybe SocketRunner -> AppState -> IO ()
 run installHandlers maybeRunWithSocket appState = do
   conf@AppConfig{..} <- AppState.getConfig appState
-  Workers.connectionWorker appState -- Loads the initial SchemaCache
+  AppState.connectionWorker appState -- Loads the initial SchemaCache
   installHandlers appState
   -- reload schema cache + config on NOTIFY
-  Workers.runListener conf appState
+  AppState.runListener conf appState
 
-  Workers.runAdmin conf appState $ serverSettings conf
+  Admin.runAdmin conf appState $ serverSettings conf
 
-  let app = postgrest configLogLevel appState (Workers.connectionWorker appState)
+  let app = postgrest conf appState (AppState.connectionWorker appState)
 
   case configServerUnixSocket of
     Just socket ->
@@ -95,25 +98,25 @@ serverSettings AppConfig{..} =
     & setServerName ("postgrest/" <> prettyVersion)
 
 -- | PostgREST application
-postgrest :: LogLevel -> AppState.AppState -> IO () -> Wai.Application
+postgrest :: AppConfig -> AppState.AppState -> IO () -> Wai.Application
-postgrest logLevel appState connWorker =
+postgrest conf appState connWorker =
+  Response.traceHeaderMiddleware conf .
   Cors.middleware .
   Auth.middleware appState .
-  Logger.middleware logLevel $
+  Logger.middleware (configLogLevel conf) $
     -- fromJust can be used, because the auth middleware will **always** add
     -- some AuthResult to the vault.
     \req respond -> case fromJust $ Auth.getResult req of
       Left err -> respond $ Error.errorResponseFor err
       Right authResult -> do
-        conf <- AppState.getConfig appState
+        appConf <- AppState.getConfig appState -- the config must be read again because it can reload
         maybeSchemaCache <- AppState.getSchemaCache appState
         pgVer <- AppState.getPgVersion appState
-        jsonDbS <- AppState.getJsonDbS appState
 
         let
           eitherResponse :: IO (Either Error Wai.Response)
           eitherResponse =
-            runExceptT $ postgrestResponse appState conf maybeSchemaCache jsonDbS pgVer authResult req
+            runExceptT $ postgrestResponse appState appConf maybeSchemaCache pgVer authResult req
 
         response <- either Error.errorResponseFor identity <$> eitherResponse
         -- Launch the connWorker when the connection is down.  The postgrest
@@ -129,12 +132,11 @@ postgrestResponse
   :: AppState.AppState
   -> AppConfig
   -> Maybe SchemaCache
-  -> ByteString
   -> PgVersion
   -> AuthResult
   -> Wai.Request
   -> Handler IO Wai.Response
-postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jsonDbS pgVer authResult@AuthResult{..} req = do
+postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@AuthResult{..} req = do
   sCache <-
     case maybeSchemaCache of
       Just sCache ->
@@ -146,16 +148,16 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jsonDbS pgVer aut
 
   apiRequest <-
     liftEither . mapLeft Error.ApiRequestError $
-      ApiRequest.userApiRequest conf sCache req body
+      ApiRequest.userApiRequest conf req body
 
   Response.optionalRollback conf apiRequest $
-    handleRequest authResult conf appState (Query.txMode apiRequest) (Just authRole /= configDbAnonRole) configDbPreparedStatements jsonDbS pgVer apiRequest sCache
+    handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache
 
-runDbHandler :: AppState.AppState -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
+runDbHandler :: AppState.AppState -> SQL.IsolationLevel -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
-runDbHandler appState mode authenticated prepared handler = do
+runDbHandler appState isoLvl mode authenticated prepared handler = do
-  dbResp <-
+  dbResp <- lift $ do
-    let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
+    let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction
-    lift . AppState.usePool appState . transaction SQL.ReadCommitted mode $ runExceptT handler
+    AppState.usePool appState . transaction isoLvl mode $ runExceptT handler
 
   resp <-
     liftEither . mapLeft Error.PgErr $
@@ -163,52 +165,62 @@ runDbHandler appState mode authenticated prepared handler = do
 
   liftEither resp
 
-handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> SQL.Mode -> Bool -> Bool -> ByteString -> PgVersion -> ApiRequest -> SchemaCache -> Handler IO Wai.Response
+handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> Bool -> Bool -> PgVersion -> ApiRequest -> SchemaCache -> Handler IO Wai.Response
-handleRequest AuthResult{..} conf appState mode authenticated prepared jsonDbS pgVer apiReq@ApiRequest{..} sCache =
+handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@ApiRequest{..} sCache =
   case (iAction, iTarget) of
     (ActionRead headersOnly, TargetIdent identifier) -> do
-      rPlan <- liftEither $ Plan.readPlan identifier conf sCache apiReq
+      wrPlan <- liftEither $ Plan.wrappedReadPlan identifier conf sCache apiReq
-      resultSet <- runQuery $ Query.readQuery rPlan conf apiReq
+      resultSet <- runQuery roleIsoLvl (Plan.wrTxMode wrPlan) $ Query.readQuery wrPlan conf apiReq
       return $ Response.readResponse headersOnly identifier apiReq resultSet
 
     (ActionMutate MutationCreate, TargetIdent identifier) -> do
       mrPlan <- liftEither $ Plan.mutateReadPlan MutationCreate apiReq identifier conf sCache
-      resultSet <- runQuery $ Query.createQuery mrPlan apiReq conf
+      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.createQuery mrPlan apiReq conf
       return $ Response.createResponse identifier mrPlan apiReq resultSet
 
     (ActionMutate MutationUpdate, TargetIdent identifier) -> do
       mrPlan <- liftEither $ Plan.mutateReadPlan MutationUpdate apiReq identifier conf sCache
-      resultSet <- runQuery $ Query.updateQuery mrPlan apiReq conf
+      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.updateQuery mrPlan apiReq conf
       return $ Response.updateResponse apiReq resultSet
 
     (ActionMutate MutationSingleUpsert, TargetIdent identifier) -> do
       mrPlan <- liftEither $ Plan.mutateReadPlan MutationSingleUpsert apiReq identifier conf sCache
-      resultSet <- runQuery $ Query.singleUpsertQuery mrPlan apiReq conf
+      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.singleUpsertQuery mrPlan apiReq conf
       return $ Response.singleUpsertResponse apiReq resultSet
 
     (ActionMutate MutationDelete, TargetIdent identifier) -> do
       mrPlan <- liftEither $ Plan.mutateReadPlan MutationDelete apiReq identifier conf sCache
-      resultSet <- runQuery $ Query.deleteQuery mrPlan apiReq conf
+      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.deleteQuery mrPlan apiReq conf
       return $ Response.deleteResponse apiReq resultSet
 
-    (ActionInvoke invMethod, TargetProc proc _) -> do
+    (ActionInvoke invMethod, TargetProc identifier _) -> do
-      cPlan <- liftEither $ Plan.callReadPlan proc conf sCache apiReq
+      cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq invMethod
-      resultSet <- runQuery $ Query.invokeQuery proc cPlan apiReq conf
+      resultSet <- runQuery (fromMaybe roleIsoLvl $ pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
-      return $ Response.invokeResponse invMethod proc apiReq resultSet
+      return $ Response.invokeResponse invMethod (Plan.crProc cPlan) apiReq resultSet
 
     (ActionInspect headersOnly, TargetDefaultSpec tSchema) -> do
-      oaiResult <- runQuery $ Query.openApiQuery sCache pgVer conf tSchema
+      oaiResult <- runQuery roleIsoLvl Plan.inspectPlanTxMode $ Query.openApiQuery sCache pgVer conf tSchema
-      return $ Response.openApiResponse headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
+      return $ Response.openApiResponse (T.decodeUtf8 prettyVersion, docsVersion) headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
 
-    (ActionInfo, _) ->
+    (ActionInfo, TargetIdent identifier) ->
-      return $ Response.infoResponse iTarget sCache
+      return $ Response.infoIdentResponse identifier sCache
+
+    (ActionInfo, TargetProc identifier _) -> do
+      cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq ApiRequest.InvHead
+      return $ Response.infoProcResponse (Plan.crProc cPlan)
+
+    (ActionInfo, TargetDefaultSpec _) ->
+      return Response.infoRootResponse
 
     _ ->
       -- This is unreachable as the ApiRequest.hs rejects it before
       -- TODO Refactor the Action/Target types to remove this line
       throwError $ Error.ApiRequestError ApiRequestTypes.NotFound
   where
-    runQuery query =
+    roleSettings = fromMaybe mempty (HM.lookup authRole $ configRoleSettings conf)
-      runDbHandler appState mode authenticated prepared $ do
+    roleIsoLvl = HM.findWithDefault SQL.ReadCommitted authRole $ configRoleIsoLvl conf
-        Query.setPgLocals conf authClaims authRole apiReq jsonDbS pgVer
+    runQuery isoLvl mode query =
+      runDbHandler appState isoLvl mode authenticated prepared $ do
+        Query.setPgLocals conf authClaims authRole (HM.toList roleSettings) apiReq pgVer
+        Query.runPreReq conf
         query

src/PostgREST/AppState.hs

@@ -1,75 +1,92 @@
+{-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 
 module PostgREST.AppState
   ( AppState
   , destroy
-  , flushPool
   , getConfig
   , getSchemaCache
   , getIsListenerOn
-  , getJsonDbS
   , getMainThreadId
   , getPgVersion
   , getRetryNextIn
   , getTime
-  , getWorkerSem
   , init
   , initWithPool
   , logWithZTime
-  , putConfig
   , putSchemaCache
-  , putIsListenerOn
-  , putJsonDbS
   , putPgVersion
-  , putRetryNextIn
-  , signalListener
   , usePool
-  , waitListener
+  , loadSchemaCache
+  , reReadConfig
+  , connectionWorker
+  , runListener
   ) where
 
-import qualified Hasql.Pool    as SQL
+import qualified Data.ByteString.Char8      as BS
-import qualified Hasql.Session as SQL
+import qualified Data.ByteString.Lazy       as LBS
+import           Data.Either.Combinators    (whenLeft)
+import qualified Data.Text.Encoding         as T
+import           Hasql.Connection           (acquire)
+import qualified Hasql.Notifications        as SQL
+import qualified Hasql.Pool                 as SQL
+import qualified Hasql.Session              as SQL
+import qualified Hasql.Transaction.Sessions as SQL
+import qualified PostgREST.Error            as Error
+import           PostgREST.Version          (prettyVersion)
 
 import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
                            updateAction)
+import Control.Debounce
+import Control.Retry      (RetryStatus, capDelay, exponentialBackoff,
+                           retrying, rsPreviousDelay)
 import Data.IORef         (IORef, atomicWriteIORef, newIORef,
                            readIORef)
 import Data.Time          (ZonedTime, defaultTimeLocale, formatTime,
                            getZonedTime)
 import Data.Time.Clock    (UTCTime, getCurrentTime)
 
-import PostgREST.Config           (AppConfig (..))
+import PostgREST.Config                  (AppConfig (..),
-import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
+                                          addFallbackAppName,
-import PostgREST.SchemaCache      (SchemaCache)
+                                          readAppConfig)
+import PostgREST.Config.Database         (queryDbSettings,
+                                          queryPgVersion,
+                                          queryRoleSettings)
+import PostgREST.Config.PgVersion        (PgVersion (..),
+                                          minimumPgVersion)
+import PostgREST.SchemaCache             (SchemaCache,
+                                          querySchemaCache)
+import PostgREST.SchemaCache.Identifiers (dumpQi)
 
 import Protolude
 
 
 data AppState = AppState
   -- | Database connection pool
-  { statePool         :: SQL.Pool
+  { statePool                     :: SQL.Pool
   -- | Database server version, will be updated by the connectionWorker
-  , statePgVersion    :: IORef PgVersion
+  , statePgVersion                :: IORef PgVersion
   -- | No schema cache at the start. Will be filled in by the connectionWorker
-  , stateSchemaCache  :: IORef (Maybe SchemaCache)
+  , stateSchemaCache              :: IORef (Maybe SchemaCache)
-  -- | Cached SchemaCache in json
+  -- | starts the connection worker with a debounce
-  , stateJsonDbS      :: IORef ByteString
+  , debouncedConnectionWorker     :: IO ()
-  -- | Binary semaphore to make sure just one connectionWorker can run at a time
-  , stateWorkerSem    :: MVar ()
   -- | Binary semaphore used to sync the listener(NOTIFY reload) with the connectionWorker.
-  , stateListener     :: MVar ()
+  , stateListener                 :: MVar ()
   -- | State of the LISTEN channel, used for the admin server checks
-  , stateIsListenerOn :: IORef Bool
+  , stateIsListenerOn             :: IORef Bool
   -- | Config that can change at runtime
-  , stateConf         :: IORef AppConfig
+  , stateConf                     :: IORef AppConfig
   -- | Time used for verifying JWT expiration
-  , stateGetTime      :: IO UTCTime
+  , stateGetTime                  :: IO UTCTime
   -- | Time with time zone used for worker logs
-  , stateGetZTime     :: IO ZonedTime
+  , stateGetZTime                 :: IO ZonedTime
   -- | Used for killing the main thread in case a subthread fails
-  , stateMainThreadId :: ThreadId
+  , stateMainThreadId             :: ThreadId
   -- | Keeps track of when the next retry for connecting to database is scheduled
-  , stateRetryNextIn  :: IORef Int
+  , stateRetryNextIn              :: IORef Int
+  -- | Logs a pool error with a debounce
+  , debounceLogAcquisitionTimeout :: IO ()
   }
 
 init :: AppConfig -> IO AppState
@@ -78,12 +95,11 @@ init conf = do
   initWithPool pool conf
 
 initWithPool :: SQL.Pool -> AppConfig -> IO AppState
-initWithPool pool conf =
+initWithPool pool conf = do
-  AppState pool
+  appState <- AppState pool
     <$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step
     <*> newIORef Nothing
-    <*> newIORef mempty
+    <*> pure (pure ())
-    <*> newEmptyMVar
     <*> newEmptyMVar
     <*> newIORef False
     <*> newIORef conf
@@ -91,20 +107,47 @@ initWithPool pool conf =
     <*> mkAutoUpdate defaultUpdateSettings { updateAction = getZonedTime }
     <*> myThreadId
     <*> newIORef 0
+    <*> pure (pure ())
+
+
+  debLogTimeout <-
+    let oneSecond = 1000000 in
+    mkDebounce defaultDebounceSettings
+       { debounceAction = logPgrstError appState SQL.AcquisitionTimeoutUsageError
+       , debounceFreq = 5*oneSecond
+       , debounceEdge = leadingEdge -- logs at the start and the end
+       }
+
+  debWorker <-
+    let decisecond = 100000 in
+    mkDebounce defaultDebounceSettings
+       { debounceAction = internalConnectionWorker appState
+       , debounceFreq = decisecond
+       , debounceEdge = leadingEdge -- runs the worker at the start and the end
+       }
+
+  return appState { debounceLogAcquisitionTimeout = debLogTimeout, debouncedConnectionWorker = debWorker }
 
 destroy :: AppState -> IO ()
 destroy = destroyPool
 
 initPool :: AppConfig -> IO SQL.Pool
 initPool AppConfig{..} =
-  SQL.acquire configDbPoolSize timeoutMilliseconds $ toUtf8 configDbUri
+  SQL.acquire
-  where
+    configDbPoolSize
-    timeoutMilliseconds = (* oneSecond) <$> configDbPoolAcquisitionTimeout
+    (fromIntegral configDbPoolAcquisitionTimeout)
-    oneSecond = 1000000
+    (fromIntegral configDbPoolMaxLifetime)
+    (fromIntegral configDbPoolMaxIdletime)
+    (toUtf8 $ addFallbackAppName prettyVersion configDbUri)
 
 -- | Run an action with a database connection.
 usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a)
-usePool AppState{..} = SQL.use statePool
+usePool AppState{..} x = do
+  res <- SQL.use statePool x
+  whenLeft res (\case
+    SQL.AcquisitionTimeoutUsageError -> debounceLogAcquisitionTimeout -- this can happen rapidly for many requests, so we debounce
+    _                                -> pure ())
+  return res
 
 -- | Flush the connection pool so that any future use of the pool will
 -- use connections freshly established after this call.
@@ -127,14 +170,8 @@ getSchemaCache = readIORef . stateSchemaCache
 putSchemaCache :: AppState -> Maybe SchemaCache -> IO ()
 putSchemaCache appState = atomicWriteIORef (stateSchemaCache appState)
 
-getJsonDbS :: AppState -> IO ByteString
+connectionWorker :: AppState -> IO ()
-getJsonDbS = readIORef . stateJsonDbS
+connectionWorker = debouncedConnectionWorker
-
-putJsonDbS :: AppState -> ByteString -> IO ()
-putJsonDbS appState = atomicWriteIORef (stateJsonDbS appState)
-
-getWorkerSem :: AppState -> MVar ()
-getWorkerSem = stateWorkerSem
 
 getRetryNextIn :: AppState -> IO Int
 getRetryNextIn = readIORef . stateRetryNextIn
@@ -157,6 +194,9 @@ logWithZTime appState txt = do
   zTime <- stateGetZTime appState
   hPutStrLn stderr $ toS (formatTime defaultTimeLocale "%d/%b/%Y:%T %z: " zTime) <> txt
 
+logPgrstError :: AppState -> SQL.UsageError -> IO ()
+logPgrstError appState e = logWithZTime appState . T.decodeUtf8 . LBS.toStrict $ Error.errorPayload $ Error.PgError False e
+
 getMainThreadId :: AppState -> ThreadId
 getMainThreadId = stateMainThreadId
 
@@ -176,3 +216,263 @@ getIsListenerOn = readIORef . stateIsListenerOn
 
 putIsListenerOn :: AppState -> Bool -> IO ()
 putIsListenerOn = atomicWriteIORef . stateIsListenerOn
+
+-- | Schema cache status
+data SCacheStatus
+  = SCLoaded
+  | SCOnRetry
+  | SCFatalFail
+
+-- | Load the SchemaCache by using a connection from the pool.
+loadSchemaCache :: AppState -> IO SCacheStatus
+loadSchemaCache appState = do
+  conf@AppConfig{..} <- getConfig appState
+  result <-
+    let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
+    usePool appState . transaction SQL.ReadCommitted SQL.Read $
+      querySchemaCache conf
+  case result of
+    Left e -> do
+      case checkIsFatal e of
+        Just hint -> do
+          logWithZTime appState "A fatal error ocurred when loading the schema cache"
+          logPgrstError appState e
+          logWithZTime appState hint
+          return SCFatalFail
+        Nothing -> do
+          putSchemaCache appState Nothing
+          logWithZTime appState "An error ocurred when loading the schema cache"
+          logPgrstError appState e
+          return SCOnRetry
+
+    Right sCache -> do
+      putSchemaCache appState (Just sCache)
+      logWithZTime appState "Schema cache loaded"
+      return SCLoaded
+
+-- | Current database connection status data ConnectionStatus
+data ConnectionStatus
+  = NotConnected
+  | Connected PgVersion
+  | FatalConnectionError Text
+  deriving (Eq)
+
+-- | The purpose of this worker is to obtain a healthy connection to pg and an
+-- up-to-date schema cache(SchemaCache).  This method is meant to be called
+-- multiple times by the same thread, but does nothing if the previous
+-- invocation has not terminated. In all cases this method does not halt the
+-- calling thread, the work is performed in a separate thread.
+--
+-- Background thread that does the following :
+--  1. Tries to connect to pg server and will keep trying until success.
+--  2. Checks if the pg version is supported and if it's not it kills the main
+--     program.
+--  3. Obtains the sCache. If this fails, it goes back to 1.
+internalConnectionWorker :: AppState -> IO ()
+internalConnectionWorker appState = work
+  where
+    work = do
+      AppConfig{..} <- getConfig appState
+      logWithZTime appState $ "Starting PostgREST " <> T.decodeUtf8 prettyVersion <> "..."
+      logWithZTime appState "Attempting to connect to the database..."
+      connected <- establishConnection appState
+      case connected of
+        FatalConnectionError reason ->
+          -- Fatal error when connecting
+          logWithZTime appState reason >> killThread (getMainThreadId appState)
+        NotConnected ->
+          -- Unreachable because establishConnection will keep trying to connect
+          return ()
+        Connected actualPgVersion -> do
+          -- Procede with initialization
+          putPgVersion appState actualPgVersion
+          when configDbChannelEnabled $
+            signalListener appState
+          logWithZTime appState "Connection successful"
+          -- this could be fail because the connection drops, but the loadSchemaCache will pick the error and retry again
+          -- We cannot retry after it fails immediately, because db-pre-config could have user errors. We just log the error and continue.
+          when configDbConfig $ reReadConfig False appState
+          scStatus <- loadSchemaCache appState
+          case scStatus of
+            SCLoaded ->
+              -- do nothing and proceed if the load was successful
+              return ()
+            SCOnRetry ->
+              -- retry reloading the schema cache
+              work
+            SCFatalFail ->
+              -- die if our schema cache query has an error
+              killThread $ getMainThreadId appState
+
+-- | Repeatedly flush the pool, and check if a connection from the
+-- pool allows access to the PostgreSQL database.
+--
+-- Releasing the pool is key for rapid recovery. Otherwise, the pool
+-- timeout would have to be reached for new healthy connections to be acquired.
+-- Which might not happen if the server is busy with requests. No idle
+-- connection, no pool timeout.
+--
+-- The connection tries are capped, but if the connection times out no error is
+-- thrown, just 'False' is returned.
+establishConnection :: AppState -> IO ConnectionStatus
+establishConnection appState =
+  retrying retrySettings shouldRetry $
+    const $ flushPool appState >> getConnectionStatus
+  where
+    retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
+    delayMicroseconds = 32000000 -- 32 seconds
+    backoffMicroseconds = 1000000 -- 1 second
+
+    getConnectionStatus :: IO ConnectionStatus
+    getConnectionStatus = do
+      pgVersion <- usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
+      case pgVersion of
+        Left e -> do
+          logPgrstError appState e
+          case checkIsFatal e of
+            Just reason ->
+              return $ FatalConnectionError reason
+            Nothing ->
+              return NotConnected
+        Right version ->
+          if version < minimumPgVersion then
+            return . FatalConnectionError $
+              "Cannot run in this PostgreSQL version, PostgREST needs at least "
+              <> pgvName minimumPgVersion
+          else
+            return . Connected  $ version
+
+    shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
+    shouldRetry rs isConnSucc = do
+      let
+        delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
+        itShould = NotConnected == isConnSucc
+      when itShould . logWithZTime appState $
+        "Attempting to reconnect to the database in "
+        <> (show delay::Text)
+        <> " seconds..."
+      when itShould $ putRetryNextIn appState delay
+      return itShould
+
+-- | Re-reads the config plus config options from the db
+reReadConfig :: Bool -> AppState -> IO ()
+reReadConfig startingUp appState = do
+  AppConfig{..} <- getConfig appState
+  dbSettings <-
+    if configDbConfig then do
+      qDbSettings <- usePool appState $ queryDbSettings (dumpQi <$> configDbPreConfig) configDbPreparedStatements
+      case qDbSettings of
+        Left e -> do
+          logWithZTime appState
+            "An error ocurred when trying to query database settings for the config parameters"
+          case checkIsFatal e of
+            Just hint -> do
+              logPgrstError appState e
+              logWithZTime appState hint
+              killThread (getMainThreadId appState)
+            Nothing -> do
+              logPgrstError appState e
+          pure mempty
+        Right x -> pure x
+    else
+      pure mempty
+  (roleSettings, roleIsolationLvl) <-
+    if configDbConfig then do
+      rSettings <- usePool appState $ queryRoleSettings configDbPreparedStatements
+      case rSettings of
+        Left e -> do
+          logWithZTime appState "An error ocurred when trying to query the role settings"
+          logPgrstError appState e
+          pure (mempty, mempty)
+        Right x -> pure x
+    else
+      pure mempty
+  readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings roleIsolationLvl >>= \case
+    Left err   ->
+      if startingUp then
+        panic err -- die on invalid config if the program is starting up
+      else
+        logWithZTime appState $ "Failed reloading config: " <> err
+    Right newConf -> do
+      putConfig appState newConf
+      if startingUp then
+        pass
+      else
+        logWithZTime appState "Config reloaded"
+
+
+runListener :: AppConfig -> AppState -> IO ()
+runListener AppConfig{configDbChannelEnabled} appState =
+  when configDbChannelEnabled $ listener appState
+
+-- | Starts a dedicated pg connection to LISTEN for notifications.  When a
+-- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
+-- cache.  It uses the connectionWorker in case the LISTEN connection dies.
+listener :: AppState -> IO ()
+listener appState = do
+  AppConfig{..} <- getConfig appState
+  let dbChannel = toS configDbChannel
+
+  -- The listener has to wait for a signal from the connectionWorker.
+  -- This is because when the connection to the db is lost, the listener also
+  -- tries to recover the connection, but not with the same pace as the connectionWorker.
+  -- Not waiting makes stderr quickly fill with connection retries messages from the listener.
+  waitListener appState
+
+  -- forkFinally allows to detect if the thread dies
+  void . flip forkFinally (handleFinally dbChannel) $ do
+    dbOrError <- acquire $ toUtf8 (addFallbackAppName prettyVersion configDbUri)
+    case dbOrError of
+      Right db -> do
+        logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
+        putIsListenerOn appState True
+        SQL.listen db $ SQL.toPgIdentifier dbChannel
+        SQL.waitForNotifications handleNotification db
+      _ ->
+        die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
+  where
+    handleFinally dbChannel _ = do
+      -- if the thread dies, we try to recover
+      logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
+      putIsListenerOn appState False
+      -- assume the pool connection was also lost, call the connection worker
+      connectionWorker appState
+      -- retry the listener
+      listener appState
+
+    handleNotification _ msg
+      | BS.null msg            = cacheReloader
+      | msg == "reload schema" = cacheReloader
+      | msg == "reload config" = reReadConfig False appState
+      | otherwise              = pure () -- Do nothing if anything else than an empty message is sent
+
+    cacheReloader =
+      -- reloads the schema cache + restarts pool connections
+      -- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
+      connectionWorker appState
+
+checkIsFatal :: SQL.UsageError -> Maybe Text
+checkIsFatal (SQL.ConnectionUsageError e)
+  | isAuthFailureMessage = Just $ toS failureMessage
+  | otherwise = Nothing
+  where isAuthFailureMessage =
+          ("FATAL:  password authentication failed" `isInfixOf` failureMessage) ||
+          ("no password supplied" `isInfixOf` failureMessage)
+        failureMessage = BS.unpack $ fromMaybe mempty e
+checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
+  = case serverError of
+      -- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
+      SQL.ServerError "42601" _ _ _ _
+        -> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
+      -- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
+      -- This would mean that a connection pooler in transaction mode is being used
+      -- while prepared statements are enabled in the PostgREST configuration,
+      -- both of which are incompatible with each other.
+      SQL.ServerError "42P05" _ _ _ _
+        -> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
+      -- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
+      -- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
+      SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
+        -> Just "Hint: Connection poolers in statement mode are not supported."
+      _ -> Nothing
+checkIsFatal _ = Nothing

src/PostgREST/Auth.hs

@@ -23,8 +23,8 @@ import qualified Data.Aeson                      as JSON
 import qualified Data.Aeson.Key                  as K
 import qualified Data.Aeson.KeyMap               as KM
 import qualified Data.Aeson.Types                as JSON
+import qualified Data.ByteString                 as BS
 import qualified Data.ByteString.Lazy.Char8      as LBS
-import qualified Data.Text.Encoding              as T
 import qualified Data.Vault.Lazy                 as Vault
 import qualified Data.Vector                     as V
 import qualified Network.HTTP.Types.Header       as HTTP
@@ -47,7 +47,7 @@ import Protolude
 
 data AuthResult = AuthResult
   { authClaims :: KM.KeyMap JSON.Value
-  , authRole   :: Text
+  , authRole   :: BS.ByteString
   }
 
 -- | Receives the JWT secret and audience (from config) and a JWT and returns a
@@ -63,7 +63,7 @@ parseToken AppConfig{..} token time = do
   liftEither . mapLeft jwtClaimsError $ JSON.toJSON <$> eitherClaims
   where
     validation =
-      JWT.defaultJWTValidationSettings audienceCheck & set JWT.allowedSkew 1
+      JWT.defaultJWTValidationSettings audienceCheck & set JWT.allowedSkew 30
 
     audienceCheck :: JWT.StringOrURI -> Bool
     audienceCheck = maybe (const True) (==) configJwtAudience
@@ -79,7 +79,7 @@ parseClaims AppConfig{..} jclaims@(JSON.Object mclaims) = do
   role <- liftEither . maybeToRight JwtTokenRequired $
     unquoted <$> walkJSPath (Just jclaims) configJwtRoleClaimKey <|> configDbAnonRole
   return AuthResult
-           { authClaims = mclaims & KM.insert "role" (JSON.toJSON role)
+           { authClaims = mclaims & KM.insert "role" (JSON.toJSON $ decodeUtf8 role)
            , authRole = role
            }
   where
@@ -89,9 +89,9 @@ parseClaims AppConfig{..} jclaims@(JSON.Object mclaims) = do
     walkJSPath (Just (JSON.Array ar)) (JSPIdx idx:rest) = walkJSPath (ar V.!? idx) rest
     walkJSPath _                      _                 = Nothing
 
-    unquoted :: JSON.Value -> Text
+    unquoted :: JSON.Value -> BS.ByteString
-    unquoted (JSON.String t) = t
+    unquoted (JSON.String t) = encodeUtf8 t
-    unquoted v = T.decodeUtf8 . LBS.toStrict $ JSON.encode v
+    unquoted v               = LBS.toStrict $ JSON.encode v
 -- impossible case - just added to please -Wincomplete-patterns
 parseClaims _ _ = return AuthResult { authClaims = KM.empty, authRole = mempty }
 
@@ -117,5 +117,5 @@ authResultKey = unsafePerformIO Vault.newKey
 getResult :: Wai.Request -> Maybe (Either Error AuthResult)
 getResult = Vault.lookup authResultKey . Wai.vault
 
-getRole :: Wai.Request -> Maybe Text
+getRole :: Wai.Request -> Maybe BS.ByteString
 getRole req = authRole <$> (rightToMaybe =<< getResult req)

src/PostgREST/CLI.hs

@@ -21,7 +21,6 @@ import PostgREST.AppState    (AppState)
 import PostgREST.Config      (AppConfig (..))
 import PostgREST.SchemaCache (querySchemaCache)
 import PostgREST.Version     (prettyVersion)
-import PostgREST.Workers     (reReadConfig)
 
 import qualified PostgREST.App      as App
 import qualified PostgREST.AppState as AppState
@@ -33,7 +32,7 @@ import Protolude hiding (hPutStrLn)
 main :: App.SignalHandlerInstaller -> Maybe App.SocketRunner -> CLI -> IO ()
 main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
   conf@AppConfig{..} <-
-    either panic identity <$> Config.readAppConfig mempty cliPath Nothing
+    either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty mempty
 
   -- Per https://github.com/PostgREST/postgrest/issues/268, we want to
   -- explicitly close the connections to PostgreSQL on shutdown.
@@ -43,7 +42,7 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
     AppState.destroy
     (\appState -> case cliCommand of
       CmdDumpConfig -> do
-        when configDbConfig $ reReadConfig True appState
+        when configDbConfig $ AppState.reReadConfig True appState
         putStr . Config.toText =<< AppState.getConfig appState
       CmdDumpSchema -> putStrLn =<< dumpSchema appState
       CmdRun -> App.run installSignalHandlers runAppWithSocket appState)
@@ -51,15 +50,12 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
 -- | Dump SchemaCache schema to JSON
 dumpSchema :: AppState -> IO LBS.ByteString
 dumpSchema appState = do
-  AppConfig{..} <- AppState.getConfig appState
+  conf@AppConfig{..} <- AppState.getConfig appState
   result <-
     let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
     AppState.usePool appState $
       transaction SQL.ReadCommitted SQL.Read $
-        querySchemaCache
+        querySchemaCache conf
-          (toList configDbSchemas)
-          configDbExtraSearchPath
-          configDbPreparedStatements
   case result of
     Left e -> do
       hPutStrLn stderr $ "An error ocurred when loading the schema cache:\n" <> show e
@@ -84,7 +80,7 @@ readCLIShowHelp =
   where
     prefs = O.prefs $ O.showHelpOnError <> O.showHelpOnEmpty
     opts = O.info parser $ O.fullDesc <> progDesc
-    parser = O.helper <*> exampleParser <*> cliParser
+    parser = O.helper <*> versionFlag <*> exampleParser <*> cliParser
 
     progDesc =
       O.progDesc $
@@ -92,6 +88,12 @@ readCLIShowHelp =
         <> BS.unpack prettyVersion
         <> " / create a REST API to an existing Postgres database"
 
+    versionFlag =
+      O.infoOption ("PostgREST " <> BS.unpack prettyVersion) $
+        O.long "version"
+        <> O.short 'v'
+        <> O.help "Show the version information"
+
     exampleParser =
       O.infoOption exampleConfigFile $
         O.long "example"
@@ -136,6 +138,9 @@ exampleConfigFile =
       |## Enable in-database configuration
       |db-config = true
       |
+      |## Function for in-database configuration
+      |## db-pre-config = "postgrest.pre_config"
+      |
       |## Extra schemas to add to the search_path of every request
       |db-extra-search-path = "public"
       |
@@ -151,6 +156,12 @@ exampleConfigFile =
       |## Time in seconds to wait to acquire a slot from the connection pool
       |# db-pool-acquisition-timeout = 10
       |
+      |## Time in seconds after which to recycle pool connections
+      |# db-pool-max-lifetime = 1800
+      |
+      |## Time in seconds after which to recycle unused pool connections
+      |# db-pool-max-idletime = 30
+      |
       |## Stored proc to exec immediately after auth
       |# db-pre-request = "stored_proc_name"
       |

src/PostgREST/Config.hs

@@ -24,6 +24,7 @@ module PostgREST.Config
   , readPGRSTEnvironment
   , toURI
   , parseSecret
+  , addFallbackAppName
   ) where
 
 import qualified Crypto.JOSE.Types      as JOSE
@@ -32,6 +33,7 @@ import qualified Data.Aeson             as JSON
 import qualified Data.ByteString        as BS
 import qualified Data.ByteString.Base64 as B64
 import qualified Data.ByteString.Lazy   as LBS
+import qualified Data.CaseInsensitive   as CI
 import qualified Data.Configurator      as C
 import qualified Data.Map.Strict        as M
 import qualified Data.Text              as T
@@ -46,10 +48,15 @@ import Data.List               (lookup)
 import Data.List.NonEmpty      (fromList, toList)
 import Data.Maybe              (fromJust)
 import Data.Scientific         (floatingOrInteger)
+import Network.URI             (escapeURIString,
+                                isUnescapedInURIComponent, parseURI,
+                                uriQuery)
 import Numeric                 (readOct, showOct)
 import System.Environment      (getEnvironment)
 import System.Posix.Types      (FileMode)
 
+import PostgREST.Config.Database         (RoleIsolationLvl,
+                                          RoleSettings)
 import PostgREST.Config.JSPath           (JSPath, JSPathExp (..),
                                           dumpJSPath, pRoleClaimKey)
 import PostgREST.Config.Proxy            (Proxy (..),
@@ -63,19 +70,22 @@ import Protolude hiding (Proxy, toList)
 
 data AppConfig = AppConfig
   { configAppSettings              :: [(Text, Text)]
-  , configDbAnonRole               :: Maybe Text
+  , configDbAnonRole               :: Maybe BS.ByteString
   , configDbChannel                :: Text
   , configDbChannelEnabled         :: Bool
   , configDbExtraSearchPath        :: [Text]
   , configDbMaxRows                :: Maybe Integer
   , configDbPlanEnabled            :: Bool
   , configDbPoolSize               :: Int
-  , configDbPoolAcquisitionTimeout :: Maybe Int
+  , configDbPoolAcquisitionTimeout :: Int
+  , configDbPoolMaxLifetime        :: Int
+  , configDbPoolMaxIdletime        :: Int
   , configDbPreRequest             :: Maybe QualifiedIdentifier
   , configDbPreparedStatements     :: Bool
   , configDbRootSpec               :: Maybe QualifiedIdentifier
   , configDbSchemas                :: NonEmpty Text
   , configDbConfig                 :: Bool
+  , configDbPreConfig              :: Maybe QualifiedIdentifier
   , configDbTxAllowOverride        :: Bool
   , configDbTxRollbackAll          :: Bool
   , configDbUri                    :: Text
@@ -93,9 +103,13 @@ data AppConfig = AppConfig
   , configRawMediaTypes            :: [MediaType]
   , configServerHost               :: Text
   , configServerPort               :: Int
+  , configServerTraceHeader        :: Maybe (CI.CI BS.ByteString)
   , configServerUnixSocket         :: Maybe FilePath
   , configServerUnixSocketMode     :: FileMode
   , configAdminServerPort          :: Maybe Int
+  , configRoleSettings             :: RoleSettings
+  , configRoleIsoLvl               :: RoleIsolationLvl
+  , configInternalSCSleep          :: Maybe Int32
   }
 
 data LogLevel = LogCrit | LogError | LogWarn | LogInfo
@@ -123,19 +137,22 @@ toText conf =
   where
     -- apply conf to all pgrst settings
     pgrstSettings = (\(k, v) -> (k, v conf)) <$>
-      [("db-anon-role",              q . fromMaybe "" . configDbAnonRole)
+      [("db-anon-role",              q . T.decodeUtf8 . fromMaybe "" . configDbAnonRole)
       ,("db-channel",                q . configDbChannel)
       ,("db-channel-enabled",            T.toLower . show . configDbChannelEnabled)
       ,("db-extra-search-path",      q . T.intercalate "," . configDbExtraSearchPath)
       ,("db-max-rows",                   maybe "\"\"" show . configDbMaxRows)
       ,("db-plan-enabled",               T.toLower . show . configDbPlanEnabled)
       ,("db-pool",                       show . configDbPoolSize)
-      ,("db-pool-acquisition-timeout",   maybe "\"\"" show . configDbPoolAcquisitionTimeout)
+      ,("db-pool-acquisition-timeout",   show . configDbPoolAcquisitionTimeout)
+      ,("db-pool-max-lifetime",          show . configDbPoolMaxLifetime)
+      ,("db-pool-max-idletime",          show . configDbPoolMaxIdletime)
       ,("db-pre-request",            q . maybe mempty dumpQi . configDbPreRequest)
       ,("db-prepared-statements",        T.toLower . show . configDbPreparedStatements)
       ,("db-root-spec",              q . maybe mempty dumpQi . configDbRootSpec)
       ,("db-schemas",                q . T.intercalate "," . toList . configDbSchemas)
       ,("db-config",                     T.toLower . show . configDbConfig)
+      ,("db-pre-config",             q . maybe mempty dumpQi . configDbPreConfig)
       ,("db-tx-end",                 q . showTxEnd)
       ,("db-uri",                    q . configDbUri)
       ,("db-use-legacy-gucs",            T.toLower . show . configDbUseLegacyGucs)
@@ -150,6 +167,7 @@ toText conf =
       ,("raw-media-types",           q . T.decodeUtf8 . BS.intercalate "," . fmap toMime . configRawMediaTypes)
       ,("server-host",               q . configServerHost)
       ,("server-port",                   show . configServerPort)
+      ,("server-trace-header",       q . T.decodeUtf8 . maybe mempty CI.original . configServerTraceHeader)
       ,("server-unix-socket",        q . maybe mempty T.pack . configServerUnixSocket)
       ,("server-unix-socket-mode",   q . T.pack . showSocketMode)
       ,("admin-server-port",             maybe "\"\"" show . configAdminServerPort)
@@ -186,13 +204,13 @@ instance JustIfMaybe a (Maybe a) where
 
 -- | Reads and parses the config and overrides its parameters from env vars,
 -- files or db settings.
-readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> IO (Either Text AppConfig)
+readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> RoleIsolationLvl -> IO (Either Text AppConfig)
-readAppConfig dbSettings optPath prevDbUri = do
+readAppConfig dbSettings optPath prevDbUri roleSettings roleIsolationLvl = do
   env <- readPGRSTEnvironment
   -- if no filename provided, start with an empty map to read config from environment
   conf <- maybe (return $ Right M.empty) loadConfig optPath
 
-  case C.runParser (parser optPath env dbSettings) =<< mapLeft show conf of
+  case C.runParser (parser optPath env dbSettings roleSettings roleIsolationLvl) =<< mapLeft show conf of
     Left err ->
       return . Left $ "Error in config " <> err
     Right parsedConfig ->
@@ -207,11 +225,11 @@ readAppConfig dbSettings optPath prevDbUri = do
       decodeJWKS <$>
         (decodeSecret =<< readSecretFile =<< readDbUriFile prevDbUri parsedConfig)
 
-parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> C.Parser C.Config AppConfig
+parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> RoleIsolationLvl -> C.Parser C.Config AppConfig
-parser optPath env dbSettings =
+parser optPath env dbSettings roleSettings roleIsolationLvl =
   AppConfig
     <$> parseAppSettings "app.settings"
-    <*> optString "db-anon-role"
+    <*> (fmap encodeUtf8 <$> optString "db-anon-role")
     <*> (fromMaybe "pgrst" <$> optString "db-channel")
     <*> (fromMaybe True <$> optBool "db-channel-enabled")
     <*> (maybe ["public"] splitOnCommas <$> optValue "db-extra-search-path")
@@ -219,7 +237,10 @@ parser optPath env dbSettings =
                      (optInt "max-rows")
     <*> (fromMaybe False <$> optBool "db-plan-enabled")
     <*> (fromMaybe 10 <$> optInt "db-pool")
-    <*> optInt "db-pool-acquisition-timeout"
+    <*> (fromMaybe 10 <$> optInt "db-pool-acquisition-timeout")
+    <*> (fromMaybe 1800 <$> optInt "db-pool-max-lifetime")
+    <*> (fromMaybe 30 <$> optWithAlias (optInt "db-pool-timeout")
+                                       (optInt "db-pool-max-idletime"))
     <*> (fmap toQi <$> optWithAlias (optString "db-pre-request")
                                     (optString "pre-request"))
     <*> (fromMaybe True <$> optBool "db-prepared-statements")
@@ -228,6 +249,7 @@ parser optPath env dbSettings =
     <*> (fromList . maybe ["public"] splitOnCommas <$> optWithAlias (optValue "db-schemas")
                                                                     (optValue "db-schema"))
     <*> (fromMaybe True <$> optBool "db-config")
+    <*> (fmap toQi <$> optString "db-pre-config")
     <*> parseTxEnd "db-tx-end" snd
     <*> parseTxEnd "db-tx-end" fst
     <*> (fromMaybe "postgresql://" <$> optString "db-uri")
@@ -247,9 +269,13 @@ parser optPath env dbSettings =
     <*> (maybe [] (fmap (MTOther . encodeUtf8) . splitOnCommas) <$> optValue "raw-media-types")
     <*> (fromMaybe "!4" <$> optString "server-host")
     <*> (fromMaybe 3000 <$> optInt "server-port")
+    <*> (fmap (CI.mk . encodeUtf8) <$> optString "server-trace-header")
     <*> (fmap T.unpack <$> optString "server-unix-socket")
     <*> parseSocketFileMode "server-unix-socket-mode"
     <*> optInt "admin-server-port"
+    <*> pure roleSettings
+    <*> pure roleIsolationLvl
+    <*> optInt "internal-schema-cache-sleep"
   where
     parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
     parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
@@ -344,20 +370,14 @@ parser optPath env dbSettings =
                                (C.Key -> C.Parser C.Value a -> C.Parser C.Config b) ->
                                C.Key -> (C.Value -> a) -> C.Parser C.Config b
     overrideFromDbOrEnvironment necessity key coercion =
-      case reloadableDbSetting <|> M.lookup envVarName env of
+      case dbConf <|> M.lookup envVarName env of
         Just dbOrEnvVal -> pure $ justIfMaybe $ coercion $ C.String dbOrEnvVal
-        Nothing  -> necessity key (coercion <$> C.value)
+        Nothing         -> necessity key (coercion <$> C.value)
       where
         dashToUnderscore '-' = '_'
         dashToUnderscore c   = c
         envVarName = "PGRST_" <> (toUpper . dashToUnderscore <$> toS key)
-        reloadableDbSetting =
+        dbConf = lookup (T.pack $ dashToUnderscore <$> toS key) dbSettings
-          let dbSettingName = T.pack $ dashToUnderscore <$> toS key in
-          if dbSettingName `notElem` [
-            "server_host", "server_port", "server_unix_socket", "server_unix_socket_mode", "admin_server_port", "log_level",
-            "db_uri", "db_channel_enabled", "db_channel", "db_pool", "db_pool_acquisition_timeout", "db_config"]
-          then lookup dbSettingName dbSettings
-          else Nothing
 
     coerceText :: C.Value -> Text
     coerceText (C.String s) = s
@@ -444,3 +464,41 @@ type Environment = M.Map [Char] Text
 readPGRSTEnvironment :: IO Environment
 readPGRSTEnvironment =
   M.map T.pack . M.fromList . filter (isPrefixOf "PGRST_" . fst) <$> getEnvironment
+
+-- | Adds a `fallback_application_name` value to the connection string. This allows querying the PostgREST version on pg_stat_activity.
+--
+-- >>> let ver = "11.1.0 (5a04ec7)"::ByteString
+-- >>> let strangeVer = "11'1&0@#$%,.:\"[]{}?+^()=asdfqwer"::ByteString
+--
+-- >>> addFallbackAppName ver "postgres://user:pass@host:5432/postgres"
+-- "postgres://user:pass@host:5432/postgres?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgres://user:pass@host:5432/postgres?"
+-- "postgres://user:pass@host:5432/postgres?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgres:///postgres?host=server&port=5432"
+-- "postgres:///postgres?host=server&port=5432&fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "host=localhost port=5432 dbname=postgres"
+-- "host=localhost port=5432 dbname=postgres fallback_application_name='PostgREST 11.1.0 (5a04ec7)'"
+--
+-- >>> addFallbackAppName ver "postgresql://"
+-- "postgresql://?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName strangeVer "host=localhost port=5432 dbname=postgres"
+-- "host=localhost port=5432 dbname=postgres fallback_application_name='PostgREST 11\\'1&0@#$%,.:\"[]{}?+^()=asdfqwer'"
+--
+-- >>> addFallbackAppName strangeVer "postgres:///postgres?host=server&port=5432"
+-- "postgres:///postgres?host=server&port=5432&fallback_application_name=PostgREST%2011%271%260%40%23%24%25%2C.%3A%22%5B%5D%7B%7D%3F%2B%5E%28%29%3Dasdfqwer"
+addFallbackAppName :: ByteString -> Text -> Text
+addFallbackAppName version dbUri = dbUri <>
+  case uriQuery <$> parseURI (toS dbUri) of
+    Nothing  -> " " <> keyValFmt -- Assume key/value connection string if the uri is not valid
+    Just ""  -> "?" <> uriFmt
+    Just "?" -> uriFmt
+    _        -> "&" <> uriFmt
+  where
+    uriFmt = pKeyWord <> toS (escapeURIString isUnescapedInURIComponent $ toS pgrstVer)
+    keyValFmt = pKeyWord <> "'" <> T.replace "'" "\\'" pgrstVer <> "'"
+    pKeyWord = "fallback_application_name="
+    pgrstVer = "PostgREST " <> T.decodeUtf8 version

src/PostgREST/Config/Database.hs

@@ -3,11 +3,19 @@
 module PostgREST.Config.Database
   ( pgVersionStatement
   , queryDbSettings
+  , queryRoleSettings
   , queryPgVersion
+  , RoleSettings
+  , RoleIsolationLvl
+  , toIsolationLevel
   ) where
 
+import Control.Arrow ((***))
+
 import PostgREST.Config.PgVersion (PgVersion (..))
 
+import qualified Data.HashMap.Strict as HM
+
 import qualified Hasql.Decoders             as HD
 import qualified Hasql.Encoders             as HE
 import           Hasql.Session              (Session, statement)
@@ -15,51 +23,174 @@ import qualified Hasql.Statement            as SQL
 import qualified Hasql.Transaction          as SQL
 import qualified Hasql.Transaction.Sessions as SQL
 
-import Text.InterpolatedString.Perl6 (q)
+import Text.InterpolatedString.Perl6 (q, qc)
 
 import Protolude
 
-queryPgVersion :: Session PgVersion
+type RoleSettings     = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
-queryPgVersion = statement mempty pgVersionStatement
+type RoleIsolationLvl = HM.HashMap ByteString SQL.IsolationLevel
 
-pgVersionStatement :: SQL.Statement () PgVersion
+toIsolationLevel :: (Eq a, IsString a) => a -> SQL.IsolationLevel
-pgVersionStatement = SQL.Statement sql HE.noParams versionRow False
+toIsolationLevel a = case a of
+  "repeatable read" -> SQL.RepeatableRead
+  "serializable"    -> SQL.Serializable
+  _                 -> SQL.ReadCommitted
+
+prefix :: Text
+prefix = "pgrst."
+
+-- | In-db settings names
+dbSettingsNames :: [Text]
+dbSettingsNames =
+  (prefix <>) <$>
+  ["db_anon_role"
+  ,"db_pre_config"
+  ,"db_extra_search_path"
+  ,"db_max_rows"
+  ,"db_plan_enabled"
+  ,"db_pre_request"
+  ,"db_prepared_statements"
+  ,"db_root_spec"
+  ,"db_schemas"
+  ,"db_tx_end"
+  ,"db_use_legacy_gucs"
+  ,"jwt_aud"
+  ,"jwt_role_claim_key"
+  ,"jwt_secret"
+  ,"jwt_secret_is_base64"
+  ,"openapi_mode"
+  ,"openapi_security_active"
+  ,"openapi_server_proxy_uri"
+  ,"raw_media_types"
+  ,"server_trace_header"
+  ]
+
+queryPgVersion :: Bool -> Session PgVersion
+queryPgVersion prepared = statement mempty $ pgVersionStatement prepared
+
+pgVersionStatement :: Bool -> SQL.Statement () PgVersion
+pgVersionStatement = SQL.Statement sql HE.noParams versionRow
   where
     sql = "SELECT current_setting('server_version_num')::integer, current_setting('server_version')"
     versionRow = HD.singleRow $ PgVersion <$> column HD.int4 <*> column HD.text
 
-queryDbSettings :: Bool -> Session [(Text, Text)]
+-- | Query the in-database configuration. The settings have the following priorities:
-queryDbSettings prepared =
+--
+-- 1. Role + with database-specific settings:
+--    ALTER ROLE authenticator IN DATABASE postgres SET <prefix>jwt_aud = 'val';
+-- 2. Role + with settings:
+--    ALTER ROLE authenticator SET <prefix>jwt_aud = 'overridden';
+-- 3. pre-config function:
+--    CREATE FUNCTION pre_config() .. PERFORM set_config(<prefix>jwt_aud, 'pre_config_aud'..)
+--
+-- The example above will result in <prefix>jwt_aud = 'val'
+-- A setting on the database only will have no effect: ALTER DATABASE postgres SET <prefix>jwt_aud = 'xx'
+queryDbSettings :: Maybe Text -> Bool -> Session [(Text, Text)]
+queryDbSettings preConfFunc prepared =
   let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
-  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty dbSettingsStatement
+  transaction SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared
-
--- | Get db settings from the connection role. Global settings will be overridden by database specific settings.
-dbSettingsStatement :: SQL.Statement () [(Text, Text)]
-dbSettingsStatement = SQL.Statement sql HE.noParams decodeSettings False
   where
-    sql = [q|
+    sql = [qc|
       WITH
-      role_setting (database, setting) AS (
+      role_setting AS (
-        SELECT setdatabase,
+        SELECT setdatabase as database,
-               unnest(setconfig)
+               unnest(setconfig) as setting
-          FROM pg_catalog.pg_db_role_setting
+        FROM pg_catalog.pg_db_role_setting
-         WHERE setrole = CURRENT_USER::regrole::oid
+        WHERE setrole = CURRENT_USER::regrole::oid
-           AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
+          AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
       ),
-      kv_settings (database, k, v) AS (
+      kv_settings AS (
         SELECT database,
-               substr(setting, 1, strpos(setting, '=') - 1),
+               substr(setting, 1, strpos(setting, '=') - 1) as k,
-               substr(setting, strpos(setting, '=') + 1)
+               substr(setting, strpos(setting, '=') + 1) as v
-          FROM role_setting
+        FROM role_setting
-         WHERE setting LIKE 'pgrst.%'
+        {preConfigF}
       )
       SELECT DISTINCT ON (key)
-             replace(k, 'pgrst.', '') AS key,
+             replace(k, '{prefix}', '') AS key,
              v AS value
-        FROM kv_settings
+      FROM kv_settings
-       ORDER BY key, database DESC;
+      WHERE k = ANY($1) AND v IS NOT NULL
+      ORDER BY key, database DESC NULLS LAST;
     |]
+    preConfigF = case preConfFunc of
+      Nothing   -> mempty
+      Just func -> [qc|
+          UNION
+          SELECT
+            null as database,
+            x as k,
+            current_setting(x, true) as v
+          FROM unnest($1) x
+          JOIN {func}() _ ON TRUE
+      |]::Text
     decodeSettings = HD.rowList $ (,) <$> column HD.text <*> column HD.text
 
+queryRoleSettings :: Bool -> Session (RoleSettings, RoleIsolationLvl)
+queryRoleSettings prepared =
+  let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
+  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared
+  where
+    sql = [q|
+      with
+      role_setting as (
+        select r.rolname, unnest(r.rolconfig) as setting
+        from pg_auth_members m
+        join pg_roles r on r.oid = m.roleid
+        where member = current_user::regrole::oid
+      ),
+      kv_settings AS (
+        SELECT
+          rolname,
+          substr(setting, 1, strpos(setting, '=') - 1) as key,
+          lower(substr(setting, strpos(setting, '=') + 1)) as value
+        FROM role_setting
+      ),
+      iso_setting AS (
+        SELECT rolname, value
+        FROM kv_settings
+        WHERE key = 'default_transaction_isolation'
+      )
+      select
+        kv.rolname,
+        i.value as iso_lvl,
+        array_agg(row(kv.key, kv.value)) filter (where key <> 'default_transation_isolation') as role_settings
+      from kv_settings kv
+      join pg_settings ps on ps.name = kv.key and ps.context = 'user'
+      left join iso_setting i on i.rolname = kv.rolname
+      group by kv.rolname, i.value;
+    |]
+
+    processRows :: [(Text, Maybe Text, [(Text, Text)])] -> (RoleSettings, RoleIsolationLvl)
+    processRows rs =
+      let
+        rowsWRoleSettings = [ (x, z) | (x, _, z) <- rs ]
+        rowsWIsolation    = [ (x, y) | (x, Just y, _) <- rs ]
+      in
+      ( HM.fromList $ bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>)) <$> rowsWRoleSettings
+      , HM.fromList $ (encodeUtf8 *** toIsolationLevel) <$> rowsWIsolation
+      )
+
+    rows :: HD.Result [(Text, Maybe Text, [(Text, Text)])]
+    rows = HD.rowList $ (,,) <$> column HD.text <*> nullableColumn HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
+
 column :: HD.Value a -> HD.Row a
 column = HD.column . HD.nonNullable
+
+nullableColumn :: HD.Value a -> HD.Row (Maybe a)
+nullableColumn = HD.column . HD.nullable
+
+compositeField :: HD.Value a -> HD.Composite a
+compositeField = HD.field . HD.nonNullable
+
+compositeArrayColumn :: HD.Composite a -> HD.Row [a]
+compositeArrayColumn = arrayColumn . HD.composite
+
+arrayColumn :: HD.Value a -> HD.Row [a]
+arrayColumn = column . HD.listArray . HD.nonNullable
+
+param :: HE.Value a -> HE.Params a
+param = HE.param . HE.nonNullable
+
+arrayParam :: HE.Value a -> HE.Params [a]
+arrayParam = param . HE.foldableArray . HE.nonNullable

src/PostgREST/Error.hs

@@ -11,12 +11,13 @@ module PostgREST.Error
   , PgError(..)
   , Error(..)
   , errorPayload
-  , checkIsFatal
   , singularityError
   ) where
 
 import qualified Data.Aeson                as JSON
 import qualified Data.ByteString.Char8     as BS
+import qualified Data.FuzzySet             as Fuzzy
+import qualified Data.HashMap.Strict       as HM
 import qualified Data.Text                 as T
 import qualified Data.Text.Encoding        as T
 import qualified Data.Text.Encoding.Error  as T
@@ -35,12 +36,14 @@ import           PostgREST.ApiRequest.Types (ApiRequestError (..),
 import           PostgREST.MediaType        (MediaType (..))
 import qualified PostgREST.MediaType        as MediaType
 
-import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
+import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..),
-import PostgREST.SchemaCache.Proc         (ProcDescription (..),
+                                           Schema)
-                                           ProcParam (..))
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                            Junction (..),
-                                           Relationship (..))
+                                           Relationship (..),
+                                           RelationshipsMap)
+import PostgREST.SchemaCache.Routine      (Routine (..),
+                                           RoutineParam (..))
 import Protolude
 
 
@@ -64,15 +67,19 @@ instance PgrstError ApiRequestError where
   status InvalidRpcMethod{}      = HTTP.status405
   status InvalidRange{}          = HTTP.status416
   status NotFound                = HTTP.status404
+
   status NoRelBetween{}          = HTTP.status400
   status NoRpc{}                 = HTTP.status404
   status NotEmbedded{}           = HTTP.status400
-  status ParseRequestError{}     = HTTP.status400
+  status PutLimitNotAllowedError = HTTP.status400
-  status PutRangeNotAllowedError = HTTP.status400
   status QueryParamError{}       = HTTP.status400
+  status RelatedOrderNotToOne{}  = HTTP.status400
+  status SpreadNotToOne{}        = HTTP.status400
+  status UnacceptableFilter{}    = HTTP.status400
   status UnacceptableSchema{}    = HTTP.status406
   status UnsupportedMethod{}     = HTTP.status405
   status LimitNoOrderError       = HTTP.status400
+  status ColumnNotFound{}        = HTTP.status400
 
   headers _ = [MediaType.toContentType MTApplicationJSON]
 
@@ -100,11 +107,6 @@ instance JSON.ToJSON ApiRequestError where
                    LowerGTUpper            -> "The lower boundary must be lower than or equal to the upper boundary in the Range header."
                    OutOfBounds lower total -> "An offset of " <> lower <> " was requested, but there are only " <> total <> " rows."),
     "hint"    .= JSON.Null]
-  toJSON (ParseRequestError message details) = JSON.object [
-    "code"    .= ApiRequestErrorCode04,
-    "message" .= message,
-    "details" .= details,
-    "hint"    .= JSON.Null]
   toJSON InvalidFilters = JSON.object [
     "code"    .= ApiRequestErrorCode05,
     "message" .= ("Filters must include all and only primary key columns with 'eq' operators" :: Text),
@@ -123,7 +125,7 @@ instance JSON.ToJSON ApiRequestError where
   toJSON NotFound = JSON.object []
   toJSON (NotEmbedded resource) = JSON.object [
     "code"    .= ApiRequestErrorCode08,
-    "message" .= ("Cannot apply filter because '" <> resource <> "' is not an embedded resource in this request" :: Text),
+    "message" .= ("'" <> resource <> "' is not an embedded resource in this request" :: Text),
     "details" .= JSON.Null,
     "hint"    .= ("Verify that '" <> resource <> "' is included in the 'select' query parameter." :: Text)]
 
@@ -139,9 +141,9 @@ instance JSON.ToJSON ApiRequestError where
     "details" .= JSON.Null,
     "hint"    .= JSON.Null]
 
-  toJSON PutRangeNotAllowedError = JSON.object [
+  toJSON PutLimitNotAllowedError = JSON.object [
     "code"    .= ApiRequestErrorCode14,
-    "message" .= ("Range header and limit/offset querystring parameters are not allowed for PUT" :: Text),
+    "message" .= ("limit/offset querystring parameters are not allowed for PUT" :: Text),
     "details" .= JSON.Null,
     "hint"    .= JSON.Null]
 
@@ -151,35 +153,165 @@ instance JSON.ToJSON ApiRequestError where
     "details" .= JSON.Null,
     "hint"    .= JSON.Null]
 
-  toJSON (NoRelBetween parent child schema) = JSON.object [
+  toJSON (RelatedOrderNotToOne origin target) = JSON.object [
+    "code"    .= ApiRequestErrorCode18,
+    "message" .= ("A related order on '" <> target <> "' is not possible" :: Text),
+    "details" .= ("'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship" :: Text),
+    "hint"    .= JSON.Null]
+
+  toJSON (SpreadNotToOne origin target) = JSON.object [
+    "code"    .= ApiRequestErrorCode19,
+    "message" .= ("A spread operation on '" <> target <> "' is not possible" :: Text),
+    "details" .= ("'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship" :: Text),
+    "hint"    .= JSON.Null]
+
+  toJSON (UnacceptableFilter target) = JSON.object [
+    "code"    .= ApiRequestErrorCode20,
+    "message" .= ("Bad operator on the '" <> target <> "' embedded resource":: Text),
+    "details" .= ("Only is null or not is null filters are allowed on embedded resources":: Text),
+    "hint"    .= JSON.Null]
+
+  toJSON (NoRelBetween parent child embedHint schema allRels) = JSON.object [
     "code"    .= SchemaCacheErrorCode00,
     "message" .= ("Could not find a relationship between '" <> parent <> "' and '" <> child <> "' in the schema cache" :: Text),
-    "details" .= JSON.Null,
+    "details" .= ("Searched for a foreign key relationship between '" <> parent <> "' and '" <> child <> maybe mempty ("' using the hint '" <>) embedHint <> "' in the schema '" <> schema <> "', but no matches were found."),
-    "hint"    .= ("Verify that '" <> parent <> "' and '" <> child <> "' exist in the schema '" <> schema <> "' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache." :: Text)]
+    "hint"    .= noRelBetweenHint parent child schema allRels]
+
   toJSON (AmbiguousRelBetween parent child rels) = JSON.object [
     "code"    .= SchemaCacheErrorCode01,
     "message" .= ("Could not embed because more than one relationship was found for '" <> parent <> "' and '" <> child <> "'" :: Text),
     "details" .= (compressedRel <$> rels),
     "hint"    .= ("Try changing '" <> child <> "' to one of the following: " <> relHint rels <> ". Find the desired relationship in the 'details' key." :: Text)]
-  toJSON (NoRpc schema procName argumentKeys hasPreferSingleObject contentType isInvPost)  =
+  toJSON (NoRpc schema procName argumentKeys hasPreferSingleObject contentType isInvPost allProcs overloadedProcs)  =
-    let prms = "(" <> T.intercalate ", " argumentKeys <> ")" in JSON.object [
+    let func = schema <> "." <> procName
+        prms = T.intercalate ", " argumentKeys
+        prmsMsg = "(" <> prms <> ")"
+        prmsDet = " with parameter" <> (if length argumentKeys > 1 then "s " else " ") <> prms
+        fmtPrms p = if null argumentKeys then " without parameters" else p
+        onlySingleParams = hasPreferSingleObject || (isInvPost && contentType `elem` [MTTextPlain, MTTextXML, MTOctetStream])
+    in JSON.object [
     "code"    .= SchemaCacheErrorCode02,
-    "message" .= ("Could not find the " <> schema <> "." <> procName <>
+    "message" .= ("Could not find the function " <> func <> (if onlySingleParams then "" else fmtPrms prmsMsg) <> " in the schema cache"),
+    "details" .= ("Searched for the function " <> func <>
       (case (hasPreferSingleObject, isInvPost, contentType) of
-        (True, _, _)                 -> " function with a single json or jsonb parameter"
+        (True, _, _)                 -> " with a single json/jsonb parameter"
-        (_, True, MTTextPlain)       -> " function with a single unnamed text parameter"
+        (_, True, MTTextPlain)       -> " with a single unnamed text parameter"
-        (_, True, MTTextXML)         -> " function with a single unnamed xml parameter"
+        (_, True, MTTextXML)         -> " with a single unnamed xml parameter"
-        (_, True, MTOctetStream)     -> " function with a single unnamed bytea parameter"
+        (_, True, MTOctetStream)     -> " with a single unnamed bytea parameter"
-        (_, True, MTApplicationJSON) -> prms <> " function or the " <> schema <> "." <> procName <>" function with a single unnamed json or jsonb parameter"
+        (_, True, MTApplicationJSON) -> fmtPrms prmsDet <> " or with a single unnamed json/jsonb parameter"
-        _                            -> prms <> " function") <>
+        _                            -> fmtPrms prmsDet) <>
-      " in the schema cache"),
+      ", but no matches were found in the schema cache."),
-    "details" .= JSON.Null,
+    -- The hint will be null in the case of single unnamed parameter functions
-    "hint"    .= ("If a new function was created in the database with this name and parameters, try reloading the schema cache." :: Text)]
+    "hint"    .= if onlySingleParams
+                 then Nothing
+                 else noRpcHint schema procName argumentKeys allProcs overloadedProcs ]
   toJSON (AmbiguousRpc procs)  = JSON.object [
     "code"    .= SchemaCacheErrorCode03,
     "message" .= ("Could not choose the best candidate function between: " <> T.intercalate ", " [pdSchema p <> "." <> pdName p <> "(" <> T.intercalate ", " [ppName a <> " => " <> ppType a | a <- pdParams p] <> ")" | p <- procs]),
     "details" .= JSON.Null,
     "hint"    .= ("Try renaming the parameters or the function itself in the database so function overloading can be resolved" :: Text)]
+  toJSON (ColumnNotFound relName colName) = JSON.object [
+    "code"    .= SchemaCacheErrorCode04,
+    "message" .= ("Column '" <> colName <> "' of relation '" <> relName <> "' does not exist" :: Text),
+    "details" .= JSON.Null,
+    "hint"    .= JSON.Null]
+
+-- |
+-- If no relationship is found then:
+--
+-- Looks for parent suggestions if parent not found
+-- Looks for child suggestions if parent is found but child is not
+-- Gives no suggestions if both are found (it means that there is a problem with the embed hint)
+--
+-- >>> :set -Wno-missing-fields
+-- >>> let qi t = QualifiedIdentifier "api" t
+-- >>> let rel ft = Relationship{relForeignTable = qi ft}
+-- >>> let rels = HM.fromList [((qi "films", "api"), [rel "directors", rel "roles", rel "actors"])]
+--
+-- >>> noRelBetweenHint "film" "directors" "api" rels
+-- Just "Perhaps you meant 'films' instead of 'film'."
+--
+-- >>> noRelBetweenHint "films" "role" "api" rels
+-- Just "Perhaps you meant 'roles' instead of 'role'."
+--
+-- >>> noRelBetweenHint "films" "role" "api" rels
+-- Just "Perhaps you meant 'roles' instead of 'role'."
+--
+-- >>> noRelBetweenHint "films" "actors" "api" rels
+-- Nothing
+--
+-- >>> noRelBetweenHint "noclosealternative" "roles" "api" rels
+-- Nothing
+--
+-- >>> noRelBetweenHint "films" "noclosealternative" "api" rels
+-- Nothing
+--
+-- >>> noRelBetweenHint "films" "noclosealternative" "noclosealternative" rels
+-- Nothing
+--
+noRelBetweenHint :: Text -> Text -> Schema -> RelationshipsMap -> Maybe Text
+noRelBetweenHint parent child schema allRels = ("Perhaps you meant '" <>) <$>
+  if isJust findParent
+    then (<> "' instead of '" <> child <> "'.") <$> suggestChild
+    else (<> "' instead of '" <> parent <> "'.") <$> suggestParent
+  where
+    findParent = HM.lookup (QualifiedIdentifier schema parent, schema) allRels
+    fuzzySetOfParents  = Fuzzy.fromList [qiName (fst p) | p <- HM.keys allRels, snd p == schema]
+    fuzzySetOfChildren = Fuzzy.fromList [qiName (relForeignTable c) | c <- fromMaybe [] findParent]
+    suggestParent = Fuzzy.getOne fuzzySetOfParents parent
+    -- Do not give suggestion if the child is found in the relations (weight = 1.0)
+    suggestChild  = headMay [snd k | k <- Fuzzy.get fuzzySetOfChildren child, fst k < 1.0]
+
+-- |
+-- If no function is found with the given name, it does a fuzzy search to all the functions
+-- in the same schema and shows the best match as hint.
+--
+-- >>> :set -Wno-missing-fields
+-- >>> let procs = [(QualifiedIdentifier "api" "test"), (QualifiedIdentifier "api" "another"), (QualifiedIdentifier "private" "other")]
+--
+-- >>> noRpcHint "api" "testt" ["val", "param", "name"] procs []
+-- Just "Perhaps you meant to call the function api.test"
+--
+-- >>> noRpcHint "api" "other" [] procs []
+-- Just "Perhaps you meant to call the function api.another"
+--
+-- >>> noRpcHint "api" "noclosealternative" [] procs []
+-- Nothing
+--
+-- If a function is found with the given name, but no params match, then it does a fuzzy search
+-- to all the overloaded functions' params using the form "param1, param2, param3, ..."
+-- and shows the best match as hint.
+--
+-- >>> let procsDesc = [Function {pdParams = [RoutineParam {ppName="val"}, RoutineParam {ppName="param"}, RoutineParam {ppName="name"}]}, Function {pdParams = [RoutineParam {ppName="id"}, RoutineParam {ppName="attr"}]}]
+--
+-- >>> noRpcHint "api" "test" ["vall", "pqaram", "nam"] procs procsDesc
+-- Just "Perhaps you meant to call the function api.test(name, param, val)"
+--
+-- >>> noRpcHint "api" "test" ["val", "param"] procs procsDesc
+-- Just "Perhaps you meant to call the function api.test(name, param, val)"
+--
+-- >>> noRpcHint "api" "test" ["id", "attrs"] procs procsDesc
+-- Just "Perhaps you meant to call the function api.test(attr, id)"
+--
+-- >>> noRpcHint "api" "test" ["id"] procs procsDesc
+-- Just "Perhaps you meant to call the function api.test(attr, id)"
+--
+-- >>> noRpcHint "api" "test" ["noclosealternative"] procs procsDesc
+-- Nothing
+--
+noRpcHint :: Text -> Text -> [Text] -> [QualifiedIdentifier] -> [Routine] -> Maybe Text
+noRpcHint schema procName params allProcs overloadedProcs =
+  fmap (("Perhaps you meant to call the function " <> schema <> ".") <>) possibleProcs
+  where
+    fuzzySetOfProcs  = Fuzzy.fromList [qiName k | k <- allProcs, qiSchema k == schema]
+    fuzzySetOfParams = Fuzzy.fromList $ listToText <$> [[ppName prm | prm <- pdParams ov] | ov <- overloadedProcs]
+    -- Cannot do a fuzzy search like: Fuzzy.getOne [[Text]] [Text], where [[Text]] is the list of params for each
+    -- overloaded function and [Text] the given params. This converts those lists to text to make fuzzy search possible.
+    -- E.g. ["val", "param", "name"] into "(name, param, val)"
+    listToText       = ("(" <>) . (<> ")") . T.intercalate ", " . sort
+    possibleProcs
+      | null overloadedProcs = Fuzzy.getOne fuzzySetOfProcs procName
+      | otherwise            = (procName <>) <$> Fuzzy.getOne fuzzySetOfParams (listToText params)
 
 compressedRel :: Relationship -> JSON.Value
 -- An ambiguousness error cannot happen for computed relationships TODO refactor so this mempty is not needed
@@ -193,7 +325,7 @@ compressedRel Relationship{..} =
     : case relCardinality of
         M2M Junction{..} -> [
             "cardinality" .= ("many-to-many" :: Text)
-          , "relationship" .= (qiName junTable <> " using " <> junConstraint1 <> fmtEls (snd <$> junColumns1) <> " and " <> junConstraint2 <> fmtEls (snd <$> junColumns2))
+          , "relationship" .= (qiName junTable <> " using " <> junConstraint1 <> fmtEls (snd <$> junColsSource) <> " and " <> junConstraint2 <> fmtEls (snd <$> junColsTarget))
           ]
         M2O cons relColumns -> [
             "cardinality" .= ("many-to-one" :: Text)
@@ -243,7 +375,7 @@ instance JSON.ToJSON SQL.UsageError where
     "hint"    .= JSON.Null]
   toJSON (SQL.SessionUsageError e) = JSON.toJSON e -- SQL.Error
   toJSON SQL.AcquisitionTimeoutUsageError = JSON.object [
-    "code"    .= ConnectionErrorCode00,
+    "code"    .= ConnectionErrorCode03,
     "message" .= ("Timed out acquiring connection from connection pool." :: Text),
     "details" .= JSON.Null,
     "hint"    .= JSON.Null]
@@ -296,6 +428,7 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
         '5':'3':_ -> HTTP.status503 -- insufficient resources
         '5':'4':_ -> HTTP.status413 -- too complex
         '5':'5':_ -> HTTP.status500 -- obj not on prereq state
+        '5':'7':'P':'0':'1':_ -> HTTP.status503 -- terminating connection due to administrator command
         '5':'7':_ -> HTTP.status500 -- operator intervention
         '5':'8':_ -> HTTP.status500 -- system error
         'F':'0':_ -> HTTP.status500 -- conf file error
@@ -313,29 +446,6 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
 
     _                       -> HTTP.status500
 
-checkIsFatal :: PgError -> Maybe Text
-checkIsFatal (PgError _ (SQL.ConnectionUsageError e))
-  | isAuthFailureMessage = Just $ toS failureMessage
-  | otherwise = Nothing
-  where isAuthFailureMessage = "FATAL:  password authentication failed" `isPrefixOf` failureMessage
-        failureMessage = BS.unpack $ fromMaybe mempty e
-checkIsFatal (PgError _ (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError))))
-  = case serverError of
-      -- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
-      SQL.ServerError "42601" _ _ _ _
-        -> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
-      -- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
-      -- This would mean that a connection pooler in transaction mode is being used
-      -- while prepared statements are enabled in the PostgREST configuration,
-      -- both of which are incompatible with each other.
-      SQL.ServerError "42P05" _ _ _ _
-        -> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
-      -- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
-      -- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
-      SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
-        -> Just "Hint: Connection poolers in statement mode are not supported."
-      _ -> Nothing
-checkIsFatal _ = Nothing
 
 
 data Error
@@ -368,7 +478,7 @@ instance PgrstError Error where
   headers (JwtTokenInvalid m)    = [MediaType.toContentType MTApplicationJSON, invalidTokenHeader m]
   headers JwtTokenRequired       = [MediaType.toContentType MTApplicationJSON, requiredTokenHeader]
   headers (PgErr err)            = headers err
-  headers SingularityError{}     = [MediaType.toContentType MTSingularJSON]
+  headers SingularityError{}     = [MediaType.toContentType (MTSingularJSON False)]
   headers _                      = [MediaType.toContentType MTApplicationJSON]
 
 instance JSON.ToJSON Error where
@@ -420,7 +530,7 @@ instance JSON.ToJSON Error where
   toJSON (SingularityError n) = JSON.object [
     "code"    .= ApiRequestErrorCode16,
     "message" .= ("JSON object requested, multiple (or no) rows returned" :: Text),
-    "details" .= T.unwords ["Results contain", show n, "rows,", T.decodeUtf8 (MediaType.toMime MTSingularJSON), "requires 1 row"],
+    "details" .= T.unwords ["The result contains", show n, "rows"],
     "hint"    .= JSON.Null]
 
   toJSON (PgErr err) = JSON.toJSON err
@@ -442,12 +552,13 @@ data ErrorCode
   = ConnectionErrorCode00
   | ConnectionErrorCode01
   | ConnectionErrorCode02
+  | ConnectionErrorCode03
   -- API Request errors
   | ApiRequestErrorCode00
   | ApiRequestErrorCode01
   | ApiRequestErrorCode02
   | ApiRequestErrorCode03
-  | ApiRequestErrorCode04
+  | ApiRequestErrorCode04 -- no longer used (used to be mapped to ParseRequestError)
   | ApiRequestErrorCode05
   | ApiRequestErrorCode06
   | ApiRequestErrorCode07
@@ -461,11 +572,15 @@ data ErrorCode
   | ApiRequestErrorCode15
   | ApiRequestErrorCode16
   | ApiRequestErrorCode17
+  | ApiRequestErrorCode18
+  | ApiRequestErrorCode19
+  | ApiRequestErrorCode20
   -- Schema Cache errors
   | SchemaCacheErrorCode00
   | SchemaCacheErrorCode01
   | SchemaCacheErrorCode02
   | SchemaCacheErrorCode03
+  | SchemaCacheErrorCode04
   -- JWT authentication errors
   | JWTErrorCode00
   | JWTErrorCode01
@@ -483,6 +598,7 @@ buildErrorCode code = "PGRST" <> case code of
   ConnectionErrorCode00  -> "000"
   ConnectionErrorCode01  -> "001"
   ConnectionErrorCode02  -> "002"
+  ConnectionErrorCode03  -> "003"
 
   ApiRequestErrorCode00  -> "100"
   ApiRequestErrorCode01  -> "101"
@@ -502,11 +618,15 @@ buildErrorCode code = "PGRST" <> case code of
   ApiRequestErrorCode15  -> "115"
   ApiRequestErrorCode16  -> "116"
   ApiRequestErrorCode17  -> "117"
+  ApiRequestErrorCode18  -> "118"
+  ApiRequestErrorCode19  -> "119"
+  ApiRequestErrorCode20  -> "120"
 
   SchemaCacheErrorCode00 -> "200"
   SchemaCacheErrorCode01 -> "201"
   SchemaCacheErrorCode02 -> "202"
   SchemaCacheErrorCode03 -> "203"
+  SchemaCacheErrorCode04 -> "204"
 
   JWTErrorCode00         -> "300"
   JWTErrorCode01         -> "301"

src/PostgREST/Logger.hs

@@ -26,5 +26,5 @@ middleware logLevel = case logLevel of
       { Wai.outputFormat = Wai.ApacheWithSettings $
           Wai.defaultApacheSettings
             & Wai.setApacheRequestFilter (\_ res -> filterStatus $ Wai.responseStatus res)
-            & Wai.setApacheUserGetter (fmap encodeUtf8 . Auth.getRole)
+            & Wai.setApacheUserGetter Auth.getRole
       }

src/PostgREST/MediaType.hs

@@ -4,16 +4,13 @@ module PostgREST.MediaType
   ( MediaType(..)
   , MTPlanOption (..)
   , MTPlanFormat (..)
-  , MTPlanAttrs(..)
   , toContentType
   , toMime
   , decodeMediaType
-  , getMediaType
   ) where
 
 import qualified Data.ByteString          as BS
 import qualified Data.ByteString.Internal as BS (c2w)
-import           Data.Maybe               (fromJust)
 
 import Network.HTTP.Types.Header (Header, hContentType)
 
@@ -22,7 +19,8 @@ import Protolude
 -- | Enumeration of currently supported media types
 data MediaType
   = MTApplicationJSON
-  | MTSingularJSON
+  | MTArrayJSONStrip
+  | MTSingularJSON Bool
   | MTGeoJSON
   | MTTextCSV
   | MTTextPlain
@@ -32,18 +30,32 @@ data MediaType
   | MTOctetStream
   | MTAny
   | MTOther ByteString
-  | MTPlan MTPlanAttrs
+  -- TODO MTPlan should only have its options as [Text]. Its ResultAggregate should have the typed attributes.
-  deriving Eq
+  | MTPlan MediaType MTPlanFormat [MTPlanOption]
-
+  deriving Show
-data MTPlanAttrs = MTPlanAttrs (Maybe MediaType) MTPlanFormat [MTPlanOption]
+instance Eq MediaType where
-instance Eq MTPlanAttrs where
+  MTApplicationJSON    == MTApplicationJSON = True
-  MTPlanAttrs {} == MTPlanAttrs {} = True -- we don't care about the attributes when comparing two MTPlan media types
+  MTArrayJSONStrip     == MTArrayJSONStrip  = True
+  MTSingularJSON x     == MTSingularJSON y  = x == y
+  MTGeoJSON            == MTGeoJSON         = True
+  MTTextCSV            == MTTextCSV         = True
+  MTTextPlain          == MTTextPlain       = True
+  MTTextXML            == MTTextXML         = True
+  MTOpenAPI            == MTOpenAPI         = True
+  MTUrlEncoded         == MTUrlEncoded      = True
+  MTOctetStream        == MTOctetStream     = True
+  MTAny                == MTAny             = True
+  MTOther x            == MTOther y         = x == y
+  MTPlan{}             == MTPlan{}          = True
+  _                    == _                 = False
 
 data MTPlanOption
   = PlanAnalyze | PlanVerbose | PlanSettings | PlanBuffers | PlanWAL
+  deriving (Eq, Show)
 
 data MTPlanFormat
   = PlanJSON | PlanText
+  deriving (Eq, Show)
 
 -- | Convert MediaType to a Content-Type HTTP Header
 toContentType :: MediaType -> Header
@@ -56,20 +68,22 @@ toContentType ct = (hContentType, toMime ct <> charset)
 
 -- | Convert from MediaType to a ByteString representing the mime type
 toMime :: MediaType -> ByteString
-toMime MTApplicationJSON = "application/json"
+toMime MTApplicationJSON      = "application/json"
-toMime MTGeoJSON         = "application/geo+json"
+toMime MTArrayJSONStrip       = "application/vnd.pgrst.array+json;nulls=stripped"
-toMime MTTextCSV         = "text/csv"
+toMime MTGeoJSON              = "application/geo+json"
-toMime MTTextPlain       = "text/plain"
+toMime MTTextCSV              = "text/csv"
-toMime MTTextXML         = "text/xml"
+toMime MTTextPlain            = "text/plain"
-toMime MTOpenAPI         = "application/openapi+json"
+toMime MTTextXML              = "text/xml"
-toMime MTSingularJSON    = "application/vnd.pgrst.object+json"
+toMime MTOpenAPI              = "application/openapi+json"
-toMime MTUrlEncoded      = "application/x-www-form-urlencoded"
+toMime (MTSingularJSON True)  = "application/vnd.pgrst.object+json;nulls=stripped"
-toMime MTOctetStream     = "application/octet-stream"
+toMime (MTSingularJSON False) = "application/vnd.pgrst.object+json"
-toMime MTAny             = "*/*"
+toMime MTUrlEncoded           = "application/x-www-form-urlencoded"
-toMime (MTOther ct)      = ct
+toMime MTOctetStream          = "application/octet-stream"
-toMime (MTPlan (MTPlanAttrs mt fmt opts)) =
+toMime MTAny                  = "*/*"
+toMime (MTOther ct)           = ct
+toMime (MTPlan mt fmt opts)   =
   "application/vnd.pgrst.plan+" <> toMimePlanFormat fmt <>
-  (if isNothing mt then mempty else "; for=\"" <> toMime (fromJust mt) <> "\"") <>
+  ("; for=\"" <> toMime mt <> "\"") <>
   (if null opts then mempty else "; options=" <> BS.intercalate "|" (toMimePlanOption <$> opts))
 
 toMimePlanOption :: MTPlanOption -> ByteString
@@ -83,42 +97,72 @@ toMimePlanFormat :: MTPlanFormat -> ByteString
 toMimePlanFormat PlanJSON = "json"
 toMimePlanFormat PlanText = "text"
 
--- | Convert from ByteString to MediaType. Warning: discards MIME parameters
+-- | Convert from ByteString to MediaType.
+--
+-- >>> decodeMediaType "application/json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan;"
+-- MTPlan MTApplicationJSON PlanText []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan;for=\"application/json\""
+-- MTPlan MTApplicationJSON PlanText []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan+json;for=\"text/csv\""
+-- MTPlan MTTextCSV PlanJSON []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json;nulls=stripped"
+-- MTArrayJSONStrip
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json;nulls=stripped"
+-- MTSingularJSON True
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json"
+-- MTSingularJSON False
+
 decodeMediaType :: BS.ByteString -> MediaType
 decodeMediaType mt =
   case BS.split (BS.c2w ';') mt of
-    "application/json":_                   -> MTApplicationJSON
+    "application/json":_                     -> MTApplicationJSON
-    "application/geo+json":_               -> MTGeoJSON
+    "application/geo+json":_                 -> MTGeoJSON
-    "text/csv":_                           -> MTTextCSV
+    "text/csv":_                             -> MTTextCSV
-    "text/plain":_                         -> MTTextPlain
+    "text/plain":_                           -> MTTextPlain
-    "text/xml":_                           -> MTTextXML
+    "text/xml":_                             -> MTTextXML
-    "application/openapi+json":_           -> MTOpenAPI
+    "application/openapi+json":_             -> MTOpenAPI
-    "application/vnd.pgrst.object+json":_  -> MTSingularJSON
+    "application/x-www-form-urlencoded":_    -> MTUrlEncoded
-    "application/vnd.pgrst.object":_       -> MTSingularJSON
+    "application/octet-stream":_             -> MTOctetStream
-    "application/x-www-form-urlencoded":_  -> MTUrlEncoded
+    "application/vnd.pgrst.plan":rest        -> getPlan PlanText rest
-    "application/octet-stream":_           -> MTOctetStream
+    "application/vnd.pgrst.plan+text":rest   -> getPlan PlanText rest
-    "application/vnd.pgrst.plan":rest      -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+json":rest   -> getPlan PlanJSON rest
-    "application/vnd.pgrst.plan+text":rest -> getPlan PlanText rest
+    "application/vnd.pgrst.object+json":rest -> checkSingularNullStrip rest
-    "application/vnd.pgrst.plan+json":rest -> getPlan PlanJSON rest
+    "application/vnd.pgrst.object":rest      -> checkSingularNullStrip rest
-    "*/*":_                                -> MTAny
+    "application/vnd.pgrst.array+json":rest  -> checkArrayNullStrip rest
-    other:_                                -> MTOther other
+    "*/*":_                                  -> MTAny
-    _                                      -> MTAny
+    other:_                                  -> MTOther other
+    _                                        -> MTAny
   where
-    getPlan fmt rest =
+    checkArrayNullStrip ["nulls=stripped"] = MTArrayJSONStrip
-     let
+    checkArrayNullStrip  _                 = MTApplicationJSON
-       opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)
-       inOpts str   = str `elem` opts
-       mtFor        = decodeMediaType . dropAround (== BS.c2w '"') <$> (BS.stripPrefix "for=" =<< find (BS.isPrefixOf "for=") rest)
-       dropAround p = BS.dropWhile p . BS.dropWhileEnd p in
-     MTPlan $ MTPlanAttrs mtFor fmt $
-      [PlanAnalyze  | inOpts "analyze" ] ++
-      [PlanVerbose  | inOpts "verbose" ] ++
-      [PlanSettings | inOpts "settings"] ++
-      [PlanBuffers  | inOpts "buffers" ] ++
-      [PlanWAL      | inOpts "wal"     ]
 
-getMediaType :: MediaType -> MediaType
+    checkSingularNullStrip ["nulls=stripped"] = MTSingularJSON True
-getMediaType mt = case mt of
+    checkSingularNullStrip  _                 = MTSingularJSON False
-  MTPlan (MTPlanAttrs (Just mType) _ _) -> mType
+
-  MTPlan (MTPlanAttrs Nothing _ _)      -> MTApplicationJSON
+    getPlan fmt rest =
-  other                                 -> other
+      let
+        opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)
+        inOpts str   = str `elem` opts
+        dropAround p = BS.dropWhile p . BS.dropWhileEnd p
+        mtFor        = fromMaybe MTApplicationJSON $ do
+          foundFor    <- find (BS.isPrefixOf "for=") rest
+          strippedFor <- BS.stripPrefix "for=" foundFor
+          pure . decodeMediaType $ dropAround (== BS.c2w '"') strippedFor
+      in
+      MTPlan mtFor fmt $
+        [PlanAnalyze  | inOpts "analyze" ] ++
+        [PlanVerbose  | inOpts "verbose" ] ++
+        [PlanSettings | inOpts "settings"] ++
+        [PlanBuffers  | inOpts "buffers" ] ++
+        [PlanWAL      | inOpts "wal"     ]

src/PostgREST/Plan.hs

@@ -16,107 +16,350 @@ resource.
 {-# LANGUAGE RecordWildCards       #-}
 
 module PostgREST.Plan
-  ( readPlan
+  ( wrappedReadPlan
   , mutateReadPlan
   , callReadPlan
+  , WrappedReadPlan(..)
   , MutateReadPlan(..)
   , CallReadPlan(..)
+  , inspectPlanTxMode
   ) where
 
-import qualified Data.HashMap.Strict        as HM
+import qualified Data.ByteString.Lazy          as LBS
-import qualified Data.Set                   as S
+import qualified Data.HashMap.Strict           as HM
-import qualified PostgREST.SchemaCache.Proc as Proc
+import qualified Data.HashMap.Strict.InsOrd    as HMI
+import qualified Data.List                     as L
+import qualified Data.Set                      as S
+import qualified PostgREST.SchemaCache.Routine as Routine
 
-import Data.Either.Combinators (mapLeft)
+import Data.Either.Combinators (mapLeft, mapRight)
 import Data.List               (delete)
 import Data.Tree               (Tree (..))
 
-import PostgREST.ApiRequest               (Action (..),
+import PostgREST.ApiRequest                  (Action (..),
-                                           ApiRequest (..),
+                                              ApiRequest (..),
-                                           InvokeMethod (..),
+                                              InvokeMethod (..),
-                                           Mutation (..),
+                                              Mutation (..),
-                                           Payload (..))
+                                              Payload (..))
-import PostgREST.Config                   (AppConfig (..))
+import PostgREST.Config                      (AppConfig (..))
-import PostgREST.Error                    (Error (..))
+import PostgREST.Error                       (Error (..))
-import PostgREST.Query.SqlFragment        (sourceCTEName)
+import PostgREST.MediaType                   (MediaType (..))
-import PostgREST.RangeQuery               (NonnegRange, allRange,
+import PostgREST.Query.SqlFragment           (sourceCTEName)
-                                           convertToLimitZeroRange,
+import PostgREST.RangeQuery                  (NonnegRange, allRange,
-                                           restrictRange)
+                                              convertToLimitZeroRange,
-import PostgREST.SchemaCache              (SchemaCache (..))
+                                              restrictRange)
-import PostgREST.SchemaCache.Identifiers  (FieldName,
+import PostgREST.SchemaCache                 (SchemaCache (..))
-                                           QualifiedIdentifier (..),
+import PostgREST.SchemaCache.Identifiers     (FieldName,
-                                           Schema)
+                                              QualifiedIdentifier (..),
-import PostgREST.SchemaCache.Proc         (ProcDescription (..),
+                                              Schema)
-                                           ProcParam (..),
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
-                                           procReturnsScalar)
+                                              Junction (..),
-import PostgREST.SchemaCache.Relationship (Cardinality (..),
+                                              Relationship (..),
-                                           Junction (..),
+                                              RelationshipsMap,
-                                           Relationship (..),
+                                              relIsToOne)
-                                           RelationshipsMap)
+import PostgREST.SchemaCache.Representations (DataRepresentation (..),
-import PostgREST.SchemaCache.Table        (tablePKCols)
+                                              RepresentationsMap)
-
+import PostgREST.SchemaCache.Routine         (ResultAggregate (..),
-import PostgREST.Plan.CallPlan
+                                              Routine (..),
-import PostgREST.Plan.MutatePlan
+                                              RoutineMap,
-import PostgREST.Plan.ReadPlan   as ReadPlan
+                                              RoutineParam (..),
+                                              funcReturnsCompositeAlias,
+                                              funcReturnsScalar,
+                                              funcReturnsSetOfScalar)
+import PostgREST.SchemaCache.Table           (Column (..), Table (..),
+                                              TablesMap,
+                                              tableColumnsList,
+                                              tablePKCols)
 
 import PostgREST.ApiRequest.Preferences
 import PostgREST.ApiRequest.Types
+import PostgREST.Plan.CallPlan
+import PostgREST.Plan.MutatePlan
+import PostgREST.Plan.ReadPlan          as ReadPlan
+import PostgREST.Plan.Types
 
+import qualified Hasql.Transaction.Sessions       as SQL
 import qualified PostgREST.ApiRequest.QueryParams as QueryParams
 
 import Protolude hiding (from)
 
+-- $setup
+-- Setup for doctests
+-- >>> import Data.Ranged.Ranges (fullRange)
+
+data WrappedReadPlan = WrappedReadPlan {
+  wrReadPlan :: ReadPlanTree
+, wrTxMode   :: SQL.Mode
+, wrResAgg   :: ResultAggregate
+}
+
 data MutateReadPlan = MutateReadPlan {
   mrReadPlan   :: ReadPlanTree
 , mrMutatePlan :: MutatePlan
+, mrTxMode     :: SQL.Mode
+, mrResAgg     :: ResultAggregate
 }
 
 data CallReadPlan = CallReadPlan {
   crReadPlan :: ReadPlanTree
 , crCallPlan :: CallPlan
+, crTxMode   :: SQL.Mode
+, crProc     :: Routine
+, crResAgg   :: ResultAggregate
 }
 
+wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error WrappedReadPlan
+wrappedReadPlan  identifier conf sCache apiRequest = do
+  rPlan <- readPlan identifier conf sCache apiRequest
+  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) Nothing rPlan
+  return $ WrappedReadPlan rPlan SQL.Read $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
+
 mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error MutateReadPlan
 mutateReadPlan  mutation apiRequest identifier conf sCache = do
   rPlan <- readPlan identifier conf sCache apiRequest
+  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) Nothing rPlan
   mPlan <- mutatePlan mutation identifier apiRequest sCache rPlan
-  return $ MutateReadPlan rPlan mPlan
+  return $ MutateReadPlan rPlan mPlan SQL.Write $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
 
-callReadPlan :: ProcDescription -> AppConfig -> SchemaCache -> ApiRequest -> Either Error CallReadPlan
+callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CallReadPlan
-callReadPlan proc conf sCache apiRequest = do
+callReadPlan identifier conf sCache apiRequest invMethod = do
-  let identifier = QualifiedIdentifier (pdSchema proc) (fromMaybe (pdName proc) $ Proc.procTableName proc)
+  let paramKeys = case invMethod of
-  rPlan <- readPlan identifier conf sCache apiRequest
+        InvGet  -> S.fromList $ fst <$> qsParams'
-  let cPlan = callPlan proc apiRequest rPlan
+        InvHead -> S.fromList $ fst <$> qsParams'
-  return $ CallReadPlan rPlan cPlan
+        InvPost -> iColumns apiRequest
+  proc@Function{..} <- mapLeft ApiRequestError $
+    findProc identifier paramKeys (preferParameters == Just SingleObject) (dbRoutines sCache) (iContentMediaType apiRequest) (invMethod == InvPost)
+  let relIdentifier = QualifiedIdentifier pdSchema (fromMaybe pdName $ Routine.funcTableName proc) -- done so a set returning function can embed other relations
+  rPlan <- readPlan relIdentifier conf sCache apiRequest
+  let args = case (invMethod, iContentMediaType apiRequest) of
+        (InvGet, _)             -> jsonRpcParams proc qsParams'
+        (InvHead, _)            -> jsonRpcParams proc qsParams'
+        (InvPost, MTUrlEncoded) -> maybe mempty (jsonRpcParams proc . payArray) $ iPayload apiRequest
+        (InvPost, _)            -> maybe mempty payRaw $ iPayload apiRequest
+      txMode = case (invMethod, pdVolatility) of
+          (InvGet,  _)                 -> SQL.Read
+          (InvHead, _)                 -> SQL.Read
+          (InvPost, Routine.Stable)    -> SQL.Read
+          (InvPost, Routine.Immutable) -> SQL.Read
+          (InvPost, Routine.Volatile)  -> SQL.Write
+      cPlan = callPlan proc apiRequest paramKeys args rPlan
+  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) (Just proc) rPlan
+  return $ CallReadPlan rPlan cPlan txMode proc $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
+  where
+    Preferences{..} = iPreferences apiRequest
+    qsParams' = QueryParams.qsParams (iQueryParams apiRequest)
+
+{-|
+  Search a pg proc by matching name and arguments keys to parameters. Since a function can be overloaded,
+  the name is not enough to find it. An overloaded function can have a different volatility or even a different return type.
+-}
+findProc :: QualifiedIdentifier -> S.Set Text -> Bool -> RoutineMap -> MediaType -> Bool -> Either ApiRequestError Routine
+findProc qi argumentsKeys paramsAsSingleObject allProcs contentMediaType isInvPost =
+  case matchProc of
+    ([], [])     -> Left $ NoRpc (qiSchema qi) (qiName qi) (S.toList argumentsKeys) paramsAsSingleObject contentMediaType isInvPost (HM.keys allProcs) lookupProcName
+    -- If there are no functions with named arguments, fallback to the single unnamed argument function
+    ([], [proc]) -> Right proc
+    ([], procs)  -> Left $ AmbiguousRpc (toList procs)
+    -- Matches the functions with named arguments
+    ([proc], _)  -> Right proc
+    (procs, _)   -> Left $ AmbiguousRpc (toList procs)
+  where
+    matchProc = overloadedProcPartition lookupProcName
+    -- First find the proc by name
+    lookupProcName = HM.lookupDefault mempty qi allProcs
+    -- The partition obtained has the form (overloadedProcs,fallbackProcs)
+    -- where fallbackProcs are functions with a single unnamed parameter
+    overloadedProcPartition = foldr select ([],[])
+    select proc ~(ts,fs)
+      | matchesParams proc         = (proc:ts,fs)
+      | hasSingleUnnamedParam proc = (ts,proc:fs)
+      | otherwise                  = (ts,fs)
+    -- If the function is called with post and has a single unnamed parameter
+    -- it can be called depending on content type and the parameter type
+    hasSingleUnnamedParam Function{pdParams=[RoutineParam{ppType}]} = isInvPost && case (contentMediaType, ppType) of
+      (MTApplicationJSON, "json")  -> True
+      (MTApplicationJSON, "jsonb") -> True
+      (MTTextPlain, "text")        -> True
+      (MTTextXML, "xml")           -> True
+      (MTOctetStream, "bytea")     -> True
+      _                            -> False
+    hasSingleUnnamedParam _ = False
+    matchesParams proc =
+      let
+        params = pdParams proc
+        firstType = (ppType <$> headMay params)
+      in
+      -- exceptional case for Prefer: params=single-object
+      if paramsAsSingleObject
+        then length params == 1 && (firstType == Just "json" || firstType == Just "jsonb")
+      -- If the function has no parameters, the arguments keys must be empty as well
+      else if null params
+        then null argumentsKeys && not (isInvPost && contentMediaType `elem` [MTOctetStream, MTTextPlain, MTTextXML])
+      -- A function has optional and required parameters. Optional parameters have a default value and
+      -- don't require arguments for the function to be executed, required parameters must have an argument present.
+      else case L.partition ppReq params of
+      -- If the function only has required parameters, the arguments keys must match those parameters
+        (reqParams, [])        -> argumentsKeys == S.fromList (ppName <$> reqParams)
+      -- If the function only has optional parameters, the arguments keys can match none or any of them(a subset)
+        ([], optParams)        -> argumentsKeys `S.isSubsetOf` S.fromList (ppName <$> optParams)
+      -- If the function has required and optional parameters, the arguments keys have to match the required parameters
+      -- and can match any or none of the default parameters.
+        (reqParams, optParams) -> argumentsKeys `S.difference` S.fromList (ppName <$> optParams) == S.fromList (ppName <$> reqParams)
+
+inspectPlanTxMode :: SQL.Mode
+inspectPlanTxMode = SQL.Read
+
+-- | During planning we need to resolve Field -> CoercibleField (finding the context specific target type and map function).
+-- | ResolverContext facilitates this without the need to pass around a laundry list of parameters.
+data ResolverContext = ResolverContext
+  { tables          :: TablesMap
+  , representations :: RepresentationsMap
+  , qi              :: QualifiedIdentifier  -- ^ The table we're currently attending; changes as we recurse into joins etc.
+  , outputType      :: Text                 -- ^ The output type for the response payload; e.g. "csv", "json", "binary".
+  }
+
+resolveColumnField :: Column -> CoercibleField
+resolveColumnField col = CoercibleField (colName col) mempty False (colNominalType col) Nothing (colDefault col)
+
+resolveTableFieldName :: Table -> FieldName -> CoercibleField
+resolveTableFieldName table fieldName =
+  fromMaybe (unknownField fieldName []) $ HMI.lookup fieldName (tableColumns table) >>=
+    Just . resolveColumnField
+
+resolveTableField :: Table -> Field -> CoercibleField
+resolveTableField table (fieldName, []) = resolveTableFieldName table fieldName
+resolveTableField table (fieldName, jp) =
+  case resolveTableFieldName table fieldName of
+    -- types that are already json/jsonb don't need to be converted with `to_jsonb` for using arrow operators `data->attr`
+    -- this prevents indexes not applying https://github.com/PostgREST/postgrest/issues/2594
+    cf@CoercibleField{cfIRType="json"}  -> cf{cfJsonPath=jp}
+    cf@CoercibleField{cfIRType="jsonb"} -> cf{cfJsonPath=jp}
+    -- other types will get converted `to_jsonb(col)->attr`
+    cf                                  -> cf{cfJsonPath=jp, cfToJson=True}
+
+-- | Resolve a type within the context based on the given field name and JSON path. Although there are situations where failure to resolve a field is considered an error (see `resolveOrError`), there are also situations where we allow it (RPC calls). If it should be an error and `resolveOrError` doesn't fit, ensure to check the `cfIRType` isn't empty.
+resolveTypeOrUnknown :: ResolverContext -> Field -> CoercibleField
+resolveTypeOrUnknown ResolverContext{..} field@(fn, jp) =
+  fromMaybe (unknownField fn jp) $ HM.lookup qi tables >>=
+    Just . flip resolveTableField field
+
+-- | Install any pre-defined data representation from source to target to coerce this reference.
+--
+-- Note that we change the IR type here. This might seem unintuitive. The short of it is that for a CoercibleField without a transformer, input type == output type. A transformer maps from a -> b, so by definition the input type will be a and the output type b after. And cfIRType is the *input* type.
+--
+-- It might feel odd that once a transformer is added we 'forget' the target type (because now a /= b). You might also note there's no obvious way to stack transforms (even if there was a stack, you erased what type you're working with so it's awkward). Alas as satisfying as it would be to engineer a layered mapping system with full type information, we just don't need it.
+withTransformer :: ResolverContext -> Text -> Text -> CoercibleField -> CoercibleField
+withTransformer ResolverContext{representations} sourceType targetType field =
+  fromMaybe field $ HM.lookup (sourceType, targetType) representations >>=
+    (\fieldRepresentation -> Just field{cfIRType=sourceType, cfTransform=Just (drFunction fieldRepresentation)})
+
+-- | Map the intermediate representation type to the output type, if available.
+withOutputFormat :: ResolverContext -> CoercibleField -> CoercibleField
+withOutputFormat ctx@ResolverContext{outputType} field@CoercibleField{cfIRType} = withTransformer ctx cfIRType outputType field
+
+-- | Map text into the intermediate representation type, if available.
+withTextParse :: ResolverContext -> CoercibleField -> CoercibleField
+withTextParse ctx field@CoercibleField{cfIRType} = withTransformer ctx "text" cfIRType field
+
+-- | Map json into the intermediate representation type, if available.
+withJsonParse :: ResolverContext -> CoercibleField -> CoercibleField
+withJsonParse ctx field@CoercibleField{cfIRType} = withTransformer ctx "json" cfIRType field
+
+-- | Map the intermediate representation type to the output type defined by the resolver context (normally json), if available.
+resolveOutputField :: ResolverContext -> Field -> CoercibleField
+resolveOutputField ctx field = withOutputFormat ctx $ resolveTypeOrUnknown ctx field
+
+-- | Map the query string format of a value (text) into the intermediate representation type, if available.
+resolveQueryInputField :: ResolverContext -> Field -> CoercibleField
+resolveQueryInputField ctx field = withTextParse ctx $ resolveTypeOrUnknown ctx field
 
 -- | Builds the ReadPlan tree on a number of stages.
 -- | Adds filters, order, limits on its respective nodes.
 -- | Adds joins conditions obtained from resource embedding.
 readPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error ReadPlanTree
-readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows} SchemaCache{dbRelationships} apiRequest  =
+readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows} SchemaCache{dbTables, dbRelationships, dbRepresentations} apiRequest  =
-  mapLeft ApiRequestError $
+  let
-  treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
+    -- JSON output format hardcoded for now. In the future we might want to support other output mappings such as CSV.
-  addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
+    ctx = ResolverContext dbTables dbRepresentations qi "json"
-  addLogicTrees apiRequest =<<
+  in
-  addRanges apiRequest =<<
+    mapLeft ApiRequestError $
-  addOrders apiRequest =<<
+    treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
-  addFilters apiRequest (initReadRequest qi $ QueryParams.qsSelect $ iQueryParams apiRequest)
+    addNullEmbedFilters =<<
+    validateSpreadEmbeds =<<
+    addRelatedOrders =<<
+    addDataRepresentationAliases =<<
+    expandStarsForDataRepresentations ctx =<<
+    addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
+    addLogicTrees ctx apiRequest =<<
+    addRanges apiRequest =<<
+    addOrders ctx apiRequest =<<
+    addFilters ctx apiRequest (initReadRequest ctx $ QueryParams.qsSelect $ iQueryParams apiRequest)
 
 -- Build the initial read plan tree
-initReadRequest :: QualifiedIdentifier -> [Tree SelectItem] -> ReadPlanTree
+initReadRequest :: ResolverContext -> [Tree SelectItem] -> ReadPlanTree
-initReadRequest qi@QualifiedIdentifier{..} =
+initReadRequest ctx@ResolverContext{qi=QualifiedIdentifier{..}} =
-  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi, relName=qiName, depth=rootDepth} []
+  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi ctx, relName=qiName, depth=rootDepth} []
   where
     rootDepth = 0
-    defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing rootDepth
+    defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing False rootDepth
     treeEntry :: Depth -> Tree SelectItem -> ReadPlanTree -> ReadPlanTree
-    treeEntry depth (Node SelectRelation{..} fldForest) (Node q rForest) =
+    treeEntry depth (Node si fldForest) (Node q rForest) =
       let nxtDepth = succ depth in
-        Node q $
+      case si of
-          foldr (treeEntry nxtDepth)
+        SelectRelation{..} ->
-          (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relAlias=selAlias, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth} [])
+          Node q $
-          fldForest:rForest
+            foldr (treeEntry nxtDepth)
-    treeEntry _ (Node SelectField{..} _) (Node q rForest) = Node q{select=(selField, selCast, selAlias):select q} rForest
+            (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relAlias=selAlias, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth} [])
+            fldForest:rForest
+        SpreadRelation{..} ->
+          Node q $
+            foldr (treeEntry nxtDepth)
+            (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth, relIsSpread=True} [])
+            fldForest:rForest
+        SelectField{..} ->
+          Node q{select=(resolveOutputField ctx{qi=from q} selField, selCast, selAlias):select q} rForest
+
+-- | Preserve the original field name if data representation is used to coerce the value.
+addDataRepresentationAliases :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addDataRepresentationAliases rPlanTree = Right $ fmap (\rPlan@ReadPlan{select=sel} -> rPlan{select=map aliasSelectItem sel}) rPlanTree
+  where
+    aliasSelectItem :: (CoercibleField, Maybe Cast, Maybe Alias) -> (CoercibleField, Maybe Cast, Maybe Alias)
+    -- If there already is an alias, don't overwrite it.
+    aliasSelectItem (fld@(CoercibleField{cfName=fieldName, cfTransform=(Just _)}), Nothing, Nothing) = (fld, Nothing, Just fieldName)
+    aliasSelectItem fld = fld
+
+knownColumnsInContext :: ResolverContext -> [Column]
+knownColumnsInContext ResolverContext{..} =
+  fromMaybe [] $ HM.lookup qi tables >>=
+  Just . tableColumnsList
+
+-- | Expand "select *" into explicit field names of the table, if necessary to apply data representations.
+expandStarsForDataRepresentations :: ResolverContext -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+expandStarsForDataRepresentations ctx@ResolverContext{qi} rPlanTree = Right $ fmap expandStars rPlanTree
+  where
+    expandStars :: ReadPlan -> ReadPlan
+    -- When the schema is "" and the table is the source CTE, we assume the true source table is given in the from
+    -- alias and belongs to the request schema. See the bit in `addRels` with `newFrom = ...`.
+    expandStars rPlan@ReadPlan{from=(QualifiedIdentifier "" "pgrst_source"), fromAlias=(Just tblAlias)} =
+      expandStarsForTable ctx{qi=qi{qiName=tblAlias}} rPlan
+    expandStars rPlan@ReadPlan{from=fromTable} =
+      expandStarsForTable ctx{qi=fromTable} rPlan
+
+expandStarsForTable :: ResolverContext -> ReadPlan -> ReadPlan
+expandStarsForTable ctx@ResolverContext{representations, outputType} rplan@ReadPlan{select=selectItems} =
+  -- If we have a '*' select AND the target table has at least one data representation, expand.
+  if ("*" `elem` map (\(field, _, _) -> cfName field) selectItems) && any hasOutputRep knownColumns
+    then rplan{select=concatMap (expandStarSelectItem knownColumns) selectItems}
+    else rplan
+  where
+    knownColumns = knownColumnsInContext ctx
+
+    hasOutputRep :: Column -> Bool
+    hasOutputRep col = HM.member (colNominalType col, outputType) representations
+
+    expandStarSelectItem :: [Column] -> (CoercibleField, Maybe Cast, Maybe Alias) -> [(CoercibleField, Maybe Cast, Maybe Alias)]
+    expandStarSelectItem columns (CoercibleField{cfName="*", cfJsonPath=[]}, b, c) = map (\col -> (withOutputFormat ctx $ resolveColumnField col, b, c)) columns
+    expandStarSelectItem _ selectItem = [selectItem]
 
 -- | Enforces the `max-rows` config on the result
 treeRestrictRange :: Maybe Integer -> Action -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -152,7 +395,7 @@ addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,re
       Node <$> newReadPlan <*> (updateForest . hush $ Node <$> newReadPlan <*> pure forest)
     Nothing -> -- root case
       let
-        newFrom  = QualifiedIdentifier mempty $ decodeUtf8 sourceCTEName
+        newFrom  = QualifiedIdentifier mempty sourceCTEName
         newAlias = Just (qiName $ from rPlan)
         newReadPlan = case action of
           -- the CTE for mutations/rpc is used as WITH sourceCTEName .. SELECT .. FROM sourceCTEName as alias,
@@ -196,7 +439,7 @@ getJoinConditions tblAlias parentAlias Relationship{relTable=qi,relForeignTable=
 findRel :: Schema -> RelationshipsMap -> NodeName -> NodeName -> Maybe Hint -> Either ApiRequestError Relationship
 findRel schema allRels origin target hint =
   case rels of
-    []  -> Left $ NoRelBetween origin target schema
+    []  -> Left $ NoRelBetween origin target hint schema allRels
     [r] -> Right r
     rs  -> Left $ AmbiguousRelBetween origin target rs
   where
@@ -242,7 +485,9 @@ findRel schema allRels origin target hint =
             target == qiName relForeignTable && isO2M relCardinality
             && matchFKRefSingleCol hnt relCardinality -- auditor
         else case hint of
+          -- DEPRECATED(remove after 2 major releases since v11.1.0): remove target
           -- target = table / view / constraint / column-from-origin (constraint/column-from-origin can only come from tables https://github.com/PostgREST/postgrest/issues/2277)
+          -- DEPRECATED(remove after 2 major releases since v11.1.0): remove hint as table/view/columns and only leave it as constraint
           -- hint   = table / view / constraint / column-from-origin / column-from-target (hint can take table / view values to aid in finding the junction in an m2m relationship)
           Nothing ->
               -- /projects?select=clients(*)
@@ -271,25 +516,23 @@ findRel schema allRels origin target hint =
             )
       ) $ fromMaybe mempty $ HM.lookup (QualifiedIdentifier schema origin, schema) allRels
 
-addFilters :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addFilters ApiRequest{..} rReq =
+addFilters ctx ApiRequest{..} rReq =
   foldr addFilterToNode (Right rReq) flts
   where
     QueryParams.QueryParams{..} = iQueryParams
     flts =
       case iAction of
-        ActionInvoke InvGet  -> qsFilters
+        ActionInvoke _ -> qsFilters
-        ActionInvoke InvHead -> qsFilters
+        ActionRead _   -> qsFilters
-        ActionInvoke _       -> qsFilters
+        _              -> qsFiltersNotRoot
-        ActionRead _         -> qsFilters
-        _                    -> qsFiltersNotRoot
 
     addFilterToNode :: (EmbedPath, Filter) -> Either ApiRequestError ReadPlanTree ->  Either ApiRequestError ReadPlanTree
     addFilterToNode =
-      updateNode (\flt (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest flt lf}  f)
+      updateNode (\flt (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest (resolveFilter ctx{qi=fromTable} flt) lf}  f)
 
-addOrders :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addOrders :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addOrders ApiRequest{..} rReq =
+addOrders ctx ApiRequest{..} rReq =
   case iAction of
     ActionMutate _ -> Right rReq
     _              -> foldr addOrderToNode (Right rReq) qsOrder
@@ -297,7 +540,115 @@ addOrders ApiRequest{..} rReq =
     QueryParams.QueryParams{..} = iQueryParams
 
     addOrderToNode :: (EmbedPath, [OrderTerm]) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
-    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=o} f)
+    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=resolveOrder ctx <$> o} f)
+
+resolveOrder :: ResolverContext -> OrderTerm -> CoercibleOrderTerm
+resolveOrder _ (OrderRelationTerm a b c d) = CoercibleOrderRelationTerm a b c d
+resolveOrder ctx (OrderTerm fld dir nulls) = CoercibleOrderTerm (resolveTypeOrUnknown ctx fld) dir nulls
+
+-- Validates that the related resource on the order is an embedded resource,
+-- e.g. if `clients` is inside the `select` in /projects?order=clients(id)&select=*,clients(*),
+-- and if it's a to-one relationship, it adds the right alias to the OrderRelationTerm so the generated query can succeed.
+addRelatedOrders :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
+  newOrder <- newRelOrder `traverse` order
+  Node rp{order=newOrder} <$> addRelatedOrders `traverse` forest
+  where
+    newRelOrder cot@CoercibleOrderTerm{}                   = Right cot
+    newRelOrder cot@CoercibleOrderRelationTerm{coRelation} =
+      let foundRP = rootLabel <$> find (\(Node ReadPlan{relName, relAlias} _) -> coRelation == fromMaybe relName relAlias) forest in
+      case foundRP of
+        Just ReadPlan{relName,relAlias,relAggAlias,relToParent} ->
+          let isToOne = relIsToOne <$> relToParent
+              name    = fromMaybe relName relAlias in
+          if isToOne == Just True
+            then Right $ cot{coRelation=relAggAlias}
+            else Left $ RelatedOrderNotToOne (qiName from) name
+        Nothing ->
+          Left $ NotEmbedded coRelation
+
+-- | Searches for null filters on embeds, e.g. `projects=not.is.null` on `GET /clients?select=*,projects(*)&projects=not.is.null`
+--
+-- (It doesn't err but uses an Either ApiRequestError type so it can combine with the other functions that modify the read plan tree)
+--
+-- Setup:
+--
+-- >>> let nullOp    = OpExpr True (Is TriNull)
+-- >>> let nonNullOp = OpExpr False (Is TriNull)
+-- >>> let notEqOp   = OpExpr True (Op OpNotEqual "val")
+-- >>> :{
+-- -- this represents the `projects(*)` part on `/clients?select=*,projects(*)`
+-- let
+-- subForestPlan =
+--   [
+--     Node {
+--       rootLabel = ReadPlan {
+--         select = [], -- there will be fields at this stage but we just omit them for brevity
+--         from = QualifiedIdentifier {qiSchema = "test", qiName = "projects"},
+--         fromAlias = Just "projects_1", where_ = [], order = [], range_ = fullRange,
+--         relName = "projects",
+--         relToParent = Nothing,
+--         relJoinConds = [],
+--         relAlias = Nothing, relAggAlias = "clients_projects_1", relHint = Nothing, relJoinType = Nothing, relIsSpread = False, depth = 1
+--       },
+--       subForest = []
+--     }
+--   ]
+-- :}
+--
+-- >>> :{
+-- -- this represents the full URL `/clients?select=*,projects(*)&projects=not.is.null`, if subForst takes the above subForestPlan and nullOp
+-- let
+-- readPlanTree op subForst =
+--   Node {
+--     rootLabel = ReadPlan {
+--       select = [], -- there will be fields at this stage but we just omit them for brevity
+--       from = QualifiedIdentifier { qiSchema = "test", qiName = "clients"},
+--       fromAlias = Nothing,
+--       where_ = [
+--         CoercibleStmnt (
+--           CoercibleFilter {
+--            field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson=False, cfIRType = "", cfTransform = Nothing, cfDefault = Nothing},
+--            opExpr = op
+--           }
+--         )
+--       ],
+--       order = [], range_ = fullRange, relName = "clients", relToParent = Nothing, relJoinConds = [], relAlias = Nothing, relAggAlias = "", relHint = Nothing,
+--       relJoinType = Nothing, relIsSpread = False, depth = 0
+--     },
+--     subForest = subForst
+--   }
+-- :}
+--
+-- Don't do anything to the filter if there's no embedding (a subtree) on projects. Assume it's a normal filter.
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp [])
+-- Right [CoercibleStmnt (CoercibleFilter {field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson = False, cfIRType = "", cfTransform = Nothing, cfDefault = Nothing}, opExpr = OpExpr True (Is TriNull)})]
+--
+-- If there's an embedding on projects, then change the filter to use the internal aggregate name (`clients_projects_1`) so the filter can succeed later.
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp subForestPlan)
+-- Right [CoercibleStmnt (CoercibleFilterNullEmbed True "clients_projects_1")]
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nonNullOp subForestPlan)
+-- Right [CoercibleStmnt (CoercibleFilterNullEmbed False "clients_projects_1")]
+addNullEmbedFilters :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addNullEmbedFilters (Node rp@ReadPlan{where_=curLogic} forest) = do
+  let forestReadPlans = rootLabel <$> forest
+  newLogic <- newNullFilters forestReadPlans `traverse` curLogic
+  Node rp{ReadPlan.where_= newLogic} <$> (addNullEmbedFilters `traverse` forest)
+  where
+    newNullFilters :: [ReadPlan] -> CoercibleLogicTree -> Either ApiRequestError CoercibleLogicTree
+    newNullFilters rPlans = \case
+      (CoercibleExpr b lOp trees) ->
+        CoercibleExpr b lOp <$> (newNullFilters rPlans `traverse` trees)
+      flt@(CoercibleStmnt (CoercibleFilter (CoercibleField fld [] _ _ _ _) opExpr)) ->
+        let foundRP = find (\ReadPlan{relName, relAlias} -> fld == fromMaybe relName relAlias) rPlans in
+        case (foundRP, opExpr) of
+          (Just ReadPlan{relAggAlias}, OpExpr b (Is TriNull)) -> Right $ CoercibleStmnt $ CoercibleFilterNullEmbed b relAggAlias
+          _                                                   -> Right flt
+      flt@(CoercibleStmnt _) ->
+        Right flt
 
 addRanges :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRanges ApiRequest{..} rReq =
@@ -311,14 +662,30 @@ addRanges ApiRequest{..} rReq =
     addRangeToNode :: (EmbedPath, NonnegRange) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
     addRangeToNode = updateNode (\r (Node q f) -> Node q{range_=r} f)
 
-addLogicTrees :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addLogicTrees ApiRequest{..} rReq =
+addLogicTrees ctx ApiRequest{..} rReq =
   foldr addLogicTreeToNode (Right rReq) qsLogic
   where
     QueryParams.QueryParams{..} = iQueryParams
 
     addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
-    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=t:lf} f)
+    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)
+
+resolveLogicTree :: ResolverContext -> LogicTree -> CoercibleLogicTree
+resolveLogicTree ctx (Stmnt flt) = CoercibleStmnt $ resolveFilter ctx flt
+resolveLogicTree ctx (Expr b op lts) = CoercibleExpr b op (map (resolveLogicTree ctx) lts)
+
+resolveFilter :: ResolverContext -> Filter -> CoercibleFilter
+resolveFilter ctx (Filter fld opExpr) = CoercibleFilter{field=resolveQueryInputField ctx fld, opExpr=opExpr}
+
+-- Validates that spread embeds are only done on to-one relationships
+validateSpreadEmbeds :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+validateSpreadEmbeds (Node rp@ReadPlan{relToParent=Nothing} forest) = Node rp <$> validateSpreadEmbeds `traverse` forest
+validateSpreadEmbeds (Node rp@ReadPlan{relIsSpread,relToParent=Just rel,relName} forest) = do
+  validRP <- if relIsSpread && not (relIsToOne rel)
+    then Left $ SpreadNotToOne (qiName $ relTable rel) relName -- TODO using relTable is not entirely right because ReadPlan might have an alias, need to store the parent alias on ReadPlan
+    else Right rp
+  Node validRP <$> validateSpreadEmbeds `traverse` forest
 
 -- Find a Node of the Tree and apply a function to it
 updateNode :: (a -> ReadPlanTree -> ReadPlanTree) -> (EmbedPath, a) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -335,61 +702,75 @@ updateNode f (targetNodeName:remainingPath, a) (Right (Node rootNode forest)) =
     findNode = find (\(Node ReadPlan{relName, relAlias} _) -> relName == targetNodeName || relAlias == Just targetNodeName) forest
 
 mutatePlan :: Mutation -> QualifiedIdentifier -> ApiRequest -> SchemaCache -> ReadPlanTree -> Either Error MutatePlan
-mutatePlan mutation qi ApiRequest{..} sCache readReq = mapLeft ApiRequestError $
+mutatePlan mutation qi ApiRequest{iPreferences=Preferences{..}, ..} SchemaCache{dbTables, dbRepresentations} readReq = mapLeft ApiRequestError $
   case mutation of
     MutationCreate ->
-      Right $ Insert qi iColumns body ((,) <$> iPreferResolution <*> Just confCols) [] returnings pkCols
+      mapRight (\typedColumns -> Insert qi typedColumns body ((,) <$> preferResolution <*> Just confCols) [] returnings pkCols applyDefaults) typedColumnsOrError
-    MutationUpdate -> Right $ Update qi iColumns body combinedLogic iTopLevelRange rootOrder returnings
+    MutationUpdate ->
+      mapRight (\typedColumns -> Update qi typedColumns body combinedLogic iTopLevelRange rootOrder returnings applyDefaults) typedColumnsOrError
     MutationSingleUpsert ->
         if null qsLogic &&
            qsFilterFields == S.fromList pkCols &&
            not (null (S.fromList pkCols)) &&
            all (\case
-              Filter _ (OpExpr False (Op OpEqual _)) -> True
+              Filter _ (OpExpr False (OpQuant OpEqual Nothing _)) -> True
-              _                                      -> False) qsFiltersRoot
+              _                                                   -> False) qsFiltersRoot
-          then Right $ Insert qi iColumns body (Just (MergeDuplicates, pkCols)) combinedLogic returnings mempty
+          then mapRight (\typedColumns -> Insert qi typedColumns body (Just (MergeDuplicates, pkCols)) combinedLogic returnings mempty False) typedColumnsOrError
         else
           Left InvalidFilters
     MutationDelete -> Right $ Delete qi combinedLogic iTopLevelRange rootOrder returnings
   where
+    ctx = ResolverContext dbTables dbRepresentations qi "json"
     confCols = fromMaybe pkCols qsOnConflict
     QueryParams.QueryParams{..} = iQueryParams
     returnings =
-      if iPreferRepresentation == None
+      if preferRepresentation == Just None || isNothing preferRepresentation
         then []
-        else returningCols readReq pkCols
+        else inferColsEmbedNeeds readReq pkCols
-    pkCols = maybe mempty tablePKCols $ HM.lookup qi $ dbTables sCache
+    tbl = HM.lookup qi dbTables
-    logic = map snd qsLogic
+    pkCols = maybe mempty tablePKCols tbl
-    rootOrder = maybe [] snd $ find (\(x, _) -> null x) qsOrder
+    logic = map (resolveLogicTree ctx . snd) qsLogic
-    combinedLogic = foldr addFilterToLogicForest logic qsFiltersRoot
+    rootOrder = resolveOrder ctx <$> maybe [] snd (find (\(x, _) -> null x) qsOrder)
+    combinedLogic = foldr (addFilterToLogicForest . resolveFilter ctx) logic qsFiltersRoot
     body = payRaw <$> iPayload -- the body is assumed to be json at this stage(ApiRequest validates)
+    applyDefaults = preferMissing == Just ApplyDefaults
+    typedColumnsOrError = resolveOrError ctx tbl `traverse` S.toList iColumns
 
-callPlan :: ProcDescription -> ApiRequest -> ReadPlanTree -> CallPlan
+resolveOrError :: ResolverContext -> Maybe Table -> FieldName -> Either ApiRequestError CoercibleField
-callPlan proc apiReq readReq = FunctionCall {
+resolveOrError _ Nothing _ = Left NotFound
+resolveOrError ctx (Just table) field =
+  case resolveTableFieldName table field of
+    CoercibleField{cfIRType=""} -> Left $ ColumnNotFound (tableName table) field
+    cf                          -> Right $ withJsonParse ctx cf
+
+callPlan :: Routine -> ApiRequest -> S.Set FieldName -> LBS.ByteString -> ReadPlanTree -> CallPlan
+callPlan proc ApiRequest{iPreferences=Preferences{..}} paramKeys args readReq = FunctionCall {
   funCQi = QualifiedIdentifier (pdSchema proc) (pdName proc)
 , funCParams = callParams
-, funCArgs = payRaw <$> iPayload apiReq
+, funCArgs = Just args
-, funCScalar = procReturnsScalar proc
+, funCScalar = funcReturnsScalar proc
-, funCMultipleCall = iPreferParameters apiReq == Just MultipleObjects
+, funCSetOfScalar = funcReturnsSetOfScalar proc
-, funCReturning = returningCols readReq []
+, funCRetCompositeAlias = funcReturnsCompositeAlias proc
+, funCReturning = inferColsEmbedNeeds readReq []
 }
   where
-    paramsAsSingleObject = iPreferParameters apiReq == Just SingleObject
+    paramsAsSingleObject = preferParameters == Just SingleObject
+    specifiedParams = filter (\x -> ppName x `S.member` paramKeys)
     callParams = case pdParams proc of
       [prm] | paramsAsSingleObject -> OnePosParam prm
             | ppName prm == mempty -> OnePosParam prm
             | otherwise            -> KeyParams $ specifiedParams [prm]
       prms  -> KeyParams $ specifiedParams prms
-    specifiedParams = filter (\x -> ppName x `S.member` iColumns apiReq)
 
-returningCols :: ReadPlanTree -> [FieldName] -> [FieldName]
+-- | Infers the columns needed for an embed to be successful after a mutation or a function call.
-returningCols rr@(Node _ forest) pkCols
+inferColsEmbedNeeds :: ReadPlanTree -> [FieldName] -> [FieldName]
+inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols
   -- if * is part of the select, we must not add pk or fk columns manually -
   -- otherwise those would be selected and output twice
   | "*" `elem` fldNames = ["*"]
   | otherwise           = returnings
   where
-    fldNames = fstFieldNames rr
+    fldNames = cfName . (\(f, _, _) -> f) <$> select
     -- Without fkCols, when a mutatePlan to
     -- /projects?select=name,clients(name) occurs, the RETURNING SQL part would
     -- be `RETURNING name`(see QueryBuilder).  This would make the embedding
@@ -403,8 +784,8 @@ returningCols rr@(Node _ forest) pkCols
           Just $ fst <$> cols
         Node ReadPlan{relToParent=Just Relationship{relCardinality=O2O _ cols}} _ ->
           Just $ fst <$> cols
-        Node ReadPlan{relToParent=Just Relationship{relCardinality=M2M Junction{junColumns1, junColumns2}}} _ ->
+        Node ReadPlan{relToParent=Just Relationship{relCardinality=M2M Junction{junColsSource=cols}}} _ ->
-          Just $ (fst <$> junColumns1) ++ (fst <$> junColumns2)
+          Just $ fst <$> cols
         Node ReadPlan{relToParent=Just ComputedRelationship{}} _ ->
           Nothing
         Node ReadPlan{relToParent=Nothing} _ ->
@@ -428,5 +809,67 @@ returningCols rr@(Node _ forest) pkCols
 
 -- Traditional filters(e.g. id=eq.1) are added as root nodes of the LogicTree
 -- they are later concatenated with AND in the QueryBuilder
-addFilterToLogicForest :: Filter -> [LogicTree] -> [LogicTree]
+addFilterToLogicForest :: CoercibleFilter -> [CoercibleLogicTree] -> [CoercibleLogicTree]
-addFilterToLogicForest flt lf = Stmnt flt : lf
+addFilterToLogicForest flt lf = CoercibleStmnt flt : lf
+
+-- | If raw(binary) output is requested, check that MediaType is one of the
+-- admitted rawMediaTypes and that`?select=...` contains only one field other
+-- than `*`
+binaryField :: AppConfig -> MediaType -> Maybe Routine -> ReadPlanTree -> Either ApiRequestError (Maybe FieldName)
+binaryField AppConfig{configRawMediaTypes} acceptMediaType proc rpTree
+  | isRawMediaType =
+    if (funcReturnsScalar <$> proc) == Just True ||
+       (funcReturnsSetOfScalar <$> proc) == Just True
+      then Right $ Just "pgrst_scalar"
+      else
+        let
+          fieldName = fstFieldName rpTree
+        in
+        case fieldName of
+          Just fld -> Right $ Just fld
+          Nothing  -> Left $ BinaryFieldError acceptMediaType
+  | otherwise =
+      Right Nothing
+  where
+    isRawMediaType = acceptMediaType `elem` configRawMediaTypes `L.union` [MTOctetStream, MTTextPlain, MTTextXML] || isRawPlan acceptMediaType
+    isRawPlan mt = case mt of
+      MTPlan MTOctetStream _ _ -> True
+      MTPlan MTTextPlain _ _   -> True
+      MTPlan MTTextXML _ _     -> True
+      _                        -> False
+
+    fstFieldName :: ReadPlanTree -> Maybe FieldName
+    fstFieldName (Node ReadPlan{select=(CoercibleField{cfName="*", cfJsonPath=[]}, _, _):_} [])  = Nothing
+    fstFieldName (Node ReadPlan{select=[(CoercibleField{cfName=fld, cfJsonPath=[]}, _, _)]} [])  = Just fld
+    fstFieldName _                                               = Nothing
+
+
+mediaToAggregate :: MediaType -> Maybe FieldName -> ApiRequest -> ResultAggregate
+mediaToAggregate mt binField apiReq@ApiRequest{iAction=act, iPreferences=Preferences{preferRepresentation=rep}} =
+  if noAgg then NoAgg
+  else case mt of
+    MTApplicationJSON     -> BuiltinAggJson
+    MTSingularJSON strip  -> BuiltinAggSingleJson strip
+    MTArrayJSONStrip      -> BuiltinAggArrayJsonStrip
+    MTGeoJSON             -> BuiltinAggGeoJson
+    MTTextCSV             -> BuiltinAggCsv
+    MTAny                 -> BuiltinAggJson
+    MTOpenAPI             -> BuiltinAggJson
+    MTUrlEncoded          -> NoAgg -- TODO: unreachable since a previous step (producedMediaTypes) whitelists the media types that can become aggregates.
+
+    -- binary types
+    MTTextPlain           -> BuiltinAggBinary binField
+    MTTextXML             -> BuiltinAggXml binField
+    MTOctetStream         -> BuiltinAggBinary binField
+    MTOther _             -> BuiltinAggBinary binField
+
+    -- Doing `Accept: application/vnd.pgrst.plan; for="application/vnd.pgrst.plan"` doesn't make sense, so we just empty the body.
+    -- TODO: fail instead to be more strict
+    MTPlan (MTPlan{}) _ _ -> NoAgg
+    MTPlan media      _ _ -> mediaToAggregate media binField apiReq
+  where
+    noAgg = case act of
+      ActionMutate _         -> rep == Just HeadersOnly || rep == Just None || isNothing rep
+      ActionRead _isHead     -> _isHead -- no need for an aggregate on HEAD https://github.com/PostgREST/postgrest/issues/2849
+      ActionInvoke invMethod -> invMethod == InvHead
+      _                      -> False

src/PostgREST/Plan/CallPlan.hs

@@ -1,25 +1,57 @@
+{-# LANGUAGE NamedFieldPuns #-}
 module PostgREST.Plan.CallPlan
   ( CallPlan(..)
   , CallParams(..)
+  , jsonRpcParams
   )
 where
 
+import qualified Data.Aeson                        as JSON
 import qualified Data.ByteString.Lazy              as LBS
+import qualified Data.HashMap.Strict               as HM
 import           PostgREST.SchemaCache.Identifiers (FieldName,
                                                     QualifiedIdentifier)
-import           PostgREST.SchemaCache.Proc        (ProcParam (..))
+import           PostgREST.SchemaCache.Routine     (Routine (..),
+                                                    RoutineParam (..))
 
 import Protolude
 
 data CallPlan = FunctionCall
-  { funCQi           :: QualifiedIdentifier
+  { funCQi                :: QualifiedIdentifier
-  , funCParams       :: CallParams
+  , funCParams            :: CallParams
-  , funCArgs         :: Maybe LBS.ByteString
+  , funCArgs              :: Maybe LBS.ByteString
-  , funCScalar       :: Bool
+  , funCScalar            :: Bool
-  , funCMultipleCall :: Bool
+  , funCSetOfScalar       :: Bool
-  , funCReturning    :: [FieldName]
+  , funCRetCompositeAlias :: Bool
+  , funCReturning         :: [FieldName]
   }
 
 data CallParams
-  = KeyParams [ProcParam] -- ^ Call with key params: func(a := val1, b:= val2)
+  = KeyParams [RoutineParam] -- ^ Call with key params: func(a := val1, b:= val2)
-  | OnePosParam ProcParam -- ^ Call with positional params(only one supported): func(val)
+  | OnePosParam RoutineParam -- ^ Call with positional params(only one supported): func(val)
+
+-- | Convert rpc params `/rpc/func?a=val1&b=val2` to json `{"a": "val1", "b": "val2"}
+jsonRpcParams :: Routine -> [(Text, Text)] -> LBS.ByteString
+jsonRpcParams proc prms =
+  if not $ pdHasVariadic proc then -- if proc has no variadic param, save steps and directly convert to json
+    JSON.encode $ HM.fromList $ second JSON.toJSON <$> prms
+  else
+    let paramsMap = HM.fromListWith mergeParams $ toRpcParamValue proc <$> prms in
+    JSON.encode paramsMap
+  where
+    mergeParams :: RpcParamValue -> RpcParamValue -> RpcParamValue
+    mergeParams (Variadic a) (Variadic b) = Variadic $ b ++ a
+    mergeParams v _                       = v -- repeated params for non-variadic parameters are not merged
+
+toRpcParamValue :: Routine -> (Text, Text) -> (Text, RpcParamValue)
+toRpcParamValue proc (k, v) | prmIsVariadic k = (k, Variadic [v])
+                            | otherwise       = (k, Fixed v)
+  where
+    prmIsVariadic prm = isJust $ find (\RoutineParam{ppName, ppVar} -> ppName == prm && ppVar) $ pdParams proc
+
+-- | RPC query param value `/rpc/func?v=<value>`, used for VARIADIC functions on form-urlencoded POST and GETs
+-- | It can be fixed `?v=1` or repeated `?v=1&v=2&v=3.
+data RpcParamValue = Fixed Text | Variadic [Text]
+instance JSON.ToJSON RpcParamValue where
+  toJSON (Fixed    v) = JSON.toJSON v
+  toJSON (Variadic v) = JSON.toJSON v

src/PostgREST/Plan/MutatePlan.hs

@@ -4,39 +4,43 @@ module PostgREST.Plan.MutatePlan
 where
 
 import qualified Data.ByteString.Lazy as LBS
-import qualified Data.Set             as S
 
 import PostgREST.ApiRequest.Preferences  (PreferResolution)
-import PostgREST.ApiRequest.Types        (LogicTree, OrderTerm)
+import PostgREST.Plan.Types              (CoercibleField,
+                                          CoercibleLogicTree,
+                                          CoercibleOrderTerm)
 import PostgREST.RangeQuery              (NonnegRange)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier)
 
+
 import Protolude
 
 data MutatePlan
   = Insert
       { in_        :: QualifiedIdentifier
-      , insCols    :: S.Set FieldName
+      , insCols    :: [CoercibleField]
       , insBody    :: Maybe LBS.ByteString
       , onConflict :: Maybe (PreferResolution, [FieldName])
-      , where_     :: [LogicTree]
+      , where_     :: [CoercibleLogicTree]
       , returning  :: [FieldName]
       , insPkCols  :: [FieldName]
+      , applyDefs  :: Bool
       }
   | Update
       { in_       :: QualifiedIdentifier
-      , updCols   :: S.Set FieldName
+      , updCols   :: [CoercibleField]
       , updBody   :: Maybe LBS.ByteString
-      , where_    :: [LogicTree]
+      , where_    :: [CoercibleLogicTree]
       , mutRange  :: NonnegRange
-      , mutOrder  :: [OrderTerm]
+      , mutOrder  :: [CoercibleOrderTerm]
       , returning :: [FieldName]
+      , applyDefs :: Bool
       }
   | Delete
       { in_       :: QualifiedIdentifier
-      , where_    :: [LogicTree]
+      , where_    :: [CoercibleLogicTree]
       , mutRange  :: NonnegRange
-      , mutOrder  :: [OrderTerm]
+      , mutOrder  :: [CoercibleOrderTerm]
       , returning :: [FieldName]
       }

src/PostgREST/Plan/ReadPlan.hs

@@ -1,16 +1,16 @@
-{-# LANGUAGE NamedFieldPuns #-}
 module PostgREST.Plan.ReadPlan
   ( ReadPlanTree
   , ReadPlan(..)
-  , fstFieldNames
   , JoinCondition(..)
   ) where
 
 import Data.Tree (Tree (..))
 
-import PostgREST.ApiRequest.Types         (Alias, Cast, Depth, Field,
+import PostgREST.ApiRequest.Types         (Alias, Cast, Depth, Hint,
-                                           Hint, JoinType, LogicTree,
+                                           JoinType, NodeName)
-                                           NodeName, OrderTerm)
+import PostgREST.Plan.Types               (CoercibleField (..),
+                                           CoercibleLogicTree,
+                                           CoercibleOrderTerm)
 import PostgREST.RangeQuery               (NonnegRange)
 import PostgREST.SchemaCache.Identifiers  (FieldName,
                                            QualifiedIdentifier)
@@ -25,14 +25,14 @@ data JoinCondition =
   JoinCondition
     (QualifiedIdentifier, FieldName)
     (QualifiedIdentifier, FieldName)
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data ReadPlan = ReadPlan
-  { select       :: [(Field, Maybe Cast, Maybe Alias)]
+  { select       :: [(CoercibleField, Maybe Cast, Maybe Alias)]
   , from         :: QualifiedIdentifier
   , fromAlias    :: Maybe Alias
-  , where_       :: [LogicTree]
+  , where_       :: [CoercibleLogicTree]
-  , order        :: [OrderTerm]
+  , order        :: [CoercibleOrderTerm]
   , range_       :: NonnegRange
   , relName      :: NodeName
   , relToParent  :: Maybe Relationship
@@ -41,12 +41,8 @@ data ReadPlan = ReadPlan
   , relAggAlias  :: Alias
   , relHint      :: Maybe Hint
   , relJoinType  :: Maybe JoinType
+  , relIsSpread  :: Bool
   , depth        :: Depth
   -- ^ used for aliasing
   }
-  deriving (Eq)
+  deriving (Eq, Show)
-
--- First level FieldNames(e.g get a,b from /table?select=a,b,other(c,d))
-fstFieldNames :: ReadPlanTree -> [FieldName]
-fstFieldNames (Node ReadPlan{select} _) =
-  fst . (\(f, _, _) -> f) <$> select

src/PostgREST/Plan/Types.hs

@@ -0,0 +1,67 @@
+module PostgREST.Plan.Types
+  ( CoercibleField(..)
+  , unknownField
+  , CoercibleLogicTree(..)
+  , CoercibleFilter(..)
+  , TransformerProc
+  , CoercibleOrderTerm(..)
+  ) where
+
+import PostgREST.ApiRequest.Types (Field, JsonPath, LogicOperator,
+                                   OpExpr, OrderDirection, OrderNulls)
+
+import PostgREST.SchemaCache.Identifiers (FieldName)
+
+import Protolude
+
+type TransformerProc = Text
+
+-- | A CoercibleField pairs the name of a query element with any type coercion information we need for some specific use case.
+-- |
+-- | As suggested by the name, it's often a reference to a field in a table but really it can be any nameable element (function parameter, calculation with an alias, etc) with a knowable type.
+-- |
+-- | In the simplest case, it allows us to parse JSON payloads with `json_to_recordset`, for which we need to know both the name and the type of each thing we'd like to extract. At a higher level, CoercibleField generalises to reflect that any value we work with in a query may need type specific handling.
+-- |
+-- | CoercibleField is the foundation for the Data Representations feature. This feature allow user-definable mappings between database types so that the same data can be presented or interpreted in various ways as needed. Sometimes the way Postgres coerces data implicitly isn't right for the job. Different mappings might be appropriate for different situations: parsing a filter from a query string requires one function (text -> field type) while parsing a payload from JSON takes another (json -> field type). And the reverse, outputting a field as JSON, requires yet a third (field type -> json). CoercibleField is that "job specific" reference to an element paired with the type we desire for that particular purpose and the function we'll use to get there, if any.
+-- |
+-- | In the planning phase, we "resolve" generic named elements into these specialised CoercibleFields. Again this is context specific: two different CoercibleFields both representing the exact same table column in the database, even in the same query, might have two different target types and mapping functions. For example, one might represent a column in a filter, and another the very same column in an output role to be sent in the response body.
+-- |
+-- | The type value is allowed to be the empty string. The analog here is soft type checking in programming languages: sometimes we don't need a variable to have a specified type and things will work anyhow. So the empty type variant is valid when we don't know and *don't need to know* about the specific type in some context. Note that this variation should not be used if it guarantees failure: in that case you should instead raise an error at the planning stage and bail out. For example, we can't parse JSON with `json_to_recordset` without knowing the types of each recipient field, and so error out. Using the empty string for the type would be incorrect and futile. On the other hand we use the empty type for RPC calls since type resolution isn't implemented for RPC, but it's fine because the query still works with Postgres' implicit coercion. In the future, hopefully we will support data representations across the board and then the empty type may be permanently retired.
+data CoercibleField = CoercibleField
+  { cfName      :: FieldName
+  , cfJsonPath  :: JsonPath
+  , cfToJson    :: Bool
+  , cfIRType    :: Text                  -- ^ The native Postgres type of the field, the intermediate (IR) type before mapping.
+  , cfTransform :: Maybe TransformerProc -- ^ The optional mapping from irType -> targetType.
+  , cfDefault   :: Maybe Text
+  } deriving (Eq, Show)
+
+unknownField :: FieldName -> JsonPath -> CoercibleField
+unknownField name path = CoercibleField name path False "" Nothing Nothing
+
+-- | Like an API request LogicTree, but with coercible field information.
+data CoercibleLogicTree
+  = CoercibleExpr Bool LogicOperator [CoercibleLogicTree]
+  | CoercibleStmnt CoercibleFilter
+  deriving (Eq, Show)
+
+data CoercibleFilter = CoercibleFilter
+  { field  :: CoercibleField
+  , opExpr :: OpExpr
+  }
+  | CoercibleFilterNullEmbed Bool FieldName
+  deriving (Eq, Show)
+
+data CoercibleOrderTerm
+  = CoercibleOrderTerm
+    { coField     :: CoercibleField
+    , coDirection :: Maybe OrderDirection
+    , coNullOrder :: Maybe OrderNulls
+    }
+  | CoercibleOrderRelationTerm
+    { coRelation  :: FieldName
+    , coRelTerm   :: Field
+    , coDirection :: Maybe OrderDirection
+    , coNullOrder :: Maybe OrderNulls
+    }
+  deriving (Eq, Show)

src/PostgREST/Query.hs

@@ -7,15 +7,16 @@ module PostgREST.Query
   , openApiQuery
   , readQuery
   , singleUpsertQuery
-  , txMode
   , updateQuery
   , setPgLocals
+  , runPreReq
   , DbHandler
   ) where
 
 import qualified Data.Aeson                        as JSON
 import qualified Data.Aeson.Key                    as K
 import qualified Data.Aeson.KeyMap                 as KM
+import qualified Data.ByteString                   as BS
 import qualified Data.ByteString.Lazy.Char8        as LBS
 import qualified Data.HashMap.Strict               as HM
 import qualified Data.Set                          as S
@@ -24,24 +25,19 @@ import qualified Hasql.Decoders                    as HD
 import qualified Hasql.DynamicStatements.Snippet   as SQL (Snippet)
 import qualified Hasql.DynamicStatements.Statement as SQL
 import qualified Hasql.Transaction                 as SQL
-import qualified Hasql.Transaction.Sessions        as SQL
 
 import qualified PostgREST.Error              as Error
 import qualified PostgREST.Query.QueryBuilder as QueryBuilder
 import qualified PostgREST.Query.Statements   as Statements
 import qualified PostgREST.RangeQuery         as RangeQuery
 import qualified PostgREST.SchemaCache        as SchemaCache
-import qualified PostgREST.SchemaCache.Proc   as Proc
 
 import Data.Scientific (FPFormat (..), formatScientific, isInteger)
 
-import PostgREST.ApiRequest              (Action (..),
+import PostgREST.ApiRequest              (ApiRequest (..))
-                                          ApiRequest (..),
-                                          InvokeMethod (..),
-                                          Target (..))
 import PostgREST.ApiRequest.Preferences  (PreferCount (..),
-                                          PreferParameters (..),
                                           PreferTransaction (..),
+                                          Preferences (..),
                                           shouldCount)
 import PostgREST.Config                  (AppConfig (..),
                                           OpenAPIMode (..))
@@ -50,42 +46,40 @@ import PostgREST.Config.PgVersion        (PgVersion (..),
 import PostgREST.Error                   (Error)
 import PostgREST.MediaType               (MediaType (..))
 import PostgREST.Plan                    (CallReadPlan (..),
-                                          MutateReadPlan (..))
+                                          MutateReadPlan (..),
+                                          WrappedReadPlan (..))
 import PostgREST.Plan.MutatePlan         (MutatePlan (..))
-import PostgREST.Plan.ReadPlan           (ReadPlanTree)
+import PostgREST.Query.SqlFragment       (escapeIdentList, fromQi,
-import PostgREST.Query.SqlFragment       (fromQi, intercalateSnippet,
+                                          intercalateSnippet,
-                                          pgFmtIdentList,
                                           setConfigLocal,
                                           setConfigLocalJson)
 import PostgREST.Query.Statements        (ResultSet (..))
 import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
                                           Schema)
-import PostgREST.SchemaCache.Proc        (ProcDescription (..),
+import PostgREST.SchemaCache.Routine     (Routine (..), RoutineMap)
-                                          ProcVolatility (..),
-                                          ProcsMap)
 import PostgREST.SchemaCache.Table       (TablesMap)
 
 import Protolude hiding (Handler)
 
 type DbHandler = ExceptT Error SQL.Transaction
 
-readQuery :: ReadPlanTree -> AppConfig -> ApiRequest -> DbHandler ResultSet
+readQuery :: WrappedReadPlan -> AppConfig -> ApiRequest -> DbHandler ResultSet
-readQuery req conf@AppConfig{..} apiReq@ApiRequest{..} = do
+readQuery WrappedReadPlan{wrReadPlan, wrResAgg} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} = do
-  let countQuery = QueryBuilder.readPlanToCountQuery req
+  let countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan
   resultSet <-
      lift . SQL.statement mempty $
       Statements.prepareRead
-        (QueryBuilder.readPlanToQuery req)
+        (QueryBuilder.readPlanToQuery wrReadPlan)
-        (if iPreferCount == Just EstimatedCount then
+        (if preferCount == Just EstimatedCount then
            -- LIMIT maxRows + 1 so we can determine below that maxRows was surpassed
            QueryBuilder.limitedQuery countQuery ((+ 1) <$> configDbMaxRows)
          else
            countQuery
         )
-        (shouldCount iPreferCount)
+        (shouldCount preferCount)
         iAcceptMediaType
-        iBinaryField
+        wrResAgg
         configDbPreparedStatements
   failNotSingular iAcceptMediaType resultSet
   optionalRollback conf apiReq
@@ -93,8 +87,8 @@ readQuery req conf@AppConfig{..} apiReq@ApiRequest{..} = do
 
 resultSetWTotal :: AppConfig -> ApiRequest -> ResultSet -> SQL.Snippet -> DbHandler ResultSet
 resultSetWTotal _ _ rs@RSPlan{} _ = return rs
-resultSetWTotal AppConfig{..} ApiRequest{..} rs@RSStandard{rsTableTotal=tableTotal} countQuery =
+resultSetWTotal AppConfig{..} ApiRequest{iPreferences=Preferences{..}} rs@RSStandard{rsTableTotal=tableTotal} countQuery =
-  case iPreferCount of
+  case preferCount of
     Just PlannedCount -> do
       total <- explain
       return rs{rsTableTotal=total}
@@ -155,63 +149,43 @@ deleteQuery mrPlan apiReq@ApiRequest{..} conf = do
   optionalRollback conf apiReq
   pure resultSet
 
-invokeQuery :: ProcDescription -> CallReadPlan -> ApiRequest -> AppConfig -> DbHandler ResultSet
+invokeQuery :: Routine -> CallReadPlan -> ApiRequest -> AppConfig -> PgVersion -> DbHandler ResultSet
-invokeQuery proc CallReadPlan{crReadPlan, crCallPlan} apiReq@ApiRequest{..} conf@AppConfig{..} = do
+invokeQuery rout CallReadPlan{crReadPlan, crCallPlan, crResAgg} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} conf@AppConfig{..} pgVer = do
   resultSet <-
     lift . SQL.statement mempty $
       Statements.prepareCall
-        (Proc.procReturnsScalar proc)
+        rout
-        (Proc.procReturnsSingle proc)
+        (QueryBuilder.callPlanToQuery crCallPlan pgVer)
-        (QueryBuilder.callPlanToQuery crCallPlan)
         (QueryBuilder.readPlanToQuery crReadPlan)
         (QueryBuilder.readPlanToCountQuery crReadPlan)
-        (shouldCount iPreferCount)
+        (shouldCount preferCount)
         iAcceptMediaType
-        (iPreferParameters == Just MultipleObjects)
+        crResAgg
-        iBinaryField
         configDbPreparedStatements
 
   optionalRollback conf apiReq
   failNotSingular iAcceptMediaType resultSet
   pure resultSet
 
-openApiQuery :: SchemaCache -> PgVersion -> AppConfig -> Schema -> DbHandler (Maybe (TablesMap, ProcsMap, Maybe Text))
+openApiQuery :: SchemaCache -> PgVersion -> AppConfig -> Schema -> DbHandler (Maybe (TablesMap, RoutineMap, Maybe Text))
 openApiQuery sCache pgVer AppConfig{..} tSchema =
   lift $ case configOpenApiMode of
     OAFollowPriv -> do
       tableAccess <- SQL.statement [tSchema] (SchemaCache.accessibleTables pgVer configDbPreparedStatements)
       Just <$> ((,,)
             (HM.filterWithKey (\qi _ -> S.member qi tableAccess) $ SchemaCache.dbTables sCache)
-        <$> SQL.statement tSchema (SchemaCache.accessibleProcs pgVer configDbPreparedStatements)
+        <$> SQL.statement tSchema (SchemaCache.accessibleFuncs pgVer configDbPreparedStatements)
         <*> SQL.statement tSchema (SchemaCache.schemaDescription configDbPreparedStatements))
     OAIgnorePriv ->
       Just <$> ((,,)
             (HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) $ SchemaCache.dbTables sCache)
-            (HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) $ SchemaCache.dbProcs sCache)
+            (HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) $ SchemaCache.dbRoutines sCache)
         <$> SQL.statement tSchema (SchemaCache.schemaDescription configDbPreparedStatements))
     OADisabled ->
       pure Nothing
 
-txMode :: ApiRequest -> SQL.Mode
-txMode ApiRequest{..} =
-  case (iAction, iTarget) of
-    (ActionRead _, _) ->
-      SQL.Read
-    (ActionInspect _, _) ->
-      SQL.Read
-    (ActionInvoke InvGet, _) ->
-      SQL.Read
-    (ActionInvoke InvHead, _) ->
-      SQL.Read
-    (ActionInvoke InvPost, TargetProc ProcDescription{pdVolatility=Stable} _) ->
-      SQL.Read
-    (ActionInvoke InvPost, TargetProc ProcDescription{pdVolatility=Immutable} _) ->
-      SQL.Read
-    _ ->
-      SQL.Write
-
 writeQuery :: MutateReadPlan -> ApiRequest -> AppConfig  -> DbHandler ResultSet
-writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq conf =
+writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan, mrResAgg} apiReq@ApiRequest{iPreferences=Preferences{..}} conf =
   let
     (isInsert, pkCols) = case mrMutatePlan of {Insert{insPkCols} -> (True, insPkCols); _ -> (False, mempty);}
   in
@@ -221,7 +195,8 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq conf =
       (QueryBuilder.mutatePlanToQuery mrMutatePlan)
       isInsert
       (iAcceptMediaType apiReq)
-      (iPreferRepresentation apiReq)
+      mrResAgg
+      preferRepresentation
       pkCols
       (configDbPreparedStatements conf)
 
@@ -231,7 +206,7 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq conf =
 failNotSingular :: MediaType -> ResultSet -> DbHandler ()
 failNotSingular _ RSPlan{} = pure ()
 failNotSingular mediaType RSStandard{rsQueryTotal=queryTotal} =
-  when (mediaType == MTSingularJSON && queryTotal /= 1) $ do
+  when (elem mediaType [MTSingularJSON True,MTSingularJSON False]  && queryTotal /= 1) $ do
     lift SQL.condemn
     throwError $ Error.singularityError queryTotal
 
@@ -245,24 +220,23 @@ failsChangesOffLimits (Just maxChanges) RSStandard{rsQueryTotal=queryTotal} =
 
 -- | Set a transaction to roll back if requested
 optionalRollback :: AppConfig -> ApiRequest -> DbHandler ()
-optionalRollback AppConfig{..} ApiRequest{..} = do
+optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} = do
   lift $ when (shouldRollback || (configDbTxRollbackAll && not shouldCommit)) $ do
     SQL.sql "SET CONSTRAINTS ALL IMMEDIATE"
     SQL.condemn
   where
     shouldCommit =
-      configDbTxAllowOverride && iPreferTransaction == Just Commit
+      configDbTxAllowOverride && preferTransaction == Just Commit
     shouldRollback =
-      configDbTxAllowOverride && iPreferTransaction == Just Rollback
+      configDbTxAllowOverride && preferTransaction == Just Rollback
 
--- | Runs local(transaction scoped) GUCs for every request, plus the pre-request function
+-- | Runs local (transaction scoped) GUCs for every request.
-setPgLocals :: AppConfig   -> KM.KeyMap JSON.Value -> Text ->
+setPgLocals :: AppConfig  -> KM.KeyMap JSON.Value -> BS.ByteString -> [(ByteString, ByteString)] ->
-               ApiRequest  -> ByteString -> PgVersion -> DbHandler ()
+               ApiRequest -> PgVersion -> DbHandler ()
-setPgLocals conf claims role req jsonDbS actualPgVersion = do
+setPgLocals AppConfig{..} claims role roleSettings req actualPgVersion = lift $
-  lift $ SQL.statement mempty $ SQL.dynamicallyParameterized
+  SQL.statement mempty $ SQL.dynamicallyParameterized
-    ("select " <> intercalateSnippet ", " (searchPathSql : roleSql ++ claimsSql ++ [methodSql, pathSql] ++ headersSql ++ cookiesSql ++ appSettingsSql ++ specSql))
+    ("select " <> intercalateSnippet ", " (searchPathSql : roleSql ++ roleSettingsSql ++ claimsSql ++ [methodSql, pathSql] ++ headersSql ++ cookiesSql ++ appSettingsSql))
-    HD.noResult (configDbPreparedStatements conf)
+    HD.noResult configDbPreparedStatements
-  lift $ traverse_ SQL.sql preReqSql
   where
     methodSql = setConfigLocal mempty ("request.method", iMethod req)
     pathSql = setConfigLocal mempty ("request.path", iPath req)
@@ -275,16 +249,13 @@ setPgLocals conf claims role req jsonDbS actualPgVersion = do
     claimsSql = if usesLegacyGucs
                   then setConfigLocal "request.jwt.claim." <$> [(toUtf8 $ K.toText c, toUtf8 $ unquoted v) | (c,v) <- KM.toList claims]
                   else [setConfigLocal mempty ("request.jwt.claims", LBS.toStrict $ JSON.encode claims)]
-    roleSql = [setConfigLocal mempty ("role", toUtf8 role)]
+    roleSql = [setConfigLocal mempty ("role", role)]
-    appSettingsSql = setConfigLocal mempty <$> (join bimap toUtf8 <$> configAppSettings conf)
+    roleSettingsSql = setConfigLocal mempty <$> roleSettings
+    appSettingsSql = setConfigLocal mempty <$> (join bimap toUtf8 <$> configAppSettings)
     searchPathSql =
-      let schemas = pgFmtIdentList (iSchema req : configDbExtraSearchPath conf) in
+      let schemas = escapeIdentList (iSchema req : configDbExtraSearchPath) in
       setConfigLocal mempty ("search_path", schemas)
-    preReqSql = (\f -> "select " <> fromQi f <> "();") <$> configDbPreRequest conf
+    usesLegacyGucs = configDbUseLegacyGucs && actualPgVersion < pgVersion140
-    specSql = case iTarget req of
-      TargetProc{tpIsRootSpec=True} -> [setConfigLocal mempty ("request.spec", jsonDbS)]
-      _                             -> mempty
-    usesLegacyGucs = configDbUseLegacyGucs conf && actualPgVersion < pgVersion140
 
     unquoted :: JSON.Value -> Text
     unquoted (JSON.String t) = t
@@ -292,3 +263,12 @@ setPgLocals conf claims role req jsonDbS actualPgVersion = do
       toS $ formatScientific Fixed (if isInteger n then Just 0 else Nothing) n
     unquoted (JSON.Bool b) = show b
     unquoted v = T.decodeUtf8 . LBS.toStrict $ JSON.encode v
+
+-- | Runs the pre-request function.
+runPreReq :: AppConfig -> DbHandler ()
+runPreReq conf = lift $ traverse_ (SQL.statement mempty . stmt) (configDbPreRequest conf)
+  where
+    stmt req = SQL.dynamicallyParameterized
+      ("select " <> fromQi req <> "()")
+      HD.noResult
+      (configDbPreparedStatements conf)

src/PostgREST/Query/QueryBuilder.hs

@@ -17,22 +17,25 @@ module PostgREST.Query.QueryBuilder
   ) where
 
 import qualified Data.ByteString.Char8           as BS
-import qualified Data.Set                        as S
 import qualified Hasql.DynamicStatements.Snippet as SQL
 
 import Data.Tree (Tree (..))
 
 import PostgREST.ApiRequest.Preferences   (PreferResolution (..))
+import PostgREST.Config.PgVersion         (PgVersion, pgVersion110,
+                                           pgVersion130)
 import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
-import PostgREST.SchemaCache.Proc         (ProcParam (..))
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                            Junction (..),
-                                           Relationship (..))
+                                           Relationship (..),
+                                           relIsToOne)
+import PostgREST.SchemaCache.Routine      (RoutineParam (..))
 
 import PostgREST.ApiRequest.Types
 import PostgREST.Plan.CallPlan
 import PostgREST.Plan.MutatePlan
 import PostgREST.Plan.ReadPlan
+import PostgREST.Plan.Types
 import PostgREST.Query.SqlFragment
 import PostgREST.RangeQuery        (allRange)
 
@@ -41,7 +44,7 @@ import Protolude
 readPlanToQuery :: ReadPlanTree -> SQL.Snippet
 readPlanToQuery (Node ReadPlan{select,from=mainQi,fromAlias,where_=logicForest,order, range_=readRange, relToParent, relJoinConds} forest) =
   "SELECT " <>
-  intercalateSnippet ", " ((pgFmtSelectItem qi <$> select) ++ selects) <> " " <>
+  intercalateSnippet ", " ((pgFmtSelectItem qi <$> (if null select && null forest then defSelect else select)) ++ selects) <> " " <>
   fromFrag <> " " <>
   intercalateSnippet " " joins <> " " <>
   (if null logicForest && null relJoinConds
@@ -52,167 +55,138 @@ readPlanToQuery (Node ReadPlan{select,from=mainQi,fromAlias,where_=logicForest,o
   where
     fromFrag = fromF relToParent mainQi fromAlias
     qi = getQualifiedIdentifier relToParent mainQi fromAlias
+    defSelect = [(unknownField "*" [], Nothing, Nothing)] -- gets all the columns in case of an empty select, ignoring/obtaining these columns is done at the aggregation stage
     (selects, joins) = foldr getSelectsJoins ([],[]) forest
 
 getSelectsJoins :: ReadPlanTree -> ([SQL.Snippet], [SQL.Snippet]) -> ([SQL.Snippet], [SQL.Snippet])
 getSelectsJoins (Node ReadPlan{relToParent=Nothing} _) _ = ([], [])
-getSelectsJoins rr@(Node ReadPlan{relName, relToParent=Just rel, relAggAlias, relAlias, relJoinType=joinType} _) (selects,joins) =
+getSelectsJoins rr@(Node ReadPlan{select, relName, relToParent=Just rel, relAggAlias, relAlias, relJoinType, relIsSpread} forest) (selects,joins) =
   let
     subquery = readPlanToQuery rr
     aliasOrName = pgFmtIdent $ fromMaybe relName relAlias
     aggAlias = pgFmtIdent relAggAlias
     correlatedSubquery sub al cond =
-      (if joinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> SQL.sql al <> " ON " <> cond
+      (if relJoinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> al <> " ON " <> cond
-    isToOne = case rel of
+    (sel, joi) = if relIsToOne rel
-      Relationship{relCardinality=M2O _ _} -> True
-      Relationship{relCardinality=O2O _ _} -> True
-      ComputedRelationship{relToOne=True}  -> True
-      _                                    -> False
-    (sel, joi) = if isToOne
       then
-        ( SQL.sql ("row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName)
+        ( if relIsSpread
+            then aggAlias <> ".*"
+            else "row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName
         , correlatedSubquery subquery aggAlias "TRUE")
       else
-        ( SQL.sql $ "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> aliasOrName
+        ( "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> aliasOrName
         , correlatedSubquery (
-            "SELECT json_agg(" <> SQL.sql aggAlias <> ") AS " <> SQL.sql aggAlias <>
+            "SELECT json_agg(" <> aggAlias <> ") AS " <> aggAlias <>
-            "FROM (" <> subquery <> " ) AS " <> SQL.sql aggAlias
+            "FROM (" <> subquery <> " ) AS " <> aggAlias
-          ) aggAlias $ if joinType == Just JTInner then SQL.sql aggAlias <> " IS NOT NULL" else "TRUE")
+          ) aggAlias $ if relJoinType == Just JTInner then aggAlias <> " IS NOT NULL" else "TRUE")
   in
-  (sel:selects, joi:joins)
+  (if null select && null forest then selects else sel:selects, joi:joins)
 
 mutatePlanToQuery :: MutatePlan -> SQL.Snippet
-mutatePlanToQuery (Insert mainQi iCols body onConflct putConditions returnings _) =
+mutatePlanToQuery (Insert mainQi iCols body onConflct putConditions returnings _ applyDefaults) =
-  "WITH " <> normalizedBody body <> " " <>
+  "INSERT INTO " <> fromQi mainQi <> (if null iCols then " " else "(" <> cols <> ") ") <>
-  "INSERT INTO " <> SQL.sql (fromQi mainQi) <> SQL.sql (if S.null iCols then " " else "(" <> cols <> ") ") <>
+  fromJsonBodyF body iCols True False applyDefaults <>
-  "SELECT " <> SQL.sql cols <> " " <>
-  SQL.sql ("FROM json_populate_recordset (null::" <> fromQi mainQi <> ", " <> selectBody <> ") _ ") <>
   -- Only used for PUT
-  (if null putConditions then mempty else "WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree (QualifiedIdentifier mempty "_") <$> putConditions)) <>
+  (if null putConditions then mempty else "WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree (QualifiedIdentifier mempty "pgrst_body") <$> putConditions)) <>
-  SQL.sql (BS.unwords [
+  maybe mempty (\(oncDo, oncCols) ->
-    maybe "" (\(oncDo, oncCols) ->
+    if null oncCols then
-      if null oncCols then
+      mempty
-        mempty
+    else
-      else
+      " ON CONFLICT(" <> intercalateSnippet ", " (pgFmtIdent <$> oncCols) <> ") " <> case oncDo of
-        " ON CONFLICT(" <> BS.intercalate ", " (pgFmtIdent <$> oncCols) <> ") " <> case oncDo of
+      IgnoreDuplicates ->
-        IgnoreDuplicates ->
+        "DO NOTHING"
-          "DO NOTHING"
+      MergeDuplicates  ->
-        MergeDuplicates  ->
+        if null iCols
-          if S.null iCols
+           then "DO NOTHING"
-             then "DO NOTHING"
+           else "DO UPDATE SET " <> intercalateSnippet ", " ((pgFmtIdent . cfName) <> const " = EXCLUDED." <> (pgFmtIdent . cfName) <$> iCols)
-             else "DO UPDATE SET " <> BS.intercalate ", " (pgFmtIdent <> const " = EXCLUDED." <> pgFmtIdent <$> S.toList iCols)
+    ) onConflct <> " " <>
-      ) onConflct,
+  returningF mainQi returnings
-    returningF mainQi returnings
-    ])
   where
-    cols = BS.intercalate ", " $ pgFmtIdent <$> S.toList iCols
+    cols = intercalateSnippet ", " $ pgFmtIdent . cfName <$> iCols
 
 -- An update without a limit is always filtered with a WHERE
-mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings)
+mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings applyDefaults)
-  | S.null uCols =
+  | null uCols =
     -- if there are no columns we cannot do UPDATE table SET {empty}, it'd be invalid syntax
     -- selecting an empty resultset from mainQi gives us the column names to prevent errors when using &select=
     -- the select has to be based on "returnings" to make computed overloaded functions not throw
-    SQL.sql $ "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"
+    "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"
 
   | range == allRange =
-    "WITH " <> normalizedBody body <> " " <>
+    "UPDATE " <> mainTbl <> " SET " <> nonRangeCols <> " " <>
-    "UPDATE " <> mainTbl <> " SET " <> SQL.sql nonRangeCols <> " " <>
+    fromJsonBodyF body uCols False False applyDefaults <>
-    "FROM (SELECT * FROM json_populate_recordset (null::" <> mainTbl <> " , " <> SQL.sql selectBody <> " )) _ " <>
     whereLogic <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings
 
   | otherwise =
-    "WITH " <> normalizedBody body <> ", " <>
+    "WITH " <>
-    "pgrst_update_body AS (SELECT * FROM json_populate_recordset (null::" <> mainTbl <> " , " <> SQL.sql selectBody <> " ) LIMIT 1), " <>
+    "pgrst_update_body AS (" <> fromJsonBodyF body uCols True True applyDefaults <> "), " <>
     "pgrst_affected_rows AS (" <>
-      "SELECT " <> SQL.sql rangeIdF <> " FROM " <> mainTbl <>
+      "SELECT " <> rangeIdF <> " FROM " <> mainTbl <>
       whereLogic <> " " <>
       orderF mainQi ordts <> " " <>
       limitOffsetF range <>
     ") " <>
-    "UPDATE " <> mainTbl <> " SET " <> SQL.sql rangeCols <>
+    "UPDATE " <> mainTbl <> " SET " <> rangeCols <>
     "FROM pgrst_affected_rows " <>
-    "WHERE " <> SQL.sql whereRangeIdF <> " " <>
+    "WHERE " <> whereRangeIdF <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings
 
   where
     whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
-    mainTbl = SQL.sql (fromQi mainQi)
+    mainTbl = fromQi mainQi
-    emptyBodyReturnedColumns = if null returnings then "NULL" else BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
+    emptyBodyReturnedColumns = if null returnings then "NULL" else intercalateSnippet ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
-    nonRangeCols = BS.intercalate ", " (pgFmtIdent <> const " = _." <> pgFmtIdent <$> S.toList uCols)
+    nonRangeCols = intercalateSnippet ", " (pgFmtIdent . cfName <> const " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_body") . cfName <$> uCols)
-    rangeCols = BS.intercalate ", " ((\col -> pgFmtIdent col <> " = (SELECT " <> pgFmtIdent col <> " FROM pgrst_update_body) ") <$> S.toList uCols)
+    rangeCols = intercalateSnippet ", " ((\col -> pgFmtIdent (cfName col) <> " = (SELECT " <> pgFmtIdent (cfName col) <> " FROM pgrst_update_body) ") <$> uCols)
-    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
+    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (cfName . coField <$> ordts)
 
 mutatePlanToQuery (Delete mainQi logicForest range ordts returnings)
   | range == allRange =
-    "DELETE FROM " <> SQL.sql (fromQi mainQi) <> " " <>
+    "DELETE FROM " <> fromQi mainQi <> " " <>
     whereLogic <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings
 
   | otherwise =
     "WITH " <>
     "pgrst_affected_rows AS (" <>
-      "SELECT " <> SQL.sql rangeIdF <> " FROM " <> SQL.sql (fromQi mainQi) <>
+      "SELECT " <> rangeIdF <> " FROM " <> fromQi mainQi <>
        whereLogic <> " " <>
       orderF mainQi ordts <> " " <>
       limitOffsetF range <>
     ") " <>
-    "DELETE FROM " <> SQL.sql (fromQi mainQi) <> " " <>
+    "DELETE FROM " <> fromQi mainQi <> " " <>
     "USING pgrst_affected_rows " <>
-    "WHERE " <> SQL.sql whereRangeIdF <> " " <>
+    "WHERE " <> whereRangeIdF <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings
 
   where
     whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
-    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
+    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (cfName . coField <$> ordts)
 
-callPlanToQuery :: CallPlan -> SQL.Snippet
+callPlanToQuery :: CallPlan -> PgVersion -> SQL.Snippet
-callPlanToQuery (FunctionCall qi params args returnsScalar multipleCall returnings) =
+callPlanToQuery (FunctionCall qi params args returnsScalar returnsSetOfScalar returnsCompositeAlias returnings) pgVer =
-  prmsCTE <> argsBody
+  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call.pgrst_scalar" else returnedColumns) <> " " <>
+  fromCall
   where
-    (prmsCTE, argFrag) = case params of
+    fromCall = case params of
-      OnePosParam prm -> ("WITH pgrst_args AS (SELECT NULL)", singleParameter args (encodeUtf8 $ ppType prm))
+      OnePosParam prm -> "FROM " <> callIt (singleParameter args $ encodeUtf8 $ ppType prm)
-      KeyParams []    -> (mempty, mempty)
+      KeyParams []    -> "FROM " <> callIt mempty
-      KeyParams prms  -> (
+      KeyParams prms  -> fromJsonBodyF args ((\p -> CoercibleField (ppName p) mempty False (ppTypeMaxLength p) Nothing Nothing) <$> prms) False True False <> ", " <>
-          "WITH " <> normalizedBody args <> ", " <>
+                         "LATERAL " <> callIt (fmtParams prms)
-          SQL.sql (
-            BS.unwords [
-            "pgrst_args AS (",
-              "SELECT * FROM json_to_recordset(" <> selectBody <> ") AS _(" <> fmtParams prms (const mempty) (\a -> " " <> encodeUtf8 (ppType a)) <> ")",
-            ")"])
-         , SQL.sql $ if multipleCall
-             then fmtParams prms varadicPrefix (\a -> " := pgrst_args." <> pgFmtIdent (ppName a))
-             else fmtParams prms varadicPrefix (\a -> " := (SELECT " <> pgFmtIdent (ppName a) <> " FROM pgrst_args LIMIT 1)")
-        )
 
-    fmtParams :: [ProcParam] -> (ProcParam -> SqlFragment) -> (ProcParam -> SqlFragment) -> SqlFragment
+    callIt :: SQL.Snippet -> SQL.Snippet
-    fmtParams prms prmFragPre prmFragSuf = BS.intercalate ", "
+    callIt argument | pgVer < pgVersion130 && pgVer >= pgVersion110 && returnsCompositeAlias = "(SELECT (" <> fromQi qi <> "(" <> argument <> ")).*) pgrst_call"
-      ((\a -> prmFragPre a <> pgFmtIdent (ppName a) <> prmFragSuf a) <$> prms)
+                    | returnsScalar || returnsSetOfScalar                                    = "(SELECT " <> fromQi qi <> "(" <> argument <> ") pgrst_scalar) pgrst_call"
+                    | otherwise                                                              = fromQi qi <> "(" <> argument <> ") pgrst_call"
 
-    varadicPrefix :: ProcParam -> SqlFragment
+    fmtParams :: [RoutineParam] -> SQL.Snippet
-    varadicPrefix a = if ppVar a then "VARIADIC " else mempty
+    fmtParams prms = intercalateSnippet ", "
-
+      ((\a -> (if ppVar a then "VARIADIC " else mempty) <> pgFmtIdent (ppName a) <> " := pgrst_body." <> pgFmtIdent (ppName a)) <$> prms)
-    argsBody :: SQL.Snippet
-    argsBody
-      | multipleCall =
-          if returnsScalar
-            then "SELECT " <> callIt <> " AS pgrst_scalar FROM pgrst_args"
-            else "SELECT pgrst_lat_args.* FROM pgrst_args, " <>
-                 "LATERAL ( SELECT " <> returnedColumns <> " FROM " <> callIt <> " ) pgrst_lat_args"
-      | otherwise =
-          if returnsScalar
-            then "SELECT " <> callIt <> " AS pgrst_scalar"
-            else "SELECT " <> returnedColumns <> " FROM " <> callIt
-
-    callIt :: SQL.Snippet
-    callIt = SQL.sql (fromQi qi) <> "(" <> argFrag <> ")"
 
     returnedColumns :: SQL.Snippet
     returnedColumns
       | null returnings = "*"
-      | otherwise       = SQL.sql $ BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName qi) <$> returnings)
+      | otherwise       = intercalateSnippet ", " (pgFmtColumn (QualifiedIdentifier mempty "pgrst_call") <$> returnings)
-
 
 -- | SQL query meant for COUNTing the root node of the Tree.
 -- It only takes WHERE into account and doesn't include LIMIT/OFFSET because it would reduce the COUNT.
@@ -254,7 +228,7 @@ getQualifiedIdentifier rel mainQi tblAlias = case rel of
 
 -- FROM clause plus implicit joins
 fromF :: Maybe Relationship -> QualifiedIdentifier -> Maybe Alias -> SQL.Snippet
-fromF rel mainQi tblAlias = SQL.sql $ "FROM " <>
+fromF rel mainQi tblAlias = "FROM " <>
   (case rel of
     Just ComputedRelationship{relFunction,relTable} -> fromQi relFunction <> "(" <> pgFmtIdent (qiName relTable) <> ")"
     _                                               -> fromQi mainQi) <>

src/PostgREST/Query/SqlFragment.hs

@@ -4,43 +4,36 @@
 {-|
 Module      : PostgREST.Query.SqlFragment
 Description : Helper functions for PostgREST.QueryBuilder.
-
-Any function that outputs a SqlFragment should be in this module.
 -}
 module PostgREST.Query.SqlFragment
   ( noLocationF
-  , SqlFragment
+  , aggF
-  , asBinaryF
-  , asCsvF
-  , asGeoJsonF
-  , asJsonF
-  , asJsonSingleF
-  , asXmlF
   , countF
   , fromQi
   , limitOffsetF
   , locationF
   , mutRangeF
-  , normalizedBody
   , orderF
   , pgFmtColumn
   , pgFmtIdent
-  , pgFmtIdentList
   , pgFmtJoinCondition
   , pgFmtLogicTree
   , pgFmtOrderTerm
   , pgFmtSelectItem
+  , fromJsonBodyF
   , responseHeadersF
   , responseStatusF
   , returningF
-  , selectBody
   , singleParameter
+  , sourceCTE
   , sourceCTEName
   , unknownEncoder
   , intercalateSnippet
   , explainF
   , setConfigLocal
   , setConfigLocalJson
+  , escapeIdent
+  , escapeIdentList
   ) where
 
 import qualified Data.Aeson                      as JSON
@@ -57,87 +50,80 @@ import Control.Arrow ((***))
 import Data.Foldable                 (foldr1)
 import Text.InterpolatedString.Perl6 (qc)
 
-import PostgREST.ApiRequest.Types        (Alias, Cast, Field,
+import PostgREST.ApiRequest.Types        (Alias, Cast,
-                                          Filter (..),
                                           FtsOperator (..),
                                           JsonOperand (..),
                                           JsonOperation (..),
                                           JsonPath,
                                           LogicOperator (..),
-                                          LogicTree (..), OpExpr (..),
+                                          OpExpr (..),
+                                          OpQuantifier (..),
                                           Operation (..),
                                           OrderDirection (..),
                                           OrderNulls (..),
-                                          OrderTerm (..),
+                                          QuantOperator (..),
                                           SimpleOperator (..),
                                           TrileanVal (..))
 import PostgREST.MediaType               (MTPlanFormat (..),
                                           MTPlanOption (..))
 import PostgREST.Plan.ReadPlan           (JoinCondition (..))
+import PostgREST.Plan.Types              (CoercibleField (..),
+                                          CoercibleFilter (..),
+                                          CoercibleLogicTree (..),
+                                          CoercibleOrderTerm (..),
+                                          unknownField)
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           rangeLimit, rangeOffset)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier (..))
+import PostgREST.SchemaCache.Routine     (ResultAggregate (..),
+                                          Routine (..),
+                                          funcReturnsScalar,
+                                          funcReturnsSetOfScalar,
+                                          funcReturnsSingleComposite)
 
 import Protolude hiding (cast)
 
-
+sourceCTEName :: Text
--- | A part of a SQL query that cannot be executed independently
-type SqlFragment = ByteString
-
-noLocationF :: SqlFragment
-noLocationF = "array[]::text[]"
-
-sourceCTEName :: SqlFragment
 sourceCTEName = "pgrst_source"
 
-singleValOperator :: SimpleOperator -> SqlFragment
+sourceCTE :: SQL.Snippet
-singleValOperator = \case
+sourceCTE = "pgrst_source"
+
+noLocationF :: SQL.Snippet
+noLocationF = "array[]::text[]"
+
+simpleOperator :: SimpleOperator -> SQL.Snippet
+simpleOperator = \case
+  OpNotEqual        -> "<>"
+  OpContains        -> "@>"
+  OpContained       -> "<@"
+  OpOverlap         -> "&&"
+  OpStrictlyLeft    -> "<<"
+  OpStrictlyRight   -> ">>"
+  OpNotExtendsRight -> "&<"
+  OpNotExtendsLeft  -> "&>"
+  OpAdjacent        -> "-|-"
+
+quantOperator :: QuantOperator -> SQL.Snippet
+quantOperator = \case
   OpEqual            -> "="
   OpGreaterThanEqual -> ">="
   OpGreaterThan      -> ">"
   OpLessThanEqual    -> "<="
   OpLessThan         -> "<"
-  OpNotEqual         -> "<>"
   OpLike             -> "like"
   OpILike            -> "ilike"
-  OpContains         -> "@>"
-  OpContained        -> "<@"
-  OpOverlap          -> "&&"
-  OpStrictlyLeft     -> "<<"
-  OpStrictlyRight    -> ">>"
-  OpNotExtendsRight  -> "&<"
-  OpNotExtendsLeft   -> "&>"
-  OpAdjacent         -> "-|-"
   OpMatch            -> "~"
   OpIMatch           -> "~*"
 
-ftsOperator :: FtsOperator -> SqlFragment
+ftsOperator :: FtsOperator -> SQL.Snippet
 ftsOperator = \case
   FilterFts          -> "@@ to_tsquery"
   FilterFtsPlain     -> "@@ plainto_tsquery"
   FilterFtsPhrase    -> "@@ phraseto_tsquery"
   FilterFtsWebsearch -> "@@ websearch_to_tsquery"
 
--- |
--- These CTEs convert a json object into a json array, this way we can use json_populate_recordset for all json payloads
--- Otherwise we'd have to use json_populate_record for json objects and json_populate_recordset for json arrays
--- We do this in SQL to avoid processing the JSON in application code
--- TODO: At this stage there shouldn't be a Maybe since ApiRequest should ensure that an INSERT/UPDATE has a body
-normalizedBody :: Maybe LBS.ByteString -> SQL.Snippet
-normalizedBody body =
-  "pgrst_payload AS (SELECT " <> jsonPlaceHolder <> " AS json_data), " <>
-  SQL.sql (BS.unwords [
-    "pgrst_body AS (",
-      "SELECT",
-        "CASE WHEN json_typeof(json_data) = 'array'",
-          "THEN json_data",
-          "ELSE json_build_array(json_data)",
-        "END AS val",
-      "FROM pgrst_payload)"])
-  where
-    jsonPlaceHolder = SQL.encoderAndParam (HE.nullable HE.jsonLazyBytes) body
-
 singleParameter :: Maybe LBS.ByteString -> ByteString -> SQL.Snippet
 singleParameter body typ =
   if typ == "bytea"
@@ -145,9 +131,6 @@ singleParameter body typ =
     then SQL.encoderAndParam (HE.nullable HE.bytea) (LBS.toStrict <$> body)
     else SQL.encoderAndParam (HE.nullable HE.unknown) (LBS.toStrict <$> body) <> "::" <> SQL.sql typ
 
-selectBody :: SqlFragment
-selectBody = "(SELECT val FROM pgrst_body)"
-
 -- Here we build the pg array literal, e.g '{"Hebdon, John","Other","Another"}', manually.
 -- This is necessary to pass an "unknown" array and let pg infer the type.
 -- There are backslashes here, but since this value is parametrized and is not a string constant
@@ -162,8 +145,21 @@ pgBuildArrayLiteral vals =
  "{" <> T.intercalate "," (escaped <$> vals) <> "}"
 
 -- TODO: refactor by following https://github.com/PostgREST/postgrest/pull/1631#issuecomment-711070833
-pgFmtIdent :: Text -> SqlFragment
+pgFmtIdent :: Text -> SQL.Snippet
-pgFmtIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
+pgFmtIdent x = SQL.sql $ escapeIdent x
+
+escapeIdent :: Text -> ByteString
+escapeIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
+
+-- Only use it if the input comes from the database itself, like on `jsonb_build_object('column_from_a_table', val)..`
+pgFmtLit :: Text -> Text
+pgFmtLit x =
+  let trimmed = trimNullChars x
+      escaped = "'" <> T.replace "'" "''" trimmed <> "'"
+      slashed = T.replace "\\" "\\\\" escaped in
+  if "\\" `T.isInfixOf` escaped
+    then "E" <> slashed
+    else slashed
 
 trimNullChars :: Text -> Text
 trimNullChars = T.takeWhile (/= '\x0')
@@ -171,12 +167,12 @@ trimNullChars = T.takeWhile (/= '\x0')
 -- |
 -- Format a list of identifiers and separate them by commas.
 --
--- >>> pgFmtIdentList ["schema_1", "schema_2", "SPECIAL \"@/\\#~_-"]
+-- >>> escapeIdentList ["schema_1", "schema_2", "SPECIAL \"@/\\#~_-"]
 -- "\"schema_1\", \"schema_2\", \"SPECIAL \"\"@/\\#~_-\""
-pgFmtIdentList :: [Text] -> SqlFragment
+escapeIdentList :: [Text] -> ByteString
-pgFmtIdentList schemas = BS.intercalate ", " $ pgFmtIdent <$> schemas
+escapeIdentList schemas = BS.intercalate ", " $ escapeIdent <$> schemas
 
-asCsvF :: SqlFragment
+asCsvF :: SQL.Snippet
 asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
   where
     asCsvHeaderF =
@@ -184,32 +180,49 @@ asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
       "  FROM (" <>
       "    SELECT json_object_keys(r)::text as k" <>
       "    FROM ( " <>
-      "      SELECT row_to_json(hh) as r from " <> sourceCTEName <> " as hh limit 1" <>
+      "      SELECT row_to_json(hh) as r from " <> sourceCTE <> " as hh limit 1" <>
       "    ) s" <>
       "  ) a" <>
       ")"
     asCsvBodyF = "coalesce(string_agg(substring(_postgrest_t::text, 2, length(_postgrest_t::text) - 2), '\n'), '')"
 
-asJsonF :: Bool -> SqlFragment
+addNullsToSnip :: Bool -> SQL.Snippet -> SQL.Snippet
-asJsonF returnsScalar
+addNullsToSnip strip snip =
-  | returnsScalar = "coalesce(json_agg(_postgrest_t.pgrst_scalar), '[]')::character varying"
+  if strip then "json_strip_nulls(" <> snip <> ")" else  snip
-  | otherwise     = "coalesce(json_agg(_postgrest_t), '[]')::character varying"
 
-asJsonSingleF :: Bool -> SqlFragment
+asJsonSingleF :: Maybe Routine -> Bool -> SQL.Snippet
-asJsonSingleF returnsScalar
+asJsonSingleF rout strip
-  | returnsScalar = "coalesce((json_agg(_postgrest_t.pgrst_scalar)->0)::text, 'null')"
+  | returnsScalar = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0"  <> ", 'null')"
-  | otherwise     = "coalesce((json_agg(_postgrest_t)->0)::text, 'null')"
+  | otherwise     = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0"  <> ", 'null')"
+  where
+    returnsScalar = maybe False funcReturnsScalar rout
 
-asXmlF :: FieldName -> SqlFragment
+asJsonF :: Maybe Routine -> Bool -> SQL.Snippet
-asXmlF fieldName = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
+asJsonF rout strip
+  | returnsSingleComposite    = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0" <> ", 'null')"
+  | returnsScalar             = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0" <> ", 'null')"
+  | returnsSetOfScalar        = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)" <> ", '[]')"
+  | otherwise                 = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)" <> ", '[]')"
+  where
+    (returnsSingleComposite, returnsScalar, returnsSetOfScalar) = case rout of
+      Just r  -> (funcReturnsSingleComposite r, funcReturnsScalar r, funcReturnsSetOfScalar r)
+      Nothing -> (False, False, False)
 
-asGeoJsonF ::  SqlFragment
+
+asXmlF :: Maybe FieldName -> SQL.Snippet
+asXmlF (Just fieldName) = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
+-- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
+asXmlF Nothing          = "coalesce(xmlagg(_postgrest_t), '')"
+
+asGeoJsonF ::  SQL.Snippet
 asGeoJsonF = "json_build_object('type', 'FeatureCollection', 'features', coalesce(json_agg(ST_AsGeoJSON(_postgrest_t)::json), '[]'))"
 
-asBinaryF :: FieldName -> SqlFragment
+asBinaryF :: Maybe FieldName -> SQL.Snippet
-asBinaryF fieldName = "coalesce(string_agg(_postgrest_t." <> pgFmtIdent fieldName <> ", ''), '')"
+asBinaryF (Just fieldName) = "coalesce(string_agg(_postgrest_t." <> pgFmtIdent fieldName <> ", ''), '')"
+-- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
+asBinaryF Nothing          = "coalesce(string_agg(_postgrest_t, ''), '')"
 
-locationF :: [Text] -> SqlFragment
+locationF :: [Text] -> SQL.Snippet
 locationF pKeys = [qc|(
   WITH data AS (SELECT row_to_json(_) AS row FROM {sourceCTEName} AS _ LIMIT 1)
   SELECT array_agg(json_data.key || '=' || coalesce('eq.' || json_data.value, 'is.null'))
@@ -219,88 +232,161 @@ locationF pKeys = [qc|(
   where
     fmtPKeys = T.intercalate "','" pKeys
 
-fromQi :: QualifiedIdentifier -> SqlFragment
+fromQi :: QualifiedIdentifier -> SQL.Snippet
 fromQi t = (if T.null s then mempty else pgFmtIdent s <> ".") <> pgFmtIdent n
   where
     n = qiName t
     s = qiSchema t
 
-pgFmtColumn :: QualifiedIdentifier -> Text -> SqlFragment
+pgFmtColumn :: QualifiedIdentifier -> Text -> SQL.Snippet
 pgFmtColumn table "*" = fromQi table <> ".*"
 pgFmtColumn table c   = fromQi table <> "." <> pgFmtIdent c
 
-pgFmtField :: QualifiedIdentifier -> Field -> SQL.Snippet
+pgFmtCallUnary :: Text -> SQL.Snippet -> SQL.Snippet
-pgFmtField table (c, []) = SQL.sql (pgFmtColumn table c)
+pgFmtCallUnary f x = SQL.sql (encodeUtf8 f) <> "(" <> x <> ")"
--- Using to_jsonb instead of to_json to avoid missing operator errors when filtering:
--- "operator does not exist: json = unknown"
-pgFmtField table (c, jp) = SQL.sql ("to_jsonb(" <> pgFmtColumn table c <> ")") <> pgFmtJsonPath jp
 
-pgFmtSelectItem :: QualifiedIdentifier -> (Field, Maybe Cast, Maybe Alias) -> SQL.Snippet
+pgFmtField :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
-pgFmtSelectItem table (f@(fName, jp), Nothing, alias) = pgFmtField table f <> SQL.sql (pgFmtAs fName jp alias)
+pgFmtField table CoercibleField{cfName=fn, cfJsonPath=[]}                                = pgFmtColumn table fn
+pgFmtField table CoercibleField{cfName=fn, cfToJson=doToJson, cfJsonPath=jp} | doToJson  = "to_jsonb(" <> pgFmtColumn table fn <> ")" <> pgFmtJsonPath jp
+                                                                             | otherwise = pgFmtColumn table fn <> pgFmtJsonPath jp
+
+-- Select the value of a named element from a table, applying its optional coercion mapping if any.
+pgFmtTableCoerce :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
+pgFmtTableCoerce table fld@(CoercibleField{cfTransform=(Just formatterProc)}) = pgFmtCallUnary formatterProc (pgFmtField table fld)
+pgFmtTableCoerce table f = pgFmtField table f
+
+-- | Like the previous but now we just have a name so no namespace or JSON paths.
+pgFmtCoerceNamed :: CoercibleField -> SQL.Snippet
+pgFmtCoerceNamed CoercibleField{cfName=fn, cfTransform=(Just formatterProc)} = pgFmtCallUnary formatterProc (pgFmtIdent fn) <> " AS " <> pgFmtIdent fn
+pgFmtCoerceNamed CoercibleField{cfName=fn} = pgFmtIdent fn
+
+pgFmtSelectItem :: QualifiedIdentifier -> (CoercibleField, Maybe Cast, Maybe Alias) -> SQL.Snippet
+pgFmtSelectItem table (fld, Nothing, alias) = pgFmtTableCoerce table fld <> pgFmtAs (cfName fld) (cfJsonPath fld) alias
 -- Ideally we'd quote the cast with "pgFmtIdent cast". However, that would invalidate common casts such as "int", "bigint", etc.
 -- Try doing: `select 1::"bigint"` - it'll err, using "int8" will work though. There's some parser magic that pg does that's invalidated when quoting.
 -- Not quoting should be fine, we validate the input on Parsers.
-pgFmtSelectItem table (f@(fName, jp), Just cast, alias) = "CAST (" <> pgFmtField table f <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )" <> SQL.sql (pgFmtAs fName jp alias)
+pgFmtSelectItem table (fld, Just cast, alias) = "CAST (" <> pgFmtTableCoerce table fld <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )" <> pgFmtAs (cfName fld) (cfJsonPath fld) alias
 
-pgFmtOrderTerm :: QualifiedIdentifier -> OrderTerm -> SQL.Snippet
+-- TODO: At this stage there shouldn't be a Maybe since ApiRequest should ensure that an INSERT/UPDATE has a body
-pgFmtOrderTerm qi ot =
+fromJsonBodyF :: Maybe LBS.ByteString -> [CoercibleField] -> Bool -> Bool -> Bool -> SQL.Snippet
-  pgFmtField qi (otTerm ot) <> " " <>
+fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
-  SQL.sql (BS.unwords [
+  (if includeSelect then "SELECT " <> namedCols <> " " else mempty) <>
-    maybe mempty direction $ otDirection ot,
+  "FROM (SELECT " <> jsonPlaceHolder <> " AS json_data) pgrst_payload, " <>
-    maybe mempty nullOrder $ otNullOrder ot])
+  -- convert a json object into a json array, this way we can use json_to_recordset for all json payloads
+  -- Otherwise we'd have to use json_to_record for json objects and json_to_recordset for json arrays
+  -- We do this in SQL to avoid processing the JSON in application code
+  "LATERAL (SELECT CASE WHEN " <> jsonTypeofF <> "(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE " <> jsonBuildArrayF <> "(pgrst_payload.json_data) END AS val) pgrst_uniform_json, " <>
+  (if includeDefaults
+    then "LATERAL (SELECT jsonb_agg(jsonb_build_object(" <> defsJsonb <> ") || elem) AS val from jsonb_array_elements(pgrst_uniform_json.val) elem) pgrst_json_defs, "
+    else mempty) <>
+  "LATERAL (SELECT " <> parsedCols <> " FROM " <>
+    (if null fields
+      -- When we are inserting no columns (e.g. using default values), we can't use our ordinary `json_to_recordset`
+      -- because it can't extract records with no columns (there's no valid syntax for the `AS (colName colType,...)`
+      -- part). But we still need to ensure as many rows are created as there are array elements.
+      then SQL.sql $ jsonArrayElementsF <> "(" <> finalBodyF <> ") _ "
+      else jsonToRecordsetF <> "(" <> SQL.sql finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
+    ) <>
+  ") pgrst_body "
   where
+    namedCols = intercalateSnippet ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . cfName <$> fields
+    parsedCols = intercalateSnippet ", " $ pgFmtCoerceNamed <$> fields
+    typedCols = intercalateSnippet ", " $ pgFmtIdent . cfName <> const " " <> SQL.sql . encodeUtf8 . cfIRType <$> fields
+    defsJsonb = SQL.sql $ BS.intercalate "," fieldsWDefaults
+    fieldsWDefaults = mapMaybe (\case
+        CoercibleField{cfName=nam, cfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
+        CoercibleField{cfDefault=Nothing} -> Nothing
+      ) fields
+    (finalBodyF, jsonTypeofF, jsonBuildArrayF, jsonArrayElementsF, jsonToRecordsetF) =
+      if includeDefaults
+        then ("pgrst_json_defs.val", "jsonb_typeof", "jsonb_build_array", "jsonb_array_elements", "jsonb_to_recordset")
+        else ("pgrst_uniform_json.val", "json_typeof", "json_build_array", "json_array_elements", "json_to_recordset")
+    jsonPlaceHolder = SQL.encoderAndParam (HE.nullable $ if includeDefaults then HE.jsonbLazyBytes else HE.jsonLazyBytes) body
+
+pgFmtOrderTerm :: QualifiedIdentifier -> CoercibleOrderTerm -> SQL.Snippet
+pgFmtOrderTerm qi ot =
+  fmtOTerm ot <> " " <>
+  SQL.sql (BS.unwords [
+    maybe mempty direction $ coDirection ot,
+    maybe mempty nullOrder $ coNullOrder ot])
+  where
+    fmtOTerm = \case
+      CoercibleOrderTerm{coField=cof}                            -> pgFmtField qi cof
+      CoercibleOrderRelationTerm{coRelation, coRelTerm=(fn, jp)} -> pgFmtField (QualifiedIdentifier mempty coRelation) (unknownField fn jp)
+
     direction OrderAsc  = "ASC"
     direction OrderDesc = "DESC"
 
     nullOrder OrderNullsFirst = "NULLS FIRST"
     nullOrder OrderNullsLast  = "NULLS LAST"
 
+-- | Interpret a literal in the way the planner indicated through the CoercibleField.
+pgFmtUnknownLiteralForField :: SQL.Snippet -> CoercibleField -> SQL.Snippet
+pgFmtUnknownLiteralForField value CoercibleField{cfTransform=(Just parserProc)} = pgFmtCallUnary parserProc value
+-- But when no transform is requested, we just use the literal as-is.
+pgFmtUnknownLiteralForField value _ = value
 
-pgFmtFilter :: QualifiedIdentifier -> Filter -> SQL.Snippet
+-- | Array version of the above, used by ANY().
-pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> case oper of
+pgFmtArrayLiteralForField :: [Text] -> CoercibleField -> SQL.Snippet
-   Op op val  -> pgFmtFieldOp op <> " " <> case op of
+-- When a transformation is requested, we need to apply the transformation to each element of the array. This could be done by just making a query with `parser(value)` for each value, but may lead to huge query lengths. Imagine `data_representations.color_from_text('...'::text)` for repeated for a hundred values. Instead we use `unnest()` to unpack a standard array literal and then apply the transformation to each element, like a map.
-     OpLike  -> unknownLiteral (T.map star val)
+-- Note the literals will be treated as text since in every case when we use ANY() the parameters are textual (coming from a query string). We want to rely on the `text->domain` parser to do the right thing.
-     OpILike -> unknownLiteral (T.map star val)
+pgFmtArrayLiteralForField values CoercibleField{cfTransform=(Just parserProc)} = SQL.sql "(SELECT " <> pgFmtCallUnary parserProc (SQL.sql "unnest(" <> unknownLiteral (pgBuildArrayLiteral values) <> "::text[])") <> ")"
-     _       -> unknownLiteral val
+-- When no transformation is requested, we don't need a subquery.
+pgFmtArrayLiteralForField values _ = unknownLiteral (pgBuildArrayLiteral values)
+
+
+pgFmtFilter :: QualifiedIdentifier -> CoercibleFilter -> SQL.Snippet
+pgFmtFilter _ (CoercibleFilterNullEmbed hasNot fld) = pgFmtIdent fld <> " IS " <> (if hasNot then "NOT" else mempty) <> " NULL"
+pgFmtFilter _ (CoercibleFilter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
+pgFmtFilter table (CoercibleFilter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
+   Op op val  -> " " <> simpleOperator op <> " " <> pgFmtUnknownLiteralForField (unknownLiteral val) fld
+
+   OpQuant op quant val -> " " <> quantOperator op <> " " <> case op of
+     OpLike  -> fmtQuant quant $ unknownLiteral (T.map star val)
+     OpILike -> fmtQuant quant $ unknownLiteral (T.map star val)
+     _       -> fmtQuant quant $ pgFmtUnknownLiteralForField (unknownLiteral val) fld
 
    -- IS cannot be prepared. `PREPARE boolplan AS SELECT * FROM projects where id IS $1` will give a syntax error.
    -- The above can be fixed by using `PREPARE boolplan AS SELECT * FROM projects where id IS NOT DISTINCT FROM $1;`
    -- However that would not accept the TRUE/FALSE/NULL/UNKNOWN keywords. See: https://stackoverflow.com/questions/6133525/proper-way-to-set-preparedstatement-parameter-to-null-under-postgres.
    -- This is why `IS` operands are whitelisted at the Parsers.hs level
-   Is triVal -> pgFmtField table fld <> " IS " <> case triVal of
+   Is triVal -> " IS " <> case triVal of
      TriTrue    -> "TRUE"
      TriFalse   -> "FALSE"
      TriNull    -> "NULL"
      TriUnknown -> "UNKNOWN"
 
+   IsDistinctFrom val -> " IS DISTINCT FROM " <> unknownLiteral val
+
    -- We don't use "IN", we use "= ANY". IN has the following disadvantages:
    -- + No way to use an empty value on IN: "col IN ()" is invalid syntax. With ANY we can do "= ANY('{}')"
    -- + Can invalidate prepared statements: multiple parameters on an IN($1, $2, $3) will lead to using different prepared statements and not take advantage of caching.
-   In vals -> pgFmtField table fld <> " " <> case vals of
+   In vals -> " " <> case vals of
       [""] -> "= ANY('{}') "
-      _    -> "= ANY (" <> unknownLiteral (pgBuildArrayLiteral vals) <> ") "
+      _    -> "= ANY (" <> pgFmtArrayLiteralForField vals fld <> ") "
 
-   Fts op lang val ->
+   Fts op lang val -> " " <> ftsOperator op <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
-     pgFmtFieldFts op <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
  where
    ftsLang = maybe mempty (\l -> unknownLiteral l <> ", ")
-   pgFmtFieldOp op = pgFmtField table fld <> " " <> SQL.sql (singleValOperator op)
-   pgFmtFieldFts op = pgFmtField table fld <> " " <> SQL.sql (ftsOperator op)
    notOp = if hasNot then "NOT" else mempty
    star c = if c == '*' then '%' else c
+   fmtQuant q val = case q of
+    Just QuantAny -> "ANY(" <> val <> ")"
+    Just QuantAll -> "ALL(" <> val <> ")"
+    Nothing       -> val
 
 pgFmtJoinCondition :: JoinCondition -> SQL.Snippet
 pgFmtJoinCondition (JoinCondition (qi1, col1) (qi2, col2)) =
-  SQL.sql $ pgFmtColumn qi1 col1 <> " = " <> pgFmtColumn qi2 col2
+  pgFmtColumn qi1 col1 <> " = " <> pgFmtColumn qi2 col2
 
-pgFmtLogicTree :: QualifiedIdentifier -> LogicTree -> SQL.Snippet
+pgFmtLogicTree :: QualifiedIdentifier -> CoercibleLogicTree -> SQL.Snippet
-pgFmtLogicTree qi (Expr hasNot op forest) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTree qi <$> forest) <> ")"
+pgFmtLogicTree qi (CoercibleExpr hasNot op forest) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTree qi <$> forest) <> ")"
   where
     notOp =  if hasNot then "NOT" else mempty
 
     opSql And = " AND "
     opSql Or  = " OR "
-pgFmtLogicTree qi (Stmnt flt) = pgFmtFilter qi flt
+pgFmtLogicTree qi (CoercibleStmnt flt) = pgFmtFilter qi flt
 
 pgFmtJsonPath :: JsonPath -> SQL.Snippet
 pgFmtJsonPath = \case
@@ -311,7 +397,7 @@ pgFmtJsonPath = \case
     pgFmtJsonOperand (JKey k) = unknownLiteral k
     pgFmtJsonOperand (JIdx i) = unknownLiteral i <> "::int"
 
-pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SqlFragment
+pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SQL.Snippet
 pgFmtAs _ [] Nothing = mempty
 pgFmtAs fName jp Nothing = case jOp <$> lastMay jp of
   Just (JKey key) -> " AS " <> pgFmtIdent key
@@ -323,7 +409,7 @@ pgFmtAs fName jp Nothing = case jOp <$> lastMay jp of
   Nothing -> mempty
 pgFmtAs _ _ (Just alias) = " AS " <> pgFmtIdent alias
 
-countF :: SQL.Snippet -> Bool -> (SQL.Snippet, SqlFragment)
+countF :: SQL.Snippet -> Bool -> (SQL.Snippet, SQL.Snippet)
 countF countQuery shouldCount =
   if shouldCount
     then (
@@ -333,11 +419,11 @@ countF countQuery shouldCount =
         mempty
       , "null::bigint")
 
-returningF :: QualifiedIdentifier -> [FieldName] -> SqlFragment
+returningF :: QualifiedIdentifier -> [FieldName] -> SQL.Snippet
 returningF qi returnings =
   if null returnings
     then "RETURNING 1" -- For mutation cases where there's no ?select, we return 1 to know how many rows were modified
-    else "RETURNING " <> BS.intercalate ", " (pgFmtColumn qi <$> returnings)
+    else "RETURNING " <> intercalateSnippet ", " (pgFmtColumn qi <$> returnings)
 
 limitOffsetF :: NonnegRange -> SQL.Snippet
 limitOffsetF range =
@@ -346,25 +432,25 @@ limitOffsetF range =
     limit = maybe "ALL" (\l -> unknownEncoder (BS.pack $ show l)) $ rangeLimit range
     offset = unknownEncoder (BS.pack . show $ rangeOffset range)
 
-responseHeadersF :: SqlFragment
+responseHeadersF :: SQL.Snippet
 responseHeadersF = currentSettingF "response.headers"
 
-responseStatusF :: SqlFragment
+responseStatusF :: SQL.Snippet
 responseStatusF = currentSettingF "response.status"
 
-currentSettingF :: SqlFragment -> SqlFragment
+currentSettingF :: SQL.Snippet -> SQL.Snippet
 currentSettingF setting =
   -- nullif is used because of https://gist.github.com/steve-chavez/8d7033ea5655096903f3b52f8ed09a15
   "nullif(current_setting('" <> setting <> "', true), '')"
 
-mutRangeF :: QualifiedIdentifier -> [FieldName] -> (SqlFragment, SqlFragment)
+mutRangeF :: QualifiedIdentifier -> [FieldName] -> (SQL.Snippet, SQL.Snippet)
 mutRangeF mainQi rangeId =
   (
-    BS.intercalate " AND " $ (\col -> pgFmtColumn mainQi col <> " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_affected_rows") col) <$> rangeId
+    intercalateSnippet " AND " $ (\col -> pgFmtColumn mainQi col <> " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_affected_rows") col) <$> rangeId
-  , BS.intercalate ", " (pgFmtColumn mainQi <$> rangeId)
+  , intercalateSnippet ", " (pgFmtColumn mainQi <$> rangeId)
   )
 
-orderF :: QualifiedIdentifier -> [OrderTerm] -> SQL.Snippet
+orderF :: QualifiedIdentifier -> [CoercibleOrderTerm] -> SQL.Snippet
 orderF _ []    = mempty
 orderF qi ordts = "ORDER BY " <> intercalateSnippet ", " (pgFmtOrderTerm qi <$> ordts)
 
@@ -392,8 +478,8 @@ explainF fmt opts snip =
     fmtPlanOpt PlanBuffers  = "BUFFERS"
     fmtPlanOpt PlanWAL      = "WAL"
 
-    fmtPlanFmt PlanJSON = "FORMAT JSON"
     fmtPlanFmt PlanText = "FORMAT TEXT"
+    fmtPlanFmt PlanJSON = "FORMAT JSON"
 
 -- | Do a pg set_config(setting, value, true) call. This is equivalent to a SET LOCAL.
 setConfigLocal :: ByteString -> (ByteString, ByteString) -> SQL.Snippet
@@ -409,3 +495,14 @@ setConfigLocalJson prefix keyVals = [setConfigLocal mempty (prefix, gucJsonVal k
     gucJsonVal = LBS.toStrict . JSON.encode . HM.fromList . arrayByteStringToText
     arrayByteStringToText :: [(ByteString, ByteString)] -> [(Text,Text)]
     arrayByteStringToText keyVal = (T.decodeUtf8 *** T.decodeUtf8) <$> keyVal
+
+aggF :: Maybe Routine -> ResultAggregate -> SQL.Snippet
+aggF rout = \case
+  BuiltinAggJson             -> asJsonF rout False
+  BuiltinAggArrayJsonStrip   -> asJsonF rout True
+  BuiltinAggSingleJson strip -> asJsonSingleF rout strip
+  BuiltinAggGeoJson          -> asGeoJsonF
+  BuiltinAggCsv              -> asCsvF
+  BuiltinAggXml    bField    -> asXmlF    bField
+  BuiltinAggBinary bField    -> asBinaryF bField
+  NoAgg                      -> "''::text"

src/PostgREST/Query/Statements.hs

@@ -23,15 +23,13 @@ import qualified Hasql.DynamicStatements.Statement as SQL
 import qualified Hasql.Statement                   as SQL
 
 import Control.Lens ((^?))
-import Data.Maybe   (fromJust)
 
 import PostgREST.ApiRequest.Preferences
-import PostgREST.MediaType               (MTPlanAttrs (..),
+import PostgREST.MediaType              (MTPlanFormat (..),
-                                          MTPlanFormat (..),
+                                         MediaType (..))
-                                          MediaType (..),
-                                          getMediaType)
 import PostgREST.Query.SqlFragment
-import PostgREST.SchemaCache.Identifiers (FieldName)
+import PostgREST.SchemaCache.Routine    (ResultAggregate (..),
+                                         Routine)
 
 import Protolude
 
@@ -55,110 +53,82 @@ data ResultSet
   | RSPlan BS.ByteString -- ^ the plan of the query
 
 
-prepareWrite :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType ->
+prepareWrite :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> ResultAggregate ->
-                PreferRepresentation -> [Text] -> Bool -> SQL.Statement () ResultSet
+                Maybe PreferRepresentation -> [Text] -> Bool -> SQL.Statement () ResultSet
-prepareWrite selectQuery mutateQuery isInsert mt rep pKeys =
+prepareWrite selectQuery mutateQuery isInsert mt rAgg rep pKeys =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
  where
   snippet =
-    "WITH " <> SQL.sql sourceCTEName <> " AS (" <> mutateQuery <> ") " <>
+    "WITH " <> sourceCTE <> " AS (" <> mutateQuery <> ") " <>
-    SQL.sql (
     "SELECT " <>
       "'' AS total_result_set, " <>
       "pg_catalog.count(_postgrest_t) AS page_total, " <>
       locF <> " AS header, " <>
-      bodyF <> " AS body, " <>
+      aggF Nothing rAgg <> " AS body, " <>
       responseHeadersF <> " AS response_headers, " <>
-      responseStatusF  <> " AS response_status "
+      responseStatusF  <> " AS response_status " <>
-    ) <>
     "FROM (" <> selectF <> ") _postgrest_t"
 
   locF =
-    if isInsert && rep == HeadersOnly
+    if isInsert && rep == Just HeadersOnly
-      then BS.unwords [
+      then
-        "CASE WHEN pg_catalog.count(_postgrest_t) = 1",
+        "CASE WHEN pg_catalog.count(_postgrest_t) = 1 " <>
-          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ")",
+          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ") " <>
-          "ELSE " <> noLocationF,
+          "ELSE " <> noLocationF <> " " <>
-        "END"]
+        "END"
       else noLocationF
 
-  bodyF
-    | rep /= Full                       = "''"
-    | getMediaType mt == MTTextCSV      = asCsvF
-    | getMediaType mt == MTGeoJSON      = asGeoJsonF
-    | getMediaType mt == MTSingularJSON = asJsonSingleF False
-    | otherwise                         = asJsonF False
-
   selectF
     -- prevent using any of the column names in ?select= when no response is returned from the CTE
-    | rep /= Full = SQL.sql ("SELECT * FROM " <> sourceCTEName)
+    | rAgg == NoAgg = "SELECT * FROM " <> sourceCTE
-    | otherwise   = selectQuery
+    | otherwise     = selectQuery
 
   decodeIt :: HD.Result ResultSet
   decodeIt = case mt of
     MTPlan{} -> planRow
     _        -> fromMaybe (RSStandard Nothing 0 mempty mempty Nothing Nothing) <$> HD.rowMaybe (standardRow False)
 
-prepareRead :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> Maybe FieldName -> Bool -> SQL.Statement () ResultSet
+prepareRead :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> ResultAggregate -> Bool -> SQL.Statement () ResultSet
-prepareRead selectQuery countQuery countTotal mt binaryField =
+prepareRead selectQuery countQuery countTotal mt rAgg =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
  where
   snippet =
-    "WITH " <>
+    "WITH " <> sourceCTE <> " AS ( " <> selectQuery <> " ) " <>
-    SQL.sql sourceCTEName <> " AS ( " <> selectQuery <> " ) " <>
     countCTEF <> " " <>
-    SQL.sql ("SELECT " <>
+    "SELECT " <>
       countResultF <> " AS total_result_set, " <>
       "pg_catalog.count(_postgrest_t) AS page_total, " <>
-      bodyF <> " AS body, " <>
+      aggF Nothing rAgg <> " AS body, " <>
       responseHeadersF <> " AS response_headers, " <>
       responseStatusF <> " AS response_status " <>
-    "FROM ( SELECT * FROM " <> sourceCTEName <> " ) _postgrest_t")
+    "FROM ( SELECT * FROM " <> sourceCTE <> " ) _postgrest_t"
 
   (countCTEF, countResultF) = countF countQuery countTotal
 
-  bodyF
-    | getMediaType mt == MTTextCSV                       = asCsvF
-    | getMediaType mt == MTSingularJSON                  = asJsonSingleF False
-    | getMediaType mt == MTGeoJSON                       = asGeoJsonF
-    | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
-    | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-    | otherwise                                          = asJsonF False
-
   decodeIt :: HD.Result ResultSet
   decodeIt = case mt of
     MTPlan{} -> planRow
     _        -> HD.singleRow $ standardRow True
 
-prepareCall :: Bool -> Bool -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
+prepareCall :: Routine -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
-               MediaType -> Bool -> Maybe FieldName -> Bool ->
+               MediaType -> ResultAggregate -> Bool ->
                SQL.Statement () ResultSet
-prepareCall returnsScalar returnsSingle callProcQuery selectQuery countQuery countTotal mt multObjects binaryField =
+prepareCall rout callProcQuery selectQuery countQuery countTotal mt rAgg =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
   where
     snippet =
-      "WITH " <> SQL.sql sourceCTEName <> " AS (" <> callProcQuery <> ") " <>
+      "WITH " <> sourceCTE <> " AS (" <> callProcQuery <> ") " <>
       countCTEF <>
-      SQL.sql (
       "SELECT " <>
         countResultF <> " AS total_result_set, " <>
         "pg_catalog.count(_postgrest_t) AS page_total, " <>
-        bodyF <> " AS body, " <>
+        aggF (Just rout) rAgg <> " AS body, " <>
         responseHeadersF <> " AS response_headers, " <>
-        responseStatusF <> " AS response_status ") <>
+        responseStatusF <> " AS response_status " <>
       "FROM (" <> selectQuery <> ") _postgrest_t"
 
     (countCTEF, countResultF) = countF countQuery countTotal
 
-    bodyF
-     | getMediaType mt == MTSingularJSON                  = asJsonSingleF returnsScalar
-     | getMediaType mt == MTTextCSV                       = asCsvF
-     | getMediaType mt == MTGeoJSON                       = asGeoJsonF
-     | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
-     | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-     | returnsSingle && not multObjects                   = asJsonSingleF returnsScalar
-     | otherwise                                          = asJsonF returnsScalar
-
     decodeIt :: HD.Result ResultSet
     decodeIt = case mt of
       MTPlan{} -> planRow
@@ -188,8 +158,8 @@ standardRow noLocation =
 
 mtSnippet :: MediaType -> SQL.Snippet -> SQL.Snippet
 mtSnippet mediaType snippet = case mediaType of
-  MTPlan (MTPlanAttrs _ fmt opts) -> explainF fmt opts snippet
+  MTPlan _ fmt opts -> explainF fmt opts snippet
-  _                               -> snippet
+  _                 -> snippet
 
 -- | We use rowList because when doing EXPLAIN (FORMAT TEXT), the result comes as many rows. FORMAT JSON comes as one.
 planRow :: HD.Result ResultSet

src/PostgREST/Response.hs

@@ -1,9 +1,15 @@
+{- |
+   Module      : PostgREST.Response
+   Description : Generate HTTP Response
+-}
 {-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 module PostgREST.Response
   ( createResponse
   , deleteResponse
-  , infoResponse
+  , infoIdentResponse
+  , infoProcResponse
+  , infoRootResponse
   , invokeResponse
   , openApiResponse
   , readResponse
@@ -12,12 +18,16 @@ module PostgREST.Response
   , addRetryHint
   , isServiceUnavailable
   , optionalRollback
+  , concatPrefAppsHeaders
+  , addPrefToHeaders
+  , traceHeaderMiddleware
   ) where
 
 import qualified Data.Aeson                as JSON
 import qualified Data.ByteString.Char8     as BS
 import qualified Data.ByteString.Lazy      as LBS
 import qualified Data.HashMap.Strict       as HM
+import qualified Data.List                 as L
 import           Data.Text.Read            (decimal)
 import qualified Network.HTTP.Types.Header as HTTP
 import qualified Network.HTTP.Types.Status as HTTP
@@ -30,10 +40,10 @@ import qualified PostgREST.RangeQuery       as RangeQuery
 import qualified PostgREST.Response.OpenAPI as OpenAPI
 
 import PostgREST.ApiRequest              (ApiRequest (..),
-                                          InvokeMethod (..),
+                                          InvokeMethod (..))
-                                          Target (..))
 import PostgREST.ApiRequest.Preferences  (PreferRepresentation (..),
                                           PreferTransaction (..),
+                                          Preferences (..),
                                           shouldCount,
                                           toAppliedHeader)
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
@@ -46,17 +56,17 @@ import PostgREST.Response.GucHeader      (GucHeader, unwrapGucHeader)
 import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
                                           Schema)
-import PostgREST.SchemaCache.Proc        (ProcDescription (..),
+import PostgREST.SchemaCache.Routine     (FuncVolatility (..),
-                                          ProcVolatility (..),
+                                          Routine (..), RoutineMap)
-                                          ProcsMap)
 import PostgREST.SchemaCache.Table       (Table (..), TablesMap)
 
-import qualified PostgREST.ApiRequest.Types as ApiRequestTypes
+import qualified PostgREST.ApiRequest.Types    as ApiRequestTypes
-import qualified PostgREST.SchemaCache.Proc as Proc
+import qualified PostgREST.SchemaCache.Routine as Routine
 
 import Protolude      hiding (Handler, toS)
 import Protolude.Conv (toS)
 
+
 readResponse :: Bool -> QualifiedIdentifier -> ApiRequest -> ResultSet -> Wai.Response
 readResponse headersOnly identifier ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
   RSStandard{..} -> do
@@ -83,7 +93,7 @@ readResponse headersOnly identifier ctxApiRequest@ApiRequest{..} resultSet = cas
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
 createResponse :: QualifiedIdentifier -> MutateReadPlan -> ApiRequest -> ResultSet -> Wai.Response
-createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}, ..} resultSet = case resultSet of
   RSStandard{..} -> do
     let
       pkCols = case mrMutatePlan of { Insert{insPkCols} -> insPkCols; _ -> mempty;}
@@ -100,89 +110,83 @@ createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiReques
                     <> HTTP.renderSimpleQuery True rsLocation
                 )
           , Just . RangeQuery.contentRangeH 1 0 $
-              if shouldCount iPreferCount then Just rsQueryTotal else Nothing
+              if shouldCount preferCount then Just rsQueryTotal else Nothing
           , if null pkCols && isNothing (qsOnConflict iQueryParams) then
               Nothing
             else
-              toAppliedHeader <$> iPreferResolution
+              toAppliedHeader <$> preferResolution
+          , toAppliedHeader <$> preferMissing
           ]
 
-    if iPreferRepresentation == Full then
+    case preferRepresentation of
-      response HTTP.status201 (headers ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
+      Just Full -> response HTTP.status201 (addPrefToHeaders headers Full ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
-    else
+      Just None -> response HTTP.status201 (addPrefToHeaders headers None) mempty
-      response HTTP.status201 headers mempty
+      Just HeadersOnly -> response HTTP.status201 (addPrefToHeaders headers HeadersOnly) mempty
+      Nothing -> response HTTP.status201 headers mempty
+
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
+
 updateResponse :: ApiRequest -> ResultSet -> Wai.Response
-updateResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+updateResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
   RSStandard{..} -> do
     let
       response = gucResponse rsGucStatus rsGucHeaders
       contentRangeHeader =
-        RangeQuery.contentRangeH 0 (rsQueryTotal - 1) $
+        Just . RangeQuery.contentRangeH 0 (rsQueryTotal - 1) $
-          if shouldCount iPreferCount then Just rsQueryTotal else Nothing
+          if shouldCount preferCount then Just rsQueryTotal else Nothing
-      headers = [contentRangeHeader]
+      headers = catMaybes [contentRangeHeader, toAppliedHeader <$> preferMissing]
 
-    if iPreferRepresentation == Full then
+    case preferRepresentation of
-        response HTTP.status200
+        Just Full -> response HTTP.status200 (addPrefToHeaders headers Full ++ contentTypeHeaders ctxApiRequest)
-          (headers ++ contentTypeHeaders ctxApiRequest)
           (LBS.fromStrict rsBody)
-      else
+        Just None -> response HTTP.status204 (addPrefToHeaders headers None) mempty
-        response HTTP.status204 headers mempty
+        _ -> response HTTP.status204 headers mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
 singleUpsertResponse :: ApiRequest -> ResultSet -> Wai.Response
-singleUpsertResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+singleUpsertResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
   RSStandard {..} -> do
     let
       response = gucResponse rsGucStatus rsGucHeaders
 
-    if iPreferRepresentation == Full then
+    case preferRepresentation of
-      response HTTP.status200 (contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
+      Just Full -> response HTTP.status200 (contentTypeHeaders ctxApiRequest ++ [toAppliedHeader Full]) (LBS.fromStrict rsBody)
-    else
+      Just None -> response HTTP.status204 [toAppliedHeader None] mempty
-      response HTTP.status204 [] mempty
+      _ -> response HTTP.status204 [] mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
 deleteResponse :: ApiRequest -> ResultSet -> Wai.Response
-deleteResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+deleteResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
   RSStandard {..} -> do
     let
       response = gucResponse rsGucStatus rsGucHeaders
       contentRangeHeader =
         RangeQuery.contentRangeH 1 0 $
-          if shouldCount iPreferCount then Just rsQueryTotal else Nothing
+          if shouldCount preferCount then Just rsQueryTotal else Nothing
       headers = [contentRangeHeader]
 
-    if iPreferRepresentation == Full then
+    case preferRepresentation of
-        response HTTP.status200
+        Just Full -> response HTTP.status200 (addPrefToHeaders headers Full ++ contentTypeHeaders ctxApiRequest)
-          (headers ++ contentTypeHeaders ctxApiRequest)
           (LBS.fromStrict rsBody)
-      else
+        Just None -> response HTTP.status204 (addPrefToHeaders headers None) mempty
-        response HTTP.status204 headers mempty
+        _ -> response HTTP.status204 headers mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
-infoResponse :: Target -> SchemaCache -> Wai.Response
+infoIdentResponse :: QualifiedIdentifier -> SchemaCache -> Wai.Response
-infoResponse target sCache =
+infoIdentResponse identifier sCache =
-  case target of
+  case HM.lookup identifier (dbTables sCache) of
-    TargetIdent identifier ->
+    Just tbl -> respondInfo $ allowH tbl
-      case HM.lookup identifier (dbTables sCache) of
+    Nothing  -> Error.errorResponseFor $ Error.ApiRequestError ApiRequestTypes.NotFound
-        Just tbl -> respondInfo $ allowH tbl
-        Nothing  -> Error.errorResponseFor $ Error.ApiRequestError ApiRequestTypes.NotFound
-    TargetProc pd _
-      | pdVolatility pd == Volatile -> respondInfo "OPTIONS,POST"
-      | otherwise                   -> respondInfo "OPTIONS,GET,HEAD,POST"
-    TargetDefaultSpec _             -> respondInfo "OPTIONS,GET,HEAD"
   where
-    respondInfo allowHeader = Wai.responseLBS HTTP.status200 [allOrigins, (HTTP.hAllow, allowHeader)] mempty
-    allOrigins = ("Access-Control-Allow-Origin", "*")
     allowH table =
       let hasPK = not . null $ tablePKCols table in
       BS.intercalate "," $
@@ -192,7 +196,19 @@ infoResponse target sCache =
           ["PATCH" | tableUpdatable table] ++
           ["DELETE" | tableDeletable table]
 
-invokeResponse :: InvokeMethod -> ProcDescription -> ApiRequest -> ResultSet -> Wai.Response
+infoProcResponse :: Routine -> Wai.Response
+infoProcResponse proc | pdVolatility proc == Volatile = respondInfo "OPTIONS,POST"
+                      | otherwise                     = respondInfo "OPTIONS,GET,HEAD,POST"
+
+infoRootResponse :: Wai.Response
+infoRootResponse = respondInfo "OPTIONS,GET,HEAD"
+
+respondInfo :: ByteString -> Wai.Response
+respondInfo allowHeader =
+  let allOrigins = ("Access-Control-Allow-Origin", "*") in
+  Wai.responseLBS HTTP.status200 [allOrigins, (HTTP.hAllow, allowHeader)] mempty
+
+invokeResponse :: InvokeMethod -> Routine -> ApiRequest -> ResultSet -> Wai.Response
 invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
   RSStandard {..} -> do
     let
@@ -205,7 +221,7 @@ invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resu
         else LBS.fromStrict rsBody
       headers = [contentRange]
 
-    if Proc.procReturnsVoid proc then
+    if Routine.funcReturnsVoid proc then
         response HTTP.status204 headers mempty
       else
         response status
@@ -215,11 +231,11 @@ invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resu
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
-openApiResponse :: Bool -> Maybe (TablesMap, ProcsMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
+openApiResponse :: (Text, Text) -> Bool -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
-openApiResponse headersOnly body conf sCache schema negotiatedByProfile =
+openApiResponse versions headersOnly body conf sCache schema negotiatedByProfile =
   Wai.responseLBS HTTP.status200
     (MediaType.toContentType MTOpenAPI : maybeToList (profileHeader schema negotiatedByProfile))
-    (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode conf sCache x y z) body)
+    (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache x y z) body)
 
 -- | Response with headers and status overridden from GUCs.
 gucResponse
@@ -263,14 +279,14 @@ isServiceUnavailable :: Wai.Response -> Bool
 isServiceUnavailable response = Wai.responseStatus response == HTTP.status503
 
 optionalRollback :: AppConfig -> ApiRequest -> ExceptT Error.Error IO Wai.Response -> ExceptT Error.Error IO Wai.Response
-optionalRollback AppConfig{..} ApiRequest{..} resp = do
+optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} resp = do
   newRes <- catchError resp $ return . Error.errorResponseFor
   return $ Wai.mapResponseHeaders preferenceApplied newRes
   where
     shouldCommit =
-      configDbTxAllowOverride && iPreferTransaction == Just Commit
+      configDbTxAllowOverride && preferTransaction == Just Commit
     shouldRollback =
-      configDbTxAllowOverride && iPreferTransaction == Just Rollback
+      configDbTxAllowOverride && preferTransaction == Just Rollback
     preferenceApplied
       | shouldCommit =
           addHeadersIfNotIncluded
@@ -284,5 +300,44 @@ optionalRollback AppConfig{..} ApiRequest{..} resp = do
 -- | Add headers not already included to allow the user to override them instead of duplicating them
 addHeadersIfNotIncluded :: [HTTP.Header] -> [HTTP.Header] -> [HTTP.Header]
 addHeadersIfNotIncluded newHeaders initialHeaders =
-  filter (\(nk, _) -> isNothing $ find (\(ik, _) -> ik == nk) initialHeaders) newHeaders ++
+  filter (\(nk, nv) -> isNothing $ find (\(ik, iv) -> ik == nk && nv == iv) initialHeaders) newHeaders ++
   initialHeaders
+
+-- |  Filters out multiple Preference-Applied Headers from the list and concatenate them into a single Preference-Applied header:
+--
+-- >>> :{
+--  concatPrefAppsHeaders
+--     [("Content-Type","application/json")
+--     , ("Preference-Applied","tx=commit")
+--     , ("Preference-Applied","return=minimal")]
+-- :}
+-- [("Content-Type","application/json"),("Preference-Applied","tx=commit, return=minimal")]
+
+concatPrefAppsHeaders :: [HTTP.Header] -> [HTTP.Header]
+concatPrefAppsHeaders headers = otherHeaders ++ [(HTTP.hPreferenceApplied, combinedPrefApps)]
+  where
+  (prefApps, otherHeaders) = L.partition (\(k, _) -> k == HTTP.hPreferenceApplied) headers
+  prefAppsValues = [ v | (_,v) <- prefApps]
+  combinedPrefApps = BS.intercalate ", " prefAppsValues
+
+-- | Given response headers and a preferRepresentation value, add
+--   preferRepresentation to Preference-Applied
+--
+-- >>> :{
+--  addPrefToHeaders
+--      [("Content-Type", "application/json")
+--      , ("Preference-Applied", "tx=commit")]
+--      None
+-- :}
+-- [("Content-Type","application/json"),("Preference-Applied","tx=commit, return=minimal")]
+
+addPrefToHeaders :: [HTTP.Header] -> PreferRepresentation -> [HTTP.Header]
+addPrefToHeaders headers pref = concatPrefAppsHeaders (headers ++ [toAppliedHeader pref])
+
+traceHeaderMiddleware :: AppConfig -> Wai.Middleware
+traceHeaderMiddleware AppConfig{configServerTraceHeader} app req respond =
+  case configServerTraceHeader of
+    Nothing -> app req respond
+    Just hdr ->
+      let hdrVal = L.lookup hdr $ Wai.requestHeaders req in
+      app req (respond . Wai.mapResponseHeaders ([(hdr, fromMaybe mempty hdrVal)] ++))

src/PostgREST/Response/OpenAPI.hs

@@ -12,7 +12,6 @@ import qualified Data.ByteString.Lazy  as LBS
 import qualified Data.HashMap.Strict   as HM
 import qualified Data.HashSet.InsOrd   as Set
 import qualified Data.Text             as T
-import qualified Data.Text.Encoding    as T
 
 import Control.Arrow              ((&&&))
 import Data.HashMap.Strict.InsOrd (InsOrdHashMap, fromList)
@@ -28,23 +27,24 @@ import PostgREST.Config                   (AppConfig (..), Proxy (..),
                                            isMalformedProxyUri, toURI)
 import PostgREST.SchemaCache              (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
-import PostgREST.SchemaCache.Proc         (ProcDescription (..),
-                                           ProcParam (..))
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                            Relationship (..),
                                            RelationshipsMap)
+import PostgREST.SchemaCache.Routine      (Routine (..),
+                                           RoutineParam (..))
 import PostgREST.SchemaCache.Table        (Column (..), Table (..),
-                                           TablesMap)
+                                           TablesMap,
-import PostgREST.Version                  (docsVersion, prettyVersion)
+                                           tableColumnsList)
 
 import PostgREST.MediaType
 
 import Protolude hiding (Proxy, get)
 
-encode :: AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [ProcDescription] -> Maybe Text -> LBS.ByteString
+encode :: (Text, Text) -> AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
-encode conf sCache tables procs schemaDescription =
+encode versions conf sCache tables procs schemaDescription =
   JSON.encode $
     postgrestSpec
+      versions
       (dbRelationships sCache)
       (concat $ HM.elems procs)
       (snd <$> HM.toList tables)
@@ -66,10 +66,22 @@ toSwaggerType "bigint"            = Just SwaggerInteger
 toSwaggerType "numeric"           = Just SwaggerNumber
 toSwaggerType "real"              = Just SwaggerNumber
 toSwaggerType "double precision"  = Just SwaggerNumber
-toSwaggerType "ARRAY"             = Just SwaggerArray
 toSwaggerType "json"              = Nothing
 toSwaggerType "jsonb"             = Nothing
-toSwaggerType _                   = Just SwaggerString
+toSwaggerType colType             = case T.takeEnd 2 colType of
+  "[]" -> Just SwaggerArray
+  _    -> Just SwaggerString
+
+typeFromArray :: Text -> Text
+typeFromArray = T.dropEnd 2
+
+toSwaggerTypeFromArray :: Text -> Maybe (SwaggerType t)
+toSwaggerTypeFromArray arrType = toSwaggerType $ typeFromArray arrType
+
+makePropertyItems :: Text -> Maybe (Referenced Schema)
+makePropertyItems arrType = case toSwaggerType arrType of
+  Just SwaggerArray -> Just $ Inline (mempty & type_ .~ toSwaggerTypeFromArray arrType)
+  _                 -> Nothing
 
 parseDefault :: Text -> Text -> Text
 parseDefault colType colDefault =
@@ -87,8 +99,8 @@ makeTableDef rels t =
       (tn, (mempty :: Schema)
         & description .~ tableDescription t
         & type_ ?~ SwaggerObject
-        & properties .~ fromList (makeProperty t rels <$> tableColumns t)
+        & properties .~ fromList (makeProperty t rels <$> tableColumnsList t)
-        & required .~ fmap colName (filter (not . colNullable) $ tableColumns t))
+        & required .~ fmap colName (filter (not . colNullable) $ tableColumnsList t))
 
 makeProperty :: Table -> RelationshipsMap -> Column -> (Text, Referenced Schema)
 makeProperty tbl rels col = (colName col, Inline s)
@@ -97,11 +109,14 @@ makeProperty tbl rels col = (colName col, Inline s)
     fk :: Maybe Text
     fk =
       let
+        searchedRels = fromMaybe mempty $ HM.lookup (QualifiedIdentifier (tableSchema tbl) (tableName tbl), tableSchema tbl) rels
+        -- Sorts the relationship list to get tables first
+        relsSortedByIsView = sortOn relFTableIsView [ r | r@Relationship{} <- searchedRels]
         -- Finds the relationship that has a single column foreign key
         rel = find (\case
           Relationship{relCardinality=(M2O _ relColumns)} -> [colName col] == (fst <$> relColumns)
           _                                               -> False
-          ) $ fromMaybe mempty $ HM.lookup (QualifiedIdentifier (tableSchema tbl) (tableName tbl), tableSchema tbl) rels
+          ) relsSortedByIsView
         fCol = (headMay . (\r -> snd <$> relColumns (relCardinality r)) =<< rel)
         fTbl = qiName . relForeignTable <$> rel
         fTblCol = (,) <$> fTbl <*> fCol
@@ -127,8 +142,9 @@ makeProperty tbl rels col = (colName col, Inline s)
         & format ?~ colType col
         & maxLength .~ (fromIntegral <$> colMaxLen col)
         & type_ .~ toSwaggerType (colType col)
+        & items .~ (SwaggerItemsObject <$> makePropertyItems (colType col))
 
-makeProcSchema :: ProcDescription -> Schema
+makeProcSchema :: Routine -> Schema
 makeProcSchema pd =
   (mempty :: Schema)
   & description .~ pdDescription pd
@@ -136,11 +152,12 @@ makeProcSchema pd =
   & properties .~ fromList (fmap makeProcProperty (pdParams pd))
   & required .~ fmap ppName (filter ppReq (pdParams pd))
 
-makeProcProperty :: ProcParam -> (Text, Referenced Schema)
+makeProcProperty :: RoutineParam -> (Text, Referenced Schema)
-makeProcProperty (ProcParam n t _ _) = (n, Inline s)
+makeProcProperty (RoutineParam n t _ _ _) = (n, Inline s)
   where
     s = (mempty :: Schema)
           & type_ .~ toSwaggerType t
+          & items .~ (SwaggerItemsObject <$> makePropertyItems t)
           & format ?~ t
 
 makePreferParam :: [Text] -> Param
@@ -152,10 +169,47 @@ makePreferParam ts =
   & schema .~ ParamOther ((mempty :: ParamOtherSchema)
     & in_ .~ ParamHeader
     & type_ ?~ SwaggerString
-    & enum_ .~ JSON.decode (JSON.encode ts))
+    & enum_ .~ JSON.decode (JSON.encode $ foldl (<>) [] (val <$> ts)))
+  where
+    val :: Text -> [Text]
+    val = \case
+      "count"      -> ["count=none"]
+      "params"     -> ["params=single-object"]
+      "return"     -> ["return=representation", "return=minimal", "return=none"]
+      "resolution" -> ["resolution=ignore-duplicates", "resolution=merge-duplicates"]
+      _            -> []
 
-makeProcParam :: ProcDescription -> [Referenced Param]
+makeProcGetParam :: RoutineParam -> Referenced Param
-makeProcParam pd =
+makeProcGetParam (RoutineParam n t _ r v) =
+  Inline $ (mempty :: Param)
+    & name .~ n
+    & required ?~ r
+    & schema .~ ParamOther fullSchema
+  where
+    fullSchema = if v then schemaMulti else schemaNotMulti
+    baseSchema = (mempty :: ParamOtherSchema)
+      & in_ .~ ParamQuery
+    schemaNotMulti = baseSchema
+      & format ?~ t
+      & type_ ?~ toParamType (toSwaggerType t)
+    schemaMulti = baseSchema
+      & type_ ?~ fromMaybe SwaggerString (toSwaggerType t)
+      & items ?~ SwaggerItemsPrimitive (Just CollectionMulti)
+        ((mempty :: ParamSchema x)
+          & type_ .~ toSwaggerTypeFromArray t
+          & format ?~ typeFromArray t)
+    toParamType paramType = case paramType of
+      -- Array uses {} in query params
+      Just SwaggerArray -> SwaggerString
+      -- Type must be specified in query params
+      Nothing           -> SwaggerString
+      _                 -> fromJust paramType
+
+makeProcGetParams :: [RoutineParam] -> [Referenced Param]
+makeProcGetParams = fmap makeProcGetParam
+
+makeProcPostParams :: Routine -> [Referenced Param]
+makeProcPostParams pd =
   [ Inline $ (mempty :: Param)
     & name     .~ "args"
     & required ?~ True
@@ -165,9 +219,11 @@ makeProcParam pd =
 
 makeParamDefs :: [Table] -> [(Text, Param)]
 makeParamDefs ti =
-  [ ("preferParams", makePreferParam ["params=single-object"])
+  -- TODO: create Prefer for each method (GET, PATCH, etc.)
-  , ("preferReturn", makePreferParam ["return=representation", "return=minimal", "return=none"])
+  [ ("preferParams", makePreferParam ["params"])
-  , ("preferCount", makePreferParam ["count=none"])
+  , ("preferReturn", makePreferParam ["return"])
+  , ("preferCount", makePreferParam ["count"])
+  , ("preferPost", makePreferParam ["return", "resolution"])
   , ("select", (mempty :: Param)
       & name        .~ "select"
       & description ?~ "Filtering Columns"
@@ -219,7 +275,7 @@ makeParamDefs ti =
         & in_ .~ ParamQuery
         & type_ ?~ SwaggerString))
   ]
-  <> concat [ makeObjectBody (tableName t) : makeRowFilters (tableName t) (tableColumns t)
+  <> concat [ makeObjectBody (tableName t) : makeRowFilters (tableName t) (tableColumnsList t)
             | t <- ti
             ]
 
@@ -267,7 +323,7 @@ makePathItem t = ("/" ++ T.unpack tn, p $ tableInsertable t || tableUpdatable t
         )
       )
     postOp = tOp
-      & parameters .~ fmap ref ["body." <> tn, "select", "preferReturn"]
+      & parameters .~ fmap ref ["body." <> tn, "select", "preferPost"]
       & at 201 ?~ "Created"
     patchOp = tOp
       & parameters .~ fmap ref (rs <> ["body." <> tn, "preferReturn"])
@@ -280,24 +336,29 @@ makePathItem t = ("/" ++ T.unpack tn, p $ tableInsertable t || tableUpdatable t
     p False = pr
     p True  = pw
     tn = tableName t
-    rs = [ T.intercalate "." ["rowFilter", tn, colName c ] | c <- tableColumns t ]
+    rs = [ T.intercalate "." ["rowFilter", tn, colName c ] | c <- tableColumnsList t ]
     ref = Ref . Reference
 
-makeProcPathItem :: ProcDescription -> (FilePath, PathItem)
+makeProcPathItem :: Routine -> (FilePath, PathItem)
 makeProcPathItem pd = ("/rpc/" ++ toS (pdName pd), pe)
   where
     -- Use first line of proc description as summary; rest as description (if present)
     -- We strip leading newlines from description so that users can include a blank line between summary and description
     (pSum, pDesc) = fmap fst &&& fmap (T.dropWhile (=='\n') . snd) $
                     T.breakOn "\n" <$> pdDescription pd
-    postOp = (mempty :: Operation)
+    procOp = (mempty :: Operation)
       & summary .~ pSum
       & description .~ mfilter (/="") pDesc
-      & parameters .~ makeProcParam pd
       & tags .~ Set.fromList ["(rpc) " <> pdName pd]
-      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON]
+      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False]
       & at 200 ?~ "OK"
-    pe = (mempty :: PathItem) & post ?~ postOp
+    getOp = procOp
+      & parameters .~ makeProcGetParams (pdParams pd)
+    postOp = procOp
+      & parameters .~ makeProcPostParams pd
+    pe = (mempty :: PathItem)
+      & get ?~ getOp
+      & post ?~ postOp
 
 makeRootPathItem :: (FilePath, PathItem)
 makeRootPathItem = ("/", p)
@@ -310,7 +371,7 @@ makeRootPathItem = ("/", p)
     pr = (mempty :: PathItem) & get ?~ getOp
     p = pr
 
-makePathItems :: [ProcDescription] -> [Table] -> InsOrdHashMap FilePath PathItem
+makePathItems :: [Routine] -> [Table] -> InsOrdHashMap FilePath PathItem
 makePathItems pds ti = fromList $ makeRootPathItem :
   fmap makePathItem ti ++ fmap makeProcPathItem pds
 
@@ -330,14 +391,14 @@ escapeHostName "*6" = "0.0.0.0"
 escapeHostName "!6" = "0.0.0.0"
 escapeHostName h    = h
 
-postgrestSpec :: RelationshipsMap -> [ProcDescription] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
+postgrestSpec :: (Text, Text) -> RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
-postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
+postgrestSpec (prettyVersion, docsVersion) rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
   & basePath ?~ T.unpack b
   & schemes ?~ [s']
   & info .~ ((mempty :: Info)
-      & version .~ T.decodeUtf8 prettyVersion
+      & version .~ prettyVersion
-      & title .~ "PostgREST API"
+      & title .~ fromMaybe "PostgREST API" dTitle
-      & description ?~ d)
+      & description ?~ fromMaybe "This is a dynamic API generated by PostgREST" dDesc)
   & externalDocs ?~ ((mempty :: ExternalDocs)
     & description ?~ "PostgREST Documentation"
     & url .~ URL ("https://postgrest.org/en/" <> docsVersion <> "/api.html"))
@@ -345,15 +406,16 @@ postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
   & definitions .~ fromList (makeTableDef rels <$> ti)
   & parameters .~ fromList (makeParamDefs ti)
   & paths .~ makePathItems pds ti
-  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
+  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False, MTTextCSV]
-  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
+  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False, MTTextCSV]
   & securityDefinitions .~ makeSecurityDefinitions securityDefName allowSecurityDef
   & security .~ [SecurityRequirement (fromList [(securityDefName, [])]) | allowSecurityDef]
     where
       s' = if s == "http" then Http else Https
       h' = Just $ Host (T.unpack $ escapeHostName h) (Just (fromInteger p))
-      d = fromMaybe "This is a dynamic API generated by PostgREST" sd
       securityDefName = "JWT"
+      (dTitle, dDesc) = fmap fst &&& fmap (T.dropWhile (=='\n') . snd) $
+                    T.breakOn "\n" <$> sd
 
 pickProxy :: Maybe Text -> Maybe Proxy
 pickProxy proxy

src/PostgREST/SchemaCache.hs

@@ -22,46 +22,56 @@ module PostgREST.SchemaCache
   ( SchemaCache(..)
   , querySchemaCache
   , accessibleTables
-  , accessibleProcs
+  , accessibleFuncs
   , schemaDescription
   ) where
 
-import qualified Data.Aeson          as JSON
+import Control.Monad.Extra (whenJust)
-import qualified Data.HashMap.Strict as HM
+
-import qualified Data.Set            as S
+import qualified Data.Aeson                 as JSON
-import qualified Hasql.Decoders      as HD
+import qualified Data.HashMap.Strict        as HM
-import qualified Hasql.Encoders      as HE
+import qualified Data.HashMap.Strict.InsOrd as HMI
-import qualified Hasql.Statement     as SQL
+import qualified Data.Set                   as S
-import qualified Hasql.Transaction   as SQL
+import qualified Hasql.Decoders             as HD
+import qualified Hasql.Encoders             as HE
+import qualified Hasql.Statement            as SQL
+import qualified Hasql.Transaction          as SQL
 
 import Contravariant.Extras          (contrazip2)
 import Text.InterpolatedString.Perl6 (q)
 
-import PostgREST.Config.Database          (pgVersionStatement)
+import PostgREST.Config                      (AppConfig (..))
-import PostgREST.Config.PgVersion         (PgVersion, pgVersion100,
+import PostgREST.Config.Database             (pgVersionStatement,
-                                           pgVersion110)
+                                              toIsolationLevel)
-import PostgREST.SchemaCache.Identifiers  (AccessSet, FieldName,
+import PostgREST.Config.PgVersion            (PgVersion, pgVersion100,
-                                           QualifiedIdentifier (..),
+                                              pgVersion110,
-                                           Schema)
+                                              pgVersion120)
-import PostgREST.SchemaCache.Proc         (PgType (..),
+import PostgREST.SchemaCache.Identifiers     (AccessSet, FieldName,
-                                           ProcDescription (..),
+                                              QualifiedIdentifier (..),
-                                           ProcParam (..),
+                                              Schema)
-                                           ProcVolatility (..),
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
-                                           ProcsMap, RetType (..))
+                                              Junction (..),
-import PostgREST.SchemaCache.Relationship (Cardinality (..),
+                                              Relationship (..),
-                                           Junction (..),
+                                              RelationshipsMap)
-                                           Relationship (..),
+import PostgREST.SchemaCache.Representations (DataRepresentation (..),
-                                           RelationshipsMap)
+                                              RepresentationsMap)
-import PostgREST.SchemaCache.Table        (Column (..), Table (..),
+import PostgREST.SchemaCache.Routine         (FuncVolatility (..),
-                                           TablesMap)
+                                              PgType (..),
+                                              RetType (..),
+                                              Routine (..),
+                                              RoutineMap,
+                                              RoutineParam (..))
+import PostgREST.SchemaCache.Table           (Column (..), ColumnMap,
+                                              Table (..), TablesMap)
 
 import Protolude
 
 
 data SchemaCache = SchemaCache
-  { dbTables        :: TablesMap
+  { dbTables          :: TablesMap
-  , dbRelationships :: RelationshipsMap
+  , dbRelationships   :: RelationshipsMap
-  , dbProcs         :: ProcsMap
+  , dbRoutines        :: RoutineMap
+  , dbRepresentations :: RepresentationsMap
   }
   deriving (Generic, JSON.ToJSON)
 
@@ -103,15 +113,19 @@ data KeyDep
 -- | A SQL query that can be executed independently
 type SqlQuery = ByteString
 
-querySchemaCache :: [Schema] -> [Schema] -> Bool -> SQL.Transaction SchemaCache
+querySchemaCache :: AppConfig -> SQL.Transaction SchemaCache
-querySchemaCache schemas extraSearchPath prepared = do
+querySchemaCache AppConfig{..} = do
   SQL.sql "set local schema ''" -- This voids the search path. The following queries need this for getting the fully qualified name(schema.name) of every db object
-  pgVer   <- SQL.statement mempty pgVersionStatement
+  pgVer   <- SQL.statement mempty $ pgVersionStatement prepared
   tabs    <- SQL.statement schemas $ allTables pgVer prepared
-  keyDeps <- SQL.statement (schemas, extraSearchPath) $ allViewsKeyDependencies prepared
+  keyDeps <- SQL.statement (schemas, configDbExtraSearchPath) $ allViewsKeyDependencies prepared
   m2oRels <- SQL.statement mempty $ allM2OandO2ORels pgVer prepared
-  procs   <- SQL.statement schemas $ allProcs pgVer prepared
+  funcs   <- SQL.statement schemas $ allFunctions pgVer prepared
   cRels   <- SQL.statement mempty $ allComputedRels prepared
+  reps    <- SQL.statement schemas $ dataRepresentations prepared
+  _       <-
+    let sleepCall = SQL.Statement "select pg_sleep($1)" (param HE.int4) HD.noResult prepared in
+    whenJust configInternalSCSleep (`SQL.statement` sleepCall) -- only used for testing
 
   let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
       rels          = addInverseRels $ addM2MRels tabsWViewsPks $ addViewM2OAndO2ORels keyDeps m2oRels
@@ -119,8 +133,12 @@ querySchemaCache schemas extraSearchPath prepared = do
   return $ removeInternal schemas $ SchemaCache {
       dbTables = tabsWViewsPks
     , dbRelationships = getOverrideRelationshipsMap rels cRels
-    , dbProcs = procs
+    , dbRoutines = funcs
+    , dbRepresentations = reps
     }
+  where
+    schemas = toList configDbSchemas
+    prepared = configDbPreparedStatements
 
 -- | overrides detected relationships with the computed relationships and gets the RelationshipsMap
 getOverrideRelationshipsMap :: [Relationship] -> [Relationship] -> RelationshipsMap
@@ -146,10 +164,11 @@ getOverrideRelationshipsMap rels cRels =
 removeInternal :: [Schema] -> SchemaCache -> SchemaCache
 removeInternal schemas dbStruct =
   SchemaCache {
-      dbTables        = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
+      dbTables          = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
-    , dbRelationships = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
+    , dbRelationships   = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
-                        HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
+                          HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
-    , dbProcs         = dbProcs dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+    , dbRoutines        = dbRoutines dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+    , dbRepresentations = dbRepresentations dbStruct -- no need to filter, not directly exposed through the API
     }
   where
     hasInternalJunction ComputedRelationship{} = False
@@ -178,15 +197,20 @@ decodeTables =
     <*> column HD.bool
     <*> column HD.bool
     <*> arrayColumn HD.text
-    <*> compositeArrayColumn
+    <*> parseCols (compositeArrayColumn
-        (Column
+      (Column
         <$> compositeField HD.text
         <*> nullableCompositeField HD.text
         <*> compositeField HD.bool
         <*> compositeField HD.text
+        <*> compositeField HD.text
         <*> nullableCompositeField HD.int4
         <*> nullableCompositeField HD.text
-        <*> compositeFieldArray HD.text)
+        <*> compositeFieldArray HD.text))
+
+
+parseCols :: HD.Row [Column] -> HD.Row ColumnMap
+parseCols = fmap (HMI.fromList . map (\col@Column{colName} -> (colName, col)))
 
 decodeRels :: HD.Result [Relationship]
 decodeRels =
@@ -222,19 +246,20 @@ viewKeyDepFromRow (s1,t1,s2,v2,cons,consType,sCols) = ViewKeyDependency (Qualifi
            | consType == "f" = FKDep
            | otherwise       = FKDepRef -- f_ref, we build this type in the query
 
-decodeProcs :: HD.Result ProcsMap
+decodeFuncs :: HD.Result RoutineMap
-decodeProcs =
+decodeFuncs =
-  -- Duplicate rows for a function means they're overloaded, order these by least args according to ProcDescription Ord instance
+  -- Duplicate rows for a function means they're overloaded, order these by least args according to Routine Ord instance
-  map sort . HM.fromListWith (++) . map ((\(x,y) -> (x, [y])) . addKey) <$> HD.rowList procRow
+  map sort . HM.fromListWith (++) . map ((\(x,y) -> (x, [y])) . addKey) <$> HD.rowList funcRow
   where
-    procRow = ProcDescription
+    funcRow = Function
               <$> column HD.text
               <*> column HD.text
               <*> nullableColumn HD.text
               <*> compositeArrayColumn
-                  (ProcParam
+                  (RoutineParam
                   <$> compositeField HD.text
                   <*> compositeField HD.text
+                  <*> compositeField HD.text
                   <*> compositeField HD.bool
                   <*> compositeField HD.bool)
               <*> (parseRetType
@@ -245,38 +270,74 @@ decodeProcs =
                   <*> column HD.bool)
               <*> (parseVolatility <$> column HD.char)
               <*> column HD.bool
+              <*> nullableColumn (toIsolationLevel <$> HD.text)
 
-    addKey :: ProcDescription -> (QualifiedIdentifier, ProcDescription)
+    addKey :: Routine -> (QualifiedIdentifier, Routine)
     addKey pd = (QualifiedIdentifier (pdSchema pd) (pdName pd), pd)
 
-    parseRetType :: Text -> Text -> Bool -> Bool -> Bool -> Maybe RetType
+    parseRetType :: Text -> Text -> Bool -> Bool -> Bool -> RetType
-    parseRetType schema name isSetOf isComposite isVoid
+    parseRetType schema name isSetOf isComposite isCompositeAlias
-      | isVoid    = Nothing
+      | isSetOf   = SetOf pgType
-      | isSetOf   = Just (SetOf pgType)
+      | otherwise = Single pgType
-      | otherwise = Just (Single pgType)
       where
         qi = QualifiedIdentifier schema name
         pgType
-          | isComposite = Composite qi
+          | isComposite = Composite qi isCompositeAlias
-          | otherwise   = Scalar
+          | otherwise   = Scalar qi
 
-    parseVolatility :: Char -> ProcVolatility
+    parseVolatility :: Char -> FuncVolatility
     parseVolatility v | v == 'i' = Immutable
                       | v == 's' = Stable
                       | otherwise = Volatile -- only 'v' can happen here
 
-allProcs :: PgVersion -> Bool -> SQL.Statement [Schema] ProcsMap
+decodeRepresentations :: HD.Result RepresentationsMap
-allProcs pgVer = SQL.Statement sql (arrayParam HE.text) decodeProcs
+decodeRepresentations =
+  HM.fromList . map (\rep@DataRepresentation{drSourceType, drTargetType} -> ((drSourceType, drTargetType), rep)) <$> HD.rowList row
   where
-    sql = procsSqlQuery pgVer <> " AND pn.nspname = ANY($1)"
+    row = DataRepresentation
+      <$> column HD.text
+      <*> column HD.text
+      <*> column HD.text
 
-accessibleProcs :: PgVersion -> Bool -> SQL.Statement Schema ProcsMap
+-- Selects all potential data representation transformations. To qualify the cast must be
-accessibleProcs pgVer = SQL.Statement sql (param HE.text) decodeProcs
+-- 1. to or from a domain
+-- 2. implicit
+-- For the time being it must also be to/from JSON or text, although one can imagine a future where we support special
+-- cases like CSV specific representations.
+dataRepresentations :: Bool -> SQL.Statement [Schema] RepresentationsMap
+dataRepresentations = SQL.Statement sql (arrayParam HE.text) decodeRepresentations
   where
-    sql = procsSqlQuery pgVer <> " AND pn.nspname = $1 AND has_function_privilege(p.oid, 'execute')"
+    sql = [q|
+    SELECT
+      c.castsource::regtype::text,
+      c.casttarget::regtype::text,
+      c.castfunc::regproc::text
+    FROM
+      pg_catalog.pg_cast c
+    JOIN pg_catalog.pg_type src_t
+      ON c.castsource::oid = src_t.oid
+    JOIN pg_catalog.pg_type dst_t
+      ON c.casttarget::oid = dst_t.oid
+    WHERE
+      c.castcontext = 'i'
+      AND c.castmethod = 'f'
+      AND has_function_privilege(c.castfunc, 'execute')
+      AND ((src_t.typtype = 'd' AND c.casttarget IN ('json'::regtype::oid , 'text'::regtype::oid))
+       OR (dst_t.typtype = 'd' AND c.castsource IN ('json'::regtype::oid , 'text'::regtype::oid)))
+    |]
 
-procsSqlQuery :: PgVersion -> SqlQuery
+allFunctions :: PgVersion -> Bool -> SQL.Statement [Schema] RoutineMap
-procsSqlQuery pgVer = [q|
+allFunctions pgVer = SQL.Statement sql (arrayParam HE.text) decodeFuncs
+  where
+    sql = funcsSqlQuery pgVer <> " AND pn.nspname = ANY($1)"
+
+accessibleFuncs :: PgVersion -> Bool -> SQL.Statement Schema RoutineMap
+accessibleFuncs pgVer = SQL.Statement sql (param HE.text) decodeFuncs
+  where
+    sql = funcsSqlQuery pgVer <> " AND pn.nspname = $1 AND has_function_privilege(p.oid, 'execute')"
+
+funcsSqlQuery :: PgVersion -> SqlQuery
+funcsSqlQuery pgVer = [q|
  -- Recursively get the base types of domains
   WITH
   base_types AS (
@@ -307,6 +368,13 @@ procsSqlQuery pgVer = [q|
       array_agg((
         COALESCE(name, ''), -- name
         type::regtype::text, -- type
+        CASE type
+          WHEN 'bit'::regtype THEN 'bit varying'
+          WHEN 'bit[]'::regtype THEN 'bit varying[]'
+          WHEN 'character'::regtype THEN 'character varying'
+          WHEN 'character[]'::regtype THEN 'character varying[]'
+          ELSE type::regtype::text
+        END, -- convert types that ignore the lenth and accept any value till maximum size
         idx <= (pronargs - pronargdefaults), -- is_required
         COALESCE(mode = 'v', FALSE) -- is_variadic
       ) ORDER BY idx) AS args,
@@ -333,9 +401,10 @@ procsSqlQuery pgVer = [q|
      -- if any TABLE, INOUT or OUT arguments present, treat as composite
      or COALESCE(proargmodes::text[] && '{t,b,o}', false)
     ) AS rettype_is_composite,
-    ('void'::regtype = t.oid) AS rettype_is_void,
+    bt.oid <> bt.base as rettype_is_composite_alias,
     p.provolatile,
-    p.provariadic > 0 as hasvariadic
+    p.provariadic > 0 as hasvariadic,
+    lower((regexp_split_to_array((regexp_split_to_array(config, '='))[2], ','))[1]) AS transaction_isolation_level
   FROM pg_proc p
   LEFT JOIN arguments a ON a.oid = p.oid
   JOIN pg_namespace pn ON pn.oid = p.pronamespace
@@ -344,6 +413,7 @@ procsSqlQuery pgVer = [q|
   JOIN pg_namespace tn ON tn.oid = t.typnamespace
   LEFT JOIN pg_class comp ON comp.oid = t.typrelid
   LEFT JOIN pg_description as d ON d.objoid = p.oid
+  LEFT JOIN LATERAL unnest(proconfig) config ON config like 'default_transaction_isolation%'
   WHERE t.oid <> 'trigger'::regtype AND COALESCE(a.callable, true)
 |] <> (if pgVer >= pgVersion110 then "AND prokind = 'f'" else "AND NOT (proisagg OR proiswindow)")
 
@@ -399,7 +469,7 @@ test         | personnages_view     | test       | actors_view      | personnage
 -}
 addViewM2OAndO2ORels :: [ViewKeyDependency] -> [Relationship] -> [Relationship]
 addViewM2OAndO2ORels keyDeps rels =
-  rels ++ concat (viewRels <$> rels)
+  rels ++ concatMap viewRels rels
   where
     isM2O card = case card of {M2O _ _ -> True; _ -> False;}
     isO2O card = case card of {O2O _ _ -> True; _ -> False;}
@@ -449,7 +519,7 @@ addViewM2OAndO2ORels keyDeps rels =
         , keyDepColsTblVw <- expandKeyDepCols $ keyDepCols tblVw ]
       else []
     viewRels _ = []
-    expandKeyDepCols kdc = zip (fst <$> kdc) <$> sequenceA (snd <$> kdc)
+    expandKeyDepCols kdc = zip (fst <$> kdc) <$> traverse snd kdc
 
 addInverseRels :: [Relationship] -> [Relationship]
 addInverseRels rels =
@@ -485,7 +555,7 @@ addViewPrimaryKeys tabs keyDeps =
     -- * We don't have any logic that requires the client to name a PK column (compared to the column hints in embedding for FKs),
     --   so we don't need to know about the other references.
     -- * We need to choose a single reference for each column, otherwise we'd output too many columns in location headers etc.
-    takeFirstPK pkCols = catMaybes $ head . snd <$> pkCols
+    takeFirstPK = mapMaybe (head . snd)
 
 allTables :: PgVersion -> Bool -> SQL.Statement [Schema] TablesMap
 allTables pgVer =
@@ -499,6 +569,8 @@ tablesSqlQuery pgVer =
   -- the tbl_constraints/key_col_usage CTEs are based on the standard "information_schema.table_constraints"/"information_schema.key_column_usage" views,
   -- we cannot use those directly as they include the following privilege filter:
   -- (pg_has_role(ss.relowner, 'USAGE'::text) OR has_column_privilege(ss.roid, a.attnum, 'SELECT, INSERT, UPDATE, REFERENCES'::text));
+  -- on the "columns" CTE, left joining on pg_depend and pg_class is used to obtain the sequence name as a column default in case there are GENERATED .. AS IDENTITY,
+  -- generated columns are only available from pg >= 10 but the query is agnostic to versions. dep.deptype = 'i' is done because there are other 'a' dependencies on PKs
   [q|
   WITH
   columns AS (
@@ -507,22 +579,21 @@ tablesSqlQuery pgVer =
           c.relname::name AS table_name,
           a.attname::name AS column_name,
           d.description AS description,
-          pg_get_expr(ad.adbin, ad.adrelid)::text AS column_default,
+  |] <> columnDefault <> [q| AS column_default,
           not (a.attnotnull OR t.typtype = 'd' AND t.typnotnull) AS is_nullable,
+          CASE
+              WHEN t.typtype = 'd' THEN
               CASE
-                  WHEN t.typtype = 'd' THEN
+                  WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
-                  CASE
+                  ELSE format_type(a.atttypid, a.atttypmod)
-                      WHEN bt.typelem <> 0::oid AND bt.typlen = (-1) THEN 'ARRAY'::text
+              END
-                      WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
+              ELSE
-                      ELSE format_type(a.atttypid, a.atttypmod)
+              CASE
-                  END
+                  WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
-                  ELSE
+                  ELSE format_type(a.atttypid, a.atttypmod)
-                  CASE
+              END
-                      WHEN t.typelem <> 0::oid AND t.typlen = (-1) THEN 'ARRAY'::text
+          END::text AS data_type,
-                      WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
+          format_type(a.atttypid, a.atttypmod)::text AS nominal_data_type,
-                      ELSE format_type(a.atttypid, a.atttypmod)
-                  END
-              END::text AS data_type,
           information_schema._pg_char_max_length(
               information_schema._pg_truetypid(a.*, t.*),
               information_schema._pg_truetypmod(a.*, t.*)
@@ -542,6 +613,12 @@ tablesSqlQuery pgVer =
               ON t.typtype = 'd' AND t.typbasetype = bt.oid
           LEFT JOIN (pg_collation co JOIN pg_namespace nco ON co.collnamespace = nco.oid)
               ON a.attcollation = co.oid AND (nco.nspname <> 'pg_catalog'::name OR co.collname <> 'default'::name)
+          LEFT JOIN pg_depend dep
+              ON dep.refobjid = a.attrelid and dep.refobjsubid = a.attnum and dep.deptype = 'i'
+          LEFT JOIN pg_class seqclass
+              ON seqclass.oid = dep.objid
+          LEFT JOIN pg_namespace seqsch
+              ON seqsch.oid = seqclass.relnamespace
       WHERE
           NOT pg_is_other_temp_schema(nc.oid)
           AND a.attnum > 0
@@ -558,6 +635,7 @@ tablesSqlQuery pgVer =
           info.description,
           info.is_nullable::boolean,
           info.data_type,
+          info.nominal_data_type,
           info.character_maximum_length,
           info.column_default,
           coalesce(enum_info.vals, '{}')) order by info.position) as columns
@@ -689,7 +767,25 @@ tablesSqlQuery pgVer =
   "ORDER BY table_schema, table_name"
   where
     relIsPartition = if pgVer >= pgVersion100 then " AND not c.relispartition " else mempty
-
+    columnDefault -- typbasetype and typdefaultbin handles `CREATE DOMAIN .. DEFAULT val`,  attidentity/attgenerated handles generated columns, pg_get_expr gets the default of a column
+      | pgVer >= pgVersion120 = [q|
+          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
+            WHEN a.attidentity  = 'd' THEN format('nextval(%s)', quote_literal(seqsch.nspname || '.' || seqclass.relname))
+            WHEN a.attgenerated = 's' THEN null
+            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
+          END|]
+      | pgVer >= pgVersion100 = [q|
+          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
+            WHEN a.attidentity = 'd' THEN format('nextval(%s)', quote_literal(seqsch.nspname || '.' || seqclass.relname))
+            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
+          END|]
+      | otherwise  = [q|
+          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
+            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
+          END|]
 
 -- | Gets many-to-one relationships and one-to-one(O2O) relationships, which are a refinement of the many-to-one's
 allM2OandO2ORels :: PgVersion -> Bool -> SQL.Statement () [Relationship]

src/PostgREST/SchemaCache/Identifiers.hs

@@ -24,7 +24,7 @@ data QualifiedIdentifier = QualifiedIdentifier
   { qiSchema :: Schema
   , qiName   :: TableName
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
 
 instance Hashable QualifiedIdentifier
 

src/PostgREST/SchemaCache/Proc.hs

@@ -1,91 +0,0 @@
-{-# LANGUAGE DeriveAnyClass #-}
-{-# LANGUAGE DeriveGeneric  #-}
-
-module PostgREST.SchemaCache.Proc
-  ( PgType(..)
-  , ProcDescription(..)
-  , ProcParam(..)
-  , ProcVolatility(..)
-  , ProcsMap
-  , RetType(..)
-  , procReturnsScalar
-  , procReturnsSingle
-  , procReturnsVoid
-  , procTableName
-  ) where
-
-import qualified Data.Aeson          as JSON
-import qualified Data.HashMap.Strict as HM
-
-import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
-                                          Schema, TableName)
-
-import Protolude
-
-data PgType
-  = Scalar
-  | Composite QualifiedIdentifier
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
-
-data RetType
-  = Single PgType
-  | SetOf PgType
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
-
-data ProcVolatility
-  = Volatile
-  | Stable
-  | Immutable
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
-
-data ProcDescription = ProcDescription
-  { pdSchema      :: Schema
-  , pdName        :: Text
-  , pdDescription :: Maybe Text
-  , pdParams      :: [ProcParam]
-  , pdReturnType  :: Maybe RetType
-  , pdVolatility  :: ProcVolatility
-  , pdHasVariadic :: Bool
-  }
-  deriving (Eq, Generic, JSON.ToJSON)
-
-data ProcParam = ProcParam
-  { ppName :: Text
-  , ppType :: Text
-  , ppReq  :: Bool
-  , ppVar  :: Bool
-  }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
-
--- Order by least number of params in the case of overloaded functions
-instance Ord ProcDescription where
-  ProcDescription schema1 name1 des1 prms1 rt1 vol1 hasVar1 `compare` ProcDescription schema2 name2 des2 prms2 rt2 vol2 hasVar2
-    | schema1 == schema2 && name1 == name2 && length prms1 < length prms2  = LT
-    | schema2 == schema2 && name1 == name2 && length prms1 > length prms2  = GT
-    | otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2)
-
--- | A map of all procs, all of which can be overloaded(one entry will have more than one ProcDescription).
--- | It uses a HashMap for a faster lookup.
-type ProcsMap = HM.HashMap QualifiedIdentifier [ProcDescription]
-
-procReturnsScalar :: ProcDescription -> Bool
-procReturnsScalar proc = case proc of
-  ProcDescription{pdReturnType = Just (Single Scalar)} -> True
-  ProcDescription{pdReturnType = Just (SetOf Scalar)}  -> True
-  _                                                    -> False
-
-procReturnsSingle :: ProcDescription -> Bool
-procReturnsSingle proc = case proc of
-  ProcDescription{pdReturnType = Just (Single _)} -> True
-  _                                               -> False
-
-procReturnsVoid :: ProcDescription -> Bool
-procReturnsVoid proc = case proc of
-  ProcDescription{pdReturnType = Nothing} -> True
-  _                                       -> False
-
-procTableName :: ProcDescription -> Maybe TableName
-procTableName proc = case pdReturnType proc of
-  Just (SetOf  (Composite qi)) -> Just $ qiName qi
-  Just (Single (Composite qi)) -> Just $ qiName qi
-  _                            -> Nothing

src/PostgREST/SchemaCache/Relationship.hs

@@ -6,6 +6,7 @@ module PostgREST.SchemaCache.Relationship
   , Relationship(..)
   , Junction(..)
   , RelationshipsMap
+  , relIsToOne
   ) where
 
 import qualified Data.Aeson          as JSON
@@ -33,7 +34,7 @@ data Relationship = Relationship
   , relToOne        :: Bool
   , relIsSelf       :: Bool
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 -- | The relationship cardinality
 -- | https://en.wikipedia.org/wiki/Cardinality_(data_modeling)
@@ -46,7 +47,7 @@ data Cardinality
   -- ^ one-to-one, this is a refinement over M2O so operating on it is pretty much the same as M2O
   | M2M Junction
   -- ^ many-to-many
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 type FKConstraint = Text
 
@@ -55,10 +56,17 @@ data Junction = Junction
   { junTable       :: QualifiedIdentifier
   , junConstraint1 :: FKConstraint
   , junConstraint2 :: FKConstraint
-  , junColumns1    :: [(FieldName, FieldName)]
+  , junColsSource  :: [(FieldName, FieldName)]
-  , junColumns2    :: [(FieldName, FieldName)]
+  , junColsTarget  :: [(FieldName, FieldName)]
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 -- | Key based on the source table and the foreign table schema
 type RelationshipsMap = HM.HashMap (QualifiedIdentifier, Schema)  [Relationship]
+
+relIsToOne :: Relationship -> Bool
+relIsToOne rel = case rel of
+  Relationship{relCardinality=M2O _ _} -> True
+  Relationship{relCardinality=O2O _ _} -> True
+  ComputedRelationship{relToOne=True}  -> True
+  _                                    -> False

src/PostgREST/SchemaCache/Representations.hs

@@ -0,0 +1,29 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric  #-}
+
+module PostgREST.SchemaCache.Representations
+  ( DataRepresentation(..)
+  , RepresentationsMap
+  ) where
+
+import qualified Data.Aeson          as JSON
+import qualified Data.HashMap.Strict as HM
+
+
+import Protolude
+
+-- | Data representations allow user customisation of how to present and receive data through APIs, per field.
+-- This structure is used for the library of available transforms. It answers questions like:
+-- - What function, if any, should be used to present a certain field that's been selected for API output?
+-- - How do we parse incoming data for a certain field type when inserting or updating?
+-- - And similarly, how do we parse textual data in a query string to be used as a filter?
+--
+-- Support for outputting special formats like CSV and binary data would fit into the same system.
+data DataRepresentation = DataRepresentation
+  { drSourceType :: Text
+  , drTargetType :: Text
+  , drFunction   :: Text
+  } deriving (Eq, Show, Generic, JSON.ToJSON, JSON.FromJSON)
+
+-- The representation map maps from (source type, target type) to a DR.
+type RepresentationsMap = HM.HashMap (Text, Text) DataRepresentation

src/PostgREST/SchemaCache/Routine.hs

@@ -0,0 +1,132 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric  #-}
+
+module PostgREST.SchemaCache.Routine
+  ( PgType(..)
+  , Routine(..)
+  , RoutineParam(..)
+  , FuncVolatility(..)
+  , RoutineMap
+  , RetType(..)
+  , funcReturnsScalar
+  , funcReturnsSetOfScalar
+  , funcReturnsSingleComposite
+  , funcReturnsVoid
+  , funcTableName
+  , funcReturnsCompositeAlias
+  , ResultAggregate(..)
+  ) where
+
+import           Data.Aeson                 ((.=))
+import qualified Data.Aeson                 as JSON
+import qualified Data.HashMap.Strict        as HM
+import qualified Hasql.Transaction.Sessions as SQL
+
+import PostgREST.SchemaCache.Identifiers (FieldName,
+                                          QualifiedIdentifier (..),
+                                          Schema, TableName)
+
+import Protolude
+
+data PgType
+  = Scalar QualifiedIdentifier
+  | Composite QualifiedIdentifier Bool -- True if the composite is a domain alias(used to work around a bug in pg 11 and 12, see QueryBuilder.hs)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
+
+data RetType
+  = Single PgType
+  | SetOf PgType
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
+
+data FuncVolatility
+  = Volatile
+  | Stable
+  | Immutable
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
+
+data Routine = Function
+  { pdSchema      :: Schema
+  , pdName        :: Text
+  , pdDescription :: Maybe Text
+  , pdParams      :: [RoutineParam]
+  , pdReturnType  :: RetType
+  , pdVolatility  :: FuncVolatility
+  , pdHasVariadic :: Bool
+  , pdIsoLvl      :: Maybe SQL.IsolationLevel
+  }
+  deriving (Eq, Show, Generic)
+-- need to define JSON manually bc SQL.IsolationLevel doesn't have a JSON instance(and we can't define one for that type without getting a compiler error)
+instance JSON.ToJSON Routine where
+  toJSON (Function sch nam desc params ret vol hasVar _) = JSON.object
+    [
+      "pdSchema"      .= sch
+    , "pdName"        .= nam
+    , "pdDescription" .= desc
+    , "pdParams"      .= JSON.toJSON params
+    , "pdReturnType"  .= JSON.toJSON ret
+    , "pdVolatility"  .= JSON.toJSON vol
+    , "pdHasVariadic" .= JSON.toJSON hasVar
+    ]
+
+data RoutineParam = RoutineParam
+  { ppName          :: Text
+  , ppType          :: Text
+  , ppTypeMaxLength :: Text
+  , ppReq           :: Bool
+  , ppVar           :: Bool
+  }
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
+
+-- Order by least number of params in the case of overloaded functions
+instance Ord Routine where
+  Function schema1 name1 des1 prms1 rt1 vol1 hasVar1 iso1 `compare` Function schema2 name2 des2 prms2 rt2 vol2 hasVar2 iso2
+    | schema1 == schema2 && name1 == name2 && length prms1 < length prms2  = LT
+    | schema2 == schema2 && name1 == name2 && length prms1 > length prms2  = GT
+    | otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1, iso1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2, iso2)
+
+-- | A map of all procs, all of which can be overloaded(one entry will have more than one Routine).
+-- | It uses a HashMap for a faster lookup.
+type RoutineMap = HM.HashMap QualifiedIdentifier [Routine]
+
+data ResultAggregate
+   = BuiltinAggJson
+   | BuiltinAggSingleJson Bool
+   | BuiltinAggArrayJsonStrip
+   | BuiltinAggGeoJson
+   | BuiltinAggCsv
+   | BuiltinAggXml    (Maybe FieldName)
+   | BuiltinAggBinary (Maybe FieldName)
+   | NoAgg
+   deriving (Eq, Show)
+
+funcReturnsScalar :: Routine -> Bool
+funcReturnsScalar proc = case proc of
+  Function{pdReturnType = Single (Scalar{})} -> True
+  _                                          -> False
+
+funcReturnsSetOfScalar :: Routine -> Bool
+funcReturnsSetOfScalar proc = case proc of
+  Function{pdReturnType = SetOf (Scalar{})} -> True
+  _                                         -> False
+
+funcReturnsCompositeAlias :: Routine -> Bool
+funcReturnsCompositeAlias proc = case proc of
+  Function{pdReturnType = Single (Composite _ True)} -> True
+  Function{pdReturnType = SetOf (Composite _ True)}  -> True
+  _                                                  -> False
+
+funcReturnsSingleComposite :: Routine -> Bool
+funcReturnsSingleComposite proc = case proc of
+  Function{pdReturnType = Single (Composite _ _)} -> True
+  _                                               -> False
+
+funcReturnsVoid :: Routine -> Bool
+funcReturnsVoid proc = case proc of
+  Function{pdReturnType = Single (Scalar (QualifiedIdentifier "pg_catalog" "void"))} -> True
+  _                                                                                  -> False
+
+funcTableName :: Routine -> Maybe TableName
+funcTableName proc = case pdReturnType proc of
+  SetOf  (Composite qi _) -> Just $ qiName qi
+  Single (Composite qi _) -> Just $ qiName qi
+  _                       -> Nothing

src/PostgREST/SchemaCache/Table.hs

@@ -1,14 +1,18 @@
-{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveAnyClass    #-}
-{-# LANGUAGE DeriveGeneric  #-}
+{-# LANGUAGE DeriveGeneric     #-}
+{-# LANGUAGE FlexibleInstances #-}
 
 module PostgREST.SchemaCache.Table
   ( Column(..)
   , Table(..)
+  , tableColumnsList
   , TablesMap
+  , ColumnMap
   ) where
 
-import qualified Data.Aeson          as JSON
+import qualified Data.Aeson                 as JSON
-import qualified Data.HashMap.Strict as HM
+import qualified Data.HashMap.Strict        as HM
+import qualified Data.HashMap.Strict.InsOrd as HMI
 
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier (..),
@@ -28,9 +32,12 @@ data Table = Table
   , tableUpdatable   :: Bool
   , tableDeletable   :: Bool
   , tablePKCols      :: [FieldName]
-  , tableColumns     :: [Column]
+  , tableColumns     :: ColumnMap
   }
-  deriving (Show, Ord, Generic, JSON.ToJSON)
+  deriving (Show, Generic, JSON.ToJSON)
+
+tableColumnsList :: Table -> [Column]
+tableColumnsList = HMI.elems . tableColumns
 
 instance Eq Table where
   Table{tableSchema=s1,tableName=n1} == Table{tableSchema=s2,tableName=n2} = s1 == s2 && n1 == n2
@@ -40,6 +47,7 @@ data Column = Column
   , colDescription :: Maybe Text
   , colNullable    :: Bool
   , colType        :: Text
+  , colNominalType :: Text
   , colMaxLen      :: Maybe Int32
   , colDefault     :: Maybe Text
   , colEnum        :: [Text]
@@ -47,3 +55,4 @@ data Column = Column
   deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 type TablesMap = HM.HashMap QualifiedIdentifier Table
+type ColumnMap = HMI.InsOrdHashMap FieldName Column

src/PostgREST/Unix.hs

@@ -14,7 +14,6 @@ import System.Posix.Files (setFileMode)
 import System.Posix.Types (FileMode)
 
 import qualified PostgREST.AppState as AppState
-import qualified PostgREST.Workers  as Workers
 
 import Protolude
 
@@ -49,10 +48,10 @@ installSignalHandlers appState = do
 
   -- The SIGUSR1 signal updates the internal 'SchemaCache' by running
   -- 'connectionWorker' exactly as before.
-  install Signals.sigUSR1 $ Workers.connectionWorker appState
+  install Signals.sigUSR1 $ AppState.connectionWorker appState
 
   -- Re-read the config on SIGUSR2
-  install Signals.sigUSR2 $ Workers.reReadConfig False appState
+  install Signals.sigUSR2 $ AppState.reReadConfig False appState
   where
     install signal handler =
       void $ Signals.installHandler signal (Signals.Catch handler) Nothing

src/PostgREST/Workers.hs

@@ -1,345 +0,0 @@
-{-# LANGUAGE LambdaCase      #-}
-{-# LANGUAGE NamedFieldPuns  #-}
-{-# LANGUAGE RecordWildCards #-}
-
-module PostgREST.Workers
-  ( connectionWorker
-  , reReadConfig
-  , runListener
-  , runAdmin
-  ) where
-
-import qualified Data.Aeson                 as JSON
-import qualified Data.ByteString            as BS
-import qualified Data.ByteString.Lazy       as LBS
-import qualified Data.Text                  as T
-import qualified Data.Text.Encoding         as T
-import qualified Hasql.Notifications        as SQL
-import qualified Hasql.Session              as SQL
-import qualified Hasql.Transaction.Sessions as SQL
-import qualified Network.HTTP.Types.Status  as HTTP
-import qualified Network.Wai                as Wai
-import qualified Network.Wai.Handler.Warp   as Warp
-
-import Control.Retry    (RetryStatus, capDelay, exponentialBackoff,
-                         retrying, rsPreviousDelay)
-import Hasql.Connection (acquire)
-
-import Network.Socket
-import Network.Socket.ByteString
-
-import PostgREST.AppState         (AppState)
-import PostgREST.Config           (AppConfig (..), readAppConfig)
-import PostgREST.Config.Database  (queryDbSettings, queryPgVersion)
-import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
-import PostgREST.Error            (PgError (PgError), checkIsFatal,
-                                   errorPayload)
-import PostgREST.SchemaCache      (querySchemaCache)
-
-import qualified PostgREST.AppState as AppState
-
-import Protolude
-
-
--- | Current database connection status data ConnectionStatus
-data ConnectionStatus
-  = NotConnected
-  | Connected PgVersion
-  | FatalConnectionError Text
-  deriving (Eq)
-
--- | Schema cache status
-data SCacheStatus
-  = SCLoaded
-  | SCOnRetry
-  | SCFatalFail
-
--- | The purpose of this worker is to obtain a healthy connection to pg and an
--- up-to-date schema cache(SchemaCache).  This method is meant to be called
--- multiple times by the same thread, but does nothing if the previous
--- invocation has not terminated. In all cases this method does not halt the
--- calling thread, the work is performed in a separate thread.
---
--- Background thread that does the following :
---  1. Tries to connect to pg server and will keep trying until success.
---  2. Checks if the pg version is supported and if it's not it kills the main
---     program.
---  3. Obtains the sCache. If this fails, it goes back to 1.
-connectionWorker :: AppState -> IO ()
-connectionWorker appState = do
-  runExclusively (AppState.getWorkerSem appState) work
-  -- Prevents multiple workers to be running at the same time. Could happen on
-  -- too many SIGUSR1s.
-  where
-    runExclusively mvar action = mask_ $ do
-      success <- tryPutMVar mvar ()
-      when success $ do
-        void $ forkIO $ action `finally` takeMVar mvar
-    work = do
-      AppConfig{..} <- AppState.getConfig appState
-      AppState.logWithZTime appState "Attempting to connect to the database..."
-      connected <- establishConnection appState
-      case connected of
-        FatalConnectionError reason ->
-          -- Fatal error when connecting
-          AppState.logWithZTime appState reason >> killThread (AppState.getMainThreadId appState)
-        NotConnected ->
-          -- Unreachable because establishConnection will keep trying to connect
-          return ()
-        Connected actualPgVersion -> do
-          -- Procede with initialization
-          AppState.putPgVersion appState actualPgVersion
-          when configDbChannelEnabled $
-            AppState.signalListener appState
-          AppState.logWithZTime appState "Connection successful"
-          -- this could be fail because the connection drops, but the
-          -- loadSchemaCache will pick the error and retry again
-          when configDbConfig $ reReadConfig False appState
-          scStatus <- loadSchemaCache appState
-          case scStatus of
-            SCLoaded ->
-              -- do nothing and proceed if the load was successful
-              return ()
-            SCOnRetry ->
-              -- retry reloading the schema cache
-              work
-            SCFatalFail ->
-              -- die if our schema cache query has an error
-              killThread $ AppState.getMainThreadId appState
-
--- | Repeatedly flush the pool, and check if a connection from the
--- pool allows access to the PostgreSQL database.
---
--- Releasing the pool is key for rapid recovery. Otherwise, the pool
--- timeout would have to be reached for new healthy connections to be acquired.
--- Which might not happen if the server is busy with requests. No idle
--- connection, no pool timeout.
---
--- The connection tries are capped, but if the connection times out no error is
--- thrown, just 'False' is returned.
-establishConnection :: AppState -> IO ConnectionStatus
-establishConnection appState =
-  retrying retrySettings shouldRetry $
-    const $ AppState.flushPool appState >> getConnectionStatus
-  where
-    retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
-    delayMicroseconds = 32000000 -- 32 seconds
-    backoffMicroseconds = 1000000 -- 1 second
-
-    getConnectionStatus :: IO ConnectionStatus
-    getConnectionStatus = do
-      pgVersion <- AppState.usePool appState queryPgVersion
-      case pgVersion of
-        Left e -> do
-          let err = PgError False e
-          AppState.logWithZTime appState . T.decodeUtf8 . LBS.toStrict $ errorPayload err
-          case checkIsFatal err of
-            Just reason ->
-              return $ FatalConnectionError reason
-            Nothing ->
-              return NotConnected
-        Right version ->
-          if version < minimumPgVersion then
-            return . FatalConnectionError $
-              "Cannot run in this PostgreSQL version, PostgREST needs at least "
-              <> pgvName minimumPgVersion
-          else
-            return . Connected  $ version
-
-    shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
-    shouldRetry rs isConnSucc = do
-      let
-        delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
-        itShould = NotConnected == isConnSucc
-      when itShould . AppState.logWithZTime appState $
-        "Attempting to reconnect to the database in "
-        <> (show delay::Text)
-        <> " seconds..."
-      when itShould $ AppState.putRetryNextIn appState delay
-      return itShould
-
--- | Load the SchemaCache by using a connection from the pool.
-loadSchemaCache :: AppState -> IO SCacheStatus
-loadSchemaCache appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  result <-
-    let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
-    AppState.usePool appState . transaction SQL.ReadCommitted SQL.Read $
-      querySchemaCache (toList configDbSchemas) configDbExtraSearchPath configDbPreparedStatements
-  case result of
-    Left e -> do
-      let
-        err = PgError False e
-        putErr = AppState.logWithZTime appState . T.decodeUtf8 . LBS.toStrict $ errorPayload err
-      case checkIsFatal err of
-        Just hint -> do
-          AppState.logWithZTime appState "A fatal error ocurred when loading the schema cache"
-          putErr
-          AppState.logWithZTime appState hint
-          return SCFatalFail
-        Nothing -> do
-          AppState.putSchemaCache appState Nothing
-          AppState.logWithZTime appState "An error ocurred when loading the schema cache"
-          putErr
-          return SCOnRetry
-
-    Right sCache -> do
-      AppState.putSchemaCache appState (Just sCache)
-      when (isJust configDbRootSpec) .
-        AppState.putJsonDbS appState . LBS.toStrict $ JSON.encode sCache
-      AppState.logWithZTime appState "Schema cache loaded"
-      return SCLoaded
-
-runListener :: AppConfig -> AppState -> IO ()
-runListener AppConfig{configDbChannelEnabled} appState =
-  when configDbChannelEnabled $ listener appState
-
--- | Starts a dedicated pg connection to LISTEN for notifications.  When a
--- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
--- cache.  It uses the connectionWorker in case the LISTEN connection dies.
-listener :: AppState -> IO ()
-listener appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  let dbChannel = toS configDbChannel
-
-  -- The listener has to wait for a signal from the connectionWorker.
-  -- This is because when the connection to the db is lost, the listener also
-  -- tries to recover the connection, but not with the same pace as the connectionWorker.
-  -- Not waiting makes stderr quickly fill with connection retries messages from the listener.
-  AppState.waitListener appState
-
-  -- forkFinally allows to detect if the thread dies
-  void . flip forkFinally (handleFinally dbChannel) $ do
-    dbOrError <- acquire $ toUtf8 configDbUri
-    case dbOrError of
-      Right db -> do
-        AppState.logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
-        AppState.putIsListenerOn appState True
-        SQL.listen db $ SQL.toPgIdentifier dbChannel
-        SQL.waitForNotifications handleNotification db
-      _ ->
-        die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
-  where
-    handleFinally dbChannel _ = do
-      -- if the thread dies, we try to recover
-      AppState.logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
-      AppState.putIsListenerOn appState False
-      -- assume the pool connection was also lost, call the connection worker
-      connectionWorker appState
-      -- retry the listener
-      listener appState
-
-    handleNotification _ msg
-      | BS.null msg            = scLoader -- reload the schema cache
-      | msg == "reload schema" = scLoader -- reload the schema cache
-      | msg == "reload config" = reReadConfig False appState -- reload the config
-      | otherwise              = pure () -- Do nothing if anything else than an empty message is sent
-
-    scLoader =
-      -- It's not necessary to check the loadSchemaCache success
-      -- here. If the connection drops, the thread will die and
-      -- proceed to recover.
-      void $ loadSchemaCache appState
-
--- | Re-reads the config plus config options from the db
-reReadConfig :: Bool -> AppState -> IO ()
-reReadConfig startingUp appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  dbSettings <-
-    if configDbConfig then do
-      qDbSettings <- AppState.usePool appState $ queryDbSettings configDbPreparedStatements
-      case qDbSettings of
-        Left e -> do
-          let
-            err = PgError False e
-            putErr = AppState.logWithZTime appState . T.decodeUtf8 . LBS.toStrict $ errorPayload err
-          AppState.logWithZTime appState
-            "An error ocurred when trying to query database settings for the config parameters"
-          case checkIsFatal err of
-            Just hint -> do
-              putErr
-              AppState.logWithZTime appState hint
-              killThread (AppState.getMainThreadId appState)
-            Nothing -> do
-              putErr
-          pure []
-        Right x -> pure x
-    else
-      pure mempty
-  readAppConfig dbSettings configFilePath (Just configDbUri) >>= \case
-    Left err   ->
-      if startingUp then
-        panic err -- die on invalid config if the program is starting up
-      else
-        AppState.logWithZTime appState $ "Failed reloading config: " <> err
-    Right newConf -> do
-      AppState.putConfig appState newConf
-      if startingUp then
-        pass
-      else
-        AppState.logWithZTime appState "Config reloaded"
-
-runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
-runAdmin conf@AppConfig{configAdminServerPort} appState settings =
-  whenJust configAdminServerPort $ \adminPort -> do
-    AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
-    void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
-  where
-    whenJust :: Applicative m => Maybe a -> (a -> m ()) -> m ()
-    whenJust mg f = maybe (pure ()) f mg
-    adminApp = admin appState conf
-
--- | PostgREST admin application
-admin :: AppState.AppState -> AppConfig -> Wai.Application
-admin appState appConfig req respond  = do
-  isMainAppReachable  <- any isRight <$> reachMainApp appConfig
-  isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
-  isConnectionUp      <-
-    if configDbChannelEnabled appConfig
-      then AppState.getIsListenerOn appState
-      else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
-
-  case Wai.pathInfo req of
-    ["ready"] ->
-      respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
-    ["live"] ->
-      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
-    _ ->
-      respond $ Wai.responseLBS HTTP.status404 [] mempty
-
--- Try to connect to the main app socket
--- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
--- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
--- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
-reachMainApp :: AppConfig -> IO [Either IOException ()]
-reachMainApp AppConfig{..} =
-  case configServerUnixSocket of
-    Just path ->  do
-      sock <- socket AF_UNIX Stream 0
-      (:[]) <$> try (do
-        connect sock $ SockAddrUnix path
-        withSocketsDo $ bracket (pure sock) close sendEmpty)
-    Nothing -> do
-      let
-        host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
-             | otherwise                                             = Just configServerHost
-        filterAddrs xs =
-          case configServerHost of
-              "*4" -> ipv4Addrs xs ++ ipv6Addrs xs
-              "!4" -> ipv4Addrs xs
-              "*6" -> ipv6Addrs xs ++ ipv4Addrs xs
-              "!6" -> ipv6Addrs xs
-              _    -> xs
-        ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
-        ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
-
-      addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
-      tryAddr `traverse` filterAddrs addrs
-  where
-    sendEmpty sock = void $ send sock mempty
-    tryAddr :: AddrInfo -> IO (Either IOException ())
-    tryAddr addr = do
-      sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
-      try $ do
-        connect sock $ addrAddress addr
-        withSocketsDo $ bracket (pure sock) close sendEmpty

stack.yaml

@@ -1,4 +1,4 @@
-resolver: lts-19.14 # 2022-07-01, GHC 9.0.2
+resolver: lts-20.6 # 2023-01-09, GHC 9.2.5
 
 nix:
   packages:
@@ -10,23 +10,7 @@ nix:
   pure: false
 
 extra-deps:
-  - HTTP-4000.3.16
-  - configurator-pg-0.2.6
-  - hashable-1.4.1.0
-  - hashtables-1.3
-  - hasql-1.6.1.1
-  - hasql-dynamic-statements-0.3.1.2
-  - hasql-implicits-0.1.0.5
-  - hasql-notifications-0.2.0.3
-  - hasql-pool-0.8.0.6
-  - hasql-transaction-1.0.1.2
-  - isomorphism-class-0.1.0.6
-  - lens-aeson-1.1.3
-  - optparse-applicative-0.16.1.0
-  - postgresql-binary-0.12.5
-  - protolude-0.3.2
-  - ptr-0.16.8.2
-  - text-builder-0.6.7
-  - text-builder-dev-0.3.3
   - git: https://github.com/PostgREST/postgresql-libpq.git
-    commit: 33ff97db570b5b432255f5f24a68db51453f6eb8
+    commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
+  - hasql-notifications-0.2.0.5
+  - hasql-pool-0.10

stack.yaml.lock

@@ -5,145 +5,33 @@
 
 packages:
 - completed:
-    hackage: HTTP-4000.3.16@sha256:6042643c15a0b43e522a6693f1e322f05000d519543a84149cb80aeffee34f71,5947
+    commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
-    pantry-tree:
+    git: https://github.com/PostgREST/postgresql-libpq.git
-      size: 1428
-      sha256: b73a7f6d21cf20bbf819e19039409c9010efb5000d2b72cdd8fd67a9027c14e8
-  original:
-    hackage: HTTP-4000.3.16
-- completed:
-    hackage: configurator-pg-0.2.6@sha256:cd9b06a458428e493a4d6def725af7ab1ab0fef678fbd871f9586fc7f9aa70be,2849
-    pantry-tree:
-      size: 2463
-      sha256: 97efe7a22afc93033bda5adcffdabc0f1c30dc32b2c3ba02114ce7cd74c942fd
-  original:
-    hackage: configurator-pg-0.2.6
-- completed:
-    hackage: hashable-1.4.1.0@sha256:50b2f002c68fe67730ee7a3cd8607486197dd99b084255005ad51ecd6970a41b,5019
-    pantry-tree:
-      size: 1248
-      sha256: 9af2f7a42674f7effcabbebc043f97057240783f1709338a77f58216f4a5f18c
-  original:
-    hackage: hashable-1.4.1.0
-- completed:
-    hackage: hashtables-1.3@sha256:ab21804fdafbbd8ad918b2911dabb729ae0ea891780fe66bf7804cbcd07edadf,10379
-    pantry-tree:
-      size: 2895
-      sha256: e71f113ad989dbc994e0fb52bcc219d62930de9afa8b3441bf7909e864481b33
-  original:
-    hackage: hashtables-1.3
-- completed:
-    hackage: hasql-1.6.1.1@sha256:948a2137308cc5354e4997bc3666753867124cd25db792424cb9614b1c1b44cf,6626
-    pantry-tree:
-      size: 2622
-      sha256: 28d21bf061522fc513f040e9c383b90532222b7258216cc094e07736add8be10
-  original:
-    hackage: hasql-1.6.1.1
-- completed:
-    hackage: hasql-dynamic-statements-0.3.1.2@sha256:417aa533c84f074e2fa16bb2c4d4231326aa512097dd1025d915388e56acd1eb,2675
-    pantry-tree:
-      size: 595
-      sha256: 91696d3f3e0ef3254772ae5a8e4e89be68285febb49b302ed83d85ac4037a417
-  original:
-    hackage: hasql-dynamic-statements-0.3.1.2
-- completed:
-    hackage: hasql-implicits-0.1.0.5@sha256:d16aacad6dc21428d72447d3ae8bcc03839a2f0aa1ec29c797ed9aca4609f9af,1361
-    pantry-tree:
-      size: 264
-      sha256: 0451b99a0a1d02db673d0c40acdf60d4e769e15852eed9e8dc05bffaf43efb70
-  original:
-    hackage: hasql-implicits-0.1.0.5
-- completed:
-    hackage: hasql-notifications-0.2.0.3@sha256:aca3f7ee847a8f0b7ef6f989dc48f4a094a06c1a34e92aa3c8bb230085966ea6,2027
-    pantry-tree:
-      size: 452
-      sha256: 999f0f2856a00d21f4498a8a58452bbefc4ea972fe2984fd234a68a5fe61d98b
-  original:
-    hackage: hasql-notifications-0.2.0.3
-- completed:
-    hackage: hasql-pool-0.8.0.6@sha256:b63bb83409bab5bc20ff24f5d62205e9b117701a0fc24531ddeac20ab8c2a42c,1818
-    pantry-tree:
-      size: 346
-      sha256: c4100946b7eae44375511e35a393abe2e1db0e5637c68cea8f53176b796bfd5b
-  original:
-    hackage: hasql-pool-0.8.0.6
-- completed:
-    hackage: hasql-transaction-1.0.1.2@sha256:297b158cd1f0727f9b0e175bd7d3741c1bcb725a8094956d0ee79b41aafdb30a,2890
-    pantry-tree:
-      size: 983
-      sha256: 3679e6d5c835cc17a8fa0c252b8221e282880044b7219aa1de2531bbd5c40691
-  original:
-    hackage: hasql-transaction-1.0.1.2
-- completed:
-    hackage: isomorphism-class-0.1.0.6@sha256:d93da31287359c761953b876354de28381f409c5c50e3241c572a443e50c553d,1703
-    pantry-tree:
-      size: 465
-      sha256: c97f922d1ae8f1a0db4c28fac9383d2716934879e95ff0b2b88ebb861d6fba14
-  original:
-    hackage: isomorphism-class-0.1.0.6
-- completed:
-    hackage: lens-aeson-1.1.3@sha256:52c8eaecd2d1c2a969c0762277c4a8ee72c339a686727d5785932e72ef9c3050,1764
-    pantry-tree:
-      size: 541
-      sha256: b31392b78f2a03111c805f4400007778eb93b49f998ab41dfbebaaf9b5526bad
-  original:
-    hackage: lens-aeson-1.1.3
-- completed:
-    hackage: optparse-applicative-0.16.1.0@sha256:418c22ed6a19124d457d96bc66bd22c93ac22fad0c7100fe4972bbb4ac989731,4982
-    pantry-tree:
-      size: 2979
-      sha256: dd092d843091c08691485d68a1908517079b1bc6f3d73928f37635a19dc27fc1
-  original:
-    hackage: optparse-applicative-0.16.1.0
-- completed:
-    hackage: postgresql-binary-0.12.5@sha256:de9da3cba9be541d6c75ae8da2858c33d83dc1b2e0c639b0b9781816b78a91f4,5594
-    pantry-tree:
-      size: 1619
-      sha256: b392337f91031a5b3407393e2f04dfe4e7a28019e88eae6a9370538b90e28c51
-  original:
-    hackage: postgresql-binary-0.12.5
-- completed:
-    hackage: protolude-0.3.2@sha256:2a38b3dad40d238ab644e234b692c8911423f9d3ed0e36b62287c4a698d92cd1,2240
-    pantry-tree:
-      size: 1594
-      sha256: a36d2912ac552d950ba4476de7d950b56b82dd28e48b9f4d0efee938f10bc525
-  original:
-    hackage: protolude-0.3.2
-- completed:
-    hackage: ptr-0.16.8.2@sha256:708ebb95117f2872d2c5a554eb6804cf1126e86abe793b2673f913f14e5eb1ac,3959
-    pantry-tree:
-      size: 1303
-      sha256: 557c438345de19f82bf01d676100da2a191ef06f624e7a4b90b09ac17cbb52a5
-  original:
-    hackage: ptr-0.16.8.2
-- completed:
-    hackage: text-builder-0.6.7@sha256:efbb3e06107e9c8d1cfe85c963938ca9f375a74379af03da3173be4ef5c37bcf,2364
-    pantry-tree:
-      size: 425
-      sha256: cd0ae197e6f9f3860a8ab71f5b87c4a8452ed1fce2fdfd35e36d68ded6e6648e
-  original:
-    hackage: text-builder-0.6.7
-- completed:
-    hackage: text-builder-dev-0.3.3@sha256:79ec422defcc2e5b34f94129c72b98d34b2efc1ed8bbd945ccb8f4f535a892c3,2784
-    pantry-tree:
-      size: 724
-      sha256: 8883631a132438e7892fcb13e89d6bbcdc0ac76c56fbea8df8d7aa482ce81f73
-  original:
-    hackage: text-builder-dev-0.3.3
-- completed:
     name: postgresql-libpq
-    version: 0.9.4.3
-    git: https://github.com/PostgREST/postgresql-libpq.git
     pantry-tree:
-      size: 1081
+      sha256: 074668b9669b9c49f3c522c8af5c608799a1965e203c463b188b2632995beac2
-      sha256: 0df271e48af32eb8292a45301af45e114110d54099ee73dbc609d39770e8175e
+      size: 1414
-    commit: 33ff97db570b5b432255f5f24a68db51453f6eb8
+    version: 0.9.4.3
   original:
+    commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
     git: https://github.com/PostgREST/postgresql-libpq.git
-    commit: 33ff97db570b5b432255f5f24a68db51453f6eb8
+- completed:
+    hackage: hasql-notifications-0.2.0.5@sha256:6c67d2ee42e948162e89176a2f52a2c6bb562678b33688e96e048be9b3d74122,2028
+    pantry-tree:
+      sha256: 8493755b0817a36b910a37d568769ca49ab715ef5dd935a0d09f0716aa39da51
+      size: 452
+  original:
+    hackage: hasql-notifications-0.2.0.5
+- completed:
+    hackage: hasql-pool-0.10@sha256:912197a328acb85505f98bb9700d61f366b87659ca45126c5c2d636687b801c3,2112
+    pantry-tree:
+      sha256: b655c540a49764a8d16b62941137e295b936b96edc0785eb9250972f0f92dc47
+      size: 346
+  original:
+    hackage: hasql-pool-0.10
 snapshots:
 - completed:
-    size: 618951
+    sha256: 4905c93319aa94aa53da8f41d614d7bacdbfe6c63a8c6132d32e6e62f24a9af4
-    url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/19/14.yaml
+    size: 649315
-    sha256: 4c31d4ef975b0211078862566aedf3b82b6cea569fc2cde4c72a51e5a8d236ce
+    url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/20/6.yaml
-  original: lts-19.14
+  original: lts-20.6

test/doc/Main.hs

@@ -11,8 +11,14 @@ main =
     [ "-XOverloadedStrings"
     , "-XNoImplicitPrelude"
     , "-XStandaloneDeriving"
+    , "-XDuplicateRecordFields"
     , "-isrc"
     , "src/PostgREST/Query/SqlFragment.hs"
     , "src/PostgREST/ApiRequest/Preferences.hs"
     , "src/PostgREST/ApiRequest/QueryParams.hs"
+    , "src/PostgREST/Error.hs"
+    , "src/PostgREST/MediaType.hs"
+    , "src/PostgREST/Config.hs"
+    , "src/PostgREST/Plan.hs"
+    , "src/PostgREST/Response.hs"
     ]

test/io/configs/aliases.config

@@ -1,4 +1,5 @@
 db-schema = "provided_through_alias"
+db-pool-timeout = 5
 max-rows = 1000
 pre-request = "check_alias"
 role-claim-key = ".aliased"

test/io/configs/expected/aliases.config

@@ -5,12 +5,15 @@ db-extra-search-path = "public"
 db-max-rows = 1000
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
+db-pool-max-idletime = 5
 db-pre-request = "check_alias"
 db-prepared-statements = true
 db-root-spec = "open_alias"
 db-schemas = "provided_through_alias"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/expected/boolean-numeric.config

@@ -5,12 +5,15 @@ db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
 db-pre-request = ""
 db-prepared-statements = false
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/expected/boolean-string.config

@@ -5,12 +5,15 @@ db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
 db-pre-request = ""
 db-prepared-statements = false
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/expected/defaults.config

@@ -5,12 +5,15 @@ db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
 db-pre-request = ""
 db-prepared-statements = true
 db-root-spec = ""
 db-schemas = "public"
 db-config = false
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/expected/no-defaults-with-db-other-authenticator.config

@@ -1,21 +1,24 @@
-db-anon-role = "other"
+db-anon-role = "pre_config_role"
 db-channel = "postgrest"
 db-channel-enabled = false
 db-extra-search-path = "public,extensions,other"
 db-max-rows = 100
 db-plan-enabled = true
 db-pool = 1
-db-pool-acquisition-timeout = 10
+db-pool-acquisition-timeout = 30
+db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
 db-pre-request = "test.other_custom_headers"
 db-prepared-statements = false
 db-root-spec = "other_root"
 db-schemas = "test,other_tenant1,other_tenant2"
 db-config = true
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "postgresql://"
 db-use-legacy-gucs = false
 jwt-aud = "https://otherexample.org"
-jwt-role-claim-key = ".\"other\".\"role\""
+jwt-role-claim-key = ".\"other\".\"pre_config_role\""
 jwt-secret = "ODERREALLYREALLYREALLYREALLYVERYSAFE"
 jwt-secret-is-base64 = true
 log-level = "info"
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = "https://otherexample.org/api"
 raw-media-types = "application/vnd.pgrst.other-db-config"
 server-host = "0.0.0.0"
 server-port = 80
+server-trace-header = "traceparent"
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001

test/io/configs/expected/no-defaults-with-db.config

@@ -5,12 +5,15 @@ db-extra-search-path = "public,extensions,private"
 db-max-rows = 1000
 db-plan-enabled = true
 db-pool = 1
-db-pool-acquisition-timeout = 10
+db-pool-acquisition-timeout = 30
+db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
 db-pre-request = "test.custom_headers"
 db-prepared-statements = false
 db-root-spec = "root"
 db-schemas = "test,tenant1,tenant2"
 db-config = true
+db-pre-config = "postgrest.preconf"
 db-tx-end = "commit-allow-override"
 db-uri = "postgresql://"
 db-use-legacy-gucs = false
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = "https://example.org/api"
 raw-media-types = "application/vnd.pgrst.db-config"
 server-host = "0.0.0.0"
 server-port = 80
+server-trace-header = "CF-Ray"
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001

test/io/configs/expected/no-defaults.config

@@ -5,12 +5,15 @@ db-extra-search-path = "public,test"
 db-max-rows = 1000
 db-plan-enabled = true
 db-pool = 1
-db-pool-acquisition-timeout = 10
+db-pool-acquisition-timeout = 30
+db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
 db-pre-request = "please_run_fast"
 db-prepared-statements = false
 db-root-spec = "openapi_v3"
 db-schemas = "multi,tenant,setup"
 db-config = false
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "tmp_db"
 db-use-legacy-gucs = false
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = "https://postgrest.org"
 raw-media-types = "application/vnd.pgrst.config"
 server-host = "0.0.0.0"
 server-port = 80
+server-trace-header = "X-Request-Id"
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001

test/io/configs/expected/types.config

@@ -5,12 +5,15 @@ db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
 db-pre-request = ""
 db-prepared-statements = true
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/no-defaults-env.yaml

@@ -7,15 +7,17 @@ PGRST_DB_EXTRA_SEARCH_PATH: public, test
 PGRST_DB_MAX_ROWS: 1000
 PGRST_DB_PLAN_ENABLED: true
 PGRST_DB_POOL: 1
-PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
+PGRST_DB_POOL_ACQUISITION_TIMEOUT: 30
+PGRST_DB_POOL_MAX_LIFETIME: 3600
+PGRST_DB_POOL_MAX_IDLETIME: 60
 PGRST_DB_PREPARED_STATEMENTS: false
 PGRST_DB_PRE_REQUEST: please_run_fast
 PGRST_DB_ROOT_SPEC: openapi_v3
 PGRST_DB_SCHEMAS: multi,   tenant,setup
 PGRST_DB_CONFIG: false
+PGRST_DB_PRE_CONFIG: "postgrest.pre_config"
 PGRST_DB_TX_END: rollback-allow-override
 PGRST_DB_URI: tmp_db
-PGRST_DB_EMBED_DEFAULT_JOIN: inner
 PGRST_DB_USE_LEGACY_GUCS: false
 PGRST_JWT_AUD: 'https://postgrest.org'
 PGRST_JWT_ROLE_CLAIM_KEY: '.user[0]."real-role"'
@@ -28,6 +30,7 @@ PGRST_OPENAPI_SERVER_PROXY_URI: 'https://postgrest.org'
 PGRST_RAW_MEDIA_TYPES: application/vnd.pgrst.config
 PGRST_SERVER_HOST: 0.0.0.0
 PGRST_SERVER_PORT: 80
+PGRST_SERVER_TRACE_HEADER: X-Request-Id
 PGRST_SERVER_UNIX_SOCKET: /tmp/pgrst_io_test.sock
 PGRST_SERVER_UNIX_SOCKET_MODE: 777
 PGRST_ADMIN_SERVER_PORT: 3001

test/io/configs/no-defaults.config

@@ -5,12 +5,15 @@ db-extra-search-path = "public, test"
 db-max-rows = 1000
 db-plan-enabled = true
 db-pool = 1
-db-pool-acquisition-timeout = 10
+db-pool-acquisition-timeout = 30
+db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
 db-pre-request = "please_run_fast"
 db-prepared-statements = false
 db-root-spec = "openapi_v3"
 db-schemas = "multi,   tenant,setup"
 db-config = false
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "tmp_db"
 db-use-legacy-gucs = false
@@ -25,6 +28,7 @@ openapi-server-proxy-uri = "https://postgrest.org"
 raw-media-types = "application/vnd.pgrst.config"
 server-host = "0.0.0.0"
 server-port = 80
+server-trace-header = "X-Request-Id"
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001

test/io/db_config.sql

@@ -9,6 +9,7 @@ ALTER ROLE db_config_authenticator SET pgrst.jwt_secret_is_base64 = 'false';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_role_claim_key = '."a"."role"';
 ALTER ROLE db_config_authenticator SET pgrst.db_anon_role = 'anonymous';
 ALTER ROLE db_config_authenticator SET pgrst.db_tx_end = 'commit-allow-override';
+ALTER ROLE db_config_authenticator SET pgrst.db_pre_config = 'postgrest.preconf';
 ALTER ROLE db_config_authenticator SET pgrst.db_schemas = 'test, tenant1, tenant2';
 ALTER ROLE db_config_authenticator SET pgrst.db_root_spec = 'root';
 ALTER ROLE db_config_authenticator SET pgrst.db_plan_enabled = 'true';
@@ -17,6 +18,7 @@ ALTER ROLE db_config_authenticator SET pgrst.db_pre_request = 'test.custom_heade
 ALTER ROLE db_config_authenticator SET pgrst.db_max_rows = '1000';
 ALTER ROLE db_config_authenticator SET pgrst.db_extra_search_path = 'public, extensions';
 ALTER ROLE db_config_authenticator SET pgrst.not_existing = 'should be ignored';
+ALTER ROLE db_config_authenticator SET pgrst.server_trace_header = 'CF-Ray';
 
 -- override with database specific setting
 ALTER ROLE db_config_authenticator IN DATABASE :DBNAME SET pgrst.jwt_secret = 'OVERRIDE=REALLY=REALLY=REALLY=REALLY=VERY=SAFE';
@@ -39,7 +41,10 @@ ALTER ROLE db_config_authenticator SET pgrst.db_channel_enabled = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_channel = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_pool = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_pool_timeout = 'ignored';
-ALTER ROLE db_config_authenticator SET pgrst.db_config = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_acquisition_timeout = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_lifetime = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_idletime = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_config = 'true';
 
 -- other authenticator reloadable config options
 CREATE ROLE other_authenticator LOGIN NOINHERIT;
@@ -48,9 +53,6 @@ ALTER ROLE other_authenticator SET pgrst.openapi_server_proxy_uri = 'https://oth
 ALTER ROLE other_authenticator SET pgrst.raw_media_types = 'application/vnd.pgrst.other-db-config';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret = 'ODERREALLYREALLYREALLYREALLYVERYSAFE';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret_is_base64 = 'true';
-ALTER ROLE other_authenticator SET pgrst.jwt_role_claim_key = '."other"."role"';
-ALTER ROLE other_authenticator SET pgrst.db_anon_role = 'other';
-ALTER ROLE other_authenticator SET pgrst.db_tx_end = 'rollback-allow-override';
 ALTER ROLE other_authenticator SET pgrst.db_schemas = 'test, other_tenant1, other_tenant2';
 ALTER ROLE other_authenticator SET pgrst.db_root_spec = 'other_root';
 ALTER ROLE other_authenticator SET pgrst.db_plan_enabled = 'true';
@@ -60,6 +62,33 @@ ALTER ROLE other_authenticator SET pgrst.db_max_rows = '100';
 ALTER ROLE other_authenticator SET pgrst.db_extra_search_path = 'public, extensions, other';
 ALTER ROLE other_authenticator SET pgrst.openapi_mode = 'disabled';
 ALTER ROLE other_authenticator SET pgrst.openapi_security_active = 'false';
+ALTER ROLE other_authenticator SET pgrst.server_trace_header = 'traceparent';
+ALTER ROLE other_authenticator SET pgrst.db_pre_config = 'postgrest.pre_config';
+
+create schema postgrest;
+grant usage on schema postgrest to db_config_authenticator;
+grant usage on schema postgrest to other_authenticator;
+
+-- pre-config hook
+create or replace function postgrest.pre_config()
+returns void as $$
+begin
+  if current_user = 'other_authenticator' then
+    perform
+      set_config('pgrst.jwt_role_claim_key', '."other"."pre_config_role"', true)
+    , set_config('pgrst.db_anon_role', 'pre_config_role', true)
+    , set_config('pgrst.db_schemas', 'will be overriden with the above ALTER ROLE.. db_schemas', true)
+    , set_config('pgrst.db_tx_end', 'rollback-allow-override', true);
+  else
+    null;
+  end if;
+end $$ language plpgsql;
+
+create or replace function postgrest.preconf()
+returns void as $$
+begin
+  null;
+end $$ language plpgsql;
 
 -- authenticator used for tests that manipulate statement timeout
 CREATE ROLE timeout_authenticator LOGIN NOINHERIT;

test/io/fixtures.sql

@@ -1,14 +1,27 @@
-\ir big_schema.sql
+-- \ir big_schema.sql big schema test currently skipped, see test_io.py
 \ir db_config.sql
 
 set search_path to public;
 
 CREATE ROLE postgrest_test_anonymous;
-ALTER ROLE :USER SET pgrst.db_anon_role = 'postgrest_test_anonymous';
+ALTER ROLE :PGUSER SET pgrst.db_anon_role = 'postgrest_test_anonymous';
 
 CREATE ROLE postgrest_test_author;
 
-GRANT postgrest_test_anonymous, postgrest_test_author TO :USER;
+CREATE ROLE postgrest_test_serializable;
+alter role postgrest_test_serializable set default_transaction_isolation = 'serializable';
+
+CREATE ROLE postgrest_test_repeatable_read;
+alter role postgrest_test_repeatable_read set default_transaction_isolation = 'REPEATABLE READ';
+
+CREATE ROLE postgrest_test_w_superuser_settings;
+alter role postgrest_test_w_superuser_settings set log_min_duration_statement = 1;
+alter role postgrest_test_w_superuser_settings set log_min_messages = 'fatal';
+
+GRANT
+  postgrest_test_anonymous, postgrest_test_author,
+  postgrest_test_serializable, postgrest_test_repeatable_read,
+  postgrest_test_w_superuser_settings TO :PGUSER;
 
 CREATE SCHEMA v1;
 GRANT USAGE ON SCHEMA v1 TO postgrest_test_anonymous;
@@ -17,7 +30,7 @@ CREATE TABLE authors_only ();
 GRANT SELECT ON authors_only TO postgrest_test_author;
 
 CREATE TABLE projects AS SELECT FROM generate_series(1,5);
-GRANT SELECT ON projects TO postgrest_test_anonymous;
+GRANT SELECT ON projects TO postgrest_test_anonymous, postgrest_test_w_superuser_settings;
 
 create function get_guc_value(name text) returns text as $$
   select nullif(current_setting(name), '')::text;
@@ -84,5 +97,80 @@ create or replace function sleep(seconds double precision) returns void as $$
 $$ language sql;
 
 create or replace function hello() returns text as $$
-  select 'hello';
+  select 'hello'::text;
 $$ language sql;
+
+create table cats(id uuid primary key, name text);
+grant all on cats to postgrest_test_anonymous;
+
+create function drop_change_cats() returns void
+language sql security definer
+as $$
+  drop table cats;
+  create table cats(id bigint primary key, name text);
+  grant all on table cats to postgrest_test_anonymous;
+  notify pgrst, 'reload schema';
+$$;
+
+alter role postgrest_test_anonymous set statement_timeout to '2s';
+alter role postgrest_test_author set statement_timeout to '10s';
+
+create function change_role_statement_timeout(timeout text) returns void as $_$
+begin
+  execute format($$
+    alter role current_user set statement_timeout = %L;
+  $$, timeout);
+end $_$ volatile language plpgsql ;
+
+create table items as select x as id from generate_series(1,5) x;
+
+create view items_w_isolation_level as
+select
+  id,
+  current_setting('transaction_isolation', true) as isolation_level
+from items;
+
+grant all on items_w_isolation_level to postgrest_test_anonymous, postgrest_test_repeatable_read, postgrest_test_serializable;
+
+create function default_isolation_level()
+returns text as $$
+  select current_setting('transaction_isolation', true);
+$$
+language sql;
+
+create function serializable_isolation_level()
+returns text as $$
+  select current_setting('transaction_isolation', true);
+$$
+language sql set default_transaction_isolation = 'serializable';
+
+create function repeatable_read_isolation_level()
+returns text as $$
+  select current_setting('transaction_isolation', true);
+$$
+language sql set default_transaction_isolation = 'REPEATABLE READ';
+
+create or replace function create_function() returns void as $_$
+  drop function if exists mult_them(int, int);
+  create or replace function mult_them(a int, b int) returns int as $$
+    select a*b;
+  $$ language sql;
+  notify pgrst, 'reload schema';
+$_$ language sql security definer;
+
+create or replace function migrate_function() returns void as $_$
+  drop function if exists mult_them(int, int);
+  create or replace function mult_them(c int, d int) returns int as $$
+    select c*d;
+  $$ language sql;
+  notify pgrst, 'reload schema';
+$_$ language sql security definer;
+
+create or replace function get_pgrst_version() returns text
+  language sql
+as $$
+select application_name
+from pg_stat_activity
+where application_name ilike 'postgrest%'
+limit 1;
+$$

test/io/fixtures.yaml

@@ -4,6 +4,10 @@ cli:
     args: ['--help']
   - name: help short
     args: ['-h']
+  - name: version long
+    args: ['--version']
+  - name: version short
+    args: ['-v']
   - name: example long
     args: ['--example']
   - name: example short

test/io/postgrest.py

@@ -42,6 +42,18 @@ class PostgrestProcess:
     process: object
     session: object
 
+    def read_stdout(self, nlines=1):
+        "Wait for line(s) on standard output."
+        output = []
+        for _ in range(10):
+            l = self.process.stdout.readline()
+            if l:
+                output.append(l.decode())
+                if len(output) >= nlines:
+                    break
+            time.sleep(0.1)
+        return output
+
 
 @contextlib.contextmanager
 def run(
@@ -50,7 +62,9 @@ def run(
     env=None,
     port=None,
     host=None,
+    wait_for_readiness=True,
     no_pool_connection_available=False,
+    no_startup_stdout=True,
 ):
     "Run PostgREST and yield an endpoint that is ready for connections."
 
@@ -88,9 +102,11 @@ def run(
             process.stdin.write(stdin or b"")
             process.stdin.close()
 
-            wait_until_ready(adminurl + "/ready")
+            if wait_for_readiness:
+                wait_until_ready(adminurl + "/ready")
 
-            process.stdout.read()
+            if no_startup_stdout:
+                process.stdout.read()
 
             if no_pool_connection_available:
                 sleep_pool_connection(baseurl, 10)
@@ -137,6 +153,14 @@ def freeport():
         return s.getsockname()[1]
 
 
+def wait_until_exit(postgrest):
+    "Wait for PostgREST to exit, or times out"
+    try:
+        return postgrest.process.wait(timeout=1)
+    except subprocess.TimeoutExpired:
+        raise PostgrestTimedOut()
+
+
 def wait_until_ready(url):
     "Wait for the given HTTP endpoint to return a status of 200."
     session = requests_unixsocket.Session()

test/io/test_io.py

@@ -66,6 +66,15 @@ def test_read_secret_from_stdin_dbconfig(defaultenv):
         assert response.status_code == 200
 
 
+def test_fail_with_invalid_password(defaultenv):
+    "Connecting with an invalid password should fail without retries."
+    uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
+    env = {**defaultenv, "PGRST_DB_URI": uri}
+    with run(env=env, wait_for_readiness=False) as postgrest:
+        exitCode = wait_until_exit(postgrest)
+        assert exitCode == 1
+
+
 def test_connect_with_dburi(dburi, defaultenv):
     "Connecting with db-uri instead of LIPQ* environment variables should work."
     defaultenv_without_libpq = {
@@ -299,13 +308,11 @@ def test_db_schema_reload(tmp_path, defaultenv):
 
         # reload config
         postgrest.process.send_signal(signal.SIGUSR2)
+        time.sleep(0.1)
 
         # reload schema cache to verify that the config reload actually happened
         postgrest.process.send_signal(signal.SIGUSR1)
-
+        time.sleep(0.1)
-        # takes max 1 second to load the internal cache(big_schema.sql included now)
-        # TODO this could go back to time.sleep(0.1) if the big_schema is put in another test suite
-        time.sleep(1)
 
         response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
         assert response.text == '"\\"v1\\", \\"public\\""'
@@ -325,7 +332,7 @@ def test_db_schema_notify_reload(defaultenv):
             "/rpc/change_db_schema_and_full_reload", data={"schemas": "v1"}
         )
 
-        time.sleep(0.1)
+        time.sleep(0.2)
 
         response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
         assert response.text == '"\\"v1\\", \\"public\\""'
@@ -407,14 +414,8 @@ def test_invalid_role_claim_key_notify_reload(defaultenv):
     with run(env=env) as postgrest:
         postgrest.session.post("/rpc/invalid_role_claim_key_reload")
 
-        output = None
+        output = postgrest.read_stdout()
-        for _ in range(10):
+        assert "failed to parse role-claim-key value" in output[0]
-            output = postgrest.process.stdout.readline()
-            if output:
-                break
-            time.sleep(0.1)
-
-        assert "failed to parse role-claim-key value" in output.decode()
 
         response = postgrest.session.post("/rpc/reset_invalid_role_claim_key")
         assert response.status_code == 204
@@ -526,7 +527,6 @@ def test_pool_size(defaultenv, metapostgrest):
     }
 
     with run(env=env) as postgrest:
-
         start = time.time()
         threads = []
         for i in range(4):
@@ -549,7 +549,7 @@ def test_pool_size(defaultenv, metapostgrest):
 
 
 def test_pool_acquisition_timeout(defaultenv, metapostgrest):
-    "Verify that PGRST_DB_POOL_ACQUISITON_TIMEOUT times out when the pool is empty"
+    "Verify that PGRST_DB_POOL_ACQUISITION_TIMEOUT times out when the pool is empty"
 
     env = {
         **defaultenv,
@@ -563,6 +563,11 @@ def test_pool_acquisition_timeout(defaultenv, metapostgrest):
         data = response.json()
         assert data["message"] == "Timed out acquiring connection from connection pool."
 
+        # ensure the message appears on the logs as well
+        output = sorted(postgrest.read_stdout(nlines=2))
+        assert " 504 " in output[0]
+        assert "Timed out acquiring connection from connection pool." in output[1]
+
 
 def test_change_statement_timeout_held_connection(defaultenv, metapostgrest):
     "Statement timeout changes take effect immediately, even with a request outliving the reconfiguration"
@@ -653,7 +658,6 @@ def test_admin_ready_includes_schema_cache_state(defaultenv, metapostgrest):
     }
 
     with run(env=env) as postgrest:
-
         # make it impossible to load the schema cache, by setting statement timeout to 1ms
         set_statement_timeout(metapostgrest, role, 1)
 
@@ -709,7 +713,6 @@ def test_admin_works_with_host_special_values(specialhostvalue, defaultenv):
     "Should get a success from the admin live and ready endpoints when using special host values for the main app"
 
     with run(env=defaultenv, port=freeport(), host=specialhostvalue) as postgrest:
-
         response = postgrest.admin.get("/live")
         assert response.status_code == 200
 
@@ -765,7 +768,6 @@ def test_no_pool_connection_required_on_bad_http_logic(defaultenv):
     "no pool connection should be consumed for failing on invalid http logic"
 
     with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
-
         # not found nested route shouldn't require opening a connection
         response = postgrest.session.head("/path/notfound")
         assert response.status_code == 404
@@ -781,7 +783,6 @@ def test_no_pool_connection_required_on_options(defaultenv):
     "no pool connection should be consumed for OPTIONS requests"
 
     with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
-
         # OPTIONS on a table shouldn't require opening a connection
         response = postgrest.session.options("/projects")
         assert response.status_code == 200
@@ -801,7 +802,6 @@ def test_no_pool_connection_required_on_bad_jwt_claim(defaultenv):
     env = {**defaultenv, "PGRST_JWT_SECRET": SECRET}
 
     with run(env=env, no_pool_connection_available=True) as postgrest:
-
         # A JWT with an invalid signature shouldn't open a connection
         headers = jwtauthheader({"role": "postgrest_test_author"}, "Wrong Secret")
         response = postgrest.session.get("/projects", headers=headers)
@@ -812,12 +812,176 @@ def test_no_pool_connection_required_on_bad_embedding(defaultenv):
     "no pool connection should be consumed for failing to embed"
 
     with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
-
         # OPTIONS on a table shouldn't require opening a connection
         response = postgrest.session.get("/projects?select=*,unexistent(*)")
         assert response.status_code == 400
 
 
+def test_notify_reloading_catalog_cache(defaultenv):
+    "notify should reload the connection catalog cache"
+
+    with run(env=defaultenv) as postgrest:
+        # first the id col is an uuid
+        response = postgrest.session.get(
+            "/cats?id=eq.dea27321-f988-4a57-93e4-8eeb38f3cf1e"
+        )
+        assert response.status_code == 200
+
+        # change it to a bigint
+        response = postgrest.session.post("/rpc/drop_change_cats")
+        assert response.status_code == 204
+        time.sleep(0.1)
+
+        # next request should succeed with a bigint value
+        response = postgrest.session.get("/cats?id=eq.1")
+        assert response.status_code == 200
+
+
+def test_role_settings(defaultenv):
+    "statement_timeout should be set per role"
+
+    env = {
+        **defaultenv,
+        "PGRST_JWT_SECRET": SECRET,
+    }
+
+    with run(env=env) as postgrest:
+        # statement_timeout for postgrest_test_anonymous
+        response = postgrest.session.get("/rpc/get_guc_value?name=statement_timeout")
+        assert response.text == '"2s"'
+
+        # reload statement_timeout with NOTIFY
+        response = postgrest.session.post(
+            "/rpc/change_role_statement_timeout", data={"timeout": "5s"}
+        )
+        assert response.status_code == 204
+
+        response = postgrest.session.get("/rpc/reload_pgrst_config")
+        assert response.status_code == 204
+        time.sleep(0.1)
+
+        response = postgrest.session.get("/rpc/get_guc_value?name=statement_timeout")
+        assert response.text == '"5s"'
+
+        # statement_timeout for postgrest_test_author
+        headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
+        response = postgrest.session.get(
+            "/rpc/get_guc_value?name=statement_timeout", headers=headers
+        )
+        assert response.text == '"10s"'
+
+
+def test_isolation_level(defaultenv):
+    "isolation_level should be set per role and per function"
+
+    env = {
+        **defaultenv,
+        "PGRST_JWT_SECRET": SECRET,
+    }
+
+    with run(env=env) as postgrest:
+        # default isolation level for postgrest_test_anonymous
+        response = postgrest.session.get(
+            "/items_w_isolation_level?select=isolation_level&limit=1"
+        )
+        assert response.text == '[{"isolation_level":"read committed"}]'
+
+        # isolation level for postgrest_test_repeatable_read on GET
+        headers = jwtauthheader({"role": "postgrest_test_repeatable_read"}, SECRET)
+        response = postgrest.session.get(
+            "/items_w_isolation_level?select=isolation_level&limit=1", headers=headers
+        )
+        assert response.text == '[{"isolation_level":"repeatable read"}]'
+
+        # isolation level for postgrest_test_serializable on POST
+        headers = jwtauthheader({"role": "postgrest_test_serializable"}, SECRET)
+        headers["Prefer"] = "return=representation"
+        response = postgrest.session.post(
+            "/items_w_isolation_level?select=isolation_level",
+            json={"id": "666"},
+            headers=headers,
+        )
+        assert response.text == '[{"isolation_level":"serializable"}]'
+
+        # isolation level for postgrest_test_serializable on PATCH
+        headers = jwtauthheader({"role": "postgrest_test_serializable"}, SECRET)
+        headers["Prefer"] = "return=representation"
+        response = postgrest.session.patch(
+            "/items_w_isolation_level?select=isolation_level&id=eq.666",
+            json={"id": "666"},
+            headers=headers,
+        )
+        assert response.text == '[{"isolation_level":"serializable"}]'
+
+        # isolation level for postgrest_test_serializable on DELETE
+        headers = jwtauthheader({"role": "postgrest_test_serializable"}, SECRET)
+        headers["Prefer"] = "return=representation"
+        response = postgrest.session.delete(
+            "/items_w_isolation_level?select=isolation_level&id=eq.666", headers=headers
+        )
+        assert response.text == '[{"isolation_level":"serializable"}]'
+
+        # default isolation level for function
+        response = postgrest.session.get("/rpc/default_isolation_level")
+        assert response.text == '"read committed"'
+
+        # changes with role isolation level
+        headers = jwtauthheader({"role": "postgrest_test_repeatable_read"}, SECRET)
+        response = postgrest.session.get(
+            "/rpc/default_isolation_level", headers=headers
+        )
+        assert response.text == '"repeatable read"'
+
+        # isolation level can be set per function
+        response = postgrest.session.get("/rpc/serializable_isolation_level")
+        assert response.text == '"serializable"'
+        response = postgrest.session.get("/rpc/repeatable_read_isolation_level")
+        assert response.text == '"repeatable read"'
+
+        # isolation level for a function overrides the role isolation level
+        headers = jwtauthheader({"role": "postgrest_test_repeatable_read"}, SECRET)
+        response = postgrest.session.get("/rpc/serializable_isolation_level")
+        assert response.text == '"serializable"'
+
+
+def test_schema_cache_reloading(defaultenv):
+    "schema cache should reload successfully"
+
+    # If DB_POOL=1, then the second request(/rpc/migrate_function) will just wait(PGRST_DB_POOL_ACQUISITION_TIMEOUT=10) for the schema cache reload to finish.
+    # This is bc the only pool connection will be busy with the PGRST_INTERNAL_SCHEMA_CACHE_SLEEP(does a pg_sleep)
+    # So this must be tested with a DB_POOL size of at least 2. That way the second request will pick the other pool connection and proceed.
+
+    env = {
+        **defaultenv,
+        "PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "1",
+        "PGRST_DB_CHANNEL_ENABLED": "true",
+        "PGRST_DB_POOL": "2",
+    }
+
+    internal_sleep = int(env["PGRST_INTERNAL_SCHEMA_CACHE_SLEEP"])
+
+    with run(env=env, wait_for_readiness=False) as postgrest:
+        time.sleep(2 * internal_sleep + 0.1)  # wait for readiness manually
+
+        response = postgrest.session.post("/rpc/create_function")
+        assert response.status_code == 204
+
+        time.sleep(
+            internal_sleep / 2
+        )  # wait to be inside the schema cache reload process
+
+        response = postgrest.session.post("/rpc/migrate_function")
+        assert response.status_code == 204
+
+        time.sleep(
+            2 * internal_sleep
+        )  # wait enough time to ensure the schema cache state remains
+
+        response = postgrest.session.get("/rpc/mult_them?c=3&d=4")
+        assert response.text == "12"
+        assert response.status_code == 200
+
+
 # TODO: This test fails now because of https://github.com/PostgREST/postgrest/pull/2122
 # The stack size of 1K(-with-rtsopts=-K1K) is not enough and this fails with "stack overflow"
 # A stack size of 200K seems to be enough for succeess
@@ -834,3 +998,68 @@ def test_openapi_in_big_schema(defaultenv):
     with run(env=env) as postgrest:
         response = postgrest.session.get("/")
         assert response.status_code == 200
+
+
+@pytest.mark.parametrize("dburi_type", ["no_params", "no_params_qmark", "with_params"])
+def test_get_pgrst_version_with_uri_connection_string(dburi_type, dburi, defaultenv):
+    "The fallback_application_name should be added to the db-uri if it has a URI format"
+    defaultenv_without_libpq = {
+        key: value
+        for key, value in defaultenv.items()
+        if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
+    }
+
+    env = {
+        "no_params": {**defaultenv, "PGRST_DB_URI": "postgresql://"},
+        "no_params_qmark": {**defaultenv, "PGRST_DB_URI": "postgresql://?"},
+        "with_params": {**defaultenv_without_libpq, "PGRST_DB_URI": dburi.decode()},
+    }
+
+    with run(env=env[dburi_type]) as postgrest:
+        response = postgrest.session.post("/rpc/get_pgrst_version")
+        version = '"%s"' % response.headers["Server"].replace(
+            "postgrest/", "PostgREST "
+        )
+        assert response.text == version
+
+
+def test_get_pgrst_version_with_keyval_connection_string(defaultenv):
+    "The fallback_application_name should be added to the db-uri if it has a keyword/value format"
+    uri = f'dbname={defaultenv["PGDATABASE"]} host={defaultenv["PGHOST"]} user={defaultenv["PGUSER"]}'
+    defaultenv_without_libpq = {
+        key: value
+        for key, value in defaultenv.items()
+        if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
+    }
+    env = {**defaultenv_without_libpq, "PGRST_DB_URI": uri}
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.post("/rpc/get_pgrst_version")
+        version = '"%s"' % response.headers["Server"].replace(
+            "postgrest/", "PostgREST "
+        )
+        assert response.text == version
+
+
+def test_log_postgrest_version(defaultenv):
+    "Should show the PostgREST version in the logs"
+
+    with run(env=defaultenv, no_startup_stdout=False) as postgrest:
+        version = postgrest.session.head("/").headers["Server"].split("/")[1]
+
+        assert (
+            "Starting PostgREST %s..." % version
+            in postgrest.process.stdout.readline().decode()
+        )
+
+
+def test_succeed_w_role_having_superuser_settings(defaultenv):
+    "Should succeed when having superuser settings on the impersonated role"
+
+    env = {**defaultenv, "PGRST_DB_CONFIG": "true", "PGRST_JWT_SECRET": SECRET}
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        headers = jwtauthheader({"role": "postgrest_test_w_superuser_settings"}, SECRET)
+        response = postgrest.session.get("/projects", headers=headers)
+        print(response.text)
+        assert response.status_code == 200

test/load/fixtures.sql

@@ -1,5 +1,5 @@
 CREATE ROLE postgrest_test_anonymous;
-GRANT postgrest_test_anonymous TO :USER;
+GRANT postgrest_test_anonymous TO :PGUSER;
 CREATE SCHEMA test;
 
 -- PUT+PATCH target needs one record and column to modify

test/pgbench/1567/new.sql

@@ -0,0 +1,15 @@
+INSERT INTO "test"."complex_items"("arr_data", "field-with_sep", "id", "name")
+SELECT pgrst_body."arr_data", pgrst_body."field-with_sep", pgrst_body."id", pgrst_body."name"
+FROM (
+  SELECT '[{"id": 4, "name": "Vier"}, {"id": 5, "name": "Funf", "arr_data": null}, {"id": 6, "name": "Sechs", "arr_data": [1, 2, 3], "field-with_sep": 6}]'::jsonb as json_data
+) pgrst_payload,
+LATERAL (
+  SELECT CASE WHEN jsonb_typeof(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE jsonb_build_array(pgrst_payload.json_data) END AS val
+) pgrst_uniform_json,
+LATERAL (
+  SELECT jsonb_agg(jsonb_build_object('field-with_sep', 1) || elem) AS vals from jsonb_array_elements(pgrst_uniform_json.val) elem
+) pgrst_json_defs,
+LATERAL (
+  SELECT * FROM jsonb_to_recordset (pgrst_json_defs.vals) AS _ ("arr_data" integer[], "field-with_sep" integer, "id" bigint, "name" text)
+) pgrst_body
+RETURNING "test"."complex_items".*;

test/pgbench/1567/old.sql

@@ -0,0 +1,12 @@
+INSERT INTO "test"."complex_items"("arr_data", "field-with_sep", "id", "name")
+SELECT pgrst_body."arr_data", pgrst_body."field-with_sep", pgrst_body."id", pgrst_body."name"
+FROM (
+  SELECT '[{"id": 4, "name": "Vier"}, {"id": 5, "name": "Funf", "arr_data": null}, {"id": 6, "name": "Sechs", "arr_data": [1, 2, 3], "field-with_sep": 6}]'::jsonb as json_data
+) pgrst_payload,
+LATERAL (
+  SELECT CASE WHEN jsonb_typeof(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE jsonb_build_array(pgrst_payload.json_data) END AS val
+) pgrst_uniform_json,
+LATERAL (
+  SELECT * FROM jsonb_to_recordset (pgrst_uniform_json.val) AS _ ("arr_data" integer[], "field-with_sep" integer, "id" bigint, "name" text)
+) pgrst_body
+RETURNING "test"."complex_items".*

test/pgbench/1652/new.sql

@@ -0,0 +1,20 @@
+WITH pgrst_source AS (
+  SELECT pgrst_call.*
+  FROM (
+    SELECT '{"id": 4}'::json as json_data
+  ) pgrst_payload,
+  LATERAL (
+    SELECT CASE WHEN json_typeof(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE json_build_array(pgrst_payload.json_data) END AS val
+  ) pgrst_uniform_json,
+  LATERAL (
+    SELECT * FROM json_to_recordset(pgrst_uniform_json.val) AS _("id" integer) LIMIT 1
+  ) pgrst_body,
+  LATERAL "test"."get_projects_below"("id" := pgrst_body.id) pgrst_call
+)
+SELECT
+  null::bigint AS total_result_set,
+  pg_catalog.count(_postgrest_t) AS page_total,
+  coalesce(json_agg(_postgrest_t), '[]')::character varying AS body,
+  nullif(current_setting('response.headers', true), '') AS response_headers,
+  nullif(current_setting('response.status', true), '') AS response_status
+FROM (SELECT "projects".* FROM "pgrst_source" AS "projects") _postgrest_t;

test/pgbench/1652/old.sql

@@ -0,0 +1,15 @@
+WITH pgrst_source AS (
+  WITH
+  pgrst_payload AS (SELECT '{"id": 4}'::json AS json_data),
+  pgrst_body AS ( SELECT CASE WHEN json_typeof(json_data) = 'array' THEN json_data ELSE json_build_array(json_data) END AS val FROM pgrst_payload),
+  pgrst_args AS ( SELECT * FROM json_to_recordset((SELECT val FROM pgrst_body)) AS _("id" integer) )
+  SELECT "get_projects_below".*
+  FROM "test"."get_projects_below"("id" := (SELECT "id" FROM pgrst_args LIMIT 1))
+)
+SELECT
+  null::bigint AS total_result_set,
+  pg_catalog.count(_postgrest_t) AS page_total,
+  coalesce(json_agg(_postgrest_t), '[]')::character varying AS body,
+  nullif(current_setting('response.headers', true), '') AS response_headers,
+  nullif(current_setting('response.status', true), '') AS response_status
+FROM (SELECT "projects".* FROM "pgrst_source" AS "projects") _postgrest_t;

test/pgbench/2677/new.sql

@@ -0,0 +1,12 @@
+INSERT INTO "test"."complex_items"("arr_data", "field-with_sep", "id", "name")
+SELECT pgrst_body."arr_data", pgrst_body."field-with_sep", pgrst_body."id", pgrst_body."name"
+FROM (
+  SELECT '[{"id": 4, "name": "Vier"}, {"id": 5, "name": "Funf", "arr_data": null}, {"id": 6, "name": "Sechs", "arr_data": [1, 2, 3], "field-with_sep": 6}]'::json as json_data
+) pgrst_payload,
+LATERAL (
+  SELECT CASE WHEN json_typeof(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE json_build_array(pgrst_payload.json_data) END AS val
+) pgrst_uniform_json,
+LATERAL (
+  SELECT * FROM json_to_recordset (pgrst_uniform_json.val) AS _ ("arr_data" integer[], "field-with_sep" integer, "id" bigint, "name" text)
+) pgrst_body
+RETURNING "test"."complex_items".*

test/pgbench/2677/old.sql

@@ -0,0 +1,7 @@
+WITH
+pgrst_payload AS (SELECT '[{"id": 4, "name": "Vier"}, {"id": 5, "name": "Funf", "arr_data": null}, {"id": 6, "name": "Sechs", "arr_data": [1, 2, 3], "field-with_sep": 6}]'::json AS json_data),
+pgrst_body AS ( SELECT CASE WHEN json_typeof(json_data) = 'array' THEN json_data ELSE json_build_array(json_data) END AS val FROM pgrst_payload)
+INSERT INTO "test"."complex_items"("arr_data", "field-with_sep", "id", "name")
+SELECT "arr_data", "field-with_sep", "id", "name"
+FROM json_to_recordset ((SELECT val FROM pgrst_body)) AS _ ("arr_data" integer[], "field-with_sep" integer, "id" bigint, "name" text)
+RETURNING "test"."complex_items".*

test/pgbench/README.md

@@ -0,0 +1,13 @@
+## pgbench tests
+
+Can be used as:
+
+```
+postgrest-with-postgresql-15 -f test/pgbench/fixtures.sql pgbench -n -T 10 -f test/pgbench/1567/old.sql
+
+postgrest-with-postgresql-15 -f test/pgbench/fixtures.sql pgbench -n -T 10 -f test/pgbench/1567/new.sql
+```
+
+## Directory structure
+
+The directory name is the issue number on github.

test/pgbench/fixtures.sql

@@ -0,0 +1,7 @@
+\ir ../spec/fixtures/load.sql
+
+ALTER TABLE test.complex_items
+    DROP CONSTRAINT complex_items_pkey;
+
+ALTER TABLE test.complex_items
+    ALTER COLUMN "field-with_sep" DROP NOT NULL;

test/spec/Feature/CorsSpec.hs

@@ -40,7 +40,7 @@ spec =
                              \Date, Location, Server, Transfer-Encoding, Range-Unit"]
           }
 
-    it "allows INFO body through even with CORS request headers present to postflight request" $
+    it "allows INFO body through even with CORS request headers present to postflight request" $ do
       request methodOptions "/items"
           [ ("Host", "localhost:3000")
           , ("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0")
@@ -54,3 +54,17 @@ spec =
         `shouldRespondWith`
           ""
           { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
+
+      request methodOptions "/items"
+          [ ("Accept", "application/json") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
+
+      request methodOptions "/shops"
+          [ ("Accept", "application/geo+json") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }

test/spec/Feature/NoSuperuserSpec.hs

@@ -0,0 +1,20 @@
+module Feature.NoSuperuserSpec where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+
+import Protolude
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "No Superuser" $ do
+    it "proves that the authenticator role is not a superuser" $ do
+      request methodGet "/rpc/is_superuser"
+          mempty
+          ""
+        `shouldRespondWith`
+          "false"
+          { matchStatus = 200 }

test/spec/Feature/ObservabilitySpec.hs

@@ -0,0 +1,34 @@
+module Feature.ObservabilitySpec where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+
+import Protolude
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "Observability" $ do
+    it "includes the server trace header on the response" $ do
+      request methodHead "/"
+          [ ("X-Request-Id", "1") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "X-Request-Id" <:> "1"] }
+
+      request methodHead "/projects"
+          [ ("X-Request-Id", "2") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "X-Request-Id" <:> "2"] }
+
+      request methodHead "/rpc/add_them?a=2&b=4"
+          [ ("X-Request-Id", "3") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "X-Request-Id" <:> "3"] }

test/spec/Feature/OpenApi/OpenApiSpec.hs

@@ -42,6 +42,20 @@ spec actualPgVersion = describe "OpenAPI" $ do
 
     liftIO $ docsUrl `shouldBe` Just (String ("https://postgrest.org/en/" <> docsVersion <> "/api.html"))
 
+  describe "schema" $ do
+
+    it "includes title and comments to schema" $ do
+      r <- simpleBody <$> get "/"
+
+      let childGetTitle = r ^? key "info" . key "title"
+      let childGetDescription = r ^? key "info" . key "description"
+
+      liftIO $ do
+
+        childGetTitle `shouldBe` Just "My API title"
+
+        childGetDescription `shouldBe` Just "My API description\nthat spans\nmultiple lines"
+
   describe "table" $ do
 
     it "includes paths to tables" $ do
@@ -91,7 +105,7 @@ spec actualPgVersion = describe "OpenAPI" $ do
             [
               { "$ref": "#/parameters/body.child_entities" },
               { "$ref": "#/parameters/select" },
-              { "$ref": "#/parameters/preferReturn" }
+              { "$ref": "#/parameters/preferPost" }
             ]
           |]
 
@@ -310,6 +324,23 @@ spec actualPgVersion = describe "OpenAPI" $ do
             }
           |]
 
+  describe "VIEW created for a TABLE with a O2M relationship" $ do
+
+    it "fk points to destination TABLE instead of the VIEW" $ do
+      r <- simpleBody <$> get "/"
+
+      let referralLink = r ^? key "definitions" . key "projects" . key "properties" . key "client_id"
+
+      liftIO $
+        referralLink `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "integer",
+              "type": "integer",
+              "description": "Note:\nThis is a Foreign Key to `clients.id`.<fk table='clients' column='id'/>"
+            }
+          |]
+
   describe "PostgreSQL to Swagger Type Mapping" $ do
 
     it "character varying to string" $ do
@@ -490,6 +521,117 @@ spec actualPgVersion = describe "OpenAPI" $ do
             }
           |]
 
+    it "array types to array" $ do
+      r <- simpleBody <$> get "/"
+
+      let text_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_text_arr"
+      let int_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_int_arr"
+      let bool_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_bool_arr"
+      let char_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_char_arr"
+      let varchar_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_varchar_arr"
+      let bigint_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_bigint_arr"
+      let numeric_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_numeric_arr"
+      let json_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_json_arr"
+      let jsonb_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_jsonb_arr"
+
+      liftIO $ do
+
+        text_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "text[]",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          |]
+
+        int_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "integer[]",
+              "type": "array",
+              "items": {
+                "type": "integer"
+              }
+            }
+          |]
+
+        bool_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "boolean[]",
+              "type": "array",
+              "items": {
+                "type": "boolean"
+              }
+            }
+          |]
+
+        char_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "character[]",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          |]
+
+        varchar_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "character varying[]",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          |]
+
+        bigint_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "bigint[]",
+              "type": "array",
+              "items": {
+                "type": "integer"
+              }
+            }
+          |]
+
+        numeric_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "numeric[]",
+              "type": "array",
+              "items": {
+                "type": "number"
+              }
+            }
+          |]
+
+        json_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "json[]",
+              "type": "array",
+              "items": {}
+            }
+          |]
+
+        jsonb_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "jsonb[]",
+              "type": "array",
+              "items": {}
+            }
+          |]
+
+
   describe "Detects default values" $ do
 
     it "text" $ do
@@ -566,10 +708,156 @@ spec actualPgVersion = describe "OpenAPI" $ do
 
   describe "RPC" $ do
 
-    it "includes function summary/description and body schema for arguments" $ do
+    it "includes function summary/description and query parameters for arguments in the get path item" $ do
       r <- simpleBody <$> get "/"
 
-      let method s = key "paths" . key "/rpc/varied_arguments" . key s
+      let method s = key "paths" . key "/rpc/varied_arguments_openapi" . key s
+          args = r ^? method "get" . key "parameters"
+          summary = r ^? method "get" . key "summary"
+          description = r ^? method "get" . key "description"
+
+      liftIO $ do
+
+        summary `shouldBe` Just "An RPC function"
+
+        description `shouldBe` Just "Just a test for RPC function arguments"
+
+        args `shouldBe` Just
+          [aesonQQ|
+            [
+              {
+                "format": "double precision",
+                "in": "query",
+                "name": "double",
+                "required": true,
+                "type": "number"
+              },
+              {
+                "format": "character varying",
+                "in": "query",
+                "name": "varchar",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "boolean",
+                "in": "query",
+                "name": "boolean",
+                "required": true,
+                "type": "boolean"
+              },
+              {
+                "format": "date",
+                "in": "query",
+                "name": "date",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "money",
+                "in": "query",
+                "name": "money",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "enum_menagerie_type",
+                "in": "query",
+                "name": "enum",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "text[]",
+                "in": "query",
+                "name": "text_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "integer[]",
+                "in": "query",
+                "name": "int_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "boolean[]",
+                "in": "query",
+                "name": "bool_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "character[]",
+                "in": "query",
+                "name": "char_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "character varying[]",
+                "in": "query",
+                "name": "varchar_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "bigint[]",
+                "in": "query",
+                "name": "bigint_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "numeric[]",
+                "in": "query",
+                "name": "numeric_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "json[]",
+                "in": "query",
+                "name": "json_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "jsonb[]",
+                "in": "query",
+                "name": "jsonb_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "integer",
+                "in": "query",
+                "name": "integer",
+                "required": false,
+                "type": "integer"
+              },
+              {
+                "format": "json",
+                "in": "query",
+                "name": "json",
+                "required": false,
+                "type": "string"
+              },
+              {
+                "format": "jsonb",
+                "in": "query",
+                "name": "jsonb",
+                "required": false,
+                "type": "string"
+              }
+            ]
+          |]
+
+    it "includes function summary/description and body schema for arguments in the post path item" $ do
+      r <- simpleBody <$> get "/"
+
+      let method s = key "paths" . key "/rpc/varied_arguments_openapi" . key s
           args = r ^? method "post" . key "parameters" . nth 0 . key "schema"
           summary = r ^? method "post" . key "summary"
           description = r ^? method "post" . key "description"
@@ -590,7 +878,15 @@ spec actualPgVersion = describe "OpenAPI" $ do
                 "date",
                 "money",
                 "enum",
-                "arr"
+                "text_arr",
+                "int_arr",
+                "bool_arr",
+                "char_arr",
+                "varchar_arr",
+                "bigint_arr",
+                "numeric_arr",
+                "json_arr",
+                "jsonb_arr"
               ],
               "properties": {
                 "double": {
@@ -617,9 +913,64 @@ spec actualPgVersion = describe "OpenAPI" $ do
                   "format": "enum_menagerie_type",
                   "type": "string"
                 },
-                "arr": {
+                "text_arr": {
                   "format": "text[]",
-                  "type": "string"
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                "int_arr": {
+                  "format": "integer[]",
+                  "type": "array",
+                  "items": {
+                    "type": "integer"
+                  }
+                },
+                "bool_arr": {
+                  "format": "boolean[]",
+                  "type": "array",
+                  "items": {
+                    "type": "boolean"
+                  }
+                },
+                "char_arr": {
+                  "format": "character[]",
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                "varchar_arr": {
+                  "format": "character varying[]",
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                "bigint_arr": {
+                  "format": "bigint[]",
+                  "type": "array",
+                  "items": {
+                    "type": "integer"
+                  }
+                },
+                "numeric_arr": {
+                  "format": "numeric[]",
+                  "type": "array",
+                  "items": {
+                    "type": "number"
+                  }
+                },
+                "json_arr": {
+                  "format": "json[]",
+                  "type": "array",
+                  "items": {}
+                },
+                "jsonb_arr": {
+                  "format": "jsonb[]",
+                  "type": "array",
+                  "items": {}
                 },
                 "integer": {
                   "format": "integer",
@@ -668,6 +1019,26 @@ spec actualPgVersion = describe "OpenAPI" $ do
 
       liftIO $ params `shouldBe` Just [aesonQQ|["num", "str"]|]
 
+    it "uses a multi collection format when the function has a VARIADIC parameter" $ do
+      r <- simpleBody <$> get "/"
+      let param = r ^? key "paths" . key "/rpc/variadic_param"
+                     . key "get" . key "parameters" .  nth 0
+
+      liftIO $ param `shouldBe` Just
+        [aesonQQ|
+          {
+            "collectionFormat": "multi",
+            "in": "query",
+            "items": {
+              "format": "text",
+              "type": "string"
+            },
+            "name": "v",
+            "required": false,
+            "type": "array"
+          }
+        |]
+
   describe "Security" $
     it "does not include security or security definitions by default" $ do
       r <- simpleBody <$> get "/"

test/spec/Feature/OpenApi/RootSpec.hs

@@ -9,8 +9,6 @@ import Test.Hspec.Wai.JSON
 
 import Protolude hiding (get)
 
-import SpecHelper
-
 spec :: SpecWith ((), Application)
 spec =
   describe "root spec function" $ do
@@ -22,9 +20,3 @@ spec =
            "info": {"title": "PostgREST API", "description": "This is a dynamic API generated by PostgREST"}
           }|]
         { matchHeaders = ["Content-Type" <:> "application/openapi+json; charset=utf-8"] }
-
-    it "accepts application/json" $
-      request methodGet "/"
-        [("Accept", "application/json")] "" `shouldRespondWith`
-        200
-        { matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/AndOrParamsSpec.hs

@@ -92,6 +92,9 @@ spec actualPgVersion =
               {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" },
               {"text_search_vector": "'art':4 'spass':5 'unmog':7"}
             ]|] { matchHeaders = [matchContentTypeJson] }
+        it "can handle isdistinct" $
+          get "/entities?and=(id.gte.2,arr.isdistinct.{1,2})&select=id" `shouldRespondWith`
+            [json|[{ "id": 3 }, { "id": 4 }]|] { matchHeaders = [matchContentTypeJson] }
 
         when (actualPgVersion >= pgVersion112) $
           it "can handle wfts (websearch_to_tsquery)" $
@@ -138,6 +141,8 @@ spec actualPgVersion =
             [json|[{ "id": 3 }, { "id": 4 }]|] { matchHeaders = [matchContentTypeJson] }
           get "/ranges?range=adj.(3,10]&select=id" `shouldRespondWith`
             [json|[{ "id": 1 }]|] { matchHeaders = [matchContentTypeJson] }
+          get "/ranges?range=isdistinct.[1,3]&select=id" `shouldRespondWith`
+            [json|[{ "id": 2 }, { "id": 3 }, { "id": 4 }, {"id": 5}]|] { matchHeaders = [matchContentTypeJson] }
 
         it "can handle array operators" $ do
           get "/entities?arr=eq.{1,2,3}&select=id" `shouldRespondWith`
@@ -166,6 +171,8 @@ spec actualPgVersion =
             [json|[{ "id": 3 }]|] { matchHeaders = [matchContentTypeJson] }
           get "/entities?arr=ov.{2,3}&select=id" `shouldRespondWith`
             [json|[{ "id": 2 }, { "id": 3 }]|] { matchHeaders = [matchContentTypeJson] }
+          get "/entities?arr=isdistinct.{1,2}&select=id" `shouldRespondWith`
+            [json|[{ "id": 1 }, { "id": 3 }, { "id": 4 }]|] { matchHeaders = [matchContentTypeJson] }
 
         context "operators with not" $ do
           it "eq, cs, like can be negated" $
@@ -180,6 +187,9 @@ spec actualPgVersion =
           it "gt, lte, ilike can be negated" $
             get "/entities?and=(name.not.ilike.*ITY2,or(id.not.gt.4,id.not.lte.1))&select=id" `shouldRespondWith`
               [json|[{"id": 1}, {"id": 2}, {"id": 3}]|] { matchHeaders = [matchContentTypeJson] }
+          it "isdistinct can be negated" $
+            get "/entities?and=(id.not.eq.2,arr.not.isdistinct.{1,2,3})&select=id" `shouldRespondWith`
+              [json|[{"id": 3}]|] { matchHeaders = [matchContentTypeJson] }
 
       context "and/or params with quotes" $ do
         it "eq can have quotes" $
@@ -252,35 +262,3 @@ spec actualPgVersion =
 
     it "can query columns that begin with and/or reserved words" $
       get "/grandchild_entities?or=(and_starting_col.eq.smth, or_starting_col.eq.smth)" `shouldRespondWith` 200
-
-    it "fails when using IN without () and provides meaningful error message" $
-      get "/entities?or=(id.in.1,2,id.eq.3)" `shouldRespondWith`
-        [json|{
-          "details": "unexpected \"1\" expecting \"(\"",
-          "message": "\"failed to parse logic tree ((id.in.1,2,id.eq.3))\" (line 1, column 10)",
-          "code": "PGRST100",
-          "hint": null
-        }|] { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-
-    it "fails on malformed query params and provides meaningful error message" $ do
-      get "/entities?or=)(" `shouldRespondWith`
-        [json|{
-          "details": "unexpected \")\" expecting \"(\"",
-          "message": "\"failed to parse logic tree ()()\" (line 1, column 3)",
-          "code": "PGRST100",
-          "hint": null
-        }|] { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-      get "/entities?and=(ord(id.eq.1,id.eq.1),id.eq.2)" `shouldRespondWith`
-        [json|{
-          "details": "unexpected \"d\" expecting \"(\"",
-          "message": "\"failed to parse logic tree ((ord(id.eq.1,id.eq.1),id.eq.2))\" (line 1, column 7)",
-          "code": "PGRST100",
-          "hint": null
-        }|] { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-      get "/entities?or=(id.eq.1,not.xor(id.eq.2,id.eq.3))" `shouldRespondWith`
-        [json|{
-          "details": "unexpected \"x\" expecting logic operator (and, or)",
-          "message": "\"failed to parse logic tree ((id.eq.1,not.xor(id.eq.2,id.eq.3)))\" (line 1, column 16)",
-          "code": "PGRST100",
-          "hint": null
-        }|] { matchStatus = 400, matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/ComputedRelsSpec.hs

@@ -104,6 +104,47 @@ spec = describe "computed relationships" $ do
         [json|[ {"name":"Final Fantasy I","designer":{"name":"Hironobu Sakaguchi"}} ]|]
         { matchStatus  = 200 }
 
+  it "applies data representations to response" $ do
+    -- A smoke test for data reps in the presence of computed relations.
+
+    -- The data rep here title cases the designer name before presentation. So here the lowercase version will be saved,
+    -- but the title case version returned. Pulling in a computed relation should not confuse this.
+    request methodPatch "/designers?select=name,videogames:computed_videogames(name)&id=eq.1"
+      [("Prefer", "return=representation"), ("Prefer", "tx=commit")]
+      [json| {"name": "sidney k. meier"} |]
+      `shouldRespondWith`
+      [json|[{"name":"Sidney K. Meier","videogames":[{"name":"Civilization I"}, {"name":"Civilization II"}]}]|]
+      { matchStatus = 200 }
+
+    -- Verify it was saved the way we requested (there's no text data rep for this column, so if we select with the wrong casing, it should fail.)
+    get "/designers?select=id&name=eq.Sidney%20K.%20Meier"
+      `shouldRespondWith`
+      [json|[]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
+    -- But with the right casing it works.
+    get "/designers?select=id,name&name=eq.sidney%20k.%20meier"
+      `shouldRespondWith`
+      [json|[{"id": 1, "name":"Sidney K. Meier"}]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
+
+    -- Most importantly, if you read it back even via a computed relation, the data rep should be applied.
+    get "/videogames?select=name,designer:computed_designers(*)&id=eq.1"
+      `shouldRespondWith`
+      [json|[
+        {"name":"Civilization I","designer":{"id": 1, "name":"Sidney K. Meier"}}
+      ]|] { matchHeaders = [matchContentTypeJson] }
+
+    -- reset the test fixture
+    request methodPatch "/designers?id=eq.1"
+      [("Prefer", "tx=commit")]
+      [json| {"name": "Sid Meier"} |]
+      `shouldRespondWith` 204
+    -- need to poke the second one too to prevent inherent ordering from changing
+    request methodPatch "/designers?id=eq.2"
+      [("Prefer", "tx=commit")]
+      [json| {"name": "Hironobu Sakaguchi"} |]
+      `shouldRespondWith` 204
+
   it "works with self joins" $
     get "/web_content?select=name,child_web_content(name),parent_web_content(name)&id=in.(0,1)"
     `shouldRespondWith`

test/spec/Feature/Query/DeleteSpec.hs

@@ -37,7 +37,8 @@ spec =
         request methodDelete "/items?id=eq.2" [("Prefer", "return=representation"), ("Prefer", "count=exact")] ""
           `shouldRespondWith` [json|[{"id":2}]|]
           { matchStatus  = 200
-          , matchHeaders = ["Content-Range" <:> "*/1"]
+          , matchHeaders = ["Content-Range" <:> "*/1"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "ignores ?select= when return not set or return=minimal" $ do
@@ -57,7 +58,8 @@ spec =
             ""
             { matchStatus  = 204
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       it "returns the deleted item and shapes the response" $
@@ -137,7 +139,8 @@ spec =
           `shouldRespondWith`
             ""
             { matchStatus = 204
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal" ]
             }
 
       it "suceeds deleting the row with no explicit select by default" $
@@ -154,7 +157,7 @@ spec =
       it "works with the limit and offset query params" $
         baseTable "limited_delete_items" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items?order=id&limit=1&offset=1" mempty
+        requestMutation methodDelete "/limited_delete_items?order=id&limit=1&offset=1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
@@ -164,7 +167,7 @@ spec =
       it "works with the limit query param plus a filter" $
         baseTable "limited_delete_items" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items?order=id&limit=1&id=gt.1" mempty
+        requestMutation methodDelete "/limited_delete_items?order=id&limit=1&id=gt.1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
@@ -200,7 +203,7 @@ spec =
       it "works with views with an explicit order by unique col" $
         baseTable "limited_delete_items_view" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items_view?order=id&limit=1&offset=1" mempty
+        requestMutation methodDelete "/limited_delete_items_view?order=id&limit=1&offset=1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
@@ -210,7 +213,7 @@ spec =
       it "works with views with an explicit order by composite pk" $
         baseTable "limited_delete_items_cpk_view" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items_cpk_view?order=id,name&limit=1&offset=1" mempty
+        requestMutation methodDelete "/limited_delete_items_cpk_view?order=id,name&limit=1&offset=1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
@@ -220,9 +223,53 @@ spec =
       it "works on a table without a pk by ordering by 'ctid'" $
         baseTable "limited_delete_items_no_pk" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items_no_pk?order=ctid&limit=1&offset=1" mempty
+        requestMutation methodDelete "/limited_delete_items_no_pk?order=ctid&limit=1&offset=1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
         , { "id": 3, "name": "item-3" }
         ]|]
+
+      it "ignores the Range header" $ do
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items"
+          (rangeHdrs (ByteRangeFromTo 0 0)) mempty
+         `shouldMutateInto`
+         [json|[]|]
+
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items?id=gte.2"
+          (rangeHdrs (ByteRangeFromTo 0 0)) mempty
+         `shouldMutateInto`
+         [json|[ { "id": 1, "name": "item-1" } ]|]
+
+      it "ignores the Range header and does not do a limited delete" $
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items?order=id"
+          (rangeHdrs (ByteRangeFromTo 0 0)) mempty
+         `shouldMutateInto`
+         [json|[]|]
+
+      it "ignores the Range header and does not throw an invalid range error" $
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items?order=id&limit=1&offset=1"
+          (rangeHdrs (ByteRangeFromTo 0 0)) mempty
+         `shouldMutateInto`
+         [json|[
+           { "id": 1, "name": "item-1" }
+         , { "id": 3, "name": "item-3" }
+         ]|]
+
+      it "ignores the Range header but not the limit and offset params" $
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items?order=id&limit=2&offset=1"
+          (rangeHdrs (ByteRangeFromTo 1 1)) mempty
+         `shouldMutateInto`
+         [json|[
+           { "id": 1, "name": "item-1" }
+         ]|]

test/spec/Feature/Query/EmbedDisambiguationSpec.hs

@@ -68,10 +68,8 @@ spec =
           , matchHeaders = [matchContentTypeJson]
           }
 
-      it "errs when there are more than two fks on a junction table(currently impossible to disambiguate, only choice is to split the table)" $
+      it "errs when there are more than two fks on a junction table but it can be disambiguated with spread embeds" $ do
         -- We have 4 possibilities for doing the junction JOIN here.
-        -- This could be solved by specifying two additional fks, like whatev_projects!fk1!fk2(*)
-        -- If the need arises this capability can be added later without causing a breaking change
         get "/whatev_sites?select=*,whatev_projects(*)" `shouldRespondWith`
           [json|
             {
@@ -105,6 +103,23 @@ spec =
           { matchStatus  = 300
           , matchHeaders = [matchContentTypeJson]
           }
+        -- Each of those 4 possibilities can be done with spread embeds, by following the details in the error above
+        get "/whatev_sites?select=*,whatev_jobs!site_id_1(...whatev_projects!project_id_1(*))" `shouldRespondWith` [json|[]|]
+        get "/whatev_sites?select=*,whatev_jobs!site_id_1(...whatev_projects!project_id_2(*))" `shouldRespondWith` [json|[]|]
+        get "/whatev_sites?select=*,whatev_jobs!site_id_2(...whatev_projects!project_id_1(*))" `shouldRespondWith` [json|[]|]
+        get "/whatev_sites?select=*,whatev_jobs!site_id_2(...whatev_projects!project_id_2(*))" `shouldRespondWith` [json|[]|]
+
+      it "can disambiguate a recursive m2m with spread embeds" $ do
+        get "/posters?select=*,subscribers:subscriptions!subscribed(...posters!subscriber(*))&limit=1" `shouldRespondWith`
+          [json| [ {"id":1,"name":"Mark","subscribers":[{"id":3,"name":"Bill"}, {"id":4,"name":"Jeff"}]}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }
+        get "/posters?select=*,subscriptions!subscriber(...posters!subscribed(*))&limit=1" `shouldRespondWith`
+          [json| [{"id":1,"name":"Mark","subscriptions":[{"id":2,"name":"Elon"}]}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }
 
       it "errs on an ambiguous embed that has two one-to-one relationships" $
         get "/first?select=second(*)" `shouldRespondWith`
@@ -202,10 +217,10 @@ spec =
         it "fails if the fk is not known" $
           get "/message?select=id,sender:person!space(name)&id=lt.4" `shouldRespondWith`
             [json|{
-              "hint":"Verify that 'message' and 'person' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
+              "hint":null,
               "message":"Could not find a relationship between 'message' and 'person' in the schema cache",
               "code": "PGRST200",
-              "details": null}|]
+              "details":"Searched for a foreign key relationship between 'message' and 'person' using the hint 'space' in the schema 'test', but no matches were found."}|]
             { matchStatus = 400
             , matchHeaders = [matchContentTypeJson] }
 
@@ -492,10 +507,10 @@ spec =
       it "doesn't work if the junction is only internal" $
         get "/end_1?select=end_2(*)" `shouldRespondWith`
           [json|{
-            "hint":"Verify that 'end_1' and 'end_2' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
+            "hint": null,
             "message":"Could not find a relationship between 'end_1' and 'end_2' in the schema cache",
             "code":"PGRST200",
-            "details": null}|]
+            "details": "Searched for a foreign key relationship between 'end_1' and 'end_2' in the schema 'test', but no matches were found."}|]
           { matchStatus  = 400
           , matchHeaders = [matchContentTypeJson] }
       it "shouldn't try to embed if the private junction has an exposed homonym" $
@@ -503,10 +518,10 @@ spec =
         -- Ref: https://github.com/PostgREST/postgrest/issues/1587#issuecomment-734995669
         get "/schauspieler?select=filme(*)" `shouldRespondWith`
           [json|{
-            "hint":"Verify that 'schauspieler' and 'filme' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
+            "hint":null,
             "message":"Could not find a relationship between 'schauspieler' and 'filme' in the schema cache",
             "code":"PGRST200",
-            "details": null}|]
+            "details":"Searched for a foreign key relationship between 'schauspieler' and 'filme' in the schema 'test', but no matches were found."}|]
           { matchStatus  = 400
           , matchHeaders = [matchContentTypeJson] }
 

test/spec/Feature/Query/InsertSpec.hs

@@ -11,8 +11,10 @@ import Test.Hspec.Wai
 import Test.Hspec.Wai.JSON
 import Text.Heredoc
 
-import PostgREST.Config.PgVersion (PgVersion, pgVersion110,
+import PostgREST.Config.PgVersion (PgVersion, pgVersion100,
-                                   pgVersion112, pgVersion130)
+                                   pgVersion110, pgVersion112,
+                                   pgVersion120, pgVersion130,
+                                   pgVersion140)
 
 import Protolude  hiding (get)
 import SpecHelper
@@ -41,7 +43,8 @@ spec actualPgVersion = do
           , "enum": "foo"
           }] |] `shouldRespondWith` [json|[{"integer":14,"varchar":"testing!"}]|]
           { matchStatus  = 201
-          , matchHeaders = [matchContentTypeJson]
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "ignores &select when return not set or using return=minimal" $ do
@@ -67,7 +70,8 @@ spec actualPgVersion = do
           `shouldRespondWith`
             ""
             { matchStatus  = 201
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
     context "non uniform json array" $ do
@@ -97,7 +101,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/1" ]
+                           , "Content-Range" <:> "*/1"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "can rename and cast the selected columns" $
@@ -108,7 +113,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/*" ]
+                           , "Content-Range" <:> "*/*"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "should not throw and return location header when selecting without PK" $
@@ -118,7 +124,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/*" ]
+                           , "Content-Range" <:> "*/*"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
     context "requesting headers only representation" $ do
@@ -131,7 +138,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/projects?id=eq.11"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
       when (actualPgVersion >= pgVersion110) $
@@ -144,7 +152,8 @@ spec actualPgVersion = do
               { matchStatus  = 201
               , matchHeaders = [ matchHeaderAbsent hContentType
                                , "Location" <:> "/car_models?name=eq.Enzo&year=eq.2021"
-                               , "Content-Range" <:> "*/*" ]
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
               }
 
     context "requesting no representation" $
@@ -191,7 +200,8 @@ spec actualPgVersion = do
               ""
               { matchStatus  = 201
               , matchHeaders = [ matchHeaderAbsent hContentType
-                               , "Location" <:> "/auto_incrementing_pk?id=eq.2" ]
+                               , "Location" <:> "/auto_incrementing_pk?id=eq.2"
+                               , "Preference-Applied" <:> "return=headers-only"]
               }
 
       context "into a table with simple pk" $
@@ -225,7 +235,8 @@ spec actualPgVersion = do
             `shouldRespondWith`
               [json| [{ "a":"bar", "b":"baz" }] |]
               { matchStatus  = 201
-              , matchHeaders = [matchHeaderAbsent hLocation]
+              , matchHeaders = [matchHeaderAbsent hLocation
+                               , "Preference-Applied" <:> "return=representation"]
               }
 
         it "returns empty array when no items inserted, and return=rep" $ do
@@ -389,6 +400,22 @@ spec actualPgVersion = do
           `shouldRespondWith` [json|[{ id: 20 }]|]
             { matchStatus  = 201 }
 
+    -- https://github.com/PostgREST/postgrest/issues/2861
+    context "bit and char columns with length" $ do
+      it "should insert to a bit column with length" $
+        request methodPost "/bitchar_with_length?select=bit"
+            [("Prefer", "return=representation")]
+            [json|{"bit": "10101"}|]
+          `shouldRespondWith` [json|[{ "bit": "10101" }]|]
+            { matchStatus  = 201 }
+
+      it "should insert to a char column with length" $
+        request methodPost "/bitchar_with_length?select=char"
+            [("Prefer", "return=representation")]
+            [json|{"char": "abcde"}|]
+          `shouldRespondWith` [json|[{ "char": "abcde" }]|]
+            { matchStatus  = 201 }
+
     context "POST with ?columns parameter" $ do
       it "ignores json keys not included in ?columns" $ do
         request methodPost "/articles?columns=id,body" [("Prefer", "return=representation")]
@@ -420,20 +447,134 @@ spec actualPgVersion = do
           , matchHeaders = []
           }
 
+      it "disallows ?columns which don't exist" $
+        post "/articles?columns=helicopter"
+          [json|[
+            {"id": 204, "body": "yyy"},
+            {"id": 205, "body": "zzz"}]|]
+          `shouldRespondWith`
+          [json|{"code":"PGRST204","details":null,"hint":null,"message":"Column 'helicopter' of relation 'articles' does not exist"} |]
+          { matchStatus  = 400
+          , matchHeaders = []
+          }
+
+      it "returns missing table error even if also has invalid ?columns" $
+        post "/garlic?columns=helicopter"
+          [json|[
+            {"id": 204, "body": "yyy"},
+            {"id": 205, "body": "zzz"}]|]
+          `shouldRespondWith`
+          [json|{} |]
+          { matchStatus  = 404
+          , matchHeaders = []
+          }
+
       it "disallows array elements that are not json objects" $
         post "/articles?columns=id,body"
           [json|[
             {"id": 204, "body": "yyy"},
             333,
             "asdf",
-            {"id": 205, "body": "zzz"}]|] `shouldRespondWith`
+            {"id": 205, "body": "zzz"}]|] `shouldRespondWith` 400
-          [json|{
+
-              "code": "22023",
+      context "apply defaults on missing values" $ do
-              "details": null,
+        -- inserting the array fails on pg 9.6, but the feature should work normally
-              "hint": null,
+        when (actualPgVersion >= pgVersion100) $
-              "message": "argument of json_populate_recordset must be an array of objects"}|]
+          it "inserts table default values(field-with_sep) when json keys are undefined" $
-          { matchStatus  = 400
+            request methodPost "/complex_items?columns=id,name,field-with_sep,arr_data" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
-          , matchHeaders = []
+                [json|[
+                  {"id": 4, "name": "Vier"},
+                  {"id": 5, "name": "Funf", "arr_data": null},
+                  {"id": 6, "name": "Sechs", "field-with_sep": 6, "arr_data": "{1,2,3}"}
+                ]|]
+              `shouldRespondWith`
+                [json|[
+                  {"id": 4, "name": "Vier", "field-with_sep": 1, "settings":null,"arr_data":null},
+                  {"id": 5, "name": "Funf", "field-with_sep": 1, "settings":null,"arr_data":null},
+                  {"id": 6, "name": "Sechs", "field-with_sep": 6, "settings":null,"arr_data":[1,2,3]}
+                ]|]
+                { matchStatus  = 201
+                , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+                }
+
+        it "inserts view default values(field-with_sep) when json keys are undefined" $
+          request methodPost "/complex_items_view?columns=id,name" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+              [json|[
+                {"id": 7, "name": "Sieben"},
+                {"id": 8}
+              ]|]
+            `shouldRespondWith`
+              [json|[
+                {"id": 7, "name": "Sieben", "field-with_sep": 1, "settings":null,"arr_data":null},
+                {"id": 8, "name": "Default", "field-with_sep": 1, "settings":null,"arr_data":null}
+              ]|]
+              { matchStatus  = 201
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+              }
+
+        it "doesn't insert json duplicate keys(since it uses jsonb)" $
+          request methodPost "/tbl_w_json?columns=id,data" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+              [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
+            `shouldRespondWith`
+              [json| [ { "data": { "a": 2 }, "id": 3 } ] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+              }
+
+        when (actualPgVersion >= pgVersion100) $
+          it "inserts a default on a generated by default as identity column" $
+            request methodPost "/channels?columns=id,data,slug&select=data,slug" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+                [json| { "slug": "foo" } |]
+              `shouldRespondWith`
+                [json| [{"data":{"foo": "bar"},"slug":"foo"}] |] -- id 1 was inserted here, we don't get it for idempotence in the tests
+                { matchStatus  = 201
+                , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+                }
+
+        when (actualPgVersion >= pgVersion120) $
+          it "fails with a good error message on generated always columns" $
+            request methodPost "/foo?columns=a,b" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+                [json| [
+                  {"a": "val"},
+                  {"a": "val", "b": "val"}
+                ]|]
+              `shouldRespondWith`
+                (if actualPgVersion < pgVersion140
+                  then [json| {
+                    "code": "42601",
+                    "details": "Column \"b\" is a generated column.",
+                    "hint": null,
+                    "message": "cannot insert into column \"b\""
+                  }|]
+                  else [json| {
+                    "code": "428C9",
+                    "details": "Column \"b\" is a generated column.",
+                    "hint": null,
+                    "message": "cannot insert a non-DEFAULT value into column \"b\""
+                  }|])
+                { matchStatus  = 400 }
+
+        it "inserts a default on a DOMAIN with default" $
+          request methodPost "/evil_friends?columns=id,name" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+              [json| { "name": "Lu" } |]
+            `shouldRespondWith`
+              [json| [{"id": 666, "name": "Lu"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+              }
+
+    it "inserts json that has duplicate keys" $ do
+      request methodPost "/tbl_w_json" [("Prefer", "return=representation")]
+          [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
+        `shouldRespondWith`
+          [json| [ { "data": { "a": 1, "a": 2 }, "id": 3 } ] |]
+          { matchStatus  = 201
+          }
+      request methodPost "/tbl_w_json?columns=id,data" [("Prefer", "return=representation")]
+          [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
+        `shouldRespondWith`
+          [json| [ { "data": { "a": 1, "a": 2 }, "id": 3 } ] |]
+          { matchStatus  = 201
           }
 
     context "with unicode values" $ do
@@ -587,7 +728,8 @@ spec actualPgVersion = do
         `shouldRespondWith`
           ""
           { matchStatus = 201
-          , matchHeaders = [matchHeaderAbsent hContentType]
+          , matchHeaders = [matchHeaderAbsent hContentType
+                           , "Preference-Applied" <:> "return=minimal"]
           }
 
   describe "Inserting into VIEWs" $ do
@@ -610,7 +752,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/with_multiple_pks?pk1=eq.1&pk2=eq.2"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
     context "requesting header only representation" $ do
@@ -622,7 +765,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/compound_pk_view?k1=eq.1&k2=eq.test"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
       it "should not throw and return location header when a PK is null" $
@@ -633,5 +777,120 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/test_null_pk_competitors_sponsors?id=eq.1&sponsor_id=is.null"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
+
+
+  -- Data representations for payload parsing requires Postgres 10 or above.
+  when (actualPgVersion >= pgVersion100) $ do
+    describe "Data representations" $ do
+      context "on regular table" $ do
+        it "parses values in POST body" $
+          -- we don't check that the parsing is correct here, just that it's happening. If it doesn't happen we'll get a
+          -- an "invalid input syntax for type integer:" error.
+          request methodPost "/datarep_todos" [("Prefer", "return=headers-only")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 201
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Location" <:> "/datarep_todos?id=eq.5"
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
+              }
+
+        it "parses values in POST body and formats individually selected values in return=representation" $
+          request methodPost "/datarep_todos?select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "label_color": "#001100"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "parses values in POST body and formats values in return=representation" $
+          request methodPost "/datarep_todos" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00", "icon_image": "3q2+7w", "created_at":-15, "budget": "-100000000000000.13"} |]
+            `shouldRespondWith`
+            [json| [{"id":5,"name": "party", "label_color": "#001100", "due_at":"2018-01-03T11:00:00Z", "icon_image": "3q2+7w==", "created_at":-15, "budget": "-100000000000000.13"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+      context "with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPost "/datarep_todos?columns=id,label_color&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "invalid but should be ignored"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "name":null, "label_color": "#001100", "due_at": "2018-01-01T00:00:00Z"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "fails without parsing anything if at least one specified column doesn't exist" $
+          request methodPost "/datarep_todos?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }
+
+      context "on updatable view" $ do
+        it "parses values in POST body" $
+          -- we don't check that the parsing is correct here, just that it's happening. If it doesn't happen we'll get a
+          -- an "invalid input syntax for type integer:" error.
+          request methodPost "/datarep_todos_computed" [("Prefer", "return=headers-only")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 201
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Location" <:> "/datarep_todos_computed?id=eq.5"
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
+              }
+
+        it "parses values in POST body and formats individually selected values in return=representation" $
+          request methodPost "/datarep_todos_computed?select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "label_color": "#001100"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "parses values in POST body and formats values in return=representation" $
+          request methodPost "/datarep_todos_computed" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5,"name": "party", "label_color": "#001100", "due_at":"2018-01-03T11:00:00Z", "dark_color":"#000880"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+      context "on updatable views with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPost "/datarep_todos_computed?columns=id,label_color&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "invalid but should be ignored"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "name":null, "label_color": "#001100", "due_at": "2018-01-01T00:00:00Z"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "fails without parsing anything if at least one specified column doesn't exist" $
+          request methodPost "/datarep_todos_computed?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos_computed' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }

test/spec/Feature/Query/JsonOperatorSpec.hs

@@ -103,7 +103,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
 
       it "can get array of objects" $ do
         get "/json_arr?select=data->0->>a&id=in.(5,6)" `shouldRespondWith`
-          [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
+          [json|[{"a":"A"}, {"a":"[1,2,3]"}]|]
           { matchHeaders = [matchContentTypeJson] }
         get "/json_arr?select=data->0->a->>2&id=in.(5,6)" `shouldRespondWith`
           [json| [{"a":null}, {"a":"3"}] |]
@@ -275,7 +275,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         [json| [{"data":8}, {"data":7}] |]
         { matchHeaders = [matchContentTypeJson] }
       get "/json_arr?select=data->-2->>a&id=in.(5,6)" `shouldRespondWith`
-        [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
+        [json| [{"a":"A"}, {"a":"[1,2,3]"}] |]
         { matchHeaders = [matchContentTypeJson] }
 
     it "can filter with negative indexes" $ do
@@ -292,25 +292,11 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         [json| [{"data":[{"a": [1,2,3]}, {"b": [4,5]}]}] |]
         { matchHeaders = [matchContentTypeJson] }
 
-    it "should fail on badly formed negatives" $ do
+  it "gives a meaningful error on bad syntax" $
-      get "/json_arr?select=data->>-78xy" `shouldRespondWith`
+    get "/json_arr?select=data->>--34" `shouldRespondWith`
-        [json|
+      [json|
-          {"details": "unexpected 'x' expecting digit, \"->\", \"::\", \".\", \",\" or end of input",
+        {"details": "unexpected \"-\" expecting digit",
-           "message": "\"failed to parse select parameter (data->>-78xy)\" (line 1, column 11)",
+         "message": "\"failed to parse select parameter (data->>--34)\" (line 1, column 9)",
-           "code": "PGRST100",
+         "code": "PGRST100",
-           "hint": null} |]
+         "hint": null} |]
-        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
+      { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-      get "/json_arr?select=data->>--34" `shouldRespondWith`
-        [json|
-          {"details": "unexpected \"-\" expecting digit",
-           "message": "\"failed to parse select parameter (data->>--34)\" (line 1, column 9)",
-           "code": "PGRST100",
-           "hint": null} |]
-        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-      get "/json_arr?select=data->>-xy-4" `shouldRespondWith`
-        [json|
-          {"details":"unexpected \"x\" expecting digit",
-           "message":"\"failed to parse select parameter (data->>-xy-4)\" (line 1, column 9)",
-           "code": "PGRST100",
-           "hint": null} |]
-        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/NullsStrip.hs

@@ -0,0 +1,116 @@
+module Feature.Query.NullsStrip where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import Protolude  hiding (get)
+import SpecHelper
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "Stripping null values from JSON response" $ do
+    let arrayStrip = ("Accept", "application/vnd.pgrst.array+json;nulls=stripped")
+    let singularStrip = ("Accept", "application/vnd.pgrst.object+json;nulls=stripped")
+
+    context "strip nulls from response" $ do
+      it "strips nulls when Accept: application/vnd.pgrst.array+json;nulls=stripped" $ do
+        request methodGet  "/organizations?select=*"
+          [arrayStrip]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"name":"Referee Org","manager_id":1},{"id":2,"name":"Auditor Org","manager_id":2},{"id":3,"name":"Acme","referee":1,"auditor":2,"manager_id":3},{"id":4,"name":"Umbrella","referee":1,"auditor":2,"manager_id":4},{"id":5,"name":"Cyberdyne","referee":3,"auditor":4,"manager_id":5},{"id":6,"name":"Oscorp","referee":3,"auditor":4,"manager_id":6}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPost  "/organizations?select=*"
+          [arrayStrip,("Prefer","return=representation")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7,"name":"John","manager_id":6}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPatch  "/organizations?id=eq.3&select=*"
+          [arrayStrip, ("Prefer","return=representation")]
+          [json|{"name":"John","referee":null}|]
+          `shouldRespondWith`
+          [json|[{"id":3,"name":"John","auditor":2,"manager_id":3}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.object+json;nulls=stripped" $
+        request methodGet  "/organizations?limit=1"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"id":1,"name":"Referee Org","manager_id":1}|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTSingularStrip]
+          }
+
+      it "throws error when Accept: application/vnd.pgrst.object+json;nulls=stripped and result not singular" $
+        request methodGet  "/organizations?select=*"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"details":"The result contains 6 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+          { matchStatus  = 406
+          , matchHeaders = [matchContentTypeSingular]
+          }
+
+    context "strip nulls from response even if explicitly selected" $ do
+      it "strips nulls when Accept: application/vnd.pgrst.array+json;nulls=stripped" $ do
+        request methodGet  "/organizations?select=id,referee,auditor"
+          [arrayStrip]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1},{"id":2},{"id":3,"referee":1,"auditor":2},{"id":4,"referee":1,"auditor":2},{"id":5,"referee":3,"auditor":4},{"id":6,"referee":3,"auditor":4}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPost  "/organizations?select=id,referee,auditor"
+          [arrayStrip,("Prefer","return=representation")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPatch  "/organizations?id=eq.3&select=id,name,referee,auditor"
+          [arrayStrip, ("Prefer","return=representation")]
+          [json|{"name":"John","referee":null}|]
+          `shouldRespondWith`
+          [json|[{"id":3,"name":"John","auditor":2}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.object+json;nulls=stripped" $
+        request methodGet  "/organizations?select=id,referee,auditor&limit=1"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"id":1}|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTSingularStrip]
+          }
+
+    context "doesn't strip nulls" $ do
+      it "doesn't strips nulls when Accept: application/vnd.pgrst.array+json" $
+        request methodGet  "/organizations?select=id,referee,auditor"
+          [("Accept", "application/vnd.pgrst.array+json")]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"referee":null,"auditor":null},{"id":2,"referee":null,"auditor":null},{"id":3,"referee":1,"auditor":2},{"id":4,"referee":1,"auditor":2},{"id":5,"referee":3,"auditor":4},{"id":6,"referee":3,"auditor":4}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }