bump version to 11.0.0

- Already fixed in #2618
- Adds tests and CHANGELOG

.github/actions/setup-nix/action.yaml

@@ -12,6 +12,8 @@ runs:
   using: composite
   steps:
     - uses: cachix/install-nix-action@v18
+      with:
+        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12
       with:
         name: postgrest

.github/get_cirrusci_freebsd

@@ -7,12 +7,13 @@ set -euo pipefail
 # https://docs.github.com/en/rest/reference/checks#list-check-suites-for-a-git-reference
 
 cirrus_artifact_name=bin
+gh_auth_header="Authorization: Bearer $GITHUB_TOKEN"
 gh_accept_header="Accept: application/vnd.github.v3+json"
 
 get_gh_check_runs_url() {
   gh_checks_list_url="https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$GITHUB_COMMIT/check-suites"
   >&2 echo "Getting list of check-suites from $gh_checks_list_url ..."
-  curl --fail -H "$gh_accept_header"  "$gh_checks_list_url" \
+  curl --fail -H "$gh_auth_header" -H "$gh_accept_header"  "$gh_checks_list_url" \
     | jq -r '.check_suites[] | select(.app.slug == "cirrus-ci") | .check_runs_url'
 }
 
@@ -21,7 +22,7 @@ wait_for_cirrusci() {
   >&2 echo "Waiting to CirrusCI run to complete (two hours maximum)..."
   for _ in $(seq 1 120); do
     echo "Checking for CirrusCI task status at $gh_check_runs_url ..."
-    status=$(curl --fail "$gh_check_runs_url" | jq -r '.check_runs[] | .status')
+    status=$(curl --fail -H "$gh_auth_header" "$gh_check_runs_url" | jq -r '.check_runs[] | .status')
     if [ "$status" == "completed" ]; then
       break
     else
@@ -37,7 +38,7 @@ wait_for_cirrusci() {
 get_cirrus_taskid() {
   gh_check_runs_url="$(get_gh_check_runs_url)"
   >&2 echo "Getting the CirrusCI task id from $gh_check_runs_url ..."
-  curl --fail -H "$gh_accept_header" "$gh_check_runs_url" \
+  curl --fail -H "$gh_auth_header" -H "$gh_accept_header" "$gh_check_runs_url" \
     | jq -r '.check_runs[] | .external_id'
 }
 

.github/scripts/arm/build.sh

@@ -9,6 +9,8 @@ PGRST_GITHUB_COMMIT="$1"
 SCRIPT_DIR="$2"
 
 DOCKER_BUILD_DIR="$SCRIPT_DIR/docker-env"
+CABAL_VERSION="3.6.0.0"
+GHC_VERSION="9.2.4"
 
 install_packages() {
   sudo apt-get update -y
@@ -26,13 +28,14 @@ install_ghcup() {
 
 install_cabal() {
   ghcup upgrade
-  ghcup install cabal 3.6.0.0
-  ghcup set cabal 3.6.0.0
+  ghcup install cabal $CABAL_VERSION
+  ghcup set cabal $CABAL_VERSION
 }
 
 install_ghc() {
-  ghcup install ghc 8.10.7
-  ghcup set ghc 8.10.7
+  ghcup upgrade
+  ghcup install ghc $GHC_VERSION
+  ghcup set ghc $GHC_VERSION
 }
 
 install_packages
@@ -41,8 +44,8 @@ install_packages
 [ -f ~/.ghcup/env ] && source ~/.ghcup/env
 
 ghcup --version || install_ghcup
-cabal --version || install_cabal
-ghc --version || install_ghc
+ghcup set cabal $CABAL_VERSION || install_cabal
+ghcup set ghc $GHC_VERSION || install_ghc
 
 cd ~/$SCRIPT_DIR
 

.github/workflows/cachix.yaml

@@ -0,0 +1,78 @@
+name: Cachix
+
+# This workflow serves to
+#   - keep cachix up to date with the main branch
+#   - incrementally update cachix for large dependency
+#     updates, e.g. after running postgrest-nixpkgs-upgrade,
+#     which can cause the main CI workflow to time out
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - rel-*
+    tags:
+      - v*
+
+jobs:
+  Seed-Cachix:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: Linux
+            runs-on: ubuntu-latest
+          - os: MacOS
+            runs-on: macos-latest
+    name: Seed ${{ matrix.os }}
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+        with:
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: Install cachix tooling
+        run: |
+          nix-env -f default.nix -iA devTools.pushCachix.bin
+          postgrest-push-cachix
+
+      - name: Seed dynamic postgrest build
+        run: |
+          nix-build -A postgrestPackage
+          postgrest-push-cachix
+
+      - name: Seed style tools
+        run: |
+          nix-build -A style
+          postgrest-push-cachix
+
+      - name: Seed test tools
+        run: |
+          nix-build -A tests
+          postgrest-push-cachix
+
+      - name: Seed static toolchain
+        if: matrix.os == 'Linux'
+        run: |
+          nix-build -A packagesStatic.haskellPackages.hello
+          postgrest-push-cachix
+
+      - name: Seed static postgresql build (for libpq)
+        if: matrix.os == 'Linux'
+        run: |
+          nix-build -A packagesStatic.pkgs.postgresql
+          postgrest-push-cachix
+
+      - name: Seed static postgrest build
+        if: matrix.os == 'Linux'
+        run: |
+          nix-build -A postgrestStatic
+          postgrest-push-cachix
+
+      - name: Build and push everything to Cachix
+        run: |
+          nix-build
+          postgrest-push-cachix

.github/workflows/ci.yaml

@@ -84,11 +84,7 @@ jobs:
 
       - name: Run IO tests
         if: always()
-        run: postgrest-with-postgresql-${{ matrix.pgVersion }} -f test/io/fixtures.sql postgrest-test-io
-
-      - name: Run query cost tests
-        if: always()
-        run: postgrest-with-postgresql-${{ matrix.pgVersion }} postgrest-test-querycost
+        run: postgrest-with-postgresql-${{ matrix.pgVersion }} -f test/io/fixtures.sql postgrest-test-io -vv
 
 
   Test-Memory-Nix:
@@ -112,7 +108,6 @@ jobs:
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
-          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
           tools: tests
 
       - name: Build static executable
@@ -135,12 +130,18 @@ jobs:
           path: postgrest-docker.tar.gz
           if-no-files-found: error
 
-      - name: Build and push everything to Cachix (main branch only)
-        if: ${{ github.ref == 'refs/heads/main' }}
+
+  Build-Macos-Nix:
+    name: Build MacOS (Nix)
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+
+      - name: Build everything
         run: |
           nix-build
-          nix-env -f default.nix -iA devTools
-          postgrest-push-cachix
 
 
   Build-Stack:
@@ -202,7 +203,8 @@ jobs:
       - name: Get FreeBSD executable from CirrusCI
         env:
           # GITHUB_SHA does weird things for pull request, so we roll our own:
-          GITHUB_COMMIT: ${{github.event.pull_request.head.sha || github.sha}}
+          GITHUB_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: .github/get_cirrusci_freebsd
       - name: Save executable as artifact
         uses: actions/upload-artifact@v3
@@ -214,12 +216,15 @@ jobs:
   Build-Cabal:
     strategy:
       matrix:
-        ghc: ['8.10.7', '9.2.4']
+        ghc: ['9.2.4']
       fail-fast: false
     name: Build Linux (Cabal, GHC ${{ matrix.ghc }})
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Workaround runner image issue
+        # https://github.com/actions/runner-images/issues/7061
+        run: sudo chown -R "$USER" /usr/local/.ghcup
       - name: ghcup
         run: |
           ghcup install ghc ${{ matrix.ghc }}
@@ -253,7 +258,7 @@ jobs:
       - uses: actions/checkout@v3
       - id: Remote-Dir
         name: Unique directory name for the remote build
-        run: echo "::set-output name=remotepath::postgrest-build-$(uuidgen)"
+        run: echo "remotepath=postgrest-build-$(uuidgen)" >> "$GITHUB_OUTPUT"
       - name: Copy script files to the remote server
         uses: appleboy/scp-action@master
         with:
@@ -290,7 +295,7 @@ jobs:
       - name: Extract downloaded binaries
         run: tar -xvf result.tar.xz && rm result.tar.xz
       - name: Save aarch64 executable as artifact
-        uses: actions/upload-artifact@v2.3.1
+        uses: actions/upload-artifact@v3
         with:
           name: postgrest-ubuntu-aarch64
           path: result/postgrest
@@ -326,14 +331,14 @@ jobs:
             exit 1
           else
             echo "Version to be released is $cabal_version"
-            echo "::set-output name=version::$cabal_version"
+            echo "version=$cabal_version" >> "$GITHUB_OUTPUT"
           fi
 
           if [[ "$cabal_version" != *.*.*.* ]]; then
             echo "Version is for a full release (version does not have four components)"
           else
             echo "Version is for a pre-release (version has four components, e.g., 1.1.1.1)"
-            echo "::set-output name=isprerelease::1"
+            echo "isprerelease=1" >> "$GITHUB_OUTPUT"
           fi
       - name: Identify changes from CHANGELOG.md
         run: |
@@ -424,7 +429,6 @@ jobs:
     name: Release on Docker Hub
     runs-on: ubuntu-latest
     needs:
-      - Build-Cabal-Arm
       - Prepare-Release
     env:
       GITHUB_COMMIT: ${{ github.sha }}
@@ -459,18 +463,6 @@ jobs:
           else
             echo "Skipping pushing to 'latest' tag for v$VERSION pre-release..."
           fi
-      - name: Publish images for ARM builds on Docker Hub
-        uses: appleboy/ssh-action@master
-        env:
-          REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
-        with:
-          host: ${{ secrets.SSH_ARM_HOST }}
-          username: ubuntu
-          key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
-          fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
-          script_stop: true
-          envs: GITHUB_COMMIT,DOCKER_REPO,DOCKER_USER,DOCKER_PASS,REMOTE_DIR,VERSION,ISPRERELEASE
-          script: bash ~/$REMOTE_DIR/docker-publish.sh "$GITHUB_COMMIT" "$DOCKER_REPO" "$DOCKER_USER" "$DOCKER_PASS" "$REMOTE_DIR" "$VERSION" "$ISPRERELEASE"
 # TODO: Enable dockerhub description update again, once a solution for the permission problem is found:
 # https://github.com/docker/hub-feedback/issues/1927
 #      - name: Update descriptions on Docker Hub
@@ -484,17 +476,49 @@ jobs:
 #            echo "Skipping updating description for pre-release..."
 #          fi
 
+  Release-Docker-Arm:
+    name: Release Arm Builds on Docker Hub
+    runs-on: ubuntu-latest
+    needs:
+      - Build-Cabal-Arm
+      - Prepare-Release
+      - Release-Docker
+    env:
+      GITHUB_COMMIT: ${{ github.sha }}
+      DOCKER_REPO: postgrest
+      DOCKER_USER: stevechavez
+      DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+      VERSION: ${{ needs.Prepare-Release.outputs.version }}
+      ISPRERELEASE: ${{ needs.Prepare-Release.outputs.isprerelease }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Publish images for ARM builds on Docker Hub
+        uses: appleboy/ssh-action@master
+        env:
+          REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
+        with:
+          host: ${{ secrets.SSH_ARM_HOST }}
+          username: ubuntu
+          key: ${{ secrets.SSH_ARM_PRIVATE_KEY }}
+          fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
+          script_stop: true
+          envs: GITHUB_COMMIT,DOCKER_REPO,DOCKER_USER,DOCKER_PASS,REMOTE_DIR,VERSION,ISPRERELEASE
+          script: bash ~/$REMOTE_DIR/docker-publish.sh "$GITHUB_COMMIT" "$DOCKER_REPO" "$DOCKER_USER" "$DOCKER_PASS" "$REMOTE_DIR" "$VERSION" "$ISPRERELEASE"
+
   Clean-Arm-Server:
     name: Remove copied files from server
     needs:
       - Build-Cabal-Arm
-      - Release-Docker
-    if: ${{ always() && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/rel-')) }}
+      - Release-Docker-Arm
+    if: success() ||
+      needs.Build-Cabal-Arm.result == 'failure' ||
+      needs.Build-Cabal-Arm.result == 'cancelled' ||
+      (needs.Build-Cabal-Arm.result == 'success' && !startsWith(github.ref, 'refs/tags/v'))
     runs-on: ubuntu-latest
     env:
       REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
     steps:
-      - uses: actions/checkout@v2.4.0
+      - uses: actions/checkout@v3
       - name: Remove uploaded files from server
         uses: appleboy/ssh-action@master
         with:

.github/workflows/report.yaml

@@ -22,7 +22,7 @@ jobs:
           name: loadtest.md
           path: artifacts
       - name: Upload to GitHub Checks
-        uses: LouisBrunner/checks-action@v1.5.0
+        uses: LouisBrunner/checks-action@v1.6.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sha: ${{ github.event.workflow_run.head_sha }}

BACKERS.md

@@ -46,12 +46,14 @@ PostgREST ongoing development is only possible thanks to our Sponsors and Backer
 
 ## Lead Backers
 
+- [Roboflow](https://github.com/roboflow)
 - Evans Fernandes
 - [Jan Sommer](https://github.com/nerfpops)
 - [Franz Gusenbauer](https://www.igutech.at/)
 
 ## Backers
 
+- Zac Miller
 - Tsingson Qin
 - Michel Pelletier
 - Jay Hannah

CHANGELOG.md

@@ -5,12 +5,94 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+## [11.0.0] - 2023-04-16
+
+### Added
+
+ - #1414, Add related orders - @steve-chavez
+   + On a many-to-one or one-to-one relationship, you can order a parent by a child column `/projects?select=*,clients(*)&order=clients(name).desc.nullsfirst`
+ - #1233, #1907, #2566, Allow spreading embedded resources - @steve-chavez
+   + On a many-to-one or one-to-one relationship, you can unnest a json object with `/projects?select=*,...clients(client_name:name)`
+   + Allows including the join table columns when resource embedding
+   + Allows disambiguating a recursive m2m embed
+   + Allows disambiguating an embed that has a many-to-many relationship using two foreign keys on a junction
+ - #2340, Allow embedding without selecting any column - @steve-chavez
+ - #2563, Allow `is.null` or `not.is.null` on an embedded resource - @steve-chavez
+   + Offers a more flexible replacement for `!inner`, e.g. `/projects?select=*,clients(*)&clients=not.is.null`
+   + Allows doing an anti join, e.g. `/projects?select=*,clients(*)&clients=is.null`
+   + Allows using or across related tables conditions
+ - #1100, Customizable OpenAPI title - @AnthonyFisi
+ - #2506, Add `server-trace-header` for tracing HTTP requests.  - @steve-chavez
+   + When the client sends the request header specified in the config it will be included in the response headers.
+ - #2694, Make `db-root-spec` stable. - @steve-chavez
+   + This can be used to override the OpenAPI spec with a custom database function
+ - #1567, On bulk inserts, missing values can get the column DEFAULT by using the `Prefer: missing=default` header - @steve-chavez
+ - #2501, Allow filtering by`IS DISTINCT FROM` using the `isdistinct` operator, e.g. `/people?alias=isdistinct.foo`
+ - #1569, Allow `any/all` modifiers on the `eq,like,ilike,gt,gte,lt,lte,match,imatch` operators, e.g. `/tbl?id=eq(any).{1,2,3}` - @steve-chavez
+   - This converts the input into an array type
+ - #2561, Configurable role settings - @steve-chavez
+   - Database roles that are members of the connection role get their settings applied, e.g. doing
+     `ALTER ROLE anon SET statement_timeout TO '5s'` will result in that `statement_timeout` getting applied for that role.
+   - Works when switching roles when a JWT is sent
+   - Settings can be reloaded with `NOTIFY pgrst, 'reload config'`.
+ - #2468, Configurable transaction isolation level with `default_transaction_isolation` - @steve-chavez
+   - Can be set per function `create function .. set default_transaction_isolation = 'repeatable read'`
+   - Or per role `alter role .. set default_transaction_isolation = 'serializable'`
+
+### Fixed
+
+ - #2651, Add the missing `get` path item for RPCs to the OpenAPI output - @laurenceisla
+ - #2648, Fix inaccurate error codes with new ones - @laurenceisla
+   + `PGRST204`: Column is not found
+   + `PGRST003`: Timed out when acquiring connection to db
+ - #1652, Fix function call with arguments not inlining - @steve-chavez
+ - #2705, Fix bug when using the `Range` header on `PATCH/DELETE` - @laurenceisla
+   + Fix the`"message": "syntax error at or near \"RETURNING\""` error
+   + Fix doing a limited update/delete when an `order` query parameter was present
+ - #2742, Fix db settings and pg version queries not getting prepared  - @steve-chavez
+ - #2618, Fix `PATCH` requests not recognizing embedded filters and using the top-level resource instead - @steve-chavez
+
+### Changed
+
+ - #2705, The `Range` header is now only considered on `GET` requests and is ignored for any other method - @laurenceisla
+   + Other methods should use the `limit/offset` query parameters for sub-ranges
+   + `PUT` requests no longer return an error when this header is present (using `limit/offset` still triggers the error)
+ - #2733, Remove bulk RPC call with the `Prefer: params=multiple-objects` header. A function with a JSON array or object parameter should be used instead.
+
+## [10.2.0] - 2023-04-12
+
+### Added
+
+ - #2663, Limit maximal postgresql connection lifetime - @robx
+   + New option `db-pool-max-lifetime` (default 30m)
+   + `db-pool-acquisition-timeout` is no longer optional and defaults to 10s
+   + Fixes postgresql resource leak with long-lived connections (#2638)
+
+### Fixed
+
+ - #2667, Fix `db-pool-acquisition-timeout` not logging to stderr when the timeout is reached - @steve-chavez
+
+## [10.1.2] - 2023-02-01
+
+### Fixed
+
+ - #2565, Fix bad M2M embedding on RPC - @steve-chavez
+ - #2575, Replace misleading error message when no function is found with a hint containing functions/parameters names suggestions - @laurenceisla
+ - #2582, Move explanation about "single parameters" from the `message` to the `details` in the error output - @laurenceisla
+ - #2569, Replace misleading error message when no relationship is found with a hint containing parent/child names suggestions - @laurenceisla
+ - #1405, Add the required OpenAPI items object when the parameter is an array - @laurenceisla
+ - #2592, Add upsert headers for POST requests to the OpenAPI output - @laurenceisla
+ - #2623, Fix FK pointing to VIEW instead of TABLE in OpenAPI output - @laurenceisla
+ - #2622, Consider any PostgreSQL authentication failure as fatal and exit immediately - @michivi
+ - #2620, Fix `NOTIFY pgrst` not reloading the db connections catalog cache - @steve-chavez
+
 ## [10.1.1] - 2022-11-08
 
 ### Fixed
 
  - #2548, Fix regression when embedding views with partial references to multi column FKs - @wolfgangwalther
  - #2558, Fix regression when requesting limit=0 and `db-max-row` is set - @laurenceisla
+ - #2542, Return a clear error without hitting the database when trying to update or insert an unknown column with `?columns` - @aljungberg
 
 ## [10.1.0] - 2022-10-28
 

cabal.project.non-nix

@@ -17,4 +17,4 @@ packages: .
 source-repository-package
   type: git
   location: https://github.com/PostgREST/postgresql-libpq.git
-  tag: 33ff97db570b5b432255f5f24a68db51453f6eb8
+  tag: 890a0a16cf57dd401420fdc6c7d576fb696003bc

default.nix

@@ -41,6 +41,7 @@ let
       allOverlays.postgresql-legacy
       allOverlays.postgresql-future
       (allOverlays.haskell-packages { inherit compiler; })
+      allOverlays.slocat
     ];
 
   # Evaluated expression of the Nixpkgs repository.
@@ -65,11 +66,17 @@ let
   postgrest =
     pkgs.haskell.packages."${compiler}".callCabal2nix name src { };
 
-  # Function that derives a fully static Haskell package based on
+  # Functionality that derives a fully static Haskell package based on
   # nh2/static-haskell-nix
   staticHaskellPackage =
     import nix/static-haskell-package.nix { inherit nixpkgs system compiler patches allOverlays; };
 
+  # Static executable.
+  postgrestStatic =
+    lib.justStaticExecutables (lib.dontCheck (staticHaskellPackage name src).package);
+
+  packagesStatic = (staticHaskellPackage name src).survey;
+
   # Options passed to cabal in dev tools and tests
   devCabalOptions =
     "-f dev --test-show-detail=direct";
@@ -94,10 +101,6 @@ rec {
   postgrestPackage =
     lib.dontCheck postgrest;
 
-  # Static executable.
-  postgrestStatic =
-    lib.justStaticExecutables (lib.dontCheck (staticHaskellPackage name src));
-
   # Profiled dynamic executable.
   postgrestProfiled =
     lib.enableExecutableProfiling (
@@ -119,14 +122,13 @@ rec {
   cabalTools =
     pkgs.callPackage nix/tools/cabalTools.nix { inherit devCabalOptions postgrest; };
 
+  withTools =
+    pkgs.callPackage nix/tools/withTools.nix { inherit cabalTools devCabalOptions postgresqlVersions postgrest; };
+
   # Development tools.
   devTools =
     pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie withTools; };
 
-  # Docker images and loading script.
-  docker =
-    pkgs.callPackage nix/tools/docker { postgrest = postgrestStatic; };
-
   # Load testing tools.
   loadtest =
     pkgs.callPackage nix/tools/loadtest.nix { inherit withTools; };
@@ -155,7 +157,12 @@ rec {
       inherit (pkgs.haskell.packages."${compiler}") hpc-codecov;
       inherit (pkgs.haskell.packages."${compiler}") weeder;
     };
+} // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux rec {
+  # Static executable.
+  inherit postgrestStatic;
+  inherit packagesStatic;
 
-  withTools =
-    pkgs.callPackage nix/tools/withTools.nix { inherit devCabalOptions postgresqlVersions postgrest; };
+  # Docker images and loading script.
+  docker =
+    pkgs.callPackage nix/tools/docker { postgrest = postgrestStatic; };
 }

nix/README.md

@@ -204,16 +204,16 @@ postgrest-loadtest
 # You can loadtest comparing to a different branch
 postgrest-loadtest-against master
 
+# You can simulate latency client/postgrest and postgrest/database
+PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest
+
+# You can build postgrest directly with cabal for faster iteration
+PGRST_BUILD_CABAL=1 postgrest-loadtest
+
 # Produce a markdown report to be used on CI
 postgrest-loadtest-report
 ```
 
-Our query cost tests ensure that our generated queries don't surpass a threshold EXPLAIN cost.
-
-```bash
-postgrest-test-querycost
-```
-
 doctests for some of our modules are also available:
 
 ```bash

nix/nixpkgs-version.nix

@@ -1,6 +1,6 @@
 # Pinned version of Nixpkgs, generated with postgrest-nixpkgs-upgrade.
 {
-  date = "2022-10-28";
-  rev = "f44ba1be526c8da9e79a5759feca2365204003f6";
-  tarballHash = "0npbwsdjw88py5w2pjflwh94wgi4jmnmls0k1n7q8m6h94w1y1ps";
+  date = "2023-03-25";
+  rev = "dbf5322e93bcc6cfc52268367a8ad21c09d76fea";
+  tarballHash = "0lwk4v9dkvd28xpqch0b0jrac4xl9lwm6snrnzx8k5lby72kmkng";
 }

nix/overlays/default.nix

@@ -7,4 +7,5 @@
   postgresql-default = import ./postgresql-default.nix;
   postgresql-legacy = import ./postgresql-legacy.nix;
   postgresql-future = import ./postgresql-future.nix;
+  slocat = import ./slocat.nix;
 }

nix/overlays/haskell-packages.nix

@@ -29,33 +29,33 @@ let
       # To fill in the sha256:
       #   update-nix-fetchgit nix/overlays/haskell-packages.nix
 
-      hashtables = lib.dontCheck prev.hashtables_1_3_1;
-      hasql = lib.dontCheck prev.hasql_1_6_1_4;
-      hasql-dynamic-statements = lib.dontCheck prev.hasql-dynamic-statements_0_3_1_2;
-      hasql-pool = lib.dontCheck
-        (prev.callHackageDirect
-          {
-            pkg = "hasql-pool";
-            ver = "0.8.0.6";
-            sha256 = "sha256-2u/cwPk8XfXffaDRzGeyzhL+9k2+2T4b8bGOZwz8AX0=";
-          }
-          { });
-      hasql-transaction = lib.dontCheck prev.hasql-transaction_1_0_1_2;
-      isomorphism-class = lib.unmarkBroken prev.isomorphism-class;
-      lens = lib.dontCheck prev.lens_5_2;
-      postgresql-binary = lib.dontCheck prev.postgresql-binary_0_13_1;
-      text-builder = lib.dontCheck prev.text-builder_0_6_7;
-      text-builder-dev = lib.dontCheck prev.text-builder-dev_0_3_3;
-
       postgresql-libpq = lib.dontCheck
         (prev.callCabal2nix "postgresql-libpq"
           (super.fetchFromGitHub {
             owner = "PostgREST";
             repo = "postgresql-libpq";
-            rev = "cef92cb4c07b56568dffdbf4b719258b82183119"; # master
-            sha256 = "0r59klrz47qcnd22s47h612mlz3jbg40wwalfj3f6djwg0cdyr85";
+            rev = "890a0a16cf57dd401420fdc6c7d576fb696003bc"; # master
+            sha256 = "1wmyhldk0k14y8whp1p4akrkqxf5snh8qsbm7fv5f7kz95nyffd0";
           })
           { });
+
+      hasql-notifications = lib.dontCheck
+        (prev.callHackageDirect
+          {
+            pkg = "hasql-notifications";
+            ver = "0.2.0.4";
+            sha256 = "sha256-fm1xiDyvDkb5WLOJ73/s8wrWEW23XFS7luAv2brfr8I=";
+          }
+          { });
+
+      hasql-pool = lib.dontCheck
+        (prev.callHackageDirect
+          {
+            pkg = "hasql-pool";
+            ver = "0.9";
+            sha256 = "sha256-5UshbbaBVY8eJ/9VagNVVxonRwMcd7UmGqDc35pJNFY=";
+          }
+          { });
     } // extraOverrides final prev;
 in
 {

nix/overlays/slocat.nix

@@ -0,0 +1,13 @@
+final: prev:
+{
+  slocat = prev.buildGoModule {
+    name = "slocat";
+    src = prev.fetchFromGitHub {
+      owner = "robx";
+      repo = "slocat";
+      rev = "52e7512c6029fd00483e41ccce260a3b4b9b3b64";
+      sha256 = "sha256-qn6luuh5wqREu3s8RfuMCP5PKdS2WdwPrujRYTpfzQ8=";
+    };
+    vendorSha256 = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo=";
+  };
+}

nix/static-haskell-package.nix

@@ -59,4 +59,7 @@ let
   survey =
     import "${patched-static-haskell-nix}/survey" { inherit normalPkgs compiler defaultCabalPackageVersionComingWithGhc; };
 in
-survey.haskellPackages."${name}"
+{
+  inherit survey;
+  package = survey.haskellPackages."${name}";
+}

nix/tools/cabalTools.nix

@@ -37,7 +37,7 @@ let
     checkedShellScript
       {
         name = "postgrest-run";
-        docs = "Run PostgREST after buidling it interactively with cabal-install";
+        docs = "Run PostgREST after building it interactively with cabal-install";
         args = [ "ARG_LEFTOVERS([PostgREST arguments])" ];
         inRootDir = true;
         withEnv = postgrest.env;

nix/tools/devTools.nix

@@ -77,7 +77,6 @@ let
       }
       ''
         ${tests}/bin/postgrest-test-spec
-        ${tests}/bin/postgrest-test-querycost
         ${tests}/bin/postgrest-test-doctests
         ${tests}/bin/postgrest-test-io
         ${style}/bin/postgrest-lint
@@ -165,6 +164,7 @@ let
           # The following unsets all GIT_ variables.
           unset "''${!GIT_@}"
 
+          # shellcheck disable=SC2317
           function restore () {
             ref="$(git stash list --format=format:%gD --grep "$1" -n1)"
             # this will avoid merge conflicts when applying the stash
@@ -304,4 +304,5 @@ buildToolbox
     hsieGraphModules
     hsieGraphSymbols
   ];
+  extra = { inherit pushCachix; };
 }

nix/tools/loadtest.nix

@@ -56,11 +56,14 @@ let
         export PGRST_LOG_LEVEL="crit"
 
         mkdir -p "$(dirname "$_arg_output")"
+        abs_output="$(realpath "$_arg_output")"
 
         # shellcheck disable=SC2145
         ${withTools.withPg} --fixtures "$_arg_testdir"/fixtures.sql \
+        ${withTools.withSlowPg} \
         ${withTools.withPgrst} \
-        sh -c "cd \"$_arg_testdir\" && ${runner} -targets targets.http -output \"$_arg_output\" \"''${_arg_leftovers[@]}\""
+        ${withTools.withSlowPgrst} \
+        sh -c "cd \"$_arg_testdir\" && ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
         ${vegeta}/bin/vegeta report -type=text "$_arg_output"
       '';
 

nix/tools/release/default.nix

@@ -72,14 +72,15 @@ let
         today_date="$(date '+%Y%m%d')"
         today_date_for_changelog="$(date '+%Y-%m-%d')"
         bump_pre="$major.$minor.$patch.$today_date"
+        bump_pre_minor="$major.$((minor+1)).0.$today_date"
         bump_patch="$major.$minor.$((patch+1))"
         bump_minor="$major.$((minor+1)).0"
         bump_major="$((major+1)).0.0"
 
         PS3="Please select the new version: "
-        select new_version in "$bump_pre" "$bump_patch" "$bump_minor" "$bump_major"; do
+        select new_version in "$bump_pre" "$bump_pre_minor" "$bump_patch" "$bump_minor" "$bump_major"; do
           case "$REPLY" in
-            1|2|3|4)
+            1|2|3|4|5)
               echo "Selected $new_version"
               break
               ;;
@@ -95,7 +96,7 @@ let
         echo "Committing ..."
         git add postgrest.cabal > /dev/null
 
-        if [[ "$new_version" != "$bump_pre" ]]; then
+        if [[ "$new_version" != "$bump_pre" && "$new_version" != "$bump_pre_minor" ]]; then
           echo "Updating CHANGELOG.md ..."
           sed -i -E "s/Unreleased/&\n\n## [$new_version] - $today_date_for_changelog/" CHANGELOG.md > /dev/null
           git add CHANGELOG.md > /dev/null
@@ -106,7 +107,7 @@ let
         echo "Tagging ..."
         git tag "v$new_version" > /dev/null
 
-        trap "Couldn't find remote. Please push manually ..." ERR
+        trap "echo Remote not found. Please push manually ..." ERR
         remote="$(git remote -v | grep PostgREST/postgrest | grep push | cut -f1)"
         trap "" ERR
 

nix/tools/tests.nix

@@ -32,18 +32,6 @@ let
           test:spec -- "''${_arg_leftovers[@]}"
       '';
 
-  testQuerycost =
-    checkedShellScript
-      {
-        name = "postgrest-test-querycost";
-        docs = "Run the Haskell test suite for query costs";
-        inRootDir = true;
-        withEnv = postgrest.env;
-      }
-      ''
-        ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:querycost
-      '';
-
   testDoctests =
     checkedShellScript
       {
@@ -140,7 +128,7 @@ let
           rm -rf coverage/*
 
           # build once before running all the tests
-          ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest lib:postgrest test:spec test:querycost
+          ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest lib:postgrest test:spec
 
           (
             trap 'echo Found dead code: Check file list above.' ERR ;
@@ -155,14 +143,11 @@ let
           HPCTIXFILE="$tmpdir"/spec.tix \
             ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec
 
-          HPCTIXFILE="$tmpdir"/querycost.tix \
-            ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:querycost
-
           # Note: No coverage for doctests, as doctests leverage GHCi and GHCi does not support hpc
 
           # collect all the tix files
           ${ghc}/bin/hpc sum  --union --exclude=Paths_postgrest --output="$tmpdir"/tests.tix \
-            "$tmpdir"/io*.tix "$tmpdir"/spec.tix "$tmpdir"/querycost.tix
+            "$tmpdir"/io*.tix "$tmpdir"/spec.tix
 
           # prepare the overlay
           ${ghc}/bin/hpc overlay --output="$tmpdir"/overlay.tix test/coverage.overlay
@@ -234,7 +219,6 @@ buildToolbox
   tools =
     [
       testSpec
-      testQuerycost
       testDoctests
       testSpecIdempotence
       testIO

nix/tools/withTools.nix

@@ -1,6 +1,7 @@
 { bash-completion
 , buildToolbox
 , cabal-install
+, cabalTools
 , checkedShellScript
 , curl
 , devCabalOptions
@@ -8,14 +9,16 @@
 , lib
 , postgresqlVersions
 , postgrest
+, slocat
 , writeText
 }:
 let
   withTmpDb =
     { name, postgresql }:
+    let commandName = "postgrest-with-${name}"; in
     checkedShellScript
       {
-        name = "postgrest-with-${name}";
+        name = commandName;
         docs = "Run the given command in a temporary database with ${name}";
         args =
           [
@@ -25,6 +28,7 @@ let
             "ARG_USE_ENV([PGUSER], [postgrest_test_authenticator], [Authenticator PG role])"
             "ARG_USE_ENV([PGDATABASE], [postgres], [PG database name])"
             "ARG_USE_ENV([PGRST_DB_SCHEMAS], [test], [Schema to expose])"
+            "ARG_USE_ENV([PGTZ], [utc], [Timezone to use])"
           ];
         positionalCompletion = "_command";
         inRootDir = true;
@@ -53,6 +57,11 @@ let
         export PGUSER
         export PGDATABASE
         export PGRST_DB_SCHEMAS
+        export PGTZ
+
+        HBA_FILE="$tmpdir/pg_hba.conf"
+        echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
+        echo "local $PGDATABASE all trust" >> "$HBA_FILE"
 
         log "Initializing database cluster..."
         # We try to make the database cluster as independent as possible from the host
@@ -62,9 +71,10 @@ let
 
         log "Starting the database cluster..."
         # Instead of listening on a local port, we will listen on a unix domain socket.
-        pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -k $PGHOST -c log_statement=\"all\"" \
+        pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
           >> "$setuplog"
 
+        # shellcheck disable=SC2317
         stop () {
           log "Stopping the database cluster..."
           pg_ctl stop -m i >> "$setuplog"
@@ -76,6 +86,10 @@ let
         psql -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
 
         log "Done. Running command..."
+
+        echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U $PGUSER"
+        echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log"
+
         ("$_arg_command" "''${_arg_leftovers[@]}")
       '';
 
@@ -125,6 +139,81 @@ let
 
   withPg = builtins.head withPgVersions;
 
+  withSlowPg =
+    checkedShellScript
+      {
+        name = "postgrest-with-slow-pg";
+        docs = "Run the given command with simulated high latency postgresql";
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
+            "ARG_LEFTOVERS([command arguments])"
+            "ARG_USE_ENV([PGHOST], [], [PG host (socket name)])"
+            "ARG_USE_ENV([PGDELAY], [0ms], [extra PG latency (duration)])"
+          ];
+        positionalCompletion = "_command";
+        inRootDir = true;
+        redirectTixFiles = false;
+        withTmpDir = true;
+      }
+      ''
+        delay="''${PGDELAY:-0ms}"
+        echo "delaying data to/from postgres by $delay"
+
+        REALPGHOST="$PGHOST"
+        export PGHOST="$tmpdir/socket"
+        mkdir -p "$PGHOST"
+
+        ${slocat}/bin/slocat -delay "$delay" -src "$PGHOST/.s.PGSQL.5432" -dst "$REALPGHOST/.s.PGSQL.5432" &
+        SLOCAT_PID=$!
+        # shellcheck disable=SC2317
+        stop_slocat() {
+          kill "$SLOCAT_PID" || true
+          wait "$SLOCAT_PID" || true
+        }
+        trap stop_slocat EXIT
+        sleep 1 # should wait for socket file to appear instead
+
+        ("$_arg_command" "''${_arg_leftovers[@]}")
+      '';
+
+  withSlowPgrst =
+    checkedShellScript
+      {
+        name = "postgrest-with-slow-postgrest";
+        docs = "Run the given command with simulated high latency postgrest";
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
+            "ARG_LEFTOVERS([command arguments])"
+            "ARG_USE_ENV([PGRST_SERVER_UNIX_SOCKET], [], [PostgREST host (socket name)])"
+            "ARG_USE_ENV([PGRST_DELAY], [0ms], [extra PostgREST latency (duration)])"
+          ];
+        positionalCompletion = "_command";
+        inRootDir = true;
+        redirectTixFiles = false;
+        withTmpDir = true;
+      }
+      ''
+        delay="''${PGRST_DELAY:-0ms}"
+        echo "delaying data to/from PostgREST by $delay"
+
+        REAL_PGRST_SERVER_UNIX_SOCKET="$PGRST_SERVER_UNIX_SOCKET"
+        export PGRST_SERVER_UNIX_SOCKET="$tmpdir/postgrest.socket"
+
+        ${slocat}/bin/slocat -delay "$delay" -src "$PGRST_SERVER_UNIX_SOCKET" -dst "$REAL_PGRST_SERVER_UNIX_SOCKET" &
+        SLOCAT_PID=$!
+        # shellcheck disable=SC2317
+        stop_slocat() {
+          kill "$SLOCAT_PID" || true
+          wait "$SLOCAT_PID" || true
+        }
+        trap stop_slocat EXIT
+        sleep 1 # should wait for socket file to appear instead
+
+        ("$_arg_command" "''${_arg_leftovers[@]}")
+      '';
+
   withGit =
     let
       name = "postgrest-with-git";
@@ -245,17 +334,25 @@ let
         export PGRST_SERVER_UNIX_SOCKET="$tmpdir"/postgrest.socket
 
         rm -f result
-        echo -n "Building postgrest... "
-        nix-build -A postgrestPackage > "$tmpdir"/build.log 2>&1 || {
-          echo "failed, output:"
-          cat "$tmpdir"/build.log
-          exit 1
-        }
+        if [ -z "''${PGRST_BUILD_CABAL:-}" ]; then
+          echo -n "Building postgrest (nix)... "
+          nix-build -A postgrestPackage > "$tmpdir"/build.log 2>&1 || {
+            echo "failed, output:"
+            cat "$tmpdir"/build.log
+            exit 1
+          }
+          PGRST_CMD=./result/bin/postgrest
+        else
+          echo -n "Building postgrest (cabal)... "
+          postgrest-build
+          PGRST_CMD=postgrest-run
+        fi
         echo "done."
 
         echo -n "Starting postgrest... "
-        ./result/bin/postgrest ${legacyConfig} > "$tmpdir"/run.log 2>&1 &
+        $PGRST_CMD ${legacyConfig} > "$tmpdir"/run.log 2>&1 &
         pid=$!
+        # shellcheck disable=SC2317
         cleanup() {
           kill "$pid" || true
         }
@@ -275,7 +372,7 @@ in
 buildToolbox
 {
   name = "postgrest-with";
-  tools = [ withPgAll withGit withPgrst ] ++ withPgVersions;
+  tools = [ withPgAll withGit withPgrst withSlowPg withSlowPgrst ] ++ withPgVersions;
   # make withTools available for other nix files
-  extra = { inherit withGit withPg withPgAll withPgrst; };
+  extra = { inherit withGit withPg withPgAll withPgrst withSlowPg withSlowPgrst; };
 }

postgrest.cabal

@@ -1,5 +1,5 @@
 name:               postgrest
-version:            10.1.1
+version:            11.0.0
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                     for tables, views, and functions, supporting all HTTP methods that security
@@ -46,7 +46,7 @@ library
                       PostgREST.Cors
                       PostgREST.SchemaCache
                       PostgREST.SchemaCache.Identifiers
-                      PostgREST.SchemaCache.Proc
+                      PostgREST.SchemaCache.Routine
                       PostgREST.SchemaCache.Relationship
                       PostgREST.SchemaCache.Table
                       PostgREST.Error
@@ -60,6 +60,7 @@ library
                       PostgREST.Plan.CallPlan
                       PostgREST.Plan.MutatePlan
                       PostgREST.Plan.ReadPlan
+                      PostgREST.Plan.Types
                       PostgREST.RangeQuery
                       PostgREST.ApiRequest
                       PostgREST.ApiRequest.Preferences
@@ -72,7 +73,7 @@ library
                       PostgREST.Workers
   other-modules:      Paths_postgrest
   build-depends:      base                      >= 4.9 && < 4.17
-                    , HTTP                      >= 4000.3.7 && < 4000.4
+                    , HTTP                      >= 4000.3.7 && < 4000.5
                     , Ranged-sets               >= 0.3 && < 0.5
                     , aeson                     >= 2.0.3 && < 2.2
                     , auto-update               >= 0.1.4 && < 0.2
@@ -85,11 +86,12 @@ library
                     , contravariant-extras      >= 0.3.3 && < 0.4
                     , cookie                    >= 0.4.2 && < 0.5
                     , either                    >= 4.4.1 && < 5.1
+                    , fuzzyset                  >= 0.2.3
                     , gitrev                    >= 1.2 && < 1.4
                     , hasql                     >= 1.6.1.1 && < 1.7
                     , hasql-dynamic-statements  >= 0.3.1 && < 0.4
                     , hasql-notifications       >= 0.1 && < 0.3
-                    , hasql-pool                >= 0.8.0.6 && < 0.9
+                    , hasql-pool                >= 0.9 && < 0.10
                     , hasql-transaction         >= 1.0.1 && < 1.1
                     , heredoc                   >= 0.2 && < 0.3
                     , http-types                >= 0.12.2 && < 0.13
@@ -97,11 +99,11 @@ library
                     , interpolatedstring-perl6  >= 1 && < 1.1
                     , jose                      >= 0.8.5.1 && < 0.11
                     , lens                      >= 4.14 && < 5.3
-                    , lens-aeson                >= 1.0.1 && < 1.2
+                    , lens-aeson                >= 1.0.1 && < 1.3
                     , mtl                       >= 2.2.2 && < 2.3
                     , network                   >= 2.6 && < 3.2
                     , network-uri               >= 2.6.1 && < 2.8
-                    , optparse-applicative      >= 0.13 && < 0.17
+                    , optparse-applicative      >= 0.13 && < 0.18
                     , parsec                    >= 3.1.11 && < 3.2
                     , protolude                 >= 0.3.1 && < 0.4
                     , regex-tdfa                >= 1.2.2 && < 1.4
@@ -185,6 +187,7 @@ test-suite spec
                       Feature.CorsSpec
                       Feature.ExtraSearchPathSpec
                       Feature.LegacyGucsSpec
+                      Feature.ObservabilitySpec
                       Feature.OpenApi.DisabledOpenApiSpec
                       Feature.OpenApi.IgnorePrivOpenApiSpec
                       Feature.OpenApi.OpenApiSpec
@@ -209,15 +212,16 @@ test-suite spec
                       Feature.Query.QuerySpec
                       Feature.Query.RangeSpec
                       Feature.Query.RawOutputTypesSpec
+                      Feature.Query.RelatedQueriesSpec
                       Feature.Query.RpcSpec
                       Feature.Query.SingularSpec
+                      Feature.Query.SpreadQueriesSpec
                       Feature.Query.UnicodeSpec
                       Feature.Query.UpdateSpec
                       Feature.Query.UpsertSpec
                       Feature.RollbackSpec
                       Feature.RpcPreRequestGucsSpec
                       SpecHelper
-                      TestTypes
   build-depends:      base              >= 4.9 && < 4.17
                     , aeson             >= 2.0.3 && < 2.2
                     , aeson-qq          >= 0.8.1 && < 0.9
@@ -227,69 +231,32 @@ test-suite spec
                     , bytestring        >= 0.10.8 && < 0.12
                     , case-insensitive  >= 1.2 && < 1.3
                     , containers        >= 0.5.7 && < 0.7
-                    , hasql-pool        >= 0.8.0.2 && < 0.9
+                    , hasql-pool        >= 0.9 && < 0.10
                     , hasql-transaction >= 1.0.1 && < 1.1
                     , heredoc           >= 0.2 && < 0.3
-                    , hspec             >= 2.3 && < 2.9
+                    , hspec             >= 2.3 && < 2.10
                     , hspec-wai         >= 0.10 && < 0.12
                     , hspec-wai-json    >= 0.10 && < 0.12
                     , http-types        >= 0.12.3 && < 0.13
                     , lens              >= 4.14 && < 5.3
-                    , lens-aeson        >= 1.0.1 && < 1.2
+                    , lens-aeson        >= 1.0.1 && < 1.3
                     , monad-control     >= 1.0.1 && < 1.1
                     , postgrest
                     , process           >= 1.4.2 && < 1.7
                     , protolude         >= 0.3.1 && < 0.4
                     , regex-tdfa        >= 1.2.2 && < 1.4
+                    , scientific        >= 0.3.4 && < 0.4
                     , text              >= 1.2.2 && < 1.3
                     , transformers-base >= 0.4.4 && < 0.5
                     , wai               >= 3.2.1 && < 3.3
                     , wai-extra         >= 3.0.19 && < 3.2
-  ghc-options:        -O0 -Werror -Wall -fwarn-identities
+  ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
                       -fno-warn-missing-signatures
                       -fwrite-ide-info
                       -- https://github.com/PostgREST/postgrest/issues/387
                       -with-rtsopts=-K33K
 
-test-suite querycost
-  type:               exitcode-stdio-1.0
-  default-language:   Haskell2010
-  default-extensions: OverloadedStrings
-                      QuasiQuotes
-                      NoImplicitPrelude
-  hs-source-dirs:     test/spec
-  main-is:            QueryCost.hs
-  other-modules:      SpecHelper
-  build-depends:      base              >= 4.9 && < 4.17
-                    , aeson             >= 2.0.3 && < 2.2
-                    , base64-bytestring >= 1 && < 1.3
-                    , bytestring        >= 0.10.8 && < 0.12
-                    , case-insensitive  >= 1.2 && < 1.3
-                    , containers        >= 0.5.7 && < 0.7
-                    , contravariant     >= 1.4 && < 1.6
-                    , hasql             >= 1.6 && < 1.7
-                    , hasql-dynamic-statements >= 0.3.1 && < 0.4
-                    , hasql-pool        >= 0.8.0.2 && < 0.9
-                    , hasql-transaction >= 1.0.1 && < 1.1
-                    , heredoc           >= 0.2 && < 0.3
-                    , hspec             >= 2.3 && < 2.9
-                    , hspec-wai         >= 0.10 && < 0.12
-                    , hspec-wai-json    >= 0.10 && < 0.12
-                    , http-types        >= 0.12.3 && < 0.13
-                    , lens              >= 4.14 && < 5.3
-                    , lens-aeson        >= 1.0.1 && < 1.2
-                    , postgrest
-                    , process           >= 1.4.2 && < 1.7
-                    , protolude         >= 0.3.1 && < 0.4
-                    , regex-tdfa        >= 1.2.2 && < 1.4
-                    , wai-extra         >= 3.0.19 && < 3.2
-  ghc-options:        -O0 -Werror -Wall -fwarn-identities
-                      -fno-spec-constr -optP-Wno-nonportable-include-path
-                      -fwrite-ide-info
-                      -- https://github.com/PostgREST/postgrest/issues/387
-                      -with-rtsopts=-K1K
-
 test-suite doctests
   type:               exitcode-stdio-1.0
   default-language:   Haskell2010

src/PostgREST/ApiRequest.hs

@@ -3,6 +3,7 @@ Module      : PostgREST.Request.ApiRequest
 Description : PostgREST functions to translate HTTP request to a domain type called ApiRequest.
 -}
 {-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE MultiWayIf      #-}
 {-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 
@@ -32,27 +33,22 @@ import qualified Data.Set              as S
 import qualified Data.Text.Encoding    as T
 import qualified Data.Vector           as V
 
+import Data.Either.Combinators (mapBoth)
+
 import Control.Arrow             ((***))
 import Data.Aeson.Types          (emptyArray, emptyObject)
 import Data.List                 (lookup, union)
 import Data.Ranged.Ranges        (emptyRange, rangeIntersection,
                                   rangeIsEmpty)
-import Data.Tree                 (Tree (..))
 import Network.HTTP.Types.Header (RequestHeaders, hCookie)
 import Network.HTTP.Types.URI    (parseSimpleQuery)
 import Network.Wai               (Request (..))
 import Network.Wai.Parse         (parseHttpAccept)
 import Web.Cookie                (parseCookies)
 
-import PostgREST.ApiRequest.Preferences  (PreferCount (..),
-                                          PreferParameters (..),
-                                          PreferRepresentation (..),
-                                          PreferResolution (..),
-                                          PreferTransaction (..))
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
 import PostgREST.ApiRequest.Types        (ApiRequestError (..),
-                                          RangeError (..),
-                                          SelectItem (..))
+                                          RangeError (..))
 import PostgREST.Config                  (AppConfig (..),
                                           OpenAPIMode (..))
 import PostgREST.MediaType               (MTPlanAttrs (..),
@@ -62,13 +58,9 @@ import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           convertToLimitZeroRange,
                                           hasLimitZero,
                                           rangeRequested)
-import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier (..),
                                           Schema)
-import PostgREST.SchemaCache.Proc        (ProcDescription (..),
-                                          ProcParam (..), ProcsMap,
-                                          procReturnsScalar)
 
 import qualified PostgREST.ApiRequest.Preferences as Preferences
 import qualified PostgREST.ApiRequest.QueryParams as QueryParams
@@ -90,6 +82,7 @@ data Payload
       -- ^ Keys of the object or if it's an array these keys are guaranteed to
       -- be the same across all its objects
       }
+  | ProcessedUrlEncoded { payArray  :: [(Text, Text)], payKeys :: S.Set Text }
   | RawJSON { payRaw  :: LBS.ByteString }
   | RawPay  { payRaw  :: LBS.ByteString }
 
@@ -114,41 +107,9 @@ data PathInfo
       }
 -- | The target db object of a user action
 data Target = TargetIdent QualifiedIdentifier
-            | TargetProc{tProc :: ProcDescription, tpIsRootSpec :: Bool}
+            | TargetProc{tProc :: QualifiedIdentifier, tpIsRootSpec :: Bool}
             | TargetDefaultSpec{tdsSchema :: Schema} -- The default spec offered at root "/"
 
--- | RPC query param value `/rpc/func?v=<value>`, used for VARIADIC functions on form-urlencoded POST and GETs
--- | It can be fixed `?v=1` or repeated `?v=1&v=2&v=3.
-data RpcParamValue = Fixed Text | Variadic [Text]
-instance JSON.ToJSON RpcParamValue where
-  toJSON (Fixed    v) = JSON.toJSON v
-  toJSON (Variadic v) = JSON.toJSON v
-
-toRpcParamValue :: ProcDescription -> (Text, Text) -> (Text, RpcParamValue)
-toRpcParamValue proc (k, v) | prmIsVariadic k = (k, Variadic [v])
-                            | otherwise       = (k, Fixed v)
-  where
-    prmIsVariadic prm = isJust $ find (\ProcParam{ppName, ppVar} -> ppName == prm && ppVar) $ pdParams proc
-
--- | Convert rpc params `/rpc/func?a=val1&b=val2` to json `{"a": "val1", "b": "val2"}
-jsonRpcParams :: ProcDescription -> [(Text, Text)] -> Payload
-jsonRpcParams proc prms =
-  if not $ pdHasVariadic proc then -- if proc has no variadic param, save steps and directly convert to json
-    ProcessedJSON (JSON.encode $ HM.fromList $ second JSON.toJSON <$> prms) (S.fromList $ fst <$> prms)
-  else
-    let paramsMap = HM.fromListWith mergeParams $ toRpcParamValue proc <$> prms in
-    ProcessedJSON (JSON.encode paramsMap) (S.fromList $ HM.keys paramsMap)
-  where
-    mergeParams :: RpcParamValue -> RpcParamValue -> RpcParamValue
-    mergeParams (Variadic a) (Variadic b) = Variadic $ b ++ a
-    mergeParams v _                       = v -- repeated params for non-variadic parameters are not merged
-
-targetToJsonRpcParams :: Maybe Target -> [(Text, Text)] -> Maybe Payload
-targetToJsonRpcParams target params =
-  case target of
-    Just TargetProc{tProc} -> Just $ jsonRpcParams tProc params
-    _                      -> Nothing
-
 {-|
   Describes what the user wants to do. This data type is a
   translation of the raw elements of an HTTP request into domain
@@ -157,37 +118,60 @@ targetToJsonRpcParams target params =
   if it is an action we are able to perform.
 -}
 data ApiRequest = ApiRequest {
-    iAction               :: Action                           -- ^ Similar but not identical to HTTP method, e.g. Create/Invoke both POST
-  , iRange                :: HM.HashMap Text NonnegRange      -- ^ Requested range of rows within response
-  , iTopLevelRange        :: NonnegRange                      -- ^ Requested range of rows from the top level
-  , iTarget               :: Target                           -- ^ The target, be it calling a proc or accessing a table
-  , iPayload              :: Maybe Payload                    -- ^ Data sent by client and used for mutation actions
-  , iPreferRepresentation :: PreferRepresentation             -- ^ If client wants created items echoed back
-  , iPreferParameters     :: Maybe PreferParameters           -- ^ How to pass parameters to a stored procedure
-  , iPreferCount          :: Maybe PreferCount                -- ^ Whether the client wants a result count
-  , iPreferResolution     :: Maybe PreferResolution           -- ^ Whether the client wants to UPSERT or ignore records on PK conflict
-  , iPreferTransaction    :: Maybe PreferTransaction          -- ^ Whether the clients wants to commit or rollback the transaction
-  , iQueryParams          :: QueryParams.QueryParams
-  , iColumns              :: S.Set FieldName                  -- ^ parsed colums from &columns parameter and payload
-  , iHeaders              :: [(ByteString, ByteString)]       -- ^ HTTP request headers
-  , iCookies              :: [(ByteString, ByteString)]       -- ^ Request Cookies
-  , iPath                 :: ByteString                       -- ^ Raw request path
-  , iMethod               :: ByteString                       -- ^ Raw request method
-  , iSchema               :: Schema                           -- ^ The request schema. Can vary depending on profile headers.
-  , iNegotiatedByProfile  :: Bool                             -- ^ If schema was was chosen according to the profile spec https://www.w3.org/TR/dx-prof-conneg/
-  , iAcceptMediaType      :: MediaType                        -- ^ The media type in the Accept header
-  , iBinaryField          :: Maybe FieldName                  -- ^ field used for raw output
+    iAction              :: Action                           -- ^ Similar but not identical to HTTP method, e.g. Create/Invoke both POST
+  , iRange               :: HM.HashMap Text NonnegRange      -- ^ Requested range of rows within response
+  , iTopLevelRange       :: NonnegRange                      -- ^ Requested range of rows from the top level
+  , iTarget              :: Target                           -- ^ The target, be it calling a proc or accessing a table
+  , iPayload             :: Maybe Payload                    -- ^ Data sent by client and used for mutation actions
+  , iPreferences         :: Preferences.Preferences          -- ^ Prefer header values
+  , iQueryParams         :: QueryParams.QueryParams
+  , iColumns             :: S.Set FieldName                  -- ^ parsed colums from &columns parameter and payload
+  , iHeaders             :: [(ByteString, ByteString)]       -- ^ HTTP request headers
+  , iCookies             :: [(ByteString, ByteString)]       -- ^ Request Cookies
+  , iPath                :: ByteString                       -- ^ Raw request path
+  , iMethod              :: ByteString                       -- ^ Raw request method
+  , iSchema              :: Schema                           -- ^ The request schema. Can vary depending on profile headers.
+  , iNegotiatedByProfile :: Bool                             -- ^ If schema was was chosen according to the profile spec https://www.w3.org/TR/dx-prof-conneg/
+  , iAcceptMediaType     :: MediaType                        -- ^ The media type in the Accept header
+  , iContentMediaType    :: MediaType                        -- ^ The media type in the Content-Type header
   }
 
 -- | Examines HTTP request and translates it into user intent.
-userApiRequest :: AppConfig -> SchemaCache -> Request -> RequestBody -> Either ApiRequestError ApiRequest
-userApiRequest conf sCache req reqBody = do
-  qPrms <- first QueryParamError $ QueryParams.parse $ rawQueryString req
-  pInfo <- getPathInfo conf $ pathInfo req
-  act <- getAction pInfo $ requestMethod req
-  mediaTypes <- getMediaTypes conf (requestHeaders req) act pInfo
-  negotiatedSchema <- getSchema conf (requestHeaders req) (requestMethod req)
-  apiRequest conf sCache req reqBody qPrms pInfo act mediaTypes negotiatedSchema
+userApiRequest :: AppConfig -> Request -> RequestBody -> Either ApiRequestError ApiRequest
+userApiRequest conf req reqBody = do
+  pInfo@PathInfo{..} <- getPathInfo conf $ pathInfo req
+  act <- getAction pInfo method
+  qPrms <- first QueryParamError $ QueryParams.parse (pathIsProc && act `elem` [ActionInvoke InvGet, ActionInvoke InvHead]) $ rawQueryString req
+  (acceptMediaType, contentMediaType) <- getMediaTypes conf hdrs act pInfo
+  (schema, negotiatedByProfile) <- getSchema conf hdrs method
+  (topLevelRange, ranges) <- getRanges method qPrms hdrs
+  (payload, columns) <- getPayload reqBody contentMediaType qPrms act pInfo
+  return $ ApiRequest {
+    iAction = act
+  , iTarget = if | pathIsProc    -> TargetProc (QualifiedIdentifier schema pathName) pathIsRootSpec
+                 | pathIsDefSpec -> TargetDefaultSpec schema
+                 | otherwise     -> TargetIdent $ QualifiedIdentifier schema pathName
+  , iRange = ranges
+  , iTopLevelRange = topLevelRange
+  , iPayload = payload
+  , iPreferences = Preferences.fromHeaders hdrs
+  , iQueryParams = qPrms
+  , iColumns = columns
+  , iHeaders = iHdrs
+  , iCookies = iCkies
+  , iPath = rawPathInfo req
+  , iMethod = method
+  , iSchema = schema
+  , iNegotiatedByProfile = negotiatedByProfile
+  , iAcceptMediaType = acceptMediaType
+  , iContentMediaType = contentMediaType
+  }
+  where
+    method = requestMethod req
+    hdrs = requestHeaders req
+    lookupHeader    = flip lookup hdrs
+    iHdrs = [ (CI.foldedCase k, v) | (k,v) <- hdrs, k /= hCookie]
+    iCkies = maybe [] parseCookies $ lookupHeader "Cookie"
 
 getPathInfo :: AppConfig -> [Text] -> Either ApiRequestError PathInfo
 getPathInfo AppConfig{configOpenApiMode, configDbRootSpec} path =
@@ -249,114 +233,71 @@ getSchema AppConfig{configDbSchemas} hdrs method = do
     acceptProfile = T.decodeUtf8 <$> lookupHeader "Accept-Profile"
     lookupHeader    = flip lookup hdrs
 
-apiRequest :: AppConfig -> SchemaCache -> Request -> RequestBody -> QueryParams.QueryParams -> PathInfo -> Action -> (MediaType, MediaType) -> (Schema, Bool) -> Either ApiRequestError ApiRequest
-apiRequest conf sCache req reqBody queryparams@QueryParams{..} PathInfo{pathName, pathIsProc, pathIsRootSpec, pathIsDefSpec} action (acceptMediaType, contentMediaType) (schema, negotiatedByProfile)
+getRanges :: ByteString -> QueryParams -> RequestHeaders -> Either ApiRequestError (NonnegRange, HM.HashMap Text NonnegRange)
+getRanges method QueryParams{qsOrder,qsRanges} hdrs
   | isInvalidRange = Left $ InvalidRange (if rangeIsEmpty headerRange then LowerGTUpper else NegativeLimit)
-  | shouldParsePayload && isLeft payload = either (Left . InvalidBody) witness payload
-  | not expectParams && not (L.null qsParams) = Left $ ParseRequestError "Unexpected param or filter missing operator" ("Failed to parse " <> show qsParams)
   | method `elem` ["PATCH", "DELETE"] && not (null qsRanges) && null qsOrder = Left LimitNoOrderError
-  | method == "PUT" && topLevelRange /= allRange = Left PutRangeNotAllowedError
-  | otherwise = do
-     checkedTarget <- target
-     bField <- binaryField conf acceptMediaType checkedTarget queryparams
-     return ApiRequest {
-      iAction = action
-      , iTarget = checkedTarget
-      , iRange = ranges
-      , iTopLevelRange = topLevelRange
-      , iPayload = relevantPayload
-      , iPreferRepresentation = fromMaybe None preferRepresentation
-      , iPreferParameters = preferParameters
-      , iPreferCount = preferCount
-      , iPreferResolution = preferResolution
-      , iPreferTransaction = preferTransaction
-      , iQueryParams = queryparams
-      , iColumns = payloadColumns
-      , iHeaders = [ (CI.foldedCase k, v) | (k,v) <- hdrs, k /= hCookie]
-      , iCookies = maybe [] parseCookies $ lookupHeader "Cookie"
-      , iPath = rawPathInfo req
-      , iMethod = method
-      , iSchema = schema
-      , iNegotiatedByProfile = negotiatedByProfile
-      , iAcceptMediaType = acceptMediaType
-      , iBinaryField = bField
-      }
- where
-  expectParams = pathIsProc && method /= "POST"
+  | method == "PUT" && topLevelRange /= allRange = Left PutLimitNotAllowedError
+  | otherwise = Right (topLevelRange, ranges)
+  where
+    -- According to the RFC (https://www.rfc-editor.org/rfc/rfc9110.html#name-range),
+    -- the Range header must be ignored for all methods other than GET
+    headerRange = if method == "GET" then rangeRequested hdrs else allRange
+    limitRange = fromMaybe allRange (HM.lookup "limit" qsRanges)
+    headerAndLimitRange = rangeIntersection headerRange limitRange
+    -- Bypass all the ranges and send only the limit zero range (0 <= x <= -1) if
+    -- limit=0 is present in the query params (not allowed for the Range header)
+    ranges = HM.insert "limit" (convertToLimitZeroRange limitRange headerAndLimitRange) qsRanges
+    -- The only emptyRange allowed is the limit zero range
+    isInvalidRange = topLevelRange == emptyRange && not (hasLimitZero limitRange)
+    topLevelRange = fromMaybe allRange $ HM.lookup "limit" ranges -- if no limit is specified, get all the request rows
 
-  columns = case action of
-    ActionMutate MutationCreate -> qsColumns
-    ActionMutate MutationUpdate -> qsColumns
-    ActionInvoke InvPost        -> qsColumns
-    _                           -> Nothing
+getPayload :: RequestBody -> MediaType -> QueryParams.QueryParams -> Action -> PathInfo -> Either ApiRequestError (Maybe Payload, S.Set FieldName)
+getPayload reqBody contentMediaType QueryParams{qsColumns} action PathInfo{pathIsProc}= do
+  checkedPayload <- if shouldParsePayload then payload else Right Nothing
+  let cols = case (checkedPayload, columns) of
+        (Just ProcessedJSON{payKeys}, _)       -> payKeys
+        (Just ProcessedUrlEncoded{payKeys}, _) -> payKeys
+        (Just RawJSON{}, Just cls)             -> cls
+        _                                      -> S.empty
+  return (checkedPayload, cols)
+  where
+    payload :: Either ApiRequestError (Maybe Payload)
+    payload = mapBoth InvalidBody Just $ case (contentMediaType, pathIsProc) of
+      (MTApplicationJSON, _) ->
+        if isJust columns
+          then Right $ RawJSON reqBody
+          else note "All object keys must match" . payloadAttributes reqBody
+                 =<< if LBS.null reqBody && pathIsProc
+                       then Right emptyObject
+                       else first BS.pack $ JSON.eitherDecode reqBody
+      (MTTextCSV, _) -> do
+        json <- csvToJson <$> first BS.pack (CSV.decodeByName reqBody)
+        note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json
+      (MTUrlEncoded, isProc) -> do
+        let params = (T.decodeUtf8 *** T.decodeUtf8) <$> parseSimpleQuery (LBS.toStrict reqBody)
+        if isProc
+          then Right $ ProcessedUrlEncoded params (S.fromList $ fst <$> params)
+          else
+            let paramsMap = HM.fromList $ (identity *** JSON.String) <$> params in
+            Right $ ProcessedJSON (JSON.encode paramsMap) $ S.fromList (HM.keys paramsMap)
+      (MTTextPlain, True) -> Right $ RawPay reqBody
+      (MTTextXML, True) -> Right $ RawPay reqBody
+      (MTOctetStream, True) -> Right $ RawPay reqBody
+      (ct, _) -> Left $ "Content-Type not acceptable: " <> MediaType.toMime ct
 
-  payloadColumns =
-    case (contentMediaType, action) of
-      (_, ActionInvoke InvGet)  -> S.fromList $ fst <$> qsParams
-      (_, ActionInvoke InvHead) -> S.fromList $ fst <$> qsParams
-      (MTUrlEncoded, _)         -> S.fromList $ map (T.decodeUtf8 . fst) $ parseSimpleQuery $ LBS.toStrict reqBody
-      _ -> case (relevantPayload, columns) of
-        (Just ProcessedJSON{payKeys}, _) -> payKeys
-        (Just RawJSON{}, Just cls)       -> cls
-        _                                -> S.empty
-  payload :: Either ByteString Payload
-  payload = case (contentMediaType, pathIsProc) of
-    (MTApplicationJSON, _) ->
-      if isJust columns
-        then Right $ RawJSON reqBody
-        else note "All object keys must match" . payloadAttributes reqBody
-               =<< if LBS.null reqBody && pathIsProc
-                     then Right emptyObject
-                     else first BS.pack $ JSON.eitherDecode reqBody
-    (MTTextCSV, _) -> do
-      json <- csvToJson <$> first BS.pack (CSV.decodeByName reqBody)
-      note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json
-    (MTUrlEncoded, _) ->
-      let paramsMap = HM.fromList $ (T.decodeUtf8 *** JSON.String . T.decodeUtf8) <$> parseSimpleQuery (LBS.toStrict reqBody) in
-      Right $ ProcessedJSON (JSON.encode paramsMap) $ S.fromList (HM.keys paramsMap)
-    (MTTextPlain, True) -> Right $ RawPay reqBody
-    (MTTextXML, True) -> Right $ RawPay reqBody
-    (MTOctetStream, True) -> Right $ RawPay reqBody
-    (ct, _) -> Left $ "Content-Type not acceptable: " <> MediaType.toMime ct
-  topLevelRange = fromMaybe allRange $ HM.lookup "limit" ranges -- if no limit is specified, get all the request rows
+    shouldParsePayload = case (action, contentMediaType) of
+      (ActionMutate MutationCreate, _)       -> True
+      (ActionInvoke InvPost, _)              -> True
+      (ActionMutate MutationSingleUpsert, _) -> True
+      (ActionMutate MutationUpdate, _)       -> True
+      _                                      -> False
 
-  target
-    | pathIsProc    = (`TargetProc` pathIsRootSpec) <$> callFindProc schema pathName
-    | pathIsDefSpec = Right $ TargetDefaultSpec schema
-    | otherwise     = Right $ TargetIdent $ QualifiedIdentifier schema pathName
-    where
-      callFindProc procSch procNam = findProc
-        (QualifiedIdentifier procSch procNam) payloadColumns (preferParameters == Just SingleObject) (dbProcs sCache)
-        contentMediaType (action == ActionInvoke InvPost)
-
-  shouldParsePayload = case (action, contentMediaType) of
-    (ActionMutate MutationCreate, _)       -> True
-    (ActionInvoke InvPost, MTUrlEncoded)   -> False
-    (ActionInvoke InvPost, _)              -> True
-    (ActionMutate MutationSingleUpsert, _) -> True
-    (ActionMutate MutationUpdate, _)       -> True
-    _                                      -> False
-  relevantPayload = case (contentMediaType, action) of
-    -- Though ActionInvoke GET/HEAD doesn't really have a payload, we use the payload variable as a way
-    -- to store the query string arguments to the function.
-    (_, ActionInvoke InvGet)             -> targetToJsonRpcParams (rightToMaybe target) qsParams
-    (_, ActionInvoke InvHead)            -> targetToJsonRpcParams (rightToMaybe target) qsParams
-    (MTUrlEncoded, ActionInvoke InvPost) -> targetToJsonRpcParams (rightToMaybe target) $ (T.decodeUtf8 *** T.decodeUtf8) <$> parseSimpleQuery (LBS.toStrict reqBody)
-    _ | shouldParsePayload               -> rightToMaybe payload
-      | otherwise                        -> Nothing
-  method          = requestMethod req
-  hdrs            = requestHeaders req
-  lookupHeader    = flip lookup hdrs
-  Preferences.Preferences{..} = Preferences.fromHeaders hdrs
-  headerRange = rangeRequested hdrs
-  limitRange = fromMaybe allRange (HM.lookup "limit" qsRanges)
-  headerAndLimitRange = rangeIntersection headerRange limitRange
-
-  -- Bypass all the ranges and send only the limit zero range (0 <= x <= -1) if
-  -- limit=0 is present in the query params (not allowed for the Range header)
-  ranges = HM.insert "limit" (convertToLimitZeroRange limitRange headerAndLimitRange) qsRanges
-  -- The only emptyRange allowed is the limit zero range
-  isInvalidRange = topLevelRange == emptyRange && not (hasLimitZero limitRange)
+    columns = case action of
+      ActionMutate MutationCreate -> qsColumns
+      ActionMutate MutationUpdate -> qsColumns
+      ActionInvoke InvPost        -> qsColumns
+      _                           -> Nothing
 
 {-|
   Find the best match from a list of media types accepted by the
@@ -445,90 +386,3 @@ requestMediaTypes conf action path =
       [MTApplicationJSON, MTSingularJSON, MTGeoJSON, MTTextCSV] ++
       [MTPlan $ MTPlanAttrs Nothing PlanJSON mempty | configDbPlanEnabled conf]
     rawMediaTypes = configRawMediaTypes conf `union` [MTOctetStream, MTTextPlain, MTTextXML]
-
-{-|
-  Search a pg proc by matching name and arguments keys to parameters. Since a function can be overloaded,
-  the name is not enough to find it. An overloaded function can have a different volatility or even a different return type.
--}
-findProc :: QualifiedIdentifier -> S.Set Text -> Bool -> ProcsMap -> MediaType -> Bool -> Either ApiRequestError ProcDescription
-findProc qi argumentsKeys paramsAsSingleObject allProcs contentMediaType isInvPost =
-  case matchProc of
-    ([], [])     -> Left $ NoRpc (qiSchema qi) (qiName qi) (S.toList argumentsKeys) paramsAsSingleObject contentMediaType isInvPost
-    -- If there are no functions with named arguments, fallback to the single unnamed argument function
-    ([], [proc]) -> Right proc
-    ([], procs)  -> Left $ AmbiguousRpc (toList procs)
-    -- Matches the functions with named arguments
-    ([proc], _)  -> Right proc
-    (procs, _)   -> Left $ AmbiguousRpc (toList procs)
-  where
-    matchProc = overloadedProcPartition $ HM.lookupDefault mempty qi allProcs -- first find the proc by name
-    -- The partition obtained has the form (overloadedProcs,fallbackProcs)
-    -- where fallbackProcs are functions with a single unnamed parameter
-    overloadedProcPartition = foldr select ([],[])
-    select proc ~(ts,fs)
-      | matchesParams proc         = (proc:ts,fs)
-      | hasSingleUnnamedParam proc = (ts,proc:fs)
-      | otherwise                  = (ts,fs)
-    -- If the function is called with post and has a single unnamed parameter
-    -- it can be called depending on content type and the parameter type
-    hasSingleUnnamedParam ProcDescription{pdParams=[ProcParam{ppType}]} = isInvPost && case (contentMediaType, ppType) of
-      (MTApplicationJSON, "json")  -> True
-      (MTApplicationJSON, "jsonb") -> True
-      (MTTextPlain, "text")        -> True
-      (MTTextXML, "xml")           -> True
-      (MTOctetStream, "bytea")     -> True
-      _                            -> False
-    hasSingleUnnamedParam _ = False
-    matchesParams proc =
-      let
-        params = pdParams proc
-        firstType = (ppType <$> headMay params)
-      in
-      -- exceptional case for Prefer: params=single-object
-      if paramsAsSingleObject
-        then length params == 1 && (firstType == Just "json" || firstType == Just "jsonb")
-      -- If the function has no parameters, the arguments keys must be empty as well
-      else if null params
-        then null argumentsKeys && not (isInvPost && contentMediaType `elem` [MTOctetStream, MTTextPlain, MTTextXML])
-      -- A function has optional and required parameters. Optional parameters have a default value and
-      -- don't require arguments for the function to be executed, required parameters must have an argument present.
-      else case L.partition ppReq params of
-      -- If the function only has required parameters, the arguments keys must match those parameters
-        (reqParams, [])        -> argumentsKeys == S.fromList (ppName <$> reqParams)
-      -- If the function only has optional parameters, the arguments keys can match none or any of them(a subset)
-        ([], optParams)        -> argumentsKeys `S.isSubsetOf` S.fromList (ppName <$> optParams)
-      -- If the function has required and optional parameters, the arguments keys have to match the required parameters
-      -- and can match any or none of the default parameters.
-        (reqParams, optParams) -> argumentsKeys `S.difference` S.fromList (ppName <$> optParams) == S.fromList (ppName <$> reqParams)
-
--- | If raw(binary) output is requested, check that MediaType is one of the
--- admitted rawMediaTypes and that`?select=...` contains only one field other
--- than `*`
-binaryField :: AppConfig -> MediaType -> Target -> QueryParams -> Either ApiRequestError (Maybe FieldName)
-binaryField AppConfig{configRawMediaTypes} acceptMediaType target QueryParams{qsSelect}
-  | returnsScalar target && isRawMediaType =
-      Right $ Just "pgrst_scalar"
-  | isRawMediaType =
-      let
-        fieldName = fstFieldName qsSelect
-      in
-      case fieldName of
-        Just fld -> Right $ Just fld
-        Nothing  -> Left $ BinaryFieldError acceptMediaType
-  | otherwise =
-      Right Nothing
-  where
-    isRawMediaType = acceptMediaType `elem` configRawMediaTypes `union` [MTOctetStream, MTTextPlain, MTTextXML] || isRawPlan acceptMediaType
-    isRawPlan mt = case mt of
-      MTPlan (MTPlanAttrs (Just MTOctetStream) _ _) -> True
-      MTPlan (MTPlanAttrs (Just MTTextPlain) _ _)   -> True
-      MTPlan (MTPlanAttrs (Just MTTextXML) _ _)     -> True
-      _                                             -> False
-    returnsScalar :: Target -> Bool
-    returnsScalar (TargetProc proc _) = procReturnsScalar proc
-    returnsScalar _                   = False
-
-    fstFieldName :: [Tree SelectItem] -> Maybe FieldName
-    fstFieldName [Node SelectField{selField=("*", _)} []] = Nothing
-    fstFieldName [Node SelectField{selField=(fld, _)} []] = Just fld
-    fstFieldName _                                        = Nothing

src/PostgREST/ApiRequest/Preferences.hs

@@ -9,6 +9,7 @@
 module PostgREST.ApiRequest.Preferences
   ( Preferences(..)
   , PreferCount(..)
+  , PreferMissing(..)
   , PreferParameters(..)
   , PreferRepresentation(..)
   , PreferResolution(..)
@@ -33,16 +34,18 @@ import Protolude
 -- >>> deriving instance Show PreferParameters
 -- >>> deriving instance Show PreferCount
 -- >>> deriving instance Show PreferTransaction
+-- >>> deriving instance Show PreferMissing
 -- >>> deriving instance Show Preferences
 
 -- | Preferences recognized by the application.
 data Preferences
   = Preferences
     { preferResolution     :: Maybe PreferResolution
-    , preferRepresentation :: Maybe PreferRepresentation
+    , preferRepresentation :: PreferRepresentation
     , preferParameters     :: Maybe PreferParameters
     , preferCount          :: Maybe PreferCount
     , preferTransaction    :: Maybe PreferTransaction
+    , preferMissing        :: Maybe PreferMissing
     }
 
 -- |
@@ -53,21 +56,23 @@ data Preferences
 -- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates, count=exact")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
---     , preferRepresentation = Nothing
+--     , preferRepresentation = None
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
+--     , preferMissing = Nothing
 --     }
 --
 -- Multiple headers can also be used:
 --
--- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact")]
+-- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
---     , preferRepresentation = Nothing
+--     , preferRepresentation = None
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
+--     , preferMissing = Just ApplyNulls
 --     }
 --
 -- If a preference is set more than once, only the first is used:
@@ -92,23 +97,25 @@ data Preferences
 --
 -- Preferences can be separated by arbitrary amounts of space, lower-case header is also recognized:
 --
--- >>> pPrint $ fromHeaders [("prefer", "count=exact,    tx=commit   ,return=minimal")]
+-- >>> pPrint $ fromHeaders [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default")]
 -- Preferences
 --     { preferResolution = Nothing
---     , preferRepresentation = Just None
+--     , preferRepresentation = Full
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Just Commit
+--     , preferMissing = Just ApplyDefaults
 --     }
 --
 fromHeaders :: [HTTP.Header] -> Preferences
 fromHeaders headers =
   Preferences
     { preferResolution = parsePrefs [MergeDuplicates, IgnoreDuplicates]
-    , preferRepresentation = parsePrefs [Full, None, HeadersOnly]
-    , preferParameters = parsePrefs [SingleObject, MultipleObjects]
+    , preferRepresentation = fromMaybe None $ parsePrefs [Full, None, HeadersOnly]
+    , preferParameters = parsePrefs [SingleObject]
     , preferCount = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
     , preferTransaction = parsePrefs [Commit, Rollback]
+    , preferMissing = parsePrefs [ApplyDefaults, ApplyNulls]
     }
   where
     prefHeaders = filter ((==) HTTP.hPrefer . fst) headers
@@ -169,13 +176,10 @@ instance ToHeaderValue PreferRepresentation where
 -- | How to pass parameters to stored procedures.
 data PreferParameters
   = SingleObject    -- ^ Pass all parameters as a single json object to a stored procedure.
-  | MultipleObjects -- ^ Pass an array of json objects as params to a stored procedure.
   deriving Eq
 
--- TODO: Deprecate params=multiple-objects in next major version
 instance ToHeaderValue PreferParameters where
   toHeaderValue SingleObject    = "params=single-object"
-  toHeaderValue MultipleObjects = "params=multiple-objects"
 
 -- | How to determine the count of (expected) results
 data PreferCount
@@ -204,3 +208,17 @@ instance ToHeaderValue PreferTransaction where
   toHeaderValue Rollback = "tx=rollback"
 
 instance ToAppliedHeader PreferTransaction
+
+-- |
+-- How to handle the insertion/update when the keys specified in ?columns are not present
+-- in the json body.
+data PreferMissing
+  = ApplyDefaults  -- ^ Use the default column value for missing values.
+  | ApplyNulls     -- ^ Use the null value for missing values.
+  deriving Eq
+
+instance ToHeaderValue PreferMissing where
+  toHeaderValue ApplyDefaults = "missing=default"
+  toHeaderValue ApplyNulls    = "missing=null"
+
+instance ToAppliedHeader PreferMissing

src/PostgREST/ApiRequest/QueryParams.hs

@@ -30,14 +30,13 @@ import Data.Ranged.Ranges            (Range (..))
 import Data.Tree                     (Tree (..))
 import Text.Parsec.Error             (errorMessages,
                                       showErrorMessages)
-import Text.Parsec.Prim              (parserFail)
 import Text.ParserCombinators.Parsec (GenParser, ParseError, Parser,
                                       anyChar, between, char, digit,
                                       eof, errorPos, letter,
                                       lookAhead, many1, noneOf,
                                       notFollowedBy, oneOf,
-                                      optionMaybe, sepBy1, string,
-                                      try, (<?>))
+                                      optionMaybe, sepBy, sepBy1,
+                                      string, try, (<?>))
 
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           rangeGeq, rangeLimit,
@@ -46,14 +45,16 @@ import PostgREST.SchemaCache.Identifiers (FieldName)
 
 import PostgREST.ApiRequest.Types (EmbedParam (..), EmbedPath, Field,
                                    Filter (..), FtsOperator (..),
-                                   JoinType (..), JsonOperand (..),
+                                   Hint, JoinType (..),
+                                   JsonOperand (..),
                                    JsonOperation (..), JsonPath,
                                    ListVal, LogicOperator (..),
                                    LogicTree (..), OpExpr (..),
-                                   Operation (..),
+                                   OpQuantifier (..), Operation (..),
                                    OrderDirection (..),
                                    OrderNulls (..), OrderTerm (..),
-                                   QPError (..), SelectItem (..),
+                                   QPError (..), QuantOperator (..),
+                                   SelectItem (..),
                                    SimpleOperator (..), SingleVal,
                                    TrileanVal (..))
 
@@ -66,7 +67,9 @@ import Protolude hiding (try)
 -- >>> deriving instance Show QPError
 -- >>> deriving instance Show TrileanVal
 -- >>> deriving instance Show FtsOperator
+-- >>> deriving instance Show QuantOperator
 -- >>> deriving instance Show SimpleOperator
+-- >>> deriving instance Show OpQuantifier
 -- >>> deriving instance Show Operation
 -- >>> deriving instance Show OpExpr
 -- >>> deriving instance Show JsonOperand
@@ -74,6 +77,11 @@ import Protolude hiding (try)
 -- >>> deriving instance Show Filter
 -- >>> deriving instance Show JoinType
 -- >>> deriving instance Show SelectItem
+-- >>> deriving instance Show OrderDirection
+-- >>> deriving instance Show OrderNulls
+-- >>> deriving instance Show OrderTerm
+-- >>> deriving instance Show LogicOperator
+-- >>> deriving instance Show LogicTree
 
 data QueryParams =
   QueryParams
@@ -108,39 +116,45 @@ data QueryParams =
 --
 -- The canonical representation of the query string has parameters sorted alphabetically:
 --
--- >>> qsCanonical <$> parse "a=1&c=3&b=2&d"
+-- >>> qsCanonical <$> parse True "a=1&c=3&b=2&d"
 -- Right "a=1&b=2&c=3&d="
 --
 -- 'select' is a reserved parameter that selects the fields to be returned:
 --
--- >>> qsSelect <$> parse "select=name,location"
+-- >>> qsSelect <$> parse False "select=name,location"
 -- Right [Node {rootLabel = SelectField {selField = ("name",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectField {selField = ("location",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]
 --
 -- Filters are parameters whose value contains an operator, separated by a '.' from its value:
 --
--- >>> qsFilters <$> parse "a.b=eq.0"
--- Right [(["a"],Filter {field = ("b",[]), opExpr = OpExpr False (Op OpEqual "0")})]
+-- >>> qsFilters <$> parse False "a.b=eq.0"
+-- Right [(["a"],Filter {field = ("b",[]), opExpr = OpExpr False (OpQuant OpEqual Nothing "0")})]
 --
 -- If the operator specified in a filter does not exist, parsing the query string fails:
 --
--- >>> qsFilters <$> parse "a.b=noop.0"
--- Left (QPError "\"failed to parse filter (noop.0)\" (line 1, column 6)" "unknown single value operator noop")
-parse :: ByteString -> Either QPError QueryParams
-parse qs =
-  QueryParams
-    canonical
-    params
-    ranges
-    <$> pRequestOrder `traverse` order
-    <*> pRequestLogicTree `traverse` logic
-    <*> pRequestColumns columns
-    <*> pRequestSelect select
-    <*> pRequestFilter `traverse` filters
-    <*> (fmap snd <$> (pRequestFilter `traverse` filtersRoot))
-    <*> pRequestFilter `traverse` filtersNotRoot
-    <*> pure (S.fromList (fst <$> filters))
-    <*> sequenceA (pRequestOnConflict <$> onConflict)
+-- >>> qsFilters <$> parse False "a.b=noop.0"
+-- Left (QPError "\"failed to parse filter (noop.0)\" (line 1, column 1)" "unexpected \"o\" expecting \"not\" or operator (eq, gt, ...)")
+parse :: Bool -> ByteString -> Either QPError QueryParams
+parse isRpcGet qs = do
+  rOrd                      <- pRequestOrder `traverse` order
+  rLogic                    <- pRequestLogicTree `traverse` logic
+  rCols                     <- pRequestColumns columns
+  rSel                      <- pRequestSelect select
+  (rFlts, params)           <- L.partition hasOp <$> pRequestFilter isRpcGet `traverse` filters
+  (rFltsRoot, rFltsNotRoot) <- pure $ L.partition hasRootFilter rFlts
+  rOnConflict               <- pRequestOnConflict `traverse` onConflict
+
+  let rFltsFields           = S.fromList (fst <$> filters)
+      params'               = mapMaybe (\case {(_, Filter (fld, _) (NoOpExpr v)) -> Just (fld,v); _ -> Nothing}) params
+      rFltsRoot'            = snd <$> rFltsRoot
+
+  return $ QueryParams canonical params' ranges rOrd rLogic rCols rSel rFlts rFltsRoot' rFltsNotRoot rFltsFields rOnConflict
   where
+    hasRootFilter, hasOp :: (EmbedPath, Filter) -> Bool
+    hasRootFilter ([], _) = True
+    hasRootFilter _       = False
+    hasOp (_, Filter (_, _) (NoOpExpr _)) = False
+    hasOp _                               = True
+
     logic = filter (endingIn ["and", "or"] . fst) nonemptyParams
     select = fromMaybe "*" $ lookupParam "select"
     onConflict = lookupParam "on_conflict"
@@ -167,32 +181,11 @@ parse qs =
     endingIn xx key = lastWord `elem` xx
       where lastWord = L.last $ T.split (== '.') key
 
-    (filters, params) = L.partition isParam filtersAndParams
-    isParam (k, v) = isEmbedPath k || hasOperator v || hasFtsOperator v
-
-    filtersAndParams = filter (isFilterOrParam . fst)  nonemptyParams
-    isFilterOrParam k = not (endingIn reservedEmbeddable k) && notElem k reserved
+    filters = filter (isFilter . fst) nonemptyParams
+    isFilter k = not (endingIn reservedEmbeddable k) && notElem k reserved
     reserved = ["select", "columns", "on_conflict"]
     reservedEmbeddable = ["order", "limit", "offset", "and", "or"]
 
-    (filtersNotRoot, filtersRoot) = L.partition isNotRoot filters
-    isNotRoot = flip T.isInfixOf "." . fst
-
-    -- TODO: These checks are redundant to the parsers, should use parsers to differentiate params
-    hasOperator val =
-      case T.splitOn "." val of
-        "not" : _ : _ -> True
-        "is" : _      -> True
-        "in" : _      -> True
-        x : _         -> isJust (operator x) || isJust (ftsOperator x)
-        _             -> False
-
-    hasFtsOperator val =
-      case T.splitOn "(" val of
-        x : _ : _ -> isJust $ ftsOperator x
-        _         -> False
-
-    isEmbedPath = T.isInfixOf "."
     replaceLast x s = T.intercalate "." $ L.init (T.split (=='.') s) <> [x]
 
     ranges :: HM.HashMap Text (Range Integer)
@@ -209,39 +202,31 @@ parse qs =
         offsetParams =
           HM.fromList [(k, maybe allRange rangeGeq (readMaybe v)) | (k,v) <- offsets]
 
-operator :: Text -> Maybe SimpleOperator
-operator = \case
-  "eq"     -> Just OpEqual
-  "gte"    -> Just OpGreaterThanEqual
-  "gt"     -> Just OpGreaterThan
-  "lte"    -> Just OpLessThanEqual
-  "lt"     -> Just OpLessThan
-  "neq"    -> Just OpNotEqual
-  "like"   -> Just OpLike
-  "ilike"  -> Just OpILike
-  "cs"     -> Just OpContains
-  "cd"     -> Just OpContained
-  "ov"     -> Just OpOverlap
-  "sl"     -> Just OpStrictlyLeft
-  "sr"     -> Just OpStrictlyRight
-  "nxr"    -> Just OpNotExtendsRight
-  "nxl"    -> Just OpNotExtendsLeft
-  "adj"    -> Just OpAdjacent
-  "match"  -> Just OpMatch
-  "imatch" -> Just OpIMatch
-  _        -> Nothing
-
-ftsOperator :: Text -> Maybe FtsOperator
-ftsOperator = \case
-  "fts"   -> Just FilterFts
-  "plfts" -> Just FilterFtsPlain
-  "phfts" -> Just FilterFtsPhrase
-  "wfts"  -> Just FilterFtsWebsearch
-  _       -> Nothing
-
-
--- PARSERS
+simpleOperator :: Parser SimpleOperator
+simpleOperator =
+  try (string "neq" $> OpNotEqual) <|>
+  try (string "cs" $> OpContains) <|>
+  try (string "cd" $> OpContained) <|>
+  try (string "ov" $> OpOverlap) <|>
+  try (string "sl" $> OpStrictlyLeft) <|>
+  try (string "sr" $> OpStrictlyRight) <|>
+  try (string "nxr" $> OpNotExtendsRight) <|>
+  try (string "nxl" $> OpNotExtendsLeft) <|>
+  try (string "adj" $> OpAdjacent) <?>
+  "unknown single value operator"
 
+quantOperator :: Parser QuantOperator
+quantOperator =
+  try (string "eq" $> OpEqual) <|>
+  try (string "gte" $> OpGreaterThanEqual) <|>
+  try (string "gt" $> OpGreaterThan) <|>
+  try (string "lte" $> OpLessThanEqual) <|>
+  try (string "lt" $> OpLessThan) <|>
+  try (string "like" $> OpLike) <|>
+  try (string "ilike" $> OpILike) <|>
+  try (string "match" $> OpMatch) <|>
+  try (string "imatch" $> OpIMatch) <?>
+  "unknown single value operator"
 
 pRequestSelect :: Text -> Either QPError [Tree SelectItem]
 pRequestSelect selStr =
@@ -251,11 +236,25 @@ pRequestOnConflict :: Text -> Either QPError [FieldName]
 pRequestOnConflict oncStr =
   mapError $ P.parse pColumns ("failed to parse on_conflict parameter (" <> toS oncStr <> ")") (toS oncStr)
 
-pRequestFilter :: (Text, Text) -> Either QPError (EmbedPath, Filter)
-pRequestFilter (k, v) = mapError $ (,) <$> path <*> (Filter <$> fld <*> oper)
+-- |
+-- Parse `id=eq.1`(id, eq.1) into (EmbedPath, Filter)
+--
+-- >>> pRequestFilter False ("id", "eq.1")
+-- Right ([],Filter {field = ("id",[]), opExpr = OpExpr False (OpQuant OpEqual Nothing "1")})
+--
+-- >>> pRequestFilter False ("id", "val")
+-- Left (QPError "\"failed to parse filter (val)\" (line 1, column 1)" "unexpected \"v\" expecting \"not\" or operator (eq, gt, ...)")
+--
+-- >>> pRequestFilter True ("id", "val")
+-- Right ([],Filter {field = ("id",[]), opExpr = NoOpExpr "val"})
+pRequestFilter :: Bool -> (Text, Text) -> Either QPError (EmbedPath, Filter)
+pRequestFilter isRpcGet (k, v) = mapError $ (,) <$> path <*> (Filter <$> fld <*> oper)
   where
     treePath = P.parse pTreePath ("failed to parse tree path (" ++ toS k ++ ")") $ toS k
-    oper = P.parse (pOpExpr pSingleVal) ("failed to parse filter (" ++ toS v ++ ")") $ toS v
+    oper = P.parse parseFlt ("failed to parse filter (" ++ toS v ++ ")") $ toS v
+    parseFlt = if isRpcGet
+      then pOpExpr pSingleVal <|> pure (NoOpExpr v)
+      else pOpExpr pSingleVal
     path = fst <$> treePath
     fld = snd <$> treePath
 
@@ -314,20 +313,28 @@ pTreePath = do
 -- >>> P.parse pFieldForest "" "*,client(*,nested(*))"
 -- Right [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectRelation {selRelation = "client", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectRelation {selRelation = "nested", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]}]}]
 --
+-- >>> P.parse pFieldForest "" "*,...client(*),other(*)"
+-- Right [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SpreadRelation {selRelation = "client", selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]},Node {rootLabel = SelectRelation {selRelation = "other", selAlias = Nothing, selHint = Nothing, selJoinType = Nothing}, subForest = [Node {rootLabel = SelectField {selField = ("*",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]}]
+--
+-- >>> P.parse pFieldForest "" ""
+-- Right []
+--
 -- >>> P.parse pFieldForest "" "id,clients(name[])"
 -- Left (line 1, column 16):
 -- unexpected '['
--- expecting letter, digit, "-", "!", "(", "->>", "->", "::", ")", "," or end of input
+-- expecting letter, digit, "-", "->>", "->", "::", ")", "," or end of input
+--
+-- >>> P.parse pFieldForest "" "data->>-78xy"
+-- Left (line 1, column 11):
+-- unexpected 'x'
+-- expecting digit, "->", "::", ".", "," or end of input
 pFieldForest :: Parser [Tree SelectItem]
-pFieldForest = pFieldTree `sepBy1` lexeme (char ',')
+pFieldForest = pFieldTree `sepBy` lexeme (char ',')
   where
-    pFieldTree :: Parser (Tree SelectItem)
-    pFieldTree =  try (Node <$> pRelationSelect <*> between (char '(') (char ')') pFieldForest) <|>
+    pFieldTree =  Node <$> try pSpreadRelationSelect <*> between (char '(') (char ')') pFieldForest <|>
+                  Node <$> try pRelationSelect       <*> between (char '(') (char ')') pFieldForest <|>
                   Node <$> pFieldSelect <*> pure []
 
-pStar :: Parser Text
-pStar = string "*" $> "*"
-
 -- |
 -- Parse field names
 --
@@ -393,6 +400,23 @@ pFieldName =
 --
 -- >>> P.parse pJsonPath "" "->0.desc"
 -- Right [JArrow {jOp = JIdx {jVal = "+0"}}]
+--
+-- Fails on badly formed negatives
+--
+-- >>> P.parse pJsonPath "" "->>-78xy"
+-- Left (line 1, column 7):
+-- unexpected 'x'
+-- expecting digit, "->", "::", ".", "," or end of input
+--
+-- >>> P.parse pJsonPath "" "->>--34"
+-- Left (line 1, column 5):
+-- unexpected "-"
+-- expecting digit
+--
+-- >>> P.parse pJsonPath "" "->>-xy-4"
+-- Left (line 1, column 5):
+-- unexpected "x"
+-- expecting digit
 pJsonPath :: Parser JsonPath
 pJsonPath = many pJsonOperation
   where
@@ -448,27 +472,12 @@ aliasSeparator = char ':' >> notFollowedBy (char ':')
 -- Left (line 1, column 6):
 -- unexpected '>'
 pRelationSelect :: Parser SelectItem
-pRelationSelect = lexeme $ try ( do
+pRelationSelect = lexeme $ do
     alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
     name <- pFieldName
-    prm1 <- optionMaybe pEmbedParam
-    prm2 <- optionMaybe pEmbedParam
+    (hint, jType) <- pEmbedParams
     try (void $ lookAhead (string "("))
-    return $ SelectRelation name alias (embedParamHint prm1 <|> embedParamHint prm2) (embedParamJoin prm1 <|> embedParamJoin prm2)
-  )
-  where
-    pEmbedParam :: Parser EmbedParam
-    pEmbedParam =
-      char '!' *> (
-        try (string "left"  $> EPJoinType JTLeft)  <|>
-        try (string "inner" $> EPJoinType JTInner) <|>
-        try (EPHint <$> pFieldName))
-    embedParamHint prm = case prm of
-      Just (EPHint hint) -> Just hint
-      _                  -> Nothing
-    embedParamJoin prm = case prm of
-      Just (EPJoinType jt) -> Just jt
-      _                    -> Nothing
+    return $ SelectRelation name alias hint jType
 
 -- |
 -- Parse regular fields in select
@@ -506,43 +515,123 @@ pRelationSelect = lexeme $ try ( do
 -- unexpected end of input
 -- expecting letter or digit
 pFieldSelect :: Parser SelectItem
-pFieldSelect = lexeme $
-  try (
-    do
-      alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
-      fld <- pField
-      cast' <- optionMaybe (string "::" *> pIdentifier)
-      pEnd
-      return $ SelectField fld (toS <$> cast') alias
-  )
-  <|> do
+pFieldSelect = lexeme $ try (do
     s <- pStar
     pEnd
-    return $ SelectField (s, []) Nothing Nothing
+    return $ SelectField (s, []) Nothing Nothing)
+  <|> do
+    alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
+    fld <- pField
+    cast' <- optionMaybe (string "::" *> pIdentifier)
+    pEnd
+    return $ SelectField fld (toS <$> cast') alias
   where
     pEnd = try (void $ lookAhead (string ")")) <|>
            try (void $ lookAhead (string ",")) <|>
            try eof
+    pStar = string "*" $> "*"
+
+
+-- |
+-- Parse spread relations in select
+--
+-- >>> P.parse pSpreadRelationSelect "" "...rel(*)"
+-- Right (SpreadRelation {selRelation = "rel", selHint = Nothing, selJoinType = Nothing})
+--
+-- >>> P.parse pSpreadRelationSelect "" "...rel!hint!inner(*)"
+-- Right (SpreadRelation {selRelation = "rel", selHint = Just "hint", selJoinType = Just JTInner})
+--
+-- >>> P.parse pSpreadRelationSelect "" "rel(*)"
+-- Left (line 1, column 1):
+-- unexpected "r"
+-- expecting "..."
+--
+-- >>> P.parse pSpreadRelationSelect "" "alias:...rel(*)"
+-- Left (line 1, column 1):
+-- unexpected "a"
+-- expecting "..."
+--
+-- >>> P.parse pSpreadRelationSelect "" "...rel->jsonpath(*)"
+-- Left (line 1, column 9):
+-- unexpected '>'
+pSpreadRelationSelect :: Parser SelectItem
+pSpreadRelationSelect = lexeme $ do
+    name <- string "..." >> pFieldName
+    (hint, jType) <- pEmbedParams
+    try (void $ lookAhead (string "("))
+    return $ SpreadRelation name hint jType
+
+pEmbedParams :: Parser (Maybe Hint, Maybe JoinType)
+pEmbedParams = do
+  prm1 <- optionMaybe pEmbedParam
+  prm2 <- optionMaybe pEmbedParam
+  return (embedParamHint prm1 <|> embedParamHint prm2, embedParamJoin prm1 <|> embedParamJoin prm2)
+  where
+    pEmbedParam :: Parser EmbedParam
+    pEmbedParam =
+      char '!' *> (
+        try (string "left"  $> EPJoinType JTLeft)  <|>
+        try (string "inner" $> EPJoinType JTInner) <|>
+        try (EPHint <$> pFieldName))
+    embedParamHint prm = case prm of
+      Just (EPHint hint) -> Just hint
+      _                  -> Nothing
+    embedParamJoin prm = case prm of
+      Just (EPJoinType jt) -> Just jt
+      _                    -> Nothing
 
 -- |
 -- Parse operator expression used in horizontal filtering
 --
 -- >>> P.parse (pOpExpr pSingleVal) "" "fts().value"
--- Left (line 1, column 7):
--- unknown single value operator fts()
+-- Left (line 1, column 5):
+-- unexpected ")"
+-- expecting operator (eq, gt, ...)
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "eq(any).value"
+-- Right (OpExpr False (OpQuant OpEqual (Just QuantAny) "value"))
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "eq(all).value"
+-- Right (OpExpr False (OpQuant OpEqual (Just QuantAll) "value"))
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "not.eq(all).value"
+-- Right (OpExpr True (OpQuant OpEqual (Just QuantAll) "value"))
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "eq().value"
+-- Left (line 1, column 4):
+-- unexpected ")"
+-- expecting operator (eq, gt, ...)
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "is().value"
+-- Left (line 1, column 3):
+-- unexpected "("
+-- expecting operator (eq, gt, ...)
+--
+-- >>> P.parse (pOpExpr pSingleVal) "" "in().value"
+-- Left (line 1, column 3):
+-- unexpected "("
+-- expecting operator (eq, gt, ...)
 pOpExpr :: Parser SingleVal -> Parser OpExpr
-pOpExpr pSVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)) <|> OpExpr False <$> pOperation
+pOpExpr pSVal = do
+  boolExpr <- try (string "not" *> pDelimiter $> True) <|> pure False
+  OpExpr boolExpr <$> pOperation
   where
     pOperation :: Parser Operation
-    pOperation = pIn <|> pIs <|> try pFts <|> pOp <?> "operator (eq, gt, ...)"
+    pOperation = pIn <|> pIs <|> pIsDist <|> try pFts <|> try pSimpleOp <|> try pQuantOp <?> "operator (eq, gt, ...)"
 
     pIn = In <$> (try (string "in" *> pDelimiter) *> pListVal)
     pIs = Is <$> (try (string "is" *> pDelimiter) *> pTriVal)
 
-    pOp = do
-      opStr <- try (P.manyTill anyChar (try pDelimiter))
-      op <- parseMaybe ("unknown single value operator " <> opStr) . operator $ toS opStr
-      Op op <$> pSVal
+    pIsDist = IsDistinctFrom <$> (try (string "isdistinct" *> pDelimiter) *> pSVal)
+
+    pSimpleOp = do
+      op <- simpleOperator
+      pDelimiter *> (Op op <$> pSVal)
+
+    pQuantOp = do
+      op <- quantOperator
+      quant <- optionMaybe $ try (between (char '(') (char ')') (try (string "any" $> QuantAny) <|> string "all" $> QuantAll))
+      pDelimiter *> (OpQuant op quant <$> pSVal)
 
     pTriVal = try (ciString "null"    $> TriNull)
           <|> try (ciString "unknown" $> TriUnknown)
@@ -551,15 +640,14 @@ pOpExpr pSVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)
           <?> "null or trilean value (unknown, true, false)"
 
     pFts = do
-      opStr <- try (P.many (noneOf ".("))
-      op <- parseMaybe ("unknown fts operator " <> opStr) . ftsOperator $ toS opStr
+      op <-  try (string "fts"   $> FilterFts)
+         <|> try (string "plfts" $> FilterFtsPlain)
+         <|> try (string "phfts" $> FilterFtsPhrase)
+         <|> try (string "wfts"  $> FilterFtsWebsearch)
+
       lang <- optionMaybe $ try (between (char '(') (char ')') pIdentifier)
       pDelimiter >> Fts op (toS <$> lang) <$> pSVal
 
-    parseMaybe :: [Char] -> Maybe a -> Parser a
-    parseMaybe err Nothing = parserFail err
-    parseMaybe _ (Just x)  = pure x
-
     -- case insensitive char and string
     ciChar :: Char -> GenParser Char state Char
     ciChar c = char c <|> char (toUpper c)
@@ -583,24 +671,119 @@ pQuotedValue = toS <$> (char '"' *> many pCharsOrSlashed <* char '"')
 pDelimiter :: Parser Char
 pDelimiter = char '.' <?> "delimiter (.)"
 
+-- |
+-- Parses the elements in the order query parameter
+--
+-- >>> P.parse pOrder "" "name.desc.nullsfirst"
+-- Right [OrderTerm {otTerm = ("name",[]), otDirection = Just OrderDesc, otNullOrder = Just OrderNullsFirst}]
+--
+-- >>> P.parse pOrder "" "json_col->key.asc.nullslast"
+-- Right [OrderTerm {otTerm = ("json_col",[JArrow {jOp = JKey {jVal = "key"}}]), otDirection = Just OrderAsc, otNullOrder = Just OrderNullsLast}]
+--
+-- >>> P.parse pOrder "" "clients(json_col->key).desc.nullsfirst"
+-- Right [OrderRelationTerm {otRelation = "clients", otRelTerm = ("json_col",[JArrow {jOp = JKey {jVal = "key"}}]), otDirection = Just OrderDesc, otNullOrder = Just OrderNullsFirst}]
+--
+-- >>> P.parse pOrder "" "clients(name,id)"
+-- Left (line 1, column 8):
+-- unexpected '('
+-- expecting letter, digit, "-", "->>", "->", delimiter (.), "," or end of input
+--
+-- >>> P.parse pOrder "" "name,clients(name),id"
+-- Right [OrderTerm {otTerm = ("name",[]), otDirection = Nothing, otNullOrder = Nothing},OrderRelationTerm {otRelation = "clients", otRelTerm = ("name",[]), otDirection = Nothing, otNullOrder = Nothing},OrderTerm {otTerm = ("id",[]), otDirection = Nothing, otNullOrder = Nothing}]
+--
+-- >>> P.parse pOrder "" "id.ac"
+-- Left (line 1, column 4):
+-- unexpected "c"
+-- expecting "asc", "desc", "nullsfirst" or "nullslast"
+--
+-- >>> P.parse pOrder "" "id.descc"
+-- Left (line 1, column 8):
+-- unexpected 'c'
+-- expecting delimiter (.), "," or end of input
+--
+-- >>> P.parse pOrder "" "id.nulsfist"
+-- Left (line 1, column 4):
+-- unexpected "n"
+-- expecting "asc", "desc", "nullsfirst" or "nullslast"
+--
+-- >>> P.parse pOrder "" "id.nullslasttt"
+-- Left (line 1, column 13):
+-- unexpected 't'
+-- expecting "," or end of input
+--
+-- >>> P.parse pOrder "" "id.smth34"
+-- Left (line 1, column 4):
+-- unexpected "s"
+-- expecting "asc", "desc", "nullsfirst" or "nullslast"
+--
+-- >>> P.parse pOrder "" "id.asc.nlsfst"
+-- Left (line 1, column 8):
+-- unexpected "l"
+-- expecting "nullsfirst" or "nullslast"
+--
+-- >>> P.parse pOrder "" "id.asc.nullslasttt"
+-- Left (line 1, column 17):
+-- unexpected 't'
+-- expecting "," or end of input
+--
+-- >>> P.parse pOrder "" "id.asc.smth34"
+-- Left (line 1, column 8):
+-- unexpected "s"
+-- expecting "nullsfirst" or "nullslast"
 pOrder :: Parser [OrderTerm]
-pOrder = lexeme pOrderTerm `sepBy1` char ','
-
-pOrderTerm :: Parser OrderTerm
-pOrderTerm = do
-  fld <- pField
-  dir <- optionMaybe $
-         try (pDelimiter *> string "asc" $> OrderAsc) <|>
-         try (pDelimiter *> string "desc" $> OrderDesc)
-  nls <- optionMaybe pNulls <* pEnd <|>
-         pEnd $> Nothing
-  return $ OrderTerm fld dir nls
+pOrder = lexeme (try pOrderRelationTerm <|> pOrderTerm) `sepBy1` char ','
   where
+    pOrderTerm = do
+      fld <- pField
+      dir <- optionMaybe pOrdDir
+      nls <- optionMaybe pNulls <* pEnd <|>
+             pEnd $> Nothing
+      return $ OrderTerm fld dir nls
+
+    pOrderRelationTerm = do
+      nam <- pFieldName
+      fld <- between (char '(') (char ')') pField
+      dir <- optionMaybe pOrdDir
+      nls <- optionMaybe pNulls <* pEnd <|> pEnd $> Nothing
+      return $ OrderRelationTerm nam fld dir nls
+
+    pNulls :: Parser OrderNulls
     pNulls = try (pDelimiter *> string "nullsfirst" $> OrderNullsFirst) <|>
              try (pDelimiter *> string "nullslast"  $> OrderNullsLast)
-    pEnd = try (void $ lookAhead (char ',')) <|>
-           try eof
 
+    pOrdDir :: Parser OrderDirection
+    pOrdDir = try (pDelimiter *> string "asc" $> OrderAsc) <|>
+              try (pDelimiter *> string "desc" $> OrderDesc)
+
+    pEnd = try (void $ lookAhead (char ',')) <|> try eof
+
+-- |
+-- Parses the elements inside or/and
+--
+-- >>> P.parse pLogicTree "" "or()"
+-- Left (line 1, column 4):
+-- unexpected ")"
+-- expecting field name (* or [a..z0..9_$]), negation operator (not) or logic operator (and, or)
+--
+-- >>> P.parse pLogicTree "" "or(id.in.1,2,id.eq.3)"
+-- Left (line 1, column 10):
+-- unexpected "1"
+-- expecting "("
+--
+-- >>> P.parse pLogicTree "" "or)("
+-- Left (line 1, column 3):
+-- unexpected ")"
+-- expecting "("
+--
+-- >>> P.parse pLogicTree "" "and(ord(id.eq.1,id.eq.1),id.eq.2)"
+-- Left (line 1, column 7):
+-- unexpected "d"
+-- expecting "("
+--
+-- >>> P.parse pLogicTree "" "or(id.eq.1,not.xor(id.eq.2,id.eq.3))"
+-- Left (line 1, column 16):
+-- unexpected "x"
+-- expecting logic operator (and, or)
 pLogicTree :: Parser LogicTree
 pLogicTree = Stmnt <$> try pLogicFilter
              <|> Expr <$> pNot <*> pLogicOp <*> (lexeme (char '(') *> pLogicTree `sepBy1` lexeme (char ',') <* lexeme (char ')'))

src/PostgREST/ApiRequest/Types.hs

@@ -19,6 +19,7 @@ module PostgREST.ApiRequest.Types
   , NodeName
   , OpExpr(..)
   , Operation (..)
+  , OpQuantifier(..)
   , OrderDirection(..)
   , OrderNulls(..)
   , OrderTerm(..)
@@ -27,35 +28,45 @@ module PostgREST.ApiRequest.Types
   , SingleVal
   , TrileanVal(..)
   , SimpleOperator(..)
+  , QuantOperator(..)
   , FtsOperator(..)
   , SelectItem(..)
   ) where
 
 import PostgREST.MediaType                (MediaType (..))
-import PostgREST.SchemaCache.Identifiers  (FieldName)
-import PostgREST.SchemaCache.Proc         (ProcDescription (..))
-import PostgREST.SchemaCache.Relationship (Relationship)
+import PostgREST.SchemaCache.Identifiers  (FieldName,
+                                           QualifiedIdentifier)
+import PostgREST.SchemaCache.Relationship (Relationship,
+                                           RelationshipsMap)
+import PostgREST.SchemaCache.Routine      (Routine (..))
 
 import Protolude
 
--- | The select value in `/tbl?select=alias:field::cast`
+-- | The value in `/tbl?select=alias:field::cast`
 data SelectItem
   = SelectField
     { selField :: Field
     , selCast  :: Maybe Cast
     , selAlias :: Maybe Alias
     }
+-- | The value in `/tbl?select=alias:another_tbl(*)`
   | SelectRelation
     { selRelation :: FieldName
     , selAlias    :: Maybe Alias
     , selHint     :: Maybe Hint
     , selJoinType :: Maybe JoinType
     }
+-- | The value in `/tbl?select=...another_tbl(*)`
+  | SpreadRelation
+    { selRelation :: FieldName
+    , selHint     :: Maybe Hint
+    , selJoinType :: Maybe JoinType
+    }
   deriving (Eq)
 
 data ApiRequestError
   = AmbiguousRelBetween Text Text [Relationship]
-  | AmbiguousRpc [ProcDescription]
+  | AmbiguousRpc [Routine]
   | BinaryFieldError MediaType
   | MediaTypeError [ByteString]
   | InvalidBody ByteString
@@ -64,14 +75,17 @@ data ApiRequestError
   | InvalidRpcMethod ByteString
   | LimitNoOrderError
   | NotFound
-  | NoRelBetween Text Text Text
-  | NoRpc Text Text [Text] Bool MediaType Bool
+  | NoRelBetween Text Text (Maybe Text) Text RelationshipsMap
+  | NoRpc Text Text [Text] Bool MediaType Bool [QualifiedIdentifier] [Routine]
   | NotEmbedded Text
-  | ParseRequestError Text Text
-  | PutRangeNotAllowedError
+  | PutLimitNotAllowedError
   | QueryParamError QPError
+  | RelatedOrderNotToOne Text Text
+  | SpreadNotToOne Text Text
+  | UnacceptableFilter Text
   | UnacceptableSchema [Text]
   | UnsupportedMethod ByteString
+  | ColumnNotFound Text Text
 
 data QPError = QPError Text Text
 data RangeError
@@ -82,12 +96,19 @@ data RangeError
 type NodeName = Text
 type Depth = Integer
 
-data OrderTerm = OrderTerm
-  { otTerm      :: Field
-  , otDirection :: Maybe OrderDirection
-  , otNullOrder :: Maybe OrderNulls
-  }
-  deriving (Eq)
+data OrderTerm
+  = OrderTerm
+    { otTerm      :: Field
+    , otDirection :: Maybe OrderDirection
+    , otNullOrder :: Maybe OrderNulls
+    }
+  | OrderRelationTerm
+    { otRelation  :: FieldName
+    , otRelTerm   :: Field
+    , otDirection :: Maybe OrderDirection
+    , otNullOrder :: Maybe OrderNulls
+    }
+  deriving Eq
 
 data OrderDirection
   = OrderAsc
@@ -128,7 +149,7 @@ type JsonPath = [JsonOperation]
 data JsonOperation
   = JArrow { jOp :: JsonOperand }
   | J2Arrow { jOp :: JsonOperand }
-  deriving (Eq)
+  deriving (Eq, Ord)
 
 -- | Represents the key(`->'key'`) or index(`->'1`::int`), the index is Text
 -- because we reuse our escaping functons and let pg do the casting with
@@ -136,7 +157,7 @@ data JsonOperation
 data JsonOperand
   = JKey { jVal :: Text }
   | JIdx { jVal :: Text }
-  deriving (Eq)
+  deriving (Eq, Ord)
 
 -- | Boolean logic expression tree e.g. "and(name.eq.N,or(id.eq.1,id.eq.2))" is:
 --
@@ -155,20 +176,28 @@ data LogicOperator
   | Or
   deriving Eq
 
-data Filter = Filter
+data Filter
+  = Filter
   { field  :: Field
   , opExpr :: OpExpr
   }
+  | FilterNullEmbed Bool FieldName
   deriving (Eq)
 
-data OpExpr =
-  OpExpr Bool Operation
+data OpExpr
+  = OpExpr Bool Operation
+  | NoOpExpr Text
   deriving (Eq)
 
+data OpQuantifier = QuantAny | QuantAll
+  deriving Eq
+
 data Operation
   = Op SimpleOperator SingleVal
+  | OpQuant QuantOperator (Maybe OpQuantifier) SingleVal
   | In ListVal
   | Is TrileanVal
+  | IsDistinctFrom SingleVal
   | Fts FtsOperator (Maybe Language) SingleVal
   deriving (Eq)
 
@@ -188,15 +217,21 @@ data TrileanVal
   | TriUnknown
   deriving Eq
 
-data SimpleOperator
+-- Operators that are quantifiable, i.e. they can be used with the any/all modifiers
+data QuantOperator
   = OpEqual
   | OpGreaterThanEqual
   | OpGreaterThan
   | OpLessThanEqual
   | OpLessThan
-  | OpNotEqual
   | OpLike
   | OpILike
+  | OpMatch
+  | OpIMatch
+  deriving Eq
+
+data SimpleOperator
+  = OpNotEqual
   | OpContains
   | OpContained
   | OpOverlap
@@ -205,10 +240,9 @@ data SimpleOperator
   | OpNotExtendsRight
   | OpNotExtendsLeft
   | OpAdjacent
-  | OpMatch
-  | OpIMatch
   deriving Eq
 
+--
 -- | Operators for full text search operators
 data FtsOperator
   = FilterFts

src/PostgREST/App.hs

@@ -9,6 +9,7 @@ Some of its functionality includes:
 - Producing HTTP Headers according to RFCs.
 - Content Negotiation
 -}
+{-# LANGUAGE LambdaCase      #-}
 {-# LANGUAGE RecordWildCards #-}
 module PostgREST.App
   ( SignalHandlerInstaller
@@ -19,13 +20,15 @@ module PostgREST.App
 
 
 import Control.Monad.Except     (liftEither)
-import Data.Either.Combinators  (mapLeft)
+import Data.Either.Combinators  (mapLeft, whenLeft)
 import Data.Maybe               (fromJust)
 import Data.String              (IsString (..))
 import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
                                  setServerName)
 import System.Posix.Types       (FileMode)
 
+import qualified Data.HashMap.Strict        as HM
+import qualified Hasql.Pool                 as SQL
 import qualified Hasql.Transaction.Sessions as SQL
 import qualified Network.Wai                as Wai
 import qualified Network.Wai.Handler.Warp   as Warp
@@ -42,16 +45,17 @@ import qualified PostgREST.Query            as Query
 import qualified PostgREST.Response         as Response
 import qualified PostgREST.Workers          as Workers
 
-import PostgREST.ApiRequest       (Action (..), ApiRequest (..),
-                                   Mutation (..), Target (..))
-import PostgREST.AppState         (AppState)
-import PostgREST.Auth             (AuthResult (..))
-import PostgREST.Config           (AppConfig (..), LogLevel (..))
-import PostgREST.Config.PgVersion (PgVersion (..))
-import PostgREST.Error            (Error)
-import PostgREST.Query            (DbHandler)
-import PostgREST.SchemaCache      (SchemaCache (..))
-import PostgREST.Version          (prettyVersion)
+import PostgREST.ApiRequest          (Action (..), ApiRequest (..),
+                                      Mutation (..), Target (..))
+import PostgREST.AppState            (AppState)
+import PostgREST.Auth                (AuthResult (..))
+import PostgREST.Config              (AppConfig (..))
+import PostgREST.Config.PgVersion    (PgVersion (..))
+import PostgREST.Error               (Error)
+import PostgREST.Query               (DbHandler)
+import PostgREST.SchemaCache         (SchemaCache (..))
+import PostgREST.SchemaCache.Routine (Routine (..))
+import PostgREST.Version             (prettyVersion)
 
 import Protolude hiding (Handler)
 
@@ -71,7 +75,7 @@ run installHandlers maybeRunWithSocket appState = do
 
   Workers.runAdmin conf appState $ serverSettings conf
 
-  let app = postgrest configLogLevel appState (Workers.connectionWorker appState)
+  let app = postgrest conf appState (Workers.connectionWorker appState)
 
   case configServerUnixSocket of
     Just socket ->
@@ -95,25 +99,25 @@ serverSettings AppConfig{..} =
     & setServerName ("postgrest/" <> prettyVersion)
 
 -- | PostgREST application
-postgrest :: LogLevel -> AppState.AppState -> IO () -> Wai.Application
-postgrest logLevel appState connWorker =
+postgrest :: AppConfig -> AppState.AppState -> IO () -> Wai.Application
+postgrest conf appState connWorker =
+  Response.traceHeaderMiddleware conf .
   Cors.middleware .
   Auth.middleware appState .
-  Logger.middleware logLevel $
+  Logger.middleware (configLogLevel conf) $
     -- fromJust can be used, because the auth middleware will **always** add
     -- some AuthResult to the vault.
     \req respond -> case fromJust $ Auth.getResult req of
       Left err -> respond $ Error.errorResponseFor err
       Right authResult -> do
-        conf <- AppState.getConfig appState
+        appConf <- AppState.getConfig appState -- the config must be read again because it can reload
         maybeSchemaCache <- AppState.getSchemaCache appState
         pgVer <- AppState.getPgVersion appState
-        jsonDbS <- AppState.getJsonDbS appState
 
         let
           eitherResponse :: IO (Either Error Wai.Response)
           eitherResponse =
-            runExceptT $ postgrestResponse appState conf maybeSchemaCache jsonDbS pgVer authResult req
+            runExceptT $ postgrestResponse appState appConf maybeSchemaCache pgVer authResult req
 
         response <- either Error.errorResponseFor identity <$> eitherResponse
         -- Launch the connWorker when the connection is down.  The postgrest
@@ -129,12 +133,11 @@ postgrestResponse
   :: AppState.AppState
   -> AppConfig
   -> Maybe SchemaCache
-  -> ByteString
   -> PgVersion
   -> AuthResult
   -> Wai.Request
   -> Handler IO Wai.Response
-postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jsonDbS pgVer authResult@AuthResult{..} req = do
+postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@AuthResult{..} req = do
   sCache <-
     case maybeSchemaCache of
       Just sCache ->
@@ -146,69 +149,89 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jsonDbS pgVer aut
 
   apiRequest <-
     liftEither . mapLeft Error.ApiRequestError $
-      ApiRequest.userApiRequest conf sCache req body
+      ApiRequest.userApiRequest conf req body
 
   Response.optionalRollback conf apiRequest $
-    handleRequest authResult conf appState (Query.txMode apiRequest) (Just authRole /= configDbAnonRole) configDbPreparedStatements jsonDbS pgVer apiRequest sCache
+    handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache
 
-runDbHandler :: AppState.AppState -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
-runDbHandler appState mode authenticated prepared handler = do
-  dbResp <-
-    let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
-    lift . AppState.usePool appState . transaction SQL.ReadCommitted mode $ runExceptT handler
+runDbHandler :: AppState.AppState -> Maybe Text -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
+runDbHandler appState isoLvl mode authenticated prepared handler = do
+  dbResp <- lift $ do
+    let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction
+    res <- AppState.usePool appState . transaction (toIsolationLevel isoLvl) mode $ runExceptT handler
+    whenLeft res (\case
+      SQL.AcquisitionTimeoutUsageError -> AppState.debounceLogAcquisitionTimeout appState -- this can happen rapidly for many requests, so we debounce
+      _ -> pure ())
+    return res
 
   resp <-
     liftEither . mapLeft Error.PgErr $
       mapLeft (Error.PgError authenticated) dbResp
 
   liftEither resp
+  where
+    toIsolationLevel = \case
+      Nothing                -> SQL.ReadCommitted
+      Just "repeatable read" -> SQL.RepeatableRead
+      Just "serializable"    -> SQL.Serializable
+      _                      -> SQL.ReadCommitted
 
-handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> SQL.Mode -> Bool -> Bool -> ByteString -> PgVersion -> ApiRequest -> SchemaCache -> Handler IO Wai.Response
-handleRequest AuthResult{..} conf appState mode authenticated prepared jsonDbS pgVer apiReq@ApiRequest{..} sCache =
+handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> Bool -> Bool -> PgVersion -> ApiRequest -> SchemaCache -> Handler IO Wai.Response
+handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@ApiRequest{..} sCache =
   case (iAction, iTarget) of
     (ActionRead headersOnly, TargetIdent identifier) -> do
-      rPlan <- liftEither $ Plan.readPlan identifier conf sCache apiReq
-      resultSet <- runQuery $ Query.readQuery rPlan conf apiReq
+      wrPlan <- liftEither $ Plan.wrappedReadPlan identifier conf sCache apiReq
+      resultSet <- runQuery roleIsoLvl (Plan.wrTxMode wrPlan) $ Query.readQuery wrPlan conf apiReq
       return $ Response.readResponse headersOnly identifier apiReq resultSet
 
     (ActionMutate MutationCreate, TargetIdent identifier) -> do
       mrPlan <- liftEither $ Plan.mutateReadPlan MutationCreate apiReq identifier conf sCache
-      resultSet <- runQuery $ Query.createQuery mrPlan apiReq conf
+      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.createQuery mrPlan apiReq conf
       return $ Response.createResponse identifier mrPlan apiReq resultSet
 
     (ActionMutate MutationUpdate, TargetIdent identifier) -> do
       mrPlan <- liftEither $ Plan.mutateReadPlan MutationUpdate apiReq identifier conf sCache
-      resultSet <- runQuery $ Query.updateQuery mrPlan apiReq conf
+      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.updateQuery mrPlan apiReq conf
       return $ Response.updateResponse apiReq resultSet
 
     (ActionMutate MutationSingleUpsert, TargetIdent identifier) -> do
       mrPlan <- liftEither $ Plan.mutateReadPlan MutationSingleUpsert apiReq identifier conf sCache
-      resultSet <- runQuery $ Query.singleUpsertQuery mrPlan apiReq conf
+      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.singleUpsertQuery mrPlan apiReq conf
       return $ Response.singleUpsertResponse apiReq resultSet
 
     (ActionMutate MutationDelete, TargetIdent identifier) -> do
       mrPlan <- liftEither $ Plan.mutateReadPlan MutationDelete apiReq identifier conf sCache
-      resultSet <- runQuery $ Query.deleteQuery mrPlan apiReq conf
+      resultSet <- runQuery roleIsoLvl (Plan.mrTxMode mrPlan) $ Query.deleteQuery mrPlan apiReq conf
       return $ Response.deleteResponse apiReq resultSet
 
-    (ActionInvoke invMethod, TargetProc proc _) -> do
-      cPlan <- liftEither $ Plan.callReadPlan proc conf sCache apiReq
-      resultSet <- runQuery $ Query.invokeQuery proc cPlan apiReq conf
-      return $ Response.invokeResponse invMethod proc apiReq resultSet
+    (ActionInvoke invMethod, TargetProc identifier _) -> do
+      cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq invMethod
+      resultSet <- runQuery (roleIsoLvl <|> pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
+      return $ Response.invokeResponse invMethod (Plan.crProc cPlan) apiReq resultSet
 
     (ActionInspect headersOnly, TargetDefaultSpec tSchema) -> do
-      oaiResult <- runQuery $ Query.openApiQuery sCache pgVer conf tSchema
+      oaiResult <- runQuery roleIsoLvl Plan.inspectPlanTxMode $ Query.openApiQuery sCache pgVer conf tSchema
       return $ Response.openApiResponse headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
 
-    (ActionInfo, _) ->
-      return $ Response.infoResponse iTarget sCache
+    (ActionInfo, TargetIdent identifier) ->
+      return $ Response.infoIdentResponse identifier sCache
+
+    (ActionInfo, TargetProc identifier _) -> do
+      cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq ApiRequest.InvHead
+      return $ Response.infoProcResponse (Plan.crProc cPlan)
+
+    (ActionInfo, TargetDefaultSpec _) ->
+      return Response.infoRootResponse
 
     _ ->
       -- This is unreachable as the ApiRequest.hs rejects it before
       -- TODO Refactor the Action/Target types to remove this line
       throwError $ Error.ApiRequestError ApiRequestTypes.NotFound
   where
-    runQuery query =
-      runDbHandler appState mode authenticated prepared $ do
-        Query.setPgLocals conf authClaims authRole apiReq jsonDbS pgVer
+    roleSettings = fromMaybe mempty (HM.lookup authRole $ configRoleSettings conf)
+    roleIsoLvl = decodeUtf8 <$> HM.lookup "default_transaction_isolation" roleSettings
+    runQuery isoLvl mode query =
+      runDbHandler appState isoLvl mode authenticated prepared $ do
+        Query.setPgLocals conf authClaims authRole (HM.toList roleSettings) apiReq pgVer
+        Query.runPreReq conf
         query

src/PostgREST/AppState.hs

@@ -7,7 +7,6 @@ module PostgREST.AppState
   , getConfig
   , getSchemaCache
   , getIsListenerOn
-  , getJsonDbS
   , getMainThreadId
   , getPgVersion
   , getRetryNextIn
@@ -16,22 +15,27 @@ module PostgREST.AppState
   , init
   , initWithPool
   , logWithZTime
+  , logPgrstError
   , putConfig
   , putSchemaCache
   , putIsListenerOn
-  , putJsonDbS
   , putPgVersion
   , putRetryNextIn
   , signalListener
   , usePool
   , waitListener
+  , debounceLogAcquisitionTimeout
   ) where
 
-import qualified Hasql.Pool    as SQL
-import qualified Hasql.Session as SQL
+import qualified Data.ByteString.Lazy as LBS
+import qualified Data.Text.Encoding   as T
+import qualified Hasql.Pool           as SQL
+import qualified Hasql.Session        as SQL
+import qualified PostgREST.Error      as Error
 
 import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
                            updateAction)
+import Control.Debounce
 import Data.IORef         (IORef, atomicWriteIORef, newIORef,
                            readIORef)
 import Data.Time          (ZonedTime, defaultTimeLocale, formatTime,
@@ -47,29 +51,29 @@ import Protolude
 
 data AppState = AppState
   -- | Database connection pool
-  { statePool         :: SQL.Pool
+  { statePool                     :: SQL.Pool
   -- | Database server version, will be updated by the connectionWorker
-  , statePgVersion    :: IORef PgVersion
+  , statePgVersion                :: IORef PgVersion
   -- | No schema cache at the start. Will be filled in by the connectionWorker
-  , stateSchemaCache  :: IORef (Maybe SchemaCache)
-  -- | Cached SchemaCache in json
-  , stateJsonDbS      :: IORef ByteString
+  , stateSchemaCache              :: IORef (Maybe SchemaCache)
   -- | Binary semaphore to make sure just one connectionWorker can run at a time
-  , stateWorkerSem    :: MVar ()
+  , stateWorkerSem                :: MVar ()
   -- | Binary semaphore used to sync the listener(NOTIFY reload) with the connectionWorker.
-  , stateListener     :: MVar ()
+  , stateListener                 :: MVar ()
   -- | State of the LISTEN channel, used for the admin server checks
-  , stateIsListenerOn :: IORef Bool
+  , stateIsListenerOn             :: IORef Bool
   -- | Config that can change at runtime
-  , stateConf         :: IORef AppConfig
+  , stateConf                     :: IORef AppConfig
   -- | Time used for verifying JWT expiration
-  , stateGetTime      :: IO UTCTime
+  , stateGetTime                  :: IO UTCTime
   -- | Time with time zone used for worker logs
-  , stateGetZTime     :: IO ZonedTime
+  , stateGetZTime                 :: IO ZonedTime
   -- | Used for killing the main thread in case a subthread fails
-  , stateMainThreadId :: ThreadId
+  , stateMainThreadId             :: ThreadId
   -- | Keeps track of when the next retry for connecting to database is scheduled
-  , stateRetryNextIn  :: IORef Int
+  , stateRetryNextIn              :: IORef Int
+  -- | Logs a pool error with a debounce
+  , debounceLogAcquisitionTimeout :: IO ()
   }
 
 init :: AppConfig -> IO AppState
@@ -78,11 +82,10 @@ init conf = do
   initWithPool pool conf
 
 initWithPool :: SQL.Pool -> AppConfig -> IO AppState
-initWithPool pool conf =
-  AppState pool
+initWithPool pool conf = do
+  appState <- AppState pool
     <$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step
     <*> newIORef Nothing
-    <*> newIORef mempty
     <*> newEmptyMVar
     <*> newEmptyMVar
     <*> newIORef False
@@ -91,16 +94,28 @@ initWithPool pool conf =
     <*> mkAutoUpdate defaultUpdateSettings { updateAction = getZonedTime }
     <*> myThreadId
     <*> newIORef 0
+    <*> pure (pure ())
+
+  deb <-
+    let oneSecond = 1000000 in
+    mkDebounce defaultDebounceSettings
+       { debounceAction = logPgrstError appState SQL.AcquisitionTimeoutUsageError
+       , debounceFreq = 5*oneSecond
+       , debounceEdge = leadingEdge -- logs at the start and the end
+       }
+
+  return appState { debounceLogAcquisitionTimeout = deb }
 
 destroy :: AppState -> IO ()
 destroy = destroyPool
 
 initPool :: AppConfig -> IO SQL.Pool
 initPool AppConfig{..} =
-  SQL.acquire configDbPoolSize timeoutMilliseconds $ toUtf8 configDbUri
-  where
-    timeoutMilliseconds = (* oneSecond) <$> configDbPoolAcquisitionTimeout
-    oneSecond = 1000000
+  SQL.acquire
+    configDbPoolSize
+    (fromIntegral configDbPoolAcquisitionTimeout)
+    (fromIntegral configDbPoolMaxLifetime)
+    (toUtf8 configDbUri)
 
 -- | Run an action with a database connection.
 usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a)
@@ -127,12 +142,6 @@ getSchemaCache = readIORef . stateSchemaCache
 putSchemaCache :: AppState -> Maybe SchemaCache -> IO ()
 putSchemaCache appState = atomicWriteIORef (stateSchemaCache appState)
 
-getJsonDbS :: AppState -> IO ByteString
-getJsonDbS = readIORef . stateJsonDbS
-
-putJsonDbS :: AppState -> ByteString -> IO ()
-putJsonDbS appState = atomicWriteIORef (stateJsonDbS appState)
-
 getWorkerSem :: AppState -> MVar ()
 getWorkerSem = stateWorkerSem
 
@@ -157,6 +166,9 @@ logWithZTime appState txt = do
   zTime <- stateGetZTime appState
   hPutStrLn stderr $ toS (formatTime defaultTimeLocale "%d/%b/%Y:%T %z: " zTime) <> txt
 
+logPgrstError :: AppState -> SQL.UsageError -> IO ()
+logPgrstError appState e = logWithZTime appState . T.decodeUtf8 . LBS.toStrict $ Error.errorPayload $ Error.PgError False e
+
 getMainThreadId :: AppState -> ThreadId
 getMainThreadId = stateMainThreadId
 

src/PostgREST/Auth.hs

@@ -23,8 +23,8 @@ import qualified Data.Aeson                      as JSON
 import qualified Data.Aeson.Key                  as K
 import qualified Data.Aeson.KeyMap               as KM
 import qualified Data.Aeson.Types                as JSON
+import qualified Data.ByteString                 as BS
 import qualified Data.ByteString.Lazy.Char8      as LBS
-import qualified Data.Text.Encoding              as T
 import qualified Data.Vault.Lazy                 as Vault
 import qualified Data.Vector                     as V
 import qualified Network.HTTP.Types.Header       as HTTP
@@ -47,7 +47,7 @@ import Protolude
 
 data AuthResult = AuthResult
   { authClaims :: KM.KeyMap JSON.Value
-  , authRole   :: Text
+  , authRole   :: BS.ByteString
   }
 
 -- | Receives the JWT secret and audience (from config) and a JWT and returns a
@@ -79,7 +79,7 @@ parseClaims AppConfig{..} jclaims@(JSON.Object mclaims) = do
   role <- liftEither . maybeToRight JwtTokenRequired $
     unquoted <$> walkJSPath (Just jclaims) configJwtRoleClaimKey <|> configDbAnonRole
   return AuthResult
-           { authClaims = mclaims & KM.insert "role" (JSON.toJSON role)
+           { authClaims = mclaims & KM.insert "role" (JSON.toJSON $ decodeUtf8 role)
            , authRole = role
            }
   where
@@ -89,9 +89,9 @@ parseClaims AppConfig{..} jclaims@(JSON.Object mclaims) = do
     walkJSPath (Just (JSON.Array ar)) (JSPIdx idx:rest) = walkJSPath (ar V.!? idx) rest
     walkJSPath _                      _                 = Nothing
 
-    unquoted :: JSON.Value -> Text
-    unquoted (JSON.String t) = t
-    unquoted v = T.decodeUtf8 . LBS.toStrict $ JSON.encode v
+    unquoted :: JSON.Value -> BS.ByteString
+    unquoted (JSON.String t) = encodeUtf8 t
+    unquoted v               = LBS.toStrict $ JSON.encode v
 -- impossible case - just added to please -Wincomplete-patterns
 parseClaims _ _ = return AuthResult { authClaims = KM.empty, authRole = mempty }
 
@@ -117,5 +117,5 @@ authResultKey = unsafePerformIO Vault.newKey
 getResult :: Wai.Request -> Maybe (Either Error AuthResult)
 getResult = Vault.lookup authResultKey . Wai.vault
 
-getRole :: Wai.Request -> Maybe Text
+getRole :: Wai.Request -> Maybe BS.ByteString
 getRole req = authRole <$> (rightToMaybe =<< getResult req)

src/PostgREST/CLI.hs

@@ -33,7 +33,7 @@ import Protolude hiding (hPutStrLn)
 main :: App.SignalHandlerInstaller -> Maybe App.SocketRunner -> CLI -> IO ()
 main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
   conf@AppConfig{..} <-
-    either panic identity <$> Config.readAppConfig mempty cliPath Nothing
+    either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty
 
   -- Per https://github.com/PostgREST/postgrest/issues/268, we want to
   -- explicitly close the connections to PostgreSQL on shutdown.
@@ -151,6 +151,9 @@ exampleConfigFile =
       |## Time in seconds to wait to acquire a slot from the connection pool
       |# db-pool-acquisition-timeout = 10
       |
+      |## Time in seconds after which to recycle pool connections
+      |# db-pool-max-lifetime = 1800
+      |
       |## Stored proc to exec immediately after auth
       |# db-pre-request = "stored_proc_name"
       |

src/PostgREST/Config.hs

@@ -32,6 +32,7 @@ import qualified Data.Aeson             as JSON
 import qualified Data.ByteString        as BS
 import qualified Data.ByteString.Base64 as B64
 import qualified Data.ByteString.Lazy   as LBS
+import qualified Data.CaseInsensitive   as CI
 import qualified Data.Configurator      as C
 import qualified Data.Map.Strict        as M
 import qualified Data.Text              as T
@@ -50,6 +51,7 @@ import Numeric                 (readOct, showOct)
 import System.Environment      (getEnvironment)
 import System.Posix.Types      (FileMode)
 
+import PostgREST.Config.Database         (RoleSettings)
 import PostgREST.Config.JSPath           (JSPath, JSPathExp (..),
                                           dumpJSPath, pRoleClaimKey)
 import PostgREST.Config.Proxy            (Proxy (..),
@@ -63,14 +65,15 @@ import Protolude hiding (Proxy, toList)
 
 data AppConfig = AppConfig
   { configAppSettings              :: [(Text, Text)]
-  , configDbAnonRole               :: Maybe Text
+  , configDbAnonRole               :: Maybe BS.ByteString
   , configDbChannel                :: Text
   , configDbChannelEnabled         :: Bool
   , configDbExtraSearchPath        :: [Text]
   , configDbMaxRows                :: Maybe Integer
   , configDbPlanEnabled            :: Bool
   , configDbPoolSize               :: Int
-  , configDbPoolAcquisitionTimeout :: Maybe Int
+  , configDbPoolAcquisitionTimeout :: Int
+  , configDbPoolMaxLifetime        :: Int
   , configDbPreRequest             :: Maybe QualifiedIdentifier
   , configDbPreparedStatements     :: Bool
   , configDbRootSpec               :: Maybe QualifiedIdentifier
@@ -93,9 +96,11 @@ data AppConfig = AppConfig
   , configRawMediaTypes            :: [MediaType]
   , configServerHost               :: Text
   , configServerPort               :: Int
+  , configServerTraceHeader        :: Maybe (CI.CI BS.ByteString)
   , configServerUnixSocket         :: Maybe FilePath
   , configServerUnixSocketMode     :: FileMode
   , configAdminServerPort          :: Maybe Int
+  , configRoleSettings             :: RoleSettings
   }
 
 data LogLevel = LogCrit | LogError | LogWarn | LogInfo
@@ -123,14 +128,15 @@ toText conf =
   where
     -- apply conf to all pgrst settings
     pgrstSettings = (\(k, v) -> (k, v conf)) <$>
-      [("db-anon-role",              q . fromMaybe "" . configDbAnonRole)
+      [("db-anon-role",              q . T.decodeUtf8 . fromMaybe "" . configDbAnonRole)
       ,("db-channel",                q . configDbChannel)
       ,("db-channel-enabled",            T.toLower . show . configDbChannelEnabled)
       ,("db-extra-search-path",      q . T.intercalate "," . configDbExtraSearchPath)
       ,("db-max-rows",                   maybe "\"\"" show . configDbMaxRows)
       ,("db-plan-enabled",               T.toLower . show . configDbPlanEnabled)
       ,("db-pool",                       show . configDbPoolSize)
-      ,("db-pool-acquisition-timeout",   maybe "\"\"" show . configDbPoolAcquisitionTimeout)
+      ,("db-pool-acquisition-timeout",   show . configDbPoolAcquisitionTimeout)
+      ,("db-pool-max-lifetime",          show . configDbPoolMaxLifetime)
       ,("db-pre-request",            q . maybe mempty dumpQi . configDbPreRequest)
       ,("db-prepared-statements",        T.toLower . show . configDbPreparedStatements)
       ,("db-root-spec",              q . maybe mempty dumpQi . configDbRootSpec)
@@ -150,6 +156,7 @@ toText conf =
       ,("raw-media-types",           q . T.decodeUtf8 . BS.intercalate "," . fmap toMime . configRawMediaTypes)
       ,("server-host",               q . configServerHost)
       ,("server-port",                   show . configServerPort)
+      ,("server-trace-header",       q . T.decodeUtf8 . maybe mempty CI.original . configServerTraceHeader)
       ,("server-unix-socket",        q . maybe mempty T.pack . configServerUnixSocket)
       ,("server-unix-socket-mode",   q . T.pack . showSocketMode)
       ,("admin-server-port",             maybe "\"\"" show . configAdminServerPort)
@@ -186,13 +193,13 @@ instance JustIfMaybe a (Maybe a) where
 
 -- | Reads and parses the config and overrides its parameters from env vars,
 -- files or db settings.
-readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> IO (Either Text AppConfig)
-readAppConfig dbSettings optPath prevDbUri = do
+readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> IO (Either Text AppConfig)
+readAppConfig dbSettings optPath prevDbUri roleSettings = do
   env <- readPGRSTEnvironment
   -- if no filename provided, start with an empty map to read config from environment
   conf <- maybe (return $ Right M.empty) loadConfig optPath
 
-  case C.runParser (parser optPath env dbSettings) =<< mapLeft show conf of
+  case C.runParser (parser optPath env dbSettings roleSettings) =<< mapLeft show conf of
     Left err ->
       return . Left $ "Error in config " <> err
     Right parsedConfig ->
@@ -207,11 +214,11 @@ readAppConfig dbSettings optPath prevDbUri = do
       decodeJWKS <$>
         (decodeSecret =<< readSecretFile =<< readDbUriFile prevDbUri parsedConfig)
 
-parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> C.Parser C.Config AppConfig
-parser optPath env dbSettings =
+parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> C.Parser C.Config AppConfig
+parser optPath env dbSettings roleSettings =
   AppConfig
     <$> parseAppSettings "app.settings"
-    <*> optString "db-anon-role"
+    <*> (fmap encodeUtf8 <$> optString "db-anon-role")
     <*> (fromMaybe "pgrst" <$> optString "db-channel")
     <*> (fromMaybe True <$> optBool "db-channel-enabled")
     <*> (maybe ["public"] splitOnCommas <$> optValue "db-extra-search-path")
@@ -219,7 +226,8 @@ parser optPath env dbSettings =
                      (optInt "max-rows")
     <*> (fromMaybe False <$> optBool "db-plan-enabled")
     <*> (fromMaybe 10 <$> optInt "db-pool")
-    <*> optInt "db-pool-acquisition-timeout"
+    <*> (fromMaybe 10 <$> optInt "db-pool-acquisition-timeout")
+    <*> (fromMaybe 1800 <$> optInt "db-pool-max-lifetime")
     <*> (fmap toQi <$> optWithAlias (optString "db-pre-request")
                                     (optString "pre-request"))
     <*> (fromMaybe True <$> optBool "db-prepared-statements")
@@ -247,9 +255,11 @@ parser optPath env dbSettings =
     <*> (maybe [] (fmap (MTOther . encodeUtf8) . splitOnCommas) <$> optValue "raw-media-types")
     <*> (fromMaybe "!4" <$> optString "server-host")
     <*> (fromMaybe 3000 <$> optInt "server-port")
+    <*> (fmap (CI.mk . encodeUtf8) <$> optString "server-trace-header")
     <*> (fmap T.unpack <$> optString "server-unix-socket")
     <*> parseSocketFileMode "server-unix-socket-mode"
     <*> optInt "admin-server-port"
+    <*> pure roleSettings
   where
     parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
     parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
@@ -355,7 +365,8 @@ parser optPath env dbSettings =
           let dbSettingName = T.pack $ dashToUnderscore <$> toS key in
           if dbSettingName `notElem` [
             "server_host", "server_port", "server_unix_socket", "server_unix_socket_mode", "admin_server_port", "log_level",
-            "db_uri", "db_channel_enabled", "db_channel", "db_pool", "db_pool_acquisition_timeout", "db_config"]
+            "db_uri", "db_channel_enabled", "db_channel", "db_pool", "db_pool_acquisition_timeout",
+            "db_pool_max_lifetime", "db_config"]
           then lookup dbSettingName dbSettings
           else Nothing
 

src/PostgREST/Config/Database.hs

@@ -3,11 +3,17 @@
 module PostgREST.Config.Database
   ( pgVersionStatement
   , queryDbSettings
+  , queryRoleSettings
   , queryPgVersion
+  , RoleSettings
   ) where
 
+import Control.Arrow ((***))
+
 import PostgREST.Config.PgVersion (PgVersion (..))
 
+import qualified Data.HashMap.Strict as HM
+
 import qualified Hasql.Decoders             as HD
 import qualified Hasql.Encoders             as HE
 import           Hasql.Session              (Session, statement)
@@ -19,11 +25,13 @@ import Text.InterpolatedString.Perl6 (q)
 
 import Protolude
 
-queryPgVersion :: Session PgVersion
-queryPgVersion = statement mempty pgVersionStatement
+type RoleSettings = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
 
-pgVersionStatement :: SQL.Statement () PgVersion
-pgVersionStatement = SQL.Statement sql HE.noParams versionRow False
+queryPgVersion :: Bool -> Session PgVersion
+queryPgVersion prepared = statement mempty $ pgVersionStatement prepared
+
+pgVersionStatement :: Bool -> SQL.Statement () PgVersion
+pgVersionStatement = SQL.Statement sql HE.noParams versionRow
   where
     sql = "SELECT current_setting('server_version_num')::integer, current_setting('server_version')"
     versionRow = HD.singleRow $ PgVersion <$> column HD.int4 <*> column HD.text
@@ -31,11 +39,11 @@ pgVersionStatement = SQL.Statement sql HE.noParams versionRow False
 queryDbSettings :: Bool -> Session [(Text, Text)]
 queryDbSettings prepared =
   let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
-  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty dbSettingsStatement
+  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ dbSettingsStatement prepared
 
 -- | Get db settings from the connection role. Global settings will be overridden by database specific settings.
-dbSettingsStatement :: SQL.Statement () [(Text, Text)]
-dbSettingsStatement = SQL.Statement sql HE.noParams decodeSettings False
+dbSettingsStatement :: Bool -> SQL.Statement () [(Text, Text)]
+dbSettingsStatement = SQL.Statement sql HE.noParams decodeSettings
   where
     sql = [q|
       WITH
@@ -61,5 +69,45 @@ dbSettingsStatement = SQL.Statement sql HE.noParams decodeSettings False
     |]
     decodeSettings = HD.rowList $ (,) <$> column HD.text <*> column HD.text
 
+queryRoleSettings :: Bool -> Session RoleSettings
+queryRoleSettings prepared =
+  let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
+  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ roleSettingsStatement prepared
+
+roleSettingsStatement :: Bool -> SQL.Statement () RoleSettings
+roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
+  where
+    sql = [q|
+      with
+      role_setting as (
+        select r.rolname, unnest(r.rolconfig) as setting
+        from pg_auth_members m
+        join pg_roles r on r.oid = m.roleid
+        where member = current_user::regrole::oid
+      ),
+      kv_settings AS (
+        SELECT
+          rolname,
+          substr(setting, 1, strpos(setting, '=') - 1) as key,
+          lower(substr(setting, strpos(setting, '=') + 1)) as value
+        FROM role_setting
+      )
+      select rolname, array_agg(row(key, value))
+      from kv_settings
+      group by rolname;
+    |]
+    decodeRoleSettings = HM.fromList . map (bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>))) <$> HD.rowList aRow
+    aRow :: HD.Row (Text, [(Text, Text)])
+    aRow = (,) <$> column HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
+
 column :: HD.Value a -> HD.Row a
 column = HD.column . HD.nonNullable
+
+compositeField :: HD.Value a -> HD.Composite a
+compositeField = HD.field . HD.nonNullable
+
+compositeArrayColumn :: HD.Composite a -> HD.Row [a]
+compositeArrayColumn = arrayColumn . HD.composite
+
+arrayColumn :: HD.Value a -> HD.Row [a]
+arrayColumn = column . HD.listArray . HD.nonNullable

src/PostgREST/Error.hs

@@ -17,6 +17,8 @@ module PostgREST.Error
 
 import qualified Data.Aeson                as JSON
 import qualified Data.ByteString.Char8     as BS
+import qualified Data.FuzzySet             as Fuzzy
+import qualified Data.HashMap.Strict       as HM
 import qualified Data.Text                 as T
 import qualified Data.Text.Encoding        as T
 import qualified Data.Text.Encoding.Error  as T
@@ -35,12 +37,14 @@ import           PostgREST.ApiRequest.Types (ApiRequestError (..),
 import           PostgREST.MediaType        (MediaType (..))
 import qualified PostgREST.MediaType        as MediaType
 
-import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
-import PostgREST.SchemaCache.Proc         (ProcDescription (..),
-                                           ProcParam (..))
+import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..),
+                                           Schema)
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                            Junction (..),
-                                           Relationship (..))
+                                           Relationship (..),
+                                           RelationshipsMap)
+import PostgREST.SchemaCache.Routine      (Routine (..),
+                                           RoutineParam (..))
 import Protolude
 
 
@@ -64,15 +68,19 @@ instance PgrstError ApiRequestError where
   status InvalidRpcMethod{}      = HTTP.status405
   status InvalidRange{}          = HTTP.status416
   status NotFound                = HTTP.status404
+
   status NoRelBetween{}          = HTTP.status400
   status NoRpc{}                 = HTTP.status404
   status NotEmbedded{}           = HTTP.status400
-  status ParseRequestError{}     = HTTP.status400
-  status PutRangeNotAllowedError = HTTP.status400
+  status PutLimitNotAllowedError = HTTP.status400
   status QueryParamError{}       = HTTP.status400
+  status RelatedOrderNotToOne{}  = HTTP.status400
+  status SpreadNotToOne{}        = HTTP.status400
+  status UnacceptableFilter{}    = HTTP.status400
   status UnacceptableSchema{}    = HTTP.status406
   status UnsupportedMethod{}     = HTTP.status405
   status LimitNoOrderError       = HTTP.status400
+  status ColumnNotFound{}        = HTTP.status400
 
   headers _ = [MediaType.toContentType MTApplicationJSON]
 
@@ -100,11 +108,6 @@ instance JSON.ToJSON ApiRequestError where
                    LowerGTUpper            -> "The lower boundary must be lower than or equal to the upper boundary in the Range header."
                    OutOfBounds lower total -> "An offset of " <> lower <> " was requested, but there are only " <> total <> " rows."),
     "hint"    .= JSON.Null]
-  toJSON (ParseRequestError message details) = JSON.object [
-    "code"    .= ApiRequestErrorCode04,
-    "message" .= message,
-    "details" .= details,
-    "hint"    .= JSON.Null]
   toJSON InvalidFilters = JSON.object [
     "code"    .= ApiRequestErrorCode05,
     "message" .= ("Filters must include all and only primary key columns with 'eq' operators" :: Text),
@@ -123,7 +126,7 @@ instance JSON.ToJSON ApiRequestError where
   toJSON NotFound = JSON.object []
   toJSON (NotEmbedded resource) = JSON.object [
     "code"    .= ApiRequestErrorCode08,
-    "message" .= ("Cannot apply filter because '" <> resource <> "' is not an embedded resource in this request" :: Text),
+    "message" .= ("'" <> resource <> "' is not an embedded resource in this request" :: Text),
     "details" .= JSON.Null,
     "hint"    .= ("Verify that '" <> resource <> "' is included in the 'select' query parameter." :: Text)]
 
@@ -139,9 +142,9 @@ instance JSON.ToJSON ApiRequestError where
     "details" .= JSON.Null,
     "hint"    .= JSON.Null]
 
-  toJSON PutRangeNotAllowedError = JSON.object [
+  toJSON PutLimitNotAllowedError = JSON.object [
     "code"    .= ApiRequestErrorCode14,
-    "message" .= ("Range header and limit/offset querystring parameters are not allowed for PUT" :: Text),
+    "message" .= ("limit/offset querystring parameters are not allowed for PUT" :: Text),
     "details" .= JSON.Null,
     "hint"    .= JSON.Null]
 
@@ -151,35 +154,165 @@ instance JSON.ToJSON ApiRequestError where
     "details" .= JSON.Null,
     "hint"    .= JSON.Null]
 
-  toJSON (NoRelBetween parent child schema) = JSON.object [
+  toJSON (RelatedOrderNotToOne origin target) = JSON.object [
+    "code"    .= ApiRequestErrorCode18,
+    "message" .= ("A related order on '" <> target <> "' is not possible" :: Text),
+    "details" .= ("'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship" :: Text),
+    "hint"    .= JSON.Null]
+
+  toJSON (SpreadNotToOne origin target) = JSON.object [
+    "code"    .= ApiRequestErrorCode19,
+    "message" .= ("A spread operation on '" <> target <> "' is not possible" :: Text),
+    "details" .= ("'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship" :: Text),
+    "hint"    .= JSON.Null]
+
+  toJSON (UnacceptableFilter target) = JSON.object [
+    "code"    .= ApiRequestErrorCode20,
+    "message" .= ("Bad operator on the '" <> target <> "' embedded resource":: Text),
+    "details" .= ("Only is null or not is null filters are allowed on embedded resources":: Text),
+    "hint"    .= JSON.Null]
+
+  toJSON (NoRelBetween parent child embedHint schema allRels) = JSON.object [
     "code"    .= SchemaCacheErrorCode00,
     "message" .= ("Could not find a relationship between '" <> parent <> "' and '" <> child <> "' in the schema cache" :: Text),
-    "details" .= JSON.Null,
-    "hint"    .= ("Verify that '" <> parent <> "' and '" <> child <> "' exist in the schema '" <> schema <> "' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache." :: Text)]
+    "details" .= ("Searched for a foreign key relationship between '" <> parent <> "' and '" <> child <> maybe mempty ("' using the hint '" <>) embedHint <> "' in the schema '" <> schema <> "', but no matches were found."),
+    "hint"    .= noRelBetweenHint parent child schema allRels]
+
   toJSON (AmbiguousRelBetween parent child rels) = JSON.object [
     "code"    .= SchemaCacheErrorCode01,
     "message" .= ("Could not embed because more than one relationship was found for '" <> parent <> "' and '" <> child <> "'" :: Text),
     "details" .= (compressedRel <$> rels),
     "hint"    .= ("Try changing '" <> child <> "' to one of the following: " <> relHint rels <> ". Find the desired relationship in the 'details' key." :: Text)]
-  toJSON (NoRpc schema procName argumentKeys hasPreferSingleObject contentType isInvPost)  =
-    let prms = "(" <> T.intercalate ", " argumentKeys <> ")" in JSON.object [
+  toJSON (NoRpc schema procName argumentKeys hasPreferSingleObject contentType isInvPost allProcs overloadedProcs)  =
+    let func = schema <> "." <> procName
+        prms = T.intercalate ", " argumentKeys
+        prmsMsg = "(" <> prms <> ")"
+        prmsDet = " with parameter" <> (if length argumentKeys > 1 then "s " else " ") <> prms
+        fmtPrms p = if null argumentKeys then " without parameters" else p
+        onlySingleParams = hasPreferSingleObject || (isInvPost && contentType `elem` [MTTextPlain, MTTextXML, MTOctetStream])
+    in JSON.object [
     "code"    .= SchemaCacheErrorCode02,
-    "message" .= ("Could not find the " <> schema <> "." <> procName <>
+    "message" .= ("Could not find the function " <> func <> (if onlySingleParams then "" else fmtPrms prmsMsg) <> " in the schema cache"),
+    "details" .= ("Searched for the function " <> func <>
       (case (hasPreferSingleObject, isInvPost, contentType) of
-        (True, _, _)                 -> " function with a single json or jsonb parameter"
-        (_, True, MTTextPlain)       -> " function with a single unnamed text parameter"
-        (_, True, MTTextXML)         -> " function with a single unnamed xml parameter"
-        (_, True, MTOctetStream)     -> " function with a single unnamed bytea parameter"
-        (_, True, MTApplicationJSON) -> prms <> " function or the " <> schema <> "." <> procName <>" function with a single unnamed json or jsonb parameter"
-        _                            -> prms <> " function") <>
-      " in the schema cache"),
-    "details" .= JSON.Null,
-    "hint"    .= ("If a new function was created in the database with this name and parameters, try reloading the schema cache." :: Text)]
+        (True, _, _)                 -> " with a single json/jsonb parameter"
+        (_, True, MTTextPlain)       -> " with a single unnamed text parameter"
+        (_, True, MTTextXML)         -> " with a single unnamed xml parameter"
+        (_, True, MTOctetStream)     -> " with a single unnamed bytea parameter"
+        (_, True, MTApplicationJSON) -> fmtPrms prmsDet <> " or with a single unnamed json/jsonb parameter"
+        _                            -> fmtPrms prmsDet) <>
+      ", but no matches were found in the schema cache."),
+    -- The hint will be null in the case of single unnamed parameter functions
+    "hint"    .= if onlySingleParams
+                 then Nothing
+                 else noRpcHint schema procName argumentKeys allProcs overloadedProcs ]
   toJSON (AmbiguousRpc procs)  = JSON.object [
     "code"    .= SchemaCacheErrorCode03,
     "message" .= ("Could not choose the best candidate function between: " <> T.intercalate ", " [pdSchema p <> "." <> pdName p <> "(" <> T.intercalate ", " [ppName a <> " => " <> ppType a | a <- pdParams p] <> ")" | p <- procs]),
     "details" .= JSON.Null,
     "hint"    .= ("Try renaming the parameters or the function itself in the database so function overloading can be resolved" :: Text)]
+  toJSON (ColumnNotFound relName colName) = JSON.object [
+    "code"    .= SchemaCacheErrorCode04,
+    "message" .= ("Column '" <> colName <> "' of relation '" <> relName <> "' does not exist" :: Text),
+    "details" .= JSON.Null,
+    "hint"    .= JSON.Null]
+
+-- |
+-- If no relationship is found then:
+--
+-- Looks for parent suggestions if parent not found
+-- Looks for child suggestions if parent is found but child is not
+-- Gives no suggestions if both are found (it means that there is a problem with the embed hint)
+--
+-- >>> :set -Wno-missing-fields
+-- >>> let qi t = QualifiedIdentifier "api" t
+-- >>> let rel ft = Relationship{relForeignTable = qi ft}
+-- >>> let rels = HM.fromList [((qi "films", "api"), [rel "directors", rel "roles", rel "actors"])]
+--
+-- >>> noRelBetweenHint "film" "directors" "api" rels
+-- Just "Perhaps you meant 'films' instead of 'film'."
+--
+-- >>> noRelBetweenHint "films" "role" "api" rels
+-- Just "Perhaps you meant 'roles' instead of 'role'."
+--
+-- >>> noRelBetweenHint "films" "role" "api" rels
+-- Just "Perhaps you meant 'roles' instead of 'role'."
+--
+-- >>> noRelBetweenHint "films" "actors" "api" rels
+-- Nothing
+--
+-- >>> noRelBetweenHint "noclosealternative" "roles" "api" rels
+-- Nothing
+--
+-- >>> noRelBetweenHint "films" "noclosealternative" "api" rels
+-- Nothing
+--
+-- >>> noRelBetweenHint "films" "noclosealternative" "noclosealternative" rels
+-- Nothing
+--
+noRelBetweenHint :: Text -> Text -> Schema -> RelationshipsMap -> Maybe Text
+noRelBetweenHint parent child schema allRels = ("Perhaps you meant '" <>) <$>
+  if isJust findParent
+    then (<> "' instead of '" <> child <> "'.") <$> suggestChild
+    else (<> "' instead of '" <> parent <> "'.") <$> suggestParent
+  where
+    findParent = HM.lookup (QualifiedIdentifier schema parent, schema) allRels
+    fuzzySetOfParents  = Fuzzy.fromList [qiName (fst p) | p <- HM.keys allRels, snd p == schema]
+    fuzzySetOfChildren = Fuzzy.fromList [qiName (relForeignTable c) | c <- fromMaybe [] findParent]
+    suggestParent = Fuzzy.getOne fuzzySetOfParents parent
+    -- Do not give suggestion if the child is found in the relations (weight = 1.0)
+    suggestChild  = headMay [snd k | k <- Fuzzy.get fuzzySetOfChildren child, fst k < 1.0]
+
+-- |
+-- If no function is found with the given name, it does a fuzzy search to all the functions
+-- in the same schema and shows the best match as hint.
+--
+-- >>> :set -Wno-missing-fields
+-- >>> let procs = [(QualifiedIdentifier "api" "test"), (QualifiedIdentifier "api" "another"), (QualifiedIdentifier "private" "other")]
+--
+-- >>> noRpcHint "api" "testt" ["val", "param", "name"] procs []
+-- Just "Perhaps you meant to call the function api.test"
+--
+-- >>> noRpcHint "api" "other" [] procs []
+-- Just "Perhaps you meant to call the function api.another"
+--
+-- >>> noRpcHint "api" "noclosealternative" [] procs []
+-- Nothing
+--
+-- If a function is found with the given name, but no params match, then it does a fuzzy search
+-- to all the overloaded functions' params using the form "param1, param2, param3, ..."
+-- and shows the best match as hint.
+--
+-- >>> let procsDesc = [Function {pdParams = [RoutineParam {ppName="val"}, RoutineParam {ppName="param"}, RoutineParam {ppName="name"}]}, Function {pdParams = [RoutineParam {ppName="id"}, RoutineParam {ppName="attr"}]}]
+--
+-- >>> noRpcHint "api" "test" ["vall", "pqaram", "nam"] procs procsDesc
+-- Just "Perhaps you meant to call the function api.test(name, param, val)"
+--
+-- >>> noRpcHint "api" "test" ["val", "param"] procs procsDesc
+-- Just "Perhaps you meant to call the function api.test(name, param, val)"
+--
+-- >>> noRpcHint "api" "test" ["id", "attrs"] procs procsDesc
+-- Just "Perhaps you meant to call the function api.test(attr, id)"
+--
+-- >>> noRpcHint "api" "test" ["id"] procs procsDesc
+-- Just "Perhaps you meant to call the function api.test(attr, id)"
+--
+-- >>> noRpcHint "api" "test" ["noclosealternative"] procs procsDesc
+-- Nothing
+--
+noRpcHint :: Text -> Text -> [Text] -> [QualifiedIdentifier] -> [Routine] -> Maybe Text
+noRpcHint schema procName params allProcs overloadedProcs =
+  fmap (("Perhaps you meant to call the function " <> schema <> ".") <>) possibleProcs
+  where
+    fuzzySetOfProcs  = Fuzzy.fromList [qiName k | k <- allProcs, qiSchema k == schema]
+    fuzzySetOfParams = Fuzzy.fromList $ listToText <$> [[ppName prm | prm <- pdParams ov] | ov <- overloadedProcs]
+    -- Cannot do a fuzzy search like: Fuzzy.getOne [[Text]] [Text], where [[Text]] is the list of params for each
+    -- overloaded function and [Text] the given params. This converts those lists to text to make fuzzy search possible.
+    -- E.g. ["val", "param", "name"] into "(name, param, val)"
+    listToText       = ("(" <>) . (<> ")") . T.intercalate ", " . sort
+    possibleProcs
+      | null overloadedProcs = Fuzzy.getOne fuzzySetOfProcs procName
+      | otherwise            = (procName <>) <$> Fuzzy.getOne fuzzySetOfParams (listToText params)
 
 compressedRel :: Relationship -> JSON.Value
 -- An ambiguousness error cannot happen for computed relationships TODO refactor so this mempty is not needed
@@ -193,7 +326,7 @@ compressedRel Relationship{..} =
     : case relCardinality of
         M2M Junction{..} -> [
             "cardinality" .= ("many-to-many" :: Text)
-          , "relationship" .= (qiName junTable <> " using " <> junConstraint1 <> fmtEls (snd <$> junColumns1) <> " and " <> junConstraint2 <> fmtEls (snd <$> junColumns2))
+          , "relationship" .= (qiName junTable <> " using " <> junConstraint1 <> fmtEls (snd <$> junColsSource) <> " and " <> junConstraint2 <> fmtEls (snd <$> junColsTarget))
           ]
         M2O cons relColumns -> [
             "cardinality" .= ("many-to-one" :: Text)
@@ -243,7 +376,7 @@ instance JSON.ToJSON SQL.UsageError where
     "hint"    .= JSON.Null]
   toJSON (SQL.SessionUsageError e) = JSON.toJSON e -- SQL.Error
   toJSON SQL.AcquisitionTimeoutUsageError = JSON.object [
-    "code"    .= ConnectionErrorCode00,
+    "code"    .= ConnectionErrorCode03,
     "message" .= ("Timed out acquiring connection from connection pool." :: Text),
     "details" .= JSON.Null,
     "hint"    .= JSON.Null]
@@ -313,13 +446,13 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
 
     _                       -> HTTP.status500
 
-checkIsFatal :: PgError -> Maybe Text
-checkIsFatal (PgError _ (SQL.ConnectionUsageError e))
+checkIsFatal :: SQL.UsageError -> Maybe Text
+checkIsFatal (SQL.ConnectionUsageError e)
   | isAuthFailureMessage = Just $ toS failureMessage
   | otherwise = Nothing
-  where isAuthFailureMessage = "FATAL:  password authentication failed" `isPrefixOf` failureMessage
+  where isAuthFailureMessage = "FATAL:  password authentication failed" `isInfixOf` failureMessage
         failureMessage = BS.unpack $ fromMaybe mempty e
-checkIsFatal (PgError _ (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError))))
+checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
   = case serverError of
       -- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
       SQL.ServerError "42601" _ _ _ _
@@ -442,12 +575,13 @@ data ErrorCode
   = ConnectionErrorCode00
   | ConnectionErrorCode01
   | ConnectionErrorCode02
+  | ConnectionErrorCode03
   -- API Request errors
   | ApiRequestErrorCode00
   | ApiRequestErrorCode01
   | ApiRequestErrorCode02
   | ApiRequestErrorCode03
-  | ApiRequestErrorCode04
+  | ApiRequestErrorCode04 -- no longer used (used to be mapped to ParseRequestError)
   | ApiRequestErrorCode05
   | ApiRequestErrorCode06
   | ApiRequestErrorCode07
@@ -461,11 +595,15 @@ data ErrorCode
   | ApiRequestErrorCode15
   | ApiRequestErrorCode16
   | ApiRequestErrorCode17
+  | ApiRequestErrorCode18
+  | ApiRequestErrorCode19
+  | ApiRequestErrorCode20
   -- Schema Cache errors
   | SchemaCacheErrorCode00
   | SchemaCacheErrorCode01
   | SchemaCacheErrorCode02
   | SchemaCacheErrorCode03
+  | SchemaCacheErrorCode04
   -- JWT authentication errors
   | JWTErrorCode00
   | JWTErrorCode01
@@ -483,6 +621,7 @@ buildErrorCode code = "PGRST" <> case code of
   ConnectionErrorCode00  -> "000"
   ConnectionErrorCode01  -> "001"
   ConnectionErrorCode02  -> "002"
+  ConnectionErrorCode03  -> "003"
 
   ApiRequestErrorCode00  -> "100"
   ApiRequestErrorCode01  -> "101"
@@ -502,11 +641,15 @@ buildErrorCode code = "PGRST" <> case code of
   ApiRequestErrorCode15  -> "115"
   ApiRequestErrorCode16  -> "116"
   ApiRequestErrorCode17  -> "117"
+  ApiRequestErrorCode18  -> "118"
+  ApiRequestErrorCode19  -> "119"
+  ApiRequestErrorCode20  -> "120"
 
   SchemaCacheErrorCode00 -> "200"
   SchemaCacheErrorCode01 -> "201"
   SchemaCacheErrorCode02 -> "202"
   SchemaCacheErrorCode03 -> "203"
+  SchemaCacheErrorCode04 -> "204"
 
   JWTErrorCode00         -> "300"
   JWTErrorCode01         -> "301"

src/PostgREST/Logger.hs

@@ -26,5 +26,5 @@ middleware logLevel = case logLevel of
       { Wai.outputFormat = Wai.ApacheWithSettings $
           Wai.defaultApacheSettings
             & Wai.setApacheRequestFilter (\_ res -> filterStatus $ Wai.responseStatus res)
-            & Wai.setApacheUserGetter (fmap encodeUtf8 . Auth.getRole)
+            & Wai.setApacheUserGetter Auth.getRole
       }

src/PostgREST/Plan.hs

@@ -13,21 +13,27 @@ resource.
 {-# LANGUAGE DuplicateRecordFields #-}
 {-# LANGUAGE LambdaCase            #-}
 {-# LANGUAGE NamedFieldPuns        #-}
+{-# LANGUAGE OverloadedRecordDot   #-}
 {-# LANGUAGE RecordWildCards       #-}
 
 module PostgREST.Plan
-  ( readPlan
+  ( wrappedReadPlan
   , mutateReadPlan
   , callReadPlan
+  , WrappedReadPlan(..)
   , MutateReadPlan(..)
   , CallReadPlan(..)
+  , inspectPlanTxMode
   ) where
 
-import qualified Data.HashMap.Strict        as HM
-import qualified Data.Set                   as S
-import qualified PostgREST.SchemaCache.Proc as Proc
 
-import Data.Either.Combinators (mapLeft)
+import qualified Data.ByteString.Lazy          as LBS
+import qualified Data.HashMap.Strict           as HM
+import qualified Data.List                     as L
+import qualified Data.Set                      as S
+import qualified PostgREST.SchemaCache.Routine as Routine
+
+import Data.Either.Combinators (mapLeft, mapRight)
 import Data.List               (delete)
 import Data.Tree               (Tree (..))
 
@@ -38,6 +44,8 @@ import PostgREST.ApiRequest               (Action (..),
                                            Payload (..))
 import PostgREST.Config                   (AppConfig (..))
 import PostgREST.Error                    (Error (..))
+import PostgREST.MediaType                (MTPlanAttrs (..),
+                                           MediaType (..))
 import PostgREST.Query.SqlFragment        (sourceCTEName)
 import PostgREST.RangeQuery               (NonnegRange, allRange,
                                            convertToLimitZeroRange,
@@ -46,48 +54,150 @@ import PostgREST.SchemaCache              (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers  (FieldName,
                                            QualifiedIdentifier (..),
                                            Schema)
-import PostgREST.SchemaCache.Proc         (ProcDescription (..),
-                                           ProcParam (..),
-                                           procReturnsScalar)
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                            Junction (..),
                                            Relationship (..),
-                                           RelationshipsMap)
-import PostgREST.SchemaCache.Table        (tablePKCols)
-
-import PostgREST.Plan.CallPlan
-import PostgREST.Plan.MutatePlan
-import PostgREST.Plan.ReadPlan   as ReadPlan
+                                           RelationshipsMap,
+                                           relIsToOne)
+import PostgREST.SchemaCache.Routine      (Routine (..), RoutineMap,
+                                           RoutineParam (..),
+                                           funcReturnsCompositeAlias,
+                                           funcReturnsScalar,
+                                           funcReturnsSetOfScalar)
+import PostgREST.SchemaCache.Table        (Table (tableName),
+                                           tablePKCols)
 
 import PostgREST.ApiRequest.Preferences
 import PostgREST.ApiRequest.Types
+import PostgREST.Plan.CallPlan
+import PostgREST.Plan.MutatePlan
+import PostgREST.Plan.ReadPlan          as ReadPlan
+import PostgREST.Plan.Types
 
+import qualified Hasql.Transaction.Sessions       as SQL
 import qualified PostgREST.ApiRequest.QueryParams as QueryParams
 
 import Protolude hiding (from)
 
+data WrappedReadPlan = WrappedReadPlan {
+  wrReadPlan :: ReadPlanTree
+, wrTxMode   :: SQL.Mode
+, wrBinField :: Maybe FieldName
+}
+
 data MutateReadPlan = MutateReadPlan {
   mrReadPlan   :: ReadPlanTree
 , mrMutatePlan :: MutatePlan
+, mrTxMode     :: SQL.Mode
 }
 
 data CallReadPlan = CallReadPlan {
   crReadPlan :: ReadPlanTree
 , crCallPlan :: CallPlan
+, crTxMode   :: SQL.Mode
+, crProc     :: Routine
+, crBinField :: Maybe FieldName
 }
 
+wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error WrappedReadPlan
+wrappedReadPlan  identifier conf sCache apiRequest = do
+  rPlan <- readPlan identifier conf sCache apiRequest
+  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) Nothing rPlan
+  return $ WrappedReadPlan rPlan SQL.Read binField
+
 mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error MutateReadPlan
 mutateReadPlan  mutation apiRequest identifier conf sCache = do
   rPlan <- readPlan identifier conf sCache apiRequest
   mPlan <- mutatePlan mutation identifier apiRequest sCache rPlan
-  return $ MutateReadPlan rPlan mPlan
+  return $ MutateReadPlan rPlan mPlan SQL.Write
 
-callReadPlan :: ProcDescription -> AppConfig -> SchemaCache -> ApiRequest -> Either Error CallReadPlan
-callReadPlan proc conf sCache apiRequest = do
-  let identifier = QualifiedIdentifier (pdSchema proc) (fromMaybe (pdName proc) $ Proc.procTableName proc)
-  rPlan <- readPlan identifier conf sCache apiRequest
-  let cPlan = callPlan proc apiRequest rPlan
-  return $ CallReadPlan rPlan cPlan
+callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CallReadPlan
+callReadPlan identifier conf sCache apiRequest invMethod = do
+  let paramKeys = case invMethod of
+        InvGet  -> S.fromList $ fst <$> qsParams'
+        InvHead -> S.fromList $ fst <$> qsParams'
+        InvPost -> iColumns apiRequest
+  proc@Function{..} <- mapLeft ApiRequestError $
+    findProc identifier paramKeys (preferParameters == Just SingleObject) (dbRoutines sCache) (iContentMediaType apiRequest) (invMethod == InvPost)
+  let relIdentifier = QualifiedIdentifier pdSchema (fromMaybe pdName $ Routine.funcTableName proc) -- done so a set returning function can embed other relations
+  rPlan <- readPlan relIdentifier conf sCache apiRequest
+  let args = case (invMethod, iContentMediaType apiRequest) of
+        (InvGet, _)             -> jsonRpcParams proc qsParams'
+        (InvHead, _)            -> jsonRpcParams proc qsParams'
+        (InvPost, MTUrlEncoded) -> maybe mempty (jsonRpcParams proc . payArray) $ iPayload apiRequest
+        (InvPost, _)            -> maybe mempty payRaw $ iPayload apiRequest
+      txMode = case (invMethod, pdVolatility) of
+          (InvGet,  _)                 -> SQL.Read
+          (InvHead, _)                 -> SQL.Read
+          (InvPost, Routine.Stable)    -> SQL.Read
+          (InvPost, Routine.Immutable) -> SQL.Read
+          (InvPost, Routine.Volatile)  -> SQL.Write
+      cPlan = callPlan proc apiRequest paramKeys args rPlan
+  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) (Just proc) rPlan
+  return $ CallReadPlan rPlan cPlan txMode proc binField
+  where
+    Preferences{..} = iPreferences apiRequest
+    qsParams' = QueryParams.qsParams (iQueryParams apiRequest)
+
+{-|
+  Search a pg proc by matching name and arguments keys to parameters. Since a function can be overloaded,
+  the name is not enough to find it. An overloaded function can have a different volatility or even a different return type.
+-}
+findProc :: QualifiedIdentifier -> S.Set Text -> Bool -> RoutineMap -> MediaType -> Bool -> Either ApiRequestError Routine
+findProc qi argumentsKeys paramsAsSingleObject allProcs contentMediaType isInvPost =
+  case matchProc of
+    ([], [])     -> Left $ NoRpc (qiSchema qi) (qiName qi) (S.toList argumentsKeys) paramsAsSingleObject contentMediaType isInvPost (HM.keys allProcs) lookupProcName
+    -- If there are no functions with named arguments, fallback to the single unnamed argument function
+    ([], [proc]) -> Right proc
+    ([], procs)  -> Left $ AmbiguousRpc (toList procs)
+    -- Matches the functions with named arguments
+    ([proc], _)  -> Right proc
+    (procs, _)   -> Left $ AmbiguousRpc (toList procs)
+  where
+    matchProc = overloadedProcPartition lookupProcName
+    -- First find the proc by name
+    lookupProcName = HM.lookupDefault mempty qi allProcs
+    -- The partition obtained has the form (overloadedProcs,fallbackProcs)
+    -- where fallbackProcs are functions with a single unnamed parameter
+    overloadedProcPartition = foldr select ([],[])
+    select proc ~(ts,fs)
+      | matchesParams proc         = (proc:ts,fs)
+      | hasSingleUnnamedParam proc = (ts,proc:fs)
+      | otherwise                  = (ts,fs)
+    -- If the function is called with post and has a single unnamed parameter
+    -- it can be called depending on content type and the parameter type
+    hasSingleUnnamedParam Function{pdParams=[RoutineParam{ppType}]} = isInvPost && case (contentMediaType, ppType) of
+      (MTApplicationJSON, "json")  -> True
+      (MTApplicationJSON, "jsonb") -> True
+      (MTTextPlain, "text")        -> True
+      (MTTextXML, "xml")           -> True
+      (MTOctetStream, "bytea")     -> True
+      _                            -> False
+    hasSingleUnnamedParam _ = False
+    matchesParams proc =
+      let
+        params = pdParams proc
+        firstType = (ppType <$> headMay params)
+      in
+      -- exceptional case for Prefer: params=single-object
+      if paramsAsSingleObject
+        then length params == 1 && (firstType == Just "json" || firstType == Just "jsonb")
+      -- If the function has no parameters, the arguments keys must be empty as well
+      else if null params
+        then null argumentsKeys && not (isInvPost && contentMediaType `elem` [MTOctetStream, MTTextPlain, MTTextXML])
+      -- A function has optional and required parameters. Optional parameters have a default value and
+      -- don't require arguments for the function to be executed, required parameters must have an argument present.
+      else case L.partition ppReq params of
+      -- If the function only has required parameters, the arguments keys must match those parameters
+        (reqParams, [])        -> argumentsKeys == S.fromList (ppName <$> reqParams)
+      -- If the function only has optional parameters, the arguments keys can match none or any of them(a subset)
+        ([], optParams)        -> argumentsKeys `S.isSubsetOf` S.fromList (ppName <$> optParams)
+      -- If the function has required and optional parameters, the arguments keys have to match the required parameters
+      -- and can match any or none of the default parameters.
+        (reqParams, optParams) -> argumentsKeys `S.difference` S.fromList (ppName <$> optParams) == S.fromList (ppName <$> reqParams)
+
+inspectPlanTxMode :: SQL.Mode
+inspectPlanTxMode = SQL.Read
 
 -- | Builds the ReadPlan tree on a number of stages.
 -- | Adds filters, order, limits on its respective nodes.
@@ -96,6 +206,9 @@ readPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Eit
 readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows} SchemaCache{dbRelationships} apiRequest  =
   mapLeft ApiRequestError $
   treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
+  addNullEmbedFilters =<<
+  validateSpreadEmbeds =<<
+  addRelatedOrders =<<
   addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
   addLogicTrees apiRequest =<<
   addRanges apiRequest =<<
@@ -108,15 +221,23 @@ initReadRequest qi@QualifiedIdentifier{..} =
   foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi, relName=qiName, depth=rootDepth} []
   where
     rootDepth = 0
-    defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing rootDepth
+    defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing False rootDepth
     treeEntry :: Depth -> Tree SelectItem -> ReadPlanTree -> ReadPlanTree
-    treeEntry depth (Node SelectRelation{..} fldForest) (Node q rForest) =
+    treeEntry depth (Node si fldForest) (Node q rForest) =
       let nxtDepth = succ depth in
-        Node q $
-          foldr (treeEntry nxtDepth)
-          (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relAlias=selAlias, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth} [])
-          fldForest:rForest
-    treeEntry _ (Node SelectField{..} _) (Node q rForest) = Node q{select=(selField, selCast, selAlias):select q} rForest
+      case si of
+        SelectRelation{..} ->
+          Node q $
+            foldr (treeEntry nxtDepth)
+            (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relAlias=selAlias, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth} [])
+            fldForest:rForest
+        SpreadRelation{..} ->
+          Node q $
+            foldr (treeEntry nxtDepth)
+            (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth, relIsSpread=True} [])
+            fldForest:rForest
+        SelectField{..} ->
+          Node q{select=(selField, selCast, selAlias):select q} rForest
 
 -- | Enforces the `max-rows` config on the result
 treeRestrictRange :: Maybe Integer -> Action -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -196,7 +317,7 @@ getJoinConditions tblAlias parentAlias Relationship{relTable=qi,relForeignTable=
 findRel :: Schema -> RelationshipsMap -> NodeName -> NodeName -> Maybe Hint -> Either ApiRequestError Relationship
 findRel schema allRels origin target hint =
   case rels of
-    []  -> Left $ NoRelBetween origin target schema
+    []  -> Left $ NoRelBetween origin target hint schema allRels
     [r] -> Right r
     rs  -> Left $ AmbiguousRelBetween origin target rs
   where
@@ -278,11 +399,9 @@ addFilters ApiRequest{..} rReq =
     QueryParams.QueryParams{..} = iQueryParams
     flts =
       case iAction of
-        ActionInvoke InvGet  -> qsFilters
-        ActionInvoke InvHead -> qsFilters
-        ActionInvoke _       -> qsFilters
-        ActionRead _         -> qsFilters
-        _                    -> qsFiltersNotRoot
+        ActionInvoke _ -> qsFilters
+        ActionRead _   -> qsFilters
+        _              -> qsFiltersNotRoot
 
     addFilterToNode :: (EmbedPath, Filter) -> Either ApiRequestError ReadPlanTree ->  Either ApiRequestError ReadPlanTree
     addFilterToNode =
@@ -299,6 +418,47 @@ addOrders ApiRequest{..} rReq =
     addOrderToNode :: (EmbedPath, [OrderTerm]) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
     addOrderToNode = updateNode (\o (Node q f) -> Node q{order=o} f)
 
+-- Validates that the related resource on the order is an embedded resource,
+-- e.g. if `clients` is inside the `select` in /projects?order=clients(id)&select=*,clients(*),
+-- and if it's a to-one relationship, it adds the right alias to the OrderRelationTerm so the generated query can succeed.
+-- TODO might be clearer if there's an additional intermediate type
+addRelatedOrders :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
+  newOrder <- getRelOrder `traverse` order
+  Node rp{order=newOrder} <$> addRelatedOrders `traverse` forest
+  where
+    getRelOrder ot@OrderTerm{}                   = Right ot
+    getRelOrder ot@OrderRelationTerm{otRelation} =
+      let foundRP = rootLabel <$> find (\(Node ReadPlan{relName, relAlias} _) -> otRelation == fromMaybe relName relAlias) forest in
+      case foundRP of
+        Just ReadPlan{relName,relAlias,relAggAlias,relToParent} ->
+          let isToOne = relIsToOne <$> relToParent
+              name    = fromMaybe relName relAlias in
+          if isToOne == Just True
+            then Right $ ot{otRelation=relAggAlias}
+            else Left $ RelatedOrderNotToOne (qiName from) name
+        Nothing ->
+          Left $ NotEmbedded otRelation
+
+-- Searches for null filters on embeds, e.g. `clients` on /projects?select=*,clients()&clients=not.is.null.
+-- If these are found, it changes the filter to use the internal aggregate name(`projects_clients_1`) so the filter can succeed.
+-- It fails if operators other than is.null or not.is.null are used.
+addNullEmbedFilters :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addNullEmbedFilters (Node rp@ReadPlan{where_=oldLogic} forest) = do
+  let readPlans = rootLabel <$> forest
+  newLogic <- getFilters readPlans `traverse` oldLogic
+  Node rp{ReadPlan.where_= newLogic} <$> (addNullEmbedFilters `traverse` forest)
+  where
+    getFilters :: [ReadPlan] -> LogicTree -> Either ApiRequestError LogicTree
+    getFilters rPlans (Expr b lOp trees) = Expr b lOp <$> (getFilters rPlans `traverse` trees)
+    getFilters rPlans flt@(Stmnt (Filter (fld, []) opExpr)) =
+      let foundRP = find (\ReadPlan{relName, relAlias} -> fld == fromMaybe relName relAlias) rPlans in
+      case (foundRP, opExpr) of
+        (Just ReadPlan{relAggAlias}, OpExpr b (Is TriNull)) -> Right $ Stmnt $ FilterNullEmbed b relAggAlias
+        (Just ReadPlan{relName}, _)                         -> Left $ UnacceptableFilter relName
+        _                                                   -> Right flt
+    getFilters _ flt@(Stmnt _)        = Right flt
+
 addRanges :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRanges ApiRequest{..} rReq =
   case iAction of
@@ -320,6 +480,15 @@ addLogicTrees ApiRequest{..} rReq =
     addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
     addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=t:lf} f)
 
+-- Validates that spread embeds are only done on to-one relationships
+validateSpreadEmbeds :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+validateSpreadEmbeds (Node rp@ReadPlan{relToParent=Nothing} forest) = Node rp <$> validateSpreadEmbeds `traverse` forest
+validateSpreadEmbeds (Node rp@ReadPlan{relIsSpread,relToParent=Just rel,relName} forest) = do
+  validRP <- if relIsSpread && not (relIsToOne rel)
+    then Left $ SpreadNotToOne (qiName $ relTable rel) relName -- TODO using relTable is not entirely right because ReadPlan might have an alias, need to store the parent alias on ReadPlan
+    else Right rp
+  Node validRP <$> validateSpreadEmbeds `traverse` forest
+
 -- Find a Node of the Tree and apply a function to it
 updateNode :: (a -> ReadPlanTree -> ReadPlanTree) -> (EmbedPath, a) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
 updateNode f ([], a) rr = f a <$> rr
@@ -335,19 +504,20 @@ updateNode f (targetNodeName:remainingPath, a) (Right (Node rootNode forest)) =
     findNode = find (\(Node ReadPlan{relName, relAlias} _) -> relName == targetNodeName || relAlias == Just targetNodeName) forest
 
 mutatePlan :: Mutation -> QualifiedIdentifier -> ApiRequest -> SchemaCache -> ReadPlanTree -> Either Error MutatePlan
-mutatePlan mutation qi ApiRequest{..} sCache readReq = mapLeft ApiRequestError $
+mutatePlan mutation qi ApiRequest{iPreferences=preferences, ..} sCache readReq = mapLeft ApiRequestError $
   case mutation of
     MutationCreate ->
-      Right $ Insert qi iColumns body ((,) <$> iPreferResolution <*> Just confCols) [] returnings pkCols
-    MutationUpdate -> Right $ Update qi iColumns body combinedLogic iTopLevelRange rootOrder returnings
+      mapRight (\typedColumns -> Insert qi typedColumns body ((,) <$> preferences.preferResolution <*> Just confCols) [] returnings pkCols applyDefaults) typedColumnsOrError
+    MutationUpdate ->
+      mapRight (\typedColumns -> Update qi typedColumns body combinedLogic iTopLevelRange rootOrder returnings applyDefaults) typedColumnsOrError
     MutationSingleUpsert ->
         if null qsLogic &&
            qsFilterFields == S.fromList pkCols &&
            not (null (S.fromList pkCols)) &&
            all (\case
-              Filter _ (OpExpr False (Op OpEqual _)) -> True
-              _                                      -> False) qsFiltersRoot
-          then Right $ Insert qi iColumns body (Just (MergeDuplicates, pkCols)) combinedLogic returnings mempty
+              Filter _ (OpExpr False (OpQuant OpEqual Nothing _)) -> True
+              _                                                   -> False) qsFiltersRoot
+          then mapRight (\typedColumns -> Insert qi typedColumns body (Just (MergeDuplicates, pkCols)) combinedLogic returnings mempty False) typedColumnsOrError
         else
           Left InvalidFilters
     MutationDelete -> Right $ Delete qi combinedLogic iTopLevelRange rootOrder returnings
@@ -355,41 +525,53 @@ mutatePlan mutation qi ApiRequest{..} sCache readReq = mapLeft ApiRequestError $
     confCols = fromMaybe pkCols qsOnConflict
     QueryParams.QueryParams{..} = iQueryParams
     returnings =
-      if iPreferRepresentation == None
+      if preferences.preferRepresentation == None
         then []
-        else returningCols readReq pkCols
+        else inferColsEmbedNeeds readReq pkCols
     pkCols = maybe mempty tablePKCols $ HM.lookup qi $ dbTables sCache
     logic = map snd qsLogic
     rootOrder = maybe [] snd $ find (\(x, _) -> null x) qsOrder
     combinedLogic = foldr addFilterToLogicForest logic qsFiltersRoot
     body = payRaw <$> iPayload -- the body is assumed to be json at this stage(ApiRequest validates)
+    tbl = HM.lookup qi $ dbTables sCache
+    typedColumnsOrError = resolveOrError tbl `traverse` S.toList iColumns
+    applyDefaults = preferences.preferMissing == Just ApplyDefaults
 
-callPlan :: ProcDescription -> ApiRequest -> ReadPlanTree -> CallPlan
-callPlan proc apiReq readReq = FunctionCall {
+resolveOrError :: Maybe Table -> FieldName -> Either ApiRequestError TypedField
+resolveOrError Nothing _ = Left NotFound
+resolveOrError (Just table) field =
+  case resolveTableField table field of
+    Nothing         -> Left $ ColumnNotFound (tableName table) field
+    Just typedField -> Right typedField
+
+callPlan :: Routine -> ApiRequest -> S.Set FieldName -> LBS.ByteString -> ReadPlanTree -> CallPlan
+callPlan proc ApiRequest{iPreferences=Preferences{..}} paramKeys args readReq = FunctionCall {
   funCQi = QualifiedIdentifier (pdSchema proc) (pdName proc)
 , funCParams = callParams
-, funCArgs = payRaw <$> iPayload apiReq
-, funCScalar = procReturnsScalar proc
-, funCMultipleCall = iPreferParameters apiReq == Just MultipleObjects
-, funCReturning = returningCols readReq []
+, funCArgs = Just args
+, funCScalar = funcReturnsScalar proc
+, funCSetOfScalar = funcReturnsSetOfScalar proc
+, funCRetCompositeAlias = funcReturnsCompositeAlias proc
+, funCReturning = inferColsEmbedNeeds readReq []
 }
   where
-    paramsAsSingleObject = iPreferParameters apiReq == Just SingleObject
+    paramsAsSingleObject = preferParameters == Just SingleObject
+    specifiedParams = filter (\x -> ppName x `S.member` paramKeys)
     callParams = case pdParams proc of
       [prm] | paramsAsSingleObject -> OnePosParam prm
             | ppName prm == mempty -> OnePosParam prm
             | otherwise            -> KeyParams $ specifiedParams [prm]
       prms  -> KeyParams $ specifiedParams prms
-    specifiedParams = filter (\x -> ppName x `S.member` iColumns apiReq)
 
-returningCols :: ReadPlanTree -> [FieldName] -> [FieldName]
-returningCols rr@(Node _ forest) pkCols
+-- | Infers the columns needed for an embed to be successful after a mutation or a function call.
+inferColsEmbedNeeds :: ReadPlanTree -> [FieldName] -> [FieldName]
+inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols
   -- if * is part of the select, we must not add pk or fk columns manually -
   -- otherwise those would be selected and output twice
   | "*" `elem` fldNames = ["*"]
   | otherwise           = returnings
   where
-    fldNames = fstFieldNames rr
+    fldNames = (\((fld, _), _, _) -> fld) <$> select
     -- Without fkCols, when a mutatePlan to
     -- /projects?select=name,clients(name) occurs, the RETURNING SQL part would
     -- be `RETURNING name`(see QueryBuilder).  This would make the embedding
@@ -403,8 +585,8 @@ returningCols rr@(Node _ forest) pkCols
           Just $ fst <$> cols
         Node ReadPlan{relToParent=Just Relationship{relCardinality=O2O _ cols}} _ ->
           Just $ fst <$> cols
-        Node ReadPlan{relToParent=Just Relationship{relCardinality=M2M Junction{junColumns1, junColumns2}}} _ ->
-          Just $ (fst <$> junColumns1) ++ (fst <$> junColumns2)
+        Node ReadPlan{relToParent=Just Relationship{relCardinality=M2M Junction{junColsSource=cols}}} _ ->
+          Just $ fst <$> cols
         Node ReadPlan{relToParent=Just ComputedRelationship{}} _ ->
           Nothing
         Node ReadPlan{relToParent=Nothing} _ ->
@@ -430,3 +612,34 @@ returningCols rr@(Node _ forest) pkCols
 -- they are later concatenated with AND in the QueryBuilder
 addFilterToLogicForest :: Filter -> [LogicTree] -> [LogicTree]
 addFilterToLogicForest flt lf = Stmnt flt : lf
+
+-- | If raw(binary) output is requested, check that MediaType is one of the
+-- admitted rawMediaTypes and that`?select=...` contains only one field other
+-- than `*`
+binaryField :: AppConfig -> MediaType -> Maybe Routine -> ReadPlanTree -> Either ApiRequestError (Maybe FieldName)
+binaryField AppConfig{configRawMediaTypes} acceptMediaType proc rpTree
+  | isRawMediaType =
+    if (funcReturnsScalar <$> proc) == Just True ||
+       (funcReturnsSetOfScalar <$> proc) == Just True
+      then Right $ Just "pgrst_scalar"
+      else
+        let
+          fieldName = fstFieldName rpTree
+        in
+        case fieldName of
+          Just fld -> Right $ Just fld
+          Nothing  -> Left $ BinaryFieldError acceptMediaType
+  | otherwise =
+      Right Nothing
+  where
+    isRawMediaType = acceptMediaType `elem` configRawMediaTypes `L.union` [MTOctetStream, MTTextPlain, MTTextXML] || isRawPlan acceptMediaType
+    isRawPlan mt = case mt of
+      MTPlan (MTPlanAttrs (Just MTOctetStream) _ _) -> True
+      MTPlan (MTPlanAttrs (Just MTTextPlain) _ _)   -> True
+      MTPlan (MTPlanAttrs (Just MTTextXML) _ _)     -> True
+      _                                             -> False
+
+    fstFieldName :: ReadPlanTree -> Maybe FieldName
+    fstFieldName (Node ReadPlan{select=(("*", []), _, _):_} [])  = Nothing
+    fstFieldName (Node ReadPlan{select=[((fld, []), _, _)]} []) = Just fld
+    fstFieldName _                                               = Nothing

src/PostgREST/Plan/CallPlan.hs

@@ -1,25 +1,57 @@
+{-# LANGUAGE NamedFieldPuns #-}
 module PostgREST.Plan.CallPlan
   ( CallPlan(..)
   , CallParams(..)
+  , jsonRpcParams
   )
 where
 
+import qualified Data.Aeson                        as JSON
 import qualified Data.ByteString.Lazy              as LBS
+import qualified Data.HashMap.Strict               as HM
 import           PostgREST.SchemaCache.Identifiers (FieldName,
                                                     QualifiedIdentifier)
-import           PostgREST.SchemaCache.Proc        (ProcParam (..))
+import           PostgREST.SchemaCache.Routine     (Routine (..),
+                                                    RoutineParam (..))
 
 import Protolude
 
 data CallPlan = FunctionCall
-  { funCQi           :: QualifiedIdentifier
-  , funCParams       :: CallParams
-  , funCArgs         :: Maybe LBS.ByteString
-  , funCScalar       :: Bool
-  , funCMultipleCall :: Bool
-  , funCReturning    :: [FieldName]
+  { funCQi                :: QualifiedIdentifier
+  , funCParams            :: CallParams
+  , funCArgs              :: Maybe LBS.ByteString
+  , funCScalar            :: Bool
+  , funCSetOfScalar       :: Bool
+  , funCRetCompositeAlias :: Bool
+  , funCReturning         :: [FieldName]
   }
 
 data CallParams
-  = KeyParams [ProcParam] -- ^ Call with key params: func(a := val1, b:= val2)
-  | OnePosParam ProcParam -- ^ Call with positional params(only one supported): func(val)
+  = KeyParams [RoutineParam] -- ^ Call with key params: func(a := val1, b:= val2)
+  | OnePosParam RoutineParam -- ^ Call with positional params(only one supported): func(val)
+
+-- | Convert rpc params `/rpc/func?a=val1&b=val2` to json `{"a": "val1", "b": "val2"}
+jsonRpcParams :: Routine -> [(Text, Text)] -> LBS.ByteString
+jsonRpcParams proc prms =
+  if not $ pdHasVariadic proc then -- if proc has no variadic param, save steps and directly convert to json
+    JSON.encode $ HM.fromList $ second JSON.toJSON <$> prms
+  else
+    let paramsMap = HM.fromListWith mergeParams $ toRpcParamValue proc <$> prms in
+    JSON.encode paramsMap
+  where
+    mergeParams :: RpcParamValue -> RpcParamValue -> RpcParamValue
+    mergeParams (Variadic a) (Variadic b) = Variadic $ b ++ a
+    mergeParams v _                       = v -- repeated params for non-variadic parameters are not merged
+
+toRpcParamValue :: Routine -> (Text, Text) -> (Text, RpcParamValue)
+toRpcParamValue proc (k, v) | prmIsVariadic k = (k, Variadic [v])
+                            | otherwise       = (k, Fixed v)
+  where
+    prmIsVariadic prm = isJust $ find (\RoutineParam{ppName, ppVar} -> ppName == prm && ppVar) $ pdParams proc
+
+-- | RPC query param value `/rpc/func?v=<value>`, used for VARIADIC functions on form-urlencoded POST and GETs
+-- | It can be fixed `?v=1` or repeated `?v=1&v=2&v=3.
+data RpcParamValue = Fixed Text | Variadic [Text]
+instance JSON.ToJSON RpcParamValue where
+  toJSON (Fixed    v) = JSON.toJSON v
+  toJSON (Variadic v) = JSON.toJSON v

src/PostgREST/Plan/MutatePlan.hs

@@ -4,34 +4,37 @@ module PostgREST.Plan.MutatePlan
 where
 
 import qualified Data.ByteString.Lazy as LBS
-import qualified Data.Set             as S
 
 import PostgREST.ApiRequest.Preferences  (PreferResolution)
 import PostgREST.ApiRequest.Types        (LogicTree, OrderTerm)
+import PostgREST.Plan.Types              (TypedField)
 import PostgREST.RangeQuery              (NonnegRange)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier)
 
+
 import Protolude
 
 data MutatePlan
   = Insert
       { in_        :: QualifiedIdentifier
-      , insCols    :: S.Set FieldName
+      , insCols    :: [TypedField]
       , insBody    :: Maybe LBS.ByteString
       , onConflict :: Maybe (PreferResolution, [FieldName])
       , where_     :: [LogicTree]
       , returning  :: [FieldName]
       , insPkCols  :: [FieldName]
+      , applyDefs  :: Bool
       }
   | Update
       { in_       :: QualifiedIdentifier
-      , updCols   :: S.Set FieldName
+      , updCols   :: [TypedField]
       , updBody   :: Maybe LBS.ByteString
       , where_    :: [LogicTree]
       , mutRange  :: NonnegRange
       , mutOrder  :: [OrderTerm]
       , returning :: [FieldName]
+      , applyDefs :: Bool
       }
   | Delete
       { in_       :: QualifiedIdentifier

src/PostgREST/Plan/ReadPlan.hs

@@ -1,8 +1,6 @@
-{-# LANGUAGE NamedFieldPuns #-}
 module PostgREST.Plan.ReadPlan
   ( ReadPlanTree
   , ReadPlan(..)
-  , fstFieldNames
   , JoinCondition(..)
   ) where
 
@@ -41,12 +39,8 @@ data ReadPlan = ReadPlan
   , relAggAlias  :: Alias
   , relHint      :: Maybe Hint
   , relJoinType  :: Maybe JoinType
+  , relIsSpread  :: Bool
   , depth        :: Depth
   -- ^ used for aliasing
   }
   deriving (Eq)
-
--- First level FieldNames(e.g get a,b from /table?select=a,b,other(c,d))
-fstFieldNames :: ReadPlanTree -> [FieldName]
-fstFieldNames (Node ReadPlan{select} _) =
-  fst . (\(f, _, _) -> f) <$> select

src/PostgREST/Plan/Types.hs

@@ -0,0 +1,24 @@
+module PostgREST.Plan.Types
+  ( TypedField(..)
+  , resolveTableField
+  ) where
+
+import qualified Data.HashMap.Strict.InsOrd as HMI
+
+import PostgREST.SchemaCache.Identifiers (FieldName)
+import PostgREST.SchemaCache.Table       (Column (..), Table (..))
+
+import Protolude
+
+-- | A TypedField is a field with sufficient information to be read from JSON with `json_to_recordset`.
+data TypedField = TypedField
+   { tfName    :: FieldName
+   , tfIRType  :: Text -- ^ The initial type of the field, before any casting.
+   , tfDefault :: Maybe Text
+   } deriving (Eq)
+
+resolveTableField :: Table -> FieldName -> Maybe TypedField
+resolveTableField table fieldName =
+  case HMI.lookup fieldName (tableColumns table) of
+    Just column -> Just $ TypedField (colName column) (colNominalType column) (colDefault column)
+    Nothing     -> Nothing

src/PostgREST/Query.hs

@@ -7,15 +7,16 @@ module PostgREST.Query
   , openApiQuery
   , readQuery
   , singleUpsertQuery
-  , txMode
   , updateQuery
   , setPgLocals
+  , runPreReq
   , DbHandler
   ) where
 
 import qualified Data.Aeson                        as JSON
 import qualified Data.Aeson.Key                    as K
 import qualified Data.Aeson.KeyMap                 as KM
+import qualified Data.ByteString                   as BS
 import qualified Data.ByteString.Lazy.Char8        as LBS
 import qualified Data.HashMap.Strict               as HM
 import qualified Data.Set                          as S
@@ -23,25 +24,23 @@ import qualified Data.Text.Encoding                as T
 import qualified Hasql.Decoders                    as HD
 import qualified Hasql.DynamicStatements.Snippet   as SQL (Snippet)
 import qualified Hasql.DynamicStatements.Statement as SQL
+import qualified Hasql.Encoders                    as HE
+import qualified Hasql.Statement                   as SQL
 import qualified Hasql.Transaction                 as SQL
-import qualified Hasql.Transaction.Sessions        as SQL
 
-import qualified PostgREST.Error              as Error
-import qualified PostgREST.Query.QueryBuilder as QueryBuilder
-import qualified PostgREST.Query.Statements   as Statements
-import qualified PostgREST.RangeQuery         as RangeQuery
-import qualified PostgREST.SchemaCache        as SchemaCache
-import qualified PostgREST.SchemaCache.Proc   as Proc
+import qualified PostgREST.Error               as Error
+import qualified PostgREST.Query.QueryBuilder  as QueryBuilder
+import qualified PostgREST.Query.Statements    as Statements
+import qualified PostgREST.RangeQuery          as RangeQuery
+import qualified PostgREST.SchemaCache         as SchemaCache
+import qualified PostgREST.SchemaCache.Routine as Routine
 
 import Data.Scientific (FPFormat (..), formatScientific, isInteger)
 
-import PostgREST.ApiRequest              (Action (..),
-                                          ApiRequest (..),
-                                          InvokeMethod (..),
-                                          Target (..))
+import PostgREST.ApiRequest              (ApiRequest (..))
 import PostgREST.ApiRequest.Preferences  (PreferCount (..),
-                                          PreferParameters (..),
                                           PreferTransaction (..),
+                                          Preferences (..),
                                           shouldCount)
 import PostgREST.Config                  (AppConfig (..),
                                           OpenAPIMode (..))
@@ -50,9 +49,9 @@ import PostgREST.Config.PgVersion        (PgVersion (..),
 import PostgREST.Error                   (Error)
 import PostgREST.MediaType               (MediaType (..))
 import PostgREST.Plan                    (CallReadPlan (..),
-                                          MutateReadPlan (..))
+                                          MutateReadPlan (..),
+                                          WrappedReadPlan (..))
 import PostgREST.Plan.MutatePlan         (MutatePlan (..))
-import PostgREST.Plan.ReadPlan           (ReadPlanTree)
 import PostgREST.Query.SqlFragment       (fromQi, intercalateSnippet,
                                           pgFmtIdentList,
                                           setConfigLocal,
@@ -61,31 +60,29 @@ import PostgREST.Query.Statements        (ResultSet (..))
 import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
                                           Schema)
-import PostgREST.SchemaCache.Proc        (ProcDescription (..),
-                                          ProcVolatility (..),
-                                          ProcsMap)
+import PostgREST.SchemaCache.Routine     (Routine (..), RoutineMap)
 import PostgREST.SchemaCache.Table       (TablesMap)
 
 import Protolude hiding (Handler)
 
 type DbHandler = ExceptT Error SQL.Transaction
 
-readQuery :: ReadPlanTree -> AppConfig -> ApiRequest -> DbHandler ResultSet
-readQuery req conf@AppConfig{..} apiReq@ApiRequest{..} = do
-  let countQuery = QueryBuilder.readPlanToCountQuery req
+readQuery :: WrappedReadPlan -> AppConfig -> ApiRequest -> DbHandler ResultSet
+readQuery WrappedReadPlan{wrReadPlan, wrBinField} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} = do
+  let countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan
   resultSet <-
      lift . SQL.statement mempty $
       Statements.prepareRead
-        (QueryBuilder.readPlanToQuery req)
-        (if iPreferCount == Just EstimatedCount then
+        (QueryBuilder.readPlanToQuery wrReadPlan)
+        (if preferCount == Just EstimatedCount then
            -- LIMIT maxRows + 1 so we can determine below that maxRows was surpassed
            QueryBuilder.limitedQuery countQuery ((+ 1) <$> configDbMaxRows)
          else
            countQuery
         )
-        (shouldCount iPreferCount)
+        (shouldCount preferCount)
         iAcceptMediaType
-        iBinaryField
+        wrBinField
         configDbPreparedStatements
   failNotSingular iAcceptMediaType resultSet
   optionalRollback conf apiReq
@@ -93,8 +90,8 @@ readQuery req conf@AppConfig{..} apiReq@ApiRequest{..} = do
 
 resultSetWTotal :: AppConfig -> ApiRequest -> ResultSet -> SQL.Snippet -> DbHandler ResultSet
 resultSetWTotal _ _ rs@RSPlan{} _ = return rs
-resultSetWTotal AppConfig{..} ApiRequest{..} rs@RSStandard{rsTableTotal=tableTotal} countQuery =
-  case iPreferCount of
+resultSetWTotal AppConfig{..} ApiRequest{iPreferences=Preferences{..}} rs@RSStandard{rsTableTotal=tableTotal} countQuery =
+  case preferCount of
     Just PlannedCount -> do
       total <- explain
       return rs{rsTableTotal=total}
@@ -155,63 +152,45 @@ deleteQuery mrPlan apiReq@ApiRequest{..} conf = do
   optionalRollback conf apiReq
   pure resultSet
 
-invokeQuery :: ProcDescription -> CallReadPlan -> ApiRequest -> AppConfig -> DbHandler ResultSet
-invokeQuery proc CallReadPlan{crReadPlan, crCallPlan} apiReq@ApiRequest{..} conf@AppConfig{..} = do
+invokeQuery :: Routine -> CallReadPlan -> ApiRequest -> AppConfig -> PgVersion -> DbHandler ResultSet
+invokeQuery proc CallReadPlan{crReadPlan, crCallPlan, crBinField} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} conf@AppConfig{..} pgVer = do
   resultSet <-
     lift . SQL.statement mempty $
       Statements.prepareCall
-        (Proc.procReturnsScalar proc)
-        (Proc.procReturnsSingle proc)
-        (QueryBuilder.callPlanToQuery crCallPlan)
+        (Routine.funcReturnsScalar proc)
+        (Routine.funcReturnsSingleComposite proc)
+        (Routine.funcReturnsSetOfScalar proc)
+        (QueryBuilder.callPlanToQuery crCallPlan pgVer)
         (QueryBuilder.readPlanToQuery crReadPlan)
         (QueryBuilder.readPlanToCountQuery crReadPlan)
-        (shouldCount iPreferCount)
+        (shouldCount preferCount)
         iAcceptMediaType
-        (iPreferParameters == Just MultipleObjects)
-        iBinaryField
+        crBinField
         configDbPreparedStatements
 
   optionalRollback conf apiReq
   failNotSingular iAcceptMediaType resultSet
   pure resultSet
 
-openApiQuery :: SchemaCache -> PgVersion -> AppConfig -> Schema -> DbHandler (Maybe (TablesMap, ProcsMap, Maybe Text))
+openApiQuery :: SchemaCache -> PgVersion -> AppConfig -> Schema -> DbHandler (Maybe (TablesMap, RoutineMap, Maybe Text))
 openApiQuery sCache pgVer AppConfig{..} tSchema =
   lift $ case configOpenApiMode of
     OAFollowPriv -> do
       tableAccess <- SQL.statement [tSchema] (SchemaCache.accessibleTables pgVer configDbPreparedStatements)
       Just <$> ((,,)
             (HM.filterWithKey (\qi _ -> S.member qi tableAccess) $ SchemaCache.dbTables sCache)
-        <$> SQL.statement tSchema (SchemaCache.accessibleProcs pgVer configDbPreparedStatements)
+        <$> SQL.statement tSchema (SchemaCache.accessibleFuncs pgVer configDbPreparedStatements)
         <*> SQL.statement tSchema (SchemaCache.schemaDescription configDbPreparedStatements))
     OAIgnorePriv ->
       Just <$> ((,,)
             (HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) $ SchemaCache.dbTables sCache)
-            (HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) $ SchemaCache.dbProcs sCache)
+            (HM.filterWithKey (\(QualifiedIdentifier sch _) _ ->  sch == tSchema) $ SchemaCache.dbRoutines sCache)
         <$> SQL.statement tSchema (SchemaCache.schemaDescription configDbPreparedStatements))
     OADisabled ->
       pure Nothing
 
-txMode :: ApiRequest -> SQL.Mode
-txMode ApiRequest{..} =
-  case (iAction, iTarget) of
-    (ActionRead _, _) ->
-      SQL.Read
-    (ActionInspect _, _) ->
-      SQL.Read
-    (ActionInvoke InvGet, _) ->
-      SQL.Read
-    (ActionInvoke InvHead, _) ->
-      SQL.Read
-    (ActionInvoke InvPost, TargetProc ProcDescription{pdVolatility=Stable} _) ->
-      SQL.Read
-    (ActionInvoke InvPost, TargetProc ProcDescription{pdVolatility=Immutable} _) ->
-      SQL.Read
-    _ ->
-      SQL.Write
-
 writeQuery :: MutateReadPlan -> ApiRequest -> AppConfig  -> DbHandler ResultSet
-writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq conf =
+writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferences=Preferences{..}} conf =
   let
     (isInsert, pkCols) = case mrMutatePlan of {Insert{insPkCols} -> (True, insPkCols); _ -> (False, mempty);}
   in
@@ -221,7 +200,7 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq conf =
       (QueryBuilder.mutatePlanToQuery mrMutatePlan)
       isInsert
       (iAcceptMediaType apiReq)
-      (iPreferRepresentation apiReq)
+      preferRepresentation
       pkCols
       (configDbPreparedStatements conf)
 
@@ -245,24 +224,23 @@ failsChangesOffLimits (Just maxChanges) RSStandard{rsQueryTotal=queryTotal} =
 
 -- | Set a transaction to roll back if requested
 optionalRollback :: AppConfig -> ApiRequest -> DbHandler ()
-optionalRollback AppConfig{..} ApiRequest{..} = do
+optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} = do
   lift $ when (shouldRollback || (configDbTxRollbackAll && not shouldCommit)) $ do
     SQL.sql "SET CONSTRAINTS ALL IMMEDIATE"
     SQL.condemn
   where
     shouldCommit =
-      configDbTxAllowOverride && iPreferTransaction == Just Commit
+      configDbTxAllowOverride && preferTransaction == Just Commit
     shouldRollback =
-      configDbTxAllowOverride && iPreferTransaction == Just Rollback
+      configDbTxAllowOverride && preferTransaction == Just Rollback
 
--- | Runs local(transaction scoped) GUCs for every request, plus the pre-request function
-setPgLocals :: AppConfig   -> KM.KeyMap JSON.Value -> Text ->
-               ApiRequest  -> ByteString -> PgVersion -> DbHandler ()
-setPgLocals conf claims role req jsonDbS actualPgVersion = do
-  lift $ SQL.statement mempty $ SQL.dynamicallyParameterized
-    ("select " <> intercalateSnippet ", " (searchPathSql : roleSql ++ claimsSql ++ [methodSql, pathSql] ++ headersSql ++ cookiesSql ++ appSettingsSql ++ specSql))
-    HD.noResult (configDbPreparedStatements conf)
-  lift $ traverse_ SQL.sql preReqSql
+-- | Runs local (transaction scoped) GUCs for every request.
+setPgLocals :: AppConfig  -> KM.KeyMap JSON.Value -> BS.ByteString -> [(ByteString, ByteString)] ->
+               ApiRequest -> PgVersion -> DbHandler ()
+setPgLocals AppConfig{..} claims role roleSettings req actualPgVersion = lift $
+  SQL.statement mempty $ SQL.dynamicallyParameterized
+    ("select " <> intercalateSnippet ", " (searchPathSql : roleSql ++ roleSettingsSql ++ claimsSql ++ [methodSql, pathSql] ++ headersSql ++ cookiesSql ++ appSettingsSql))
+    HD.noResult configDbPreparedStatements
   where
     methodSql = setConfigLocal mempty ("request.method", iMethod req)
     pathSql = setConfigLocal mempty ("request.path", iPath req)
@@ -275,16 +253,13 @@ setPgLocals conf claims role req jsonDbS actualPgVersion = do
     claimsSql = if usesLegacyGucs
                   then setConfigLocal "request.jwt.claim." <$> [(toUtf8 $ K.toText c, toUtf8 $ unquoted v) | (c,v) <- KM.toList claims]
                   else [setConfigLocal mempty ("request.jwt.claims", LBS.toStrict $ JSON.encode claims)]
-    roleSql = [setConfigLocal mempty ("role", toUtf8 role)]
-    appSettingsSql = setConfigLocal mempty <$> (join bimap toUtf8 <$> configAppSettings conf)
+    roleSql = [setConfigLocal mempty ("role", role)]
+    roleSettingsSql = setConfigLocal mempty <$> roleSettings
+    appSettingsSql = setConfigLocal mempty <$> (join bimap toUtf8 <$> configAppSettings)
     searchPathSql =
-      let schemas = pgFmtIdentList (iSchema req : configDbExtraSearchPath conf) in
+      let schemas = pgFmtIdentList (iSchema req : configDbExtraSearchPath) in
       setConfigLocal mempty ("search_path", schemas)
-    preReqSql = (\f -> "select " <> fromQi f <> "();") <$> configDbPreRequest conf
-    specSql = case iTarget req of
-      TargetProc{tpIsRootSpec=True} -> [setConfigLocal mempty ("request.spec", jsonDbS)]
-      _                             -> mempty
-    usesLegacyGucs = configDbUseLegacyGucs conf && actualPgVersion < pgVersion140
+    usesLegacyGucs = configDbUseLegacyGucs && actualPgVersion < pgVersion140
 
     unquoted :: JSON.Value -> Text
     unquoted (JSON.String t) = t
@@ -292,3 +267,13 @@ setPgLocals conf claims role req jsonDbS actualPgVersion = do
       toS $ formatScientific Fixed (if isInteger n then Just 0 else Nothing) n
     unquoted (JSON.Bool b) = show b
     unquoted v = T.decodeUtf8 . LBS.toStrict $ JSON.encode v
+
+-- | Runs the pre-request function.
+runPreReq :: AppConfig -> DbHandler ()
+runPreReq conf = lift $ traverse_ (SQL.statement mempty . stmt) (configDbPreRequest conf)
+  where
+    stmt req = SQL.Statement
+      ("select " <> fromQi req <> "()")
+      HE.noParams
+      HD.noResult
+      (configDbPreparedStatements conf)

src/PostgREST/Query/QueryBuilder.hs

@@ -17,22 +17,25 @@ module PostgREST.Query.QueryBuilder
   ) where
 
 import qualified Data.ByteString.Char8           as BS
-import qualified Data.Set                        as S
 import qualified Hasql.DynamicStatements.Snippet as SQL
 
 import Data.Tree (Tree (..))
 
 import PostgREST.ApiRequest.Preferences   (PreferResolution (..))
+import PostgREST.Config.PgVersion         (PgVersion, pgVersion110,
+                                           pgVersion130)
 import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
-import PostgREST.SchemaCache.Proc         (ProcParam (..))
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                            Junction (..),
-                                           Relationship (..))
+                                           Relationship (..),
+                                           relIsToOne)
+import PostgREST.SchemaCache.Routine      (RoutineParam (..))
 
 import PostgREST.ApiRequest.Types
 import PostgREST.Plan.CallPlan
 import PostgREST.Plan.MutatePlan
 import PostgREST.Plan.ReadPlan
+import PostgREST.Plan.Types
 import PostgREST.Query.SqlFragment
 import PostgREST.RangeQuery        (allRange)
 
@@ -41,7 +44,7 @@ import Protolude
 readPlanToQuery :: ReadPlanTree -> SQL.Snippet
 readPlanToQuery (Node ReadPlan{select,from=mainQi,fromAlias,where_=logicForest,order, range_=readRange, relToParent, relJoinConds} forest) =
   "SELECT " <>
-  intercalateSnippet ", " ((pgFmtSelectItem qi <$> select) ++ selects) <> " " <>
+  intercalateSnippet ", " ((pgFmtSelectItem qi <$> (if null select && null forest then defSelect else select)) ++ selects) <> " " <>
   fromFrag <> " " <>
   intercalateSnippet " " joins <> " " <>
   (if null logicForest && null relJoinConds
@@ -52,45 +55,41 @@ readPlanToQuery (Node ReadPlan{select,from=mainQi,fromAlias,where_=logicForest,o
   where
     fromFrag = fromF relToParent mainQi fromAlias
     qi = getQualifiedIdentifier relToParent mainQi fromAlias
+    defSelect = [(("*", []), Nothing, Nothing)] -- gets all the columns in case of an empty select, ignoring/obtaining these columns is done at the aggregation stage
     (selects, joins) = foldr getSelectsJoins ([],[]) forest
 
 getSelectsJoins :: ReadPlanTree -> ([SQL.Snippet], [SQL.Snippet]) -> ([SQL.Snippet], [SQL.Snippet])
 getSelectsJoins (Node ReadPlan{relToParent=Nothing} _) _ = ([], [])
-getSelectsJoins rr@(Node ReadPlan{relName, relToParent=Just rel, relAggAlias, relAlias, relJoinType=joinType} _) (selects,joins) =
+getSelectsJoins rr@(Node ReadPlan{select, relName, relToParent=Just rel, relAggAlias, relAlias, relJoinType, relIsSpread} forest) (selects,joins) =
   let
     subquery = readPlanToQuery rr
     aliasOrName = pgFmtIdent $ fromMaybe relName relAlias
     aggAlias = pgFmtIdent relAggAlias
     correlatedSubquery sub al cond =
-      (if joinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> SQL.sql al <> " ON " <> cond
-    isToOne = case rel of
-      Relationship{relCardinality=M2O _ _} -> True
-      Relationship{relCardinality=O2O _ _} -> True
-      ComputedRelationship{relToOne=True}  -> True
-      _                                    -> False
-    (sel, joi) = if isToOne
+      (if relJoinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> SQL.sql al <> " ON " <> cond
+    (sel, joi) = if relIsToOne rel
       then
-        ( SQL.sql ("row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName)
+        ( if relIsSpread
+            then SQL.sql aggAlias <> ".*"
+            else SQL.sql ("row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName)
         , correlatedSubquery subquery aggAlias "TRUE")
       else
         ( SQL.sql $ "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> aliasOrName
         , correlatedSubquery (
             "SELECT json_agg(" <> SQL.sql aggAlias <> ") AS " <> SQL.sql aggAlias <>
             "FROM (" <> subquery <> " ) AS " <> SQL.sql aggAlias
-          ) aggAlias $ if joinType == Just JTInner then SQL.sql aggAlias <> " IS NOT NULL" else "TRUE")
+          ) aggAlias $ if relJoinType == Just JTInner then SQL.sql aggAlias <> " IS NOT NULL" else "TRUE")
   in
-  (sel:selects, joi:joins)
+  (if null select && null forest then selects else sel:selects, joi:joins)
 
 mutatePlanToQuery :: MutatePlan -> SQL.Snippet
-mutatePlanToQuery (Insert mainQi iCols body onConflct putConditions returnings _) =
-  "WITH " <> normalizedBody body <> " " <>
-  "INSERT INTO " <> SQL.sql (fromQi mainQi) <> SQL.sql (if S.null iCols then " " else "(" <> cols <> ") ") <>
-  "SELECT " <> SQL.sql cols <> " " <>
-  SQL.sql ("FROM json_populate_recordset (null::" <> fromQi mainQi <> ", " <> selectBody <> ") _ ") <>
+mutatePlanToQuery (Insert mainQi iCols body onConflct putConditions returnings _ applyDefaults) =
+  "INSERT INTO " <> SQL.sql (fromQi mainQi) <> SQL.sql (if null iCols then " " else "(" <> cols <> ") ") <>
+  fromJsonBodyF body iCols True False applyDefaults <>
   -- Only used for PUT
-  (if null putConditions then mempty else "WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree (QualifiedIdentifier mempty "_") <$> putConditions)) <>
+  (if null putConditions then mempty else "WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree (QualifiedIdentifier mempty "pgrst_body") <$> putConditions)) <>
   SQL.sql (BS.unwords [
-    maybe "" (\(oncDo, oncCols) ->
+    maybe mempty (\(oncDo, oncCols) ->
       if null oncCols then
         mempty
       else
@@ -98,33 +97,32 @@ mutatePlanToQuery (Insert mainQi iCols body onConflct putConditions returnings _
         IgnoreDuplicates ->
           "DO NOTHING"
         MergeDuplicates  ->
-          if S.null iCols
+          if null iCols
              then "DO NOTHING"
-             else "DO UPDATE SET " <> BS.intercalate ", " (pgFmtIdent <> const " = EXCLUDED." <> pgFmtIdent <$> S.toList iCols)
+             else "DO UPDATE SET " <> BS.intercalate ", " ((pgFmtIdent . tfName) <> const " = EXCLUDED." <> (pgFmtIdent . tfName) <$> iCols)
       ) onConflct,
     returningF mainQi returnings
     ])
   where
-    cols = BS.intercalate ", " $ pgFmtIdent <$> S.toList iCols
+    cols = BS.intercalate ", " $ pgFmtIdent . tfName <$> iCols
 
 -- An update without a limit is always filtered with a WHERE
-mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings)
-  | S.null uCols =
+mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings applyDefaults)
+  | null uCols =
     -- if there are no columns we cannot do UPDATE table SET {empty}, it'd be invalid syntax
     -- selecting an empty resultset from mainQi gives us the column names to prevent errors when using &select=
     -- the select has to be based on "returnings" to make computed overloaded functions not throw
     SQL.sql $ "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"
 
   | range == allRange =
-    "WITH " <> normalizedBody body <> " " <>
     "UPDATE " <> mainTbl <> " SET " <> SQL.sql nonRangeCols <> " " <>
-    "FROM (SELECT * FROM json_populate_recordset (null::" <> mainTbl <> " , " <> SQL.sql selectBody <> " )) _ " <>
+    fromJsonBodyF body uCols False False applyDefaults <>
     whereLogic <> " " <>
     SQL.sql (returningF mainQi returnings)
 
   | otherwise =
-    "WITH " <> normalizedBody body <> ", " <>
-    "pgrst_update_body AS (SELECT * FROM json_populate_recordset (null::" <> mainTbl <> " , " <> SQL.sql selectBody <> " ) LIMIT 1), " <>
+    "WITH " <>
+    "pgrst_update_body AS (" <> fromJsonBodyF body uCols True True applyDefaults <> "), " <>
     "pgrst_affected_rows AS (" <>
       "SELECT " <> SQL.sql rangeIdF <> " FROM " <> mainTbl <>
       whereLogic <> " " <>
@@ -140,8 +138,8 @@ mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings)
     whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
     mainTbl = SQL.sql (fromQi mainQi)
     emptyBodyReturnedColumns = if null returnings then "NULL" else BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
-    nonRangeCols = BS.intercalate ", " (pgFmtIdent <> const " = _." <> pgFmtIdent <$> S.toList uCols)
-    rangeCols = BS.intercalate ", " ((\col -> pgFmtIdent col <> " = (SELECT " <> pgFmtIdent col <> " FROM pgrst_update_body) ") <$> S.toList uCols)
+    nonRangeCols = BS.intercalate ", " (pgFmtIdent . tfName <> const " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_body") . tfName <$> uCols)
+    rangeCols = BS.intercalate ", " ((\col -> pgFmtIdent (tfName col) <> " = (SELECT " <> pgFmtIdent (tfName col) <> " FROM pgrst_update_body) ") <$> uCols)
     (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
 
 mutatePlanToQuery (Delete mainQi logicForest range ordts returnings)
@@ -167,52 +165,29 @@ mutatePlanToQuery (Delete mainQi logicForest range ordts returnings)
     whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
     (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
 
-callPlanToQuery :: CallPlan -> SQL.Snippet
-callPlanToQuery (FunctionCall qi params args returnsScalar multipleCall returnings) =
-  prmsCTE <> argsBody
+callPlanToQuery :: CallPlan -> PgVersion -> SQL.Snippet
+callPlanToQuery (FunctionCall qi params args returnsScalar returnsSetOfScalar returnsCompositeAlias returnings) pgVer =
+  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call AS pgrst_scalar " else returnedColumns) <> " " <>
+  fromCall
   where
-    (prmsCTE, argFrag) = case params of
-      OnePosParam prm -> ("WITH pgrst_args AS (SELECT NULL)", singleParameter args (encodeUtf8 $ ppType prm))
-      KeyParams []    -> (mempty, mempty)
-      KeyParams prms  -> (
-          "WITH " <> normalizedBody args <> ", " <>
-          SQL.sql (
-            BS.unwords [
-            "pgrst_args AS (",
-              "SELECT * FROM json_to_recordset(" <> selectBody <> ") AS _(" <> fmtParams prms (const mempty) (\a -> " " <> encodeUtf8 (ppType a)) <> ")",
-            ")"])
-         , SQL.sql $ if multipleCall
-             then fmtParams prms varadicPrefix (\a -> " := pgrst_args." <> pgFmtIdent (ppName a))
-             else fmtParams prms varadicPrefix (\a -> " := (SELECT " <> pgFmtIdent (ppName a) <> " FROM pgrst_args LIMIT 1)")
-        )
+    fromCall = case params of
+      OnePosParam prm -> "FROM " <> callIt (singleParameter args $ encodeUtf8 $ ppType prm)
+      KeyParams []    -> "FROM " <> callIt mempty
+      KeyParams prms  -> fromJsonBodyF args ((\p -> TypedField (ppName p) (ppType p) Nothing) <$> prms) False True False <> ", " <>
+                         "LATERAL " <> callIt (fmtParams prms)
 
-    fmtParams :: [ProcParam] -> (ProcParam -> SqlFragment) -> (ProcParam -> SqlFragment) -> SqlFragment
-    fmtParams prms prmFragPre prmFragSuf = BS.intercalate ", "
-      ((\a -> prmFragPre a <> pgFmtIdent (ppName a) <> prmFragSuf a) <$> prms)
+    callIt :: SQL.Snippet -> SQL.Snippet
+    callIt argument | pgVer < pgVersion130 && pgVer >= pgVersion110 && returnsCompositeAlias = "(SELECT (" <> SQL.sql (fromQi qi) <> "(" <> argument <> ")).*) pgrst_call"
+                    | otherwise                                                              = SQL.sql (fromQi qi) <> "(" <> argument <> ") pgrst_call"
 
-    varadicPrefix :: ProcParam -> SqlFragment
-    varadicPrefix a = if ppVar a then "VARIADIC " else mempty
-
-    argsBody :: SQL.Snippet
-    argsBody
-      | multipleCall =
-          if returnsScalar
-            then "SELECT " <> callIt <> " AS pgrst_scalar FROM pgrst_args"
-            else "SELECT pgrst_lat_args.* FROM pgrst_args, " <>
-                 "LATERAL ( SELECT " <> returnedColumns <> " FROM " <> callIt <> " ) pgrst_lat_args"
-      | otherwise =
-          if returnsScalar
-            then "SELECT " <> callIt <> " AS pgrst_scalar"
-            else "SELECT " <> returnedColumns <> " FROM " <> callIt
-
-    callIt :: SQL.Snippet
-    callIt = SQL.sql (fromQi qi) <> "(" <> argFrag <> ")"
+    fmtParams :: [RoutineParam] -> SQL.Snippet
+    fmtParams prms = SQL.sql $ BS.intercalate ", "
+      ((\a -> (if ppVar a then "VARIADIC " else mempty) <> pgFmtIdent (ppName a) <> " := pgrst_body." <> pgFmtIdent (ppName a)) <$> prms)
 
     returnedColumns :: SQL.Snippet
     returnedColumns
       | null returnings = "*"
-      | otherwise       = SQL.sql $ BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName qi) <$> returnings)
-
+      | otherwise       = SQL.sql $ BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty "pgrst_call") <$> returnings)
 
 -- | SQL query meant for COUNTing the root node of the Tree.
 -- It only takes WHERE into account and doesn't include LIMIT/OFFSET because it would reduce the COUNT.

src/PostgREST/Query/SqlFragment.hs

@@ -21,7 +21,6 @@ module PostgREST.Query.SqlFragment
   , limitOffsetF
   , locationF
   , mutRangeF
-  , normalizedBody
   , orderF
   , pgFmtColumn
   , pgFmtIdent
@@ -30,10 +29,10 @@ module PostgREST.Query.SqlFragment
   , pgFmtLogicTree
   , pgFmtOrderTerm
   , pgFmtSelectItem
+  , fromJsonBodyF
   , responseHeadersF
   , responseStatusF
   , returningF
-  , selectBody
   , singleParameter
   , sourceCTEName
   , unknownEncoder
@@ -65,15 +64,18 @@ import PostgREST.ApiRequest.Types        (Alias, Cast, Field,
                                           JsonPath,
                                           LogicOperator (..),
                                           LogicTree (..), OpExpr (..),
+                                          OpQuantifier (..),
                                           Operation (..),
                                           OrderDirection (..),
                                           OrderNulls (..),
                                           OrderTerm (..),
+                                          QuantOperator (..),
                                           SimpleOperator (..),
                                           TrileanVal (..))
 import PostgREST.MediaType               (MTPlanFormat (..),
                                           MTPlanOption (..))
 import PostgREST.Plan.ReadPlan           (JoinCondition (..))
+import PostgREST.Plan.Types              (TypedField (..))
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           rangeLimit, rangeOffset)
 import PostgREST.SchemaCache.Identifiers (FieldName,
@@ -91,24 +93,27 @@ noLocationF = "array[]::text[]"
 sourceCTEName :: SqlFragment
 sourceCTEName = "pgrst_source"
 
-singleValOperator :: SimpleOperator -> SqlFragment
-singleValOperator = \case
+simpleOperator :: SimpleOperator -> SqlFragment
+simpleOperator = \case
+  OpNotEqual        -> "<>"
+  OpContains        -> "@>"
+  OpContained       -> "<@"
+  OpOverlap         -> "&&"
+  OpStrictlyLeft    -> "<<"
+  OpStrictlyRight   -> ">>"
+  OpNotExtendsRight -> "&<"
+  OpNotExtendsLeft  -> "&>"
+  OpAdjacent        -> "-|-"
+
+quantOperator :: QuantOperator -> SqlFragment
+quantOperator = \case
   OpEqual            -> "="
   OpGreaterThanEqual -> ">="
   OpGreaterThan      -> ">"
   OpLessThanEqual    -> "<="
   OpLessThan         -> "<"
-  OpNotEqual         -> "<>"
   OpLike             -> "like"
   OpILike            -> "ilike"
-  OpContains         -> "@>"
-  OpContained        -> "<@"
-  OpOverlap          -> "&&"
-  OpStrictlyLeft     -> "<<"
-  OpStrictlyRight    -> ">>"
-  OpNotExtendsRight  -> "&<"
-  OpNotExtendsLeft   -> "&>"
-  OpAdjacent         -> "-|-"
   OpMatch            -> "~"
   OpIMatch           -> "~*"
 
@@ -119,25 +124,6 @@ ftsOperator = \case
   FilterFtsPhrase    -> "@@ phraseto_tsquery"
   FilterFtsWebsearch -> "@@ websearch_to_tsquery"
 
--- |
--- These CTEs convert a json object into a json array, this way we can use json_populate_recordset for all json payloads
--- Otherwise we'd have to use json_populate_record for json objects and json_populate_recordset for json arrays
--- We do this in SQL to avoid processing the JSON in application code
--- TODO: At this stage there shouldn't be a Maybe since ApiRequest should ensure that an INSERT/UPDATE has a body
-normalizedBody :: Maybe LBS.ByteString -> SQL.Snippet
-normalizedBody body =
-  "pgrst_payload AS (SELECT " <> jsonPlaceHolder <> " AS json_data), " <>
-  SQL.sql (BS.unwords [
-    "pgrst_body AS (",
-      "SELECT",
-        "CASE WHEN json_typeof(json_data) = 'array'",
-          "THEN json_data",
-          "ELSE json_build_array(json_data)",
-        "END AS val",
-      "FROM pgrst_payload)"])
-  where
-    jsonPlaceHolder = SQL.encoderAndParam (HE.nullable HE.jsonLazyBytes) body
-
 singleParameter :: Maybe LBS.ByteString -> ByteString -> SQL.Snippet
 singleParameter body typ =
   if typ == "bytea"
@@ -145,9 +131,6 @@ singleParameter body typ =
     then SQL.encoderAndParam (HE.nullable HE.bytea) (LBS.toStrict <$> body)
     else SQL.encoderAndParam (HE.nullable HE.unknown) (LBS.toStrict <$> body) <> "::" <> SQL.sql typ
 
-selectBody :: SqlFragment
-selectBody = "(SELECT val FROM pgrst_body)"
-
 -- Here we build the pg array literal, e.g '{"Hebdon, John","Other","Another"}', manually.
 -- This is necessary to pass an "unknown" array and let pg infer the type.
 -- There are backslashes here, but since this value is parametrized and is not a string constant
@@ -165,6 +148,16 @@ pgBuildArrayLiteral vals =
 pgFmtIdent :: Text -> SqlFragment
 pgFmtIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
 
+-- Only use it if the input comes from the database itself, like on `jsonb_build_object('column_from_a_table', val)..`
+pgFmtLit :: Text -> Text
+pgFmtLit x =
+  let trimmed = trimNullChars x
+      escaped = "'" <> T.replace "'" "''" trimmed <> "'"
+      slashed = T.replace "\\" "\\\\" escaped in
+  if "\\" `T.isInfixOf` escaped
+    then "E" <> slashed
+    else slashed
+
 trimNullChars :: Text -> Text
 trimNullChars = T.takeWhile (/= '\x0')
 
@@ -190,15 +183,17 @@ asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
       ")"
     asCsvBodyF = "coalesce(string_agg(substring(_postgrest_t::text, 2, length(_postgrest_t::text) - 2), '\n'), '')"
 
-asJsonF :: Bool -> SqlFragment
-asJsonF returnsScalar
-  | returnsScalar = "coalesce(json_agg(_postgrest_t.pgrst_scalar), '[]')::character varying"
-  | otherwise     = "coalesce(json_agg(_postgrest_t), '[]')::character varying"
-
 asJsonSingleF :: Bool -> SqlFragment
 asJsonSingleF returnsScalar
-  | returnsScalar = "coalesce((json_agg(_postgrest_t.pgrst_scalar)->0)::text, 'null')"
-  | otherwise     = "coalesce((json_agg(_postgrest_t)->0)::text, 'null')"
+  | returnsScalar = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
+  | otherwise     = "coalesce(json_agg(_postgrest_t)->0, 'null')"
+
+asJsonF :: Bool -> Bool -> Bool -> SqlFragment
+asJsonF returnsScalar returnsSetOfScalar returnsSingleComposite
+  | returnsSingleComposite              = "coalesce(json_agg(_postgrest_t)->0, 'null')"
+  | returnsScalar                       = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
+  | returnsSetOfScalar                  = "coalesce(json_agg(_postgrest_t.pgrst_scalar), '[]')"
+  | otherwise                           = "coalesce(json_agg(_postgrest_t), '[]')"
 
 asXmlF :: FieldName -> SqlFragment
 asXmlF fieldName = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
@@ -242,13 +237,53 @@ pgFmtSelectItem table (f@(fName, jp), Nothing, alias) = pgFmtField table f <> SQ
 -- Not quoting should be fine, we validate the input on Parsers.
 pgFmtSelectItem table (f@(fName, jp), Just cast, alias) = "CAST (" <> pgFmtField table f <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )" <> SQL.sql (pgFmtAs fName jp alias)
 
+-- TODO: At this stage there shouldn't be a Maybe since ApiRequest should ensure that an INSERT/UPDATE has a body
+fromJsonBodyF :: Maybe LBS.ByteString -> [TypedField] -> Bool -> Bool -> Bool -> SQL.Snippet
+fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
+  SQL.sql
+  (if includeSelect then "SELECT " <> parsedCols <> " " else mempty) <>
+  "FROM (SELECT " <> jsonPlaceHolder <> " AS json_data) pgrst_payload, " <>
+  -- convert a json object into a json array, this way we can use json_to_recordset for all json payloads
+  -- Otherwise we'd have to use json_to_record for json objects and json_to_recordset for json arrays
+  -- We do this in SQL to avoid processing the JSON in application code
+  "LATERAL (SELECT CASE WHEN " <> jsonTypeofF <> "(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE " <> jsonBuildArrayF <> "(pgrst_payload.json_data) END AS val) pgrst_uniform_json, " <>
+  (if includeDefaults
+    then "LATERAL (SELECT jsonb_agg(jsonb_build_object(" <> defsJsonb <> ") || elem) AS val from jsonb_array_elements(pgrst_uniform_json.val) elem) pgrst_json_defs, "
+    else mempty) <>
+  "LATERAL (SELECT * FROM " <>
+    (if null fields
+      -- When we are inserting no columns (e.g. using default values), we can't use our ordinary `json_to_recordset`
+      -- because it can't extract records with no columns (there's no valid syntax for the `AS (colName colType,...)`
+      -- part). But we still need to ensure as many rows are created as there are array elements.
+      then SQL.sql $ jsonArrayElementsF <> "(" <> finalBodyF <> ") _ "
+      else SQL.sql $ jsonToRecordsetF <> "(" <> finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
+    ) <>
+  ") pgrst_body "
+  where
+    parsedCols = BS.intercalate ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . tfName <$> fields
+    typedCols = BS.intercalate ", " $ pgFmtIdent . tfName <> const " " <> encodeUtf8 . tfIRType <$> fields
+    defsJsonb = SQL.sql $ BS.intercalate "," fieldsWDefaults
+    fieldsWDefaults = mapMaybe (\case
+        TypedField{tfName=nam, tfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
+        TypedField{tfDefault=Nothing} -> Nothing
+      ) fields
+    (finalBodyF, jsonTypeofF, jsonBuildArrayF, jsonArrayElementsF, jsonToRecordsetF) =
+      if includeDefaults
+        then ("pgrst_json_defs.val", "jsonb_typeof", "jsonb_build_array", "jsonb_array_elements", "jsonb_to_recordset")
+        else ("pgrst_uniform_json.val", "json_typeof", "json_build_array", "json_array_elements", "json_to_recordset")
+    jsonPlaceHolder = SQL.encoderAndParam (HE.nullable $ if includeDefaults then HE.jsonbLazyBytes else HE.jsonLazyBytes) body
+
 pgFmtOrderTerm :: QualifiedIdentifier -> OrderTerm -> SQL.Snippet
 pgFmtOrderTerm qi ot =
-  pgFmtField qi (otTerm ot) <> " " <>
+  fmtOTerm ot <> " " <>
   SQL.sql (BS.unwords [
     maybe mempty direction $ otDirection ot,
     maybe mempty nullOrder $ otNullOrder ot])
   where
+    fmtOTerm = \case
+      OrderTerm{otTerm}                        -> pgFmtField qi otTerm
+      OrderRelationTerm{otRelation, otRelTerm} -> pgFmtField (QualifiedIdentifier mempty otRelation) otRelTerm
+
     direction OrderAsc  = "ASC"
     direction OrderDesc = "DESC"
 
@@ -257,37 +292,44 @@ pgFmtOrderTerm qi ot =
 
 
 pgFmtFilter :: QualifiedIdentifier -> Filter -> SQL.Snippet
-pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> case oper of
-   Op op val  -> pgFmtFieldOp op <> " " <> case op of
-     OpLike  -> unknownLiteral (T.map star val)
-     OpILike -> unknownLiteral (T.map star val)
-     _       -> unknownLiteral val
+pgFmtFilter _ (FilterNullEmbed hasNot fld) = SQL.sql (pgFmtIdent fld) <> " IS " <> (if hasNot then "NOT" else mempty) <> " NULL"
+pgFmtFilter _ (Filter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
+pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
+   Op op val -> " " <> SQL.sql (simpleOperator op) <> " " <> unknownLiteral val
+
+   OpQuant op quant val -> " " <> SQL.sql (quantOperator op) <> " " <> case op of
+     OpLike  -> fmtQuant quant $ unknownLiteral (T.map star val)
+     OpILike -> fmtQuant quant $ unknownLiteral (T.map star val)
+     _       -> fmtQuant quant $ unknownLiteral val
 
    -- IS cannot be prepared. `PREPARE boolplan AS SELECT * FROM projects where id IS $1` will give a syntax error.
    -- The above can be fixed by using `PREPARE boolplan AS SELECT * FROM projects where id IS NOT DISTINCT FROM $1;`
    -- However that would not accept the TRUE/FALSE/NULL/UNKNOWN keywords. See: https://stackoverflow.com/questions/6133525/proper-way-to-set-preparedstatement-parameter-to-null-under-postgres.
    -- This is why `IS` operands are whitelisted at the Parsers.hs level
-   Is triVal -> pgFmtField table fld <> " IS " <> case triVal of
+   Is triVal -> " IS " <> case triVal of
      TriTrue    -> "TRUE"
      TriFalse   -> "FALSE"
      TriNull    -> "NULL"
      TriUnknown -> "UNKNOWN"
 
+   IsDistinctFrom val -> " IS DISTINCT FROM " <> unknownLiteral val
+
    -- We don't use "IN", we use "= ANY". IN has the following disadvantages:
    -- + No way to use an empty value on IN: "col IN ()" is invalid syntax. With ANY we can do "= ANY('{}')"
    -- + Can invalidate prepared statements: multiple parameters on an IN($1, $2, $3) will lead to using different prepared statements and not take advantage of caching.
-   In vals -> pgFmtField table fld <> " " <> case vals of
+   In vals -> " " <> case vals of
       [""] -> "= ANY('{}') "
       _    -> "= ANY (" <> unknownLiteral (pgBuildArrayLiteral vals) <> ") "
 
-   Fts op lang val ->
-     pgFmtFieldFts op <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
+   Fts op lang val -> " " <> SQL.sql (ftsOperator op) <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
  where
    ftsLang = maybe mempty (\l -> unknownLiteral l <> ", ")
-   pgFmtFieldOp op = pgFmtField table fld <> " " <> SQL.sql (singleValOperator op)
-   pgFmtFieldFts op = pgFmtField table fld <> " " <> SQL.sql (ftsOperator op)
    notOp = if hasNot then "NOT" else mempty
    star c = if c == '*' then '%' else c
+   fmtQuant q val = case q of
+    Just QuantAny -> "ANY(" <> val <> ")"
+    Just QuantAll -> "ALL(" <> val <> ")"
+    Nothing       -> val
 
 pgFmtJoinCondition :: JoinCondition -> SQL.Snippet
 pgFmtJoinCondition (JoinCondition (qi1, col1) (qi2, col2)) =

src/PostgREST/Query/Statements.hs

@@ -87,7 +87,7 @@ prepareWrite selectQuery mutateQuery isInsert mt rep pKeys =
     | getMediaType mt == MTTextCSV      = asCsvF
     | getMediaType mt == MTGeoJSON      = asGeoJsonF
     | getMediaType mt == MTSingularJSON = asJsonSingleF False
-    | otherwise                         = asJsonF False
+    | otherwise                         = asJsonF False False False
 
   selectF
     -- prevent using any of the column names in ?select= when no response is returned from the CTE
@@ -123,17 +123,17 @@ prepareRead selectQuery countQuery countTotal mt binaryField =
     | getMediaType mt == MTGeoJSON                       = asGeoJsonF
     | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
     | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-    | otherwise                                          = asJsonF False
+    | otherwise                                          = asJsonF False False False
 
   decodeIt :: HD.Result ResultSet
   decodeIt = case mt of
     MTPlan{} -> planRow
     _        -> HD.singleRow $ standardRow True
 
-prepareCall :: Bool -> Bool -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
-               MediaType -> Bool -> Maybe FieldName -> Bool ->
+prepareCall :: Bool -> Bool -> Bool -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
+               MediaType -> Maybe FieldName -> Bool ->
                SQL.Statement () ResultSet
-prepareCall returnsScalar returnsSingle callProcQuery selectQuery countQuery countTotal mt multObjects binaryField =
+prepareCall returnsScalar returnsSingleComposite returnsSetOfScalar callProcQuery selectQuery countQuery countTotal mt binaryField =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
   where
     snippet =
@@ -156,8 +156,7 @@ prepareCall returnsScalar returnsSingle callProcQuery selectQuery countQuery cou
      | getMediaType mt == MTGeoJSON                       = asGeoJsonF
      | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
      | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-     | returnsSingle && not multObjects                   = asJsonSingleF returnsScalar
-     | otherwise                                          = asJsonF returnsScalar
+     | otherwise                                          = asJsonF returnsScalar returnsSetOfScalar returnsSingleComposite
 
     decodeIt :: HD.Result ResultSet
     decodeIt = case mt of

src/PostgREST/Response.hs

@@ -3,7 +3,9 @@
 module PostgREST.Response
   ( createResponse
   , deleteResponse
-  , infoResponse
+  , infoIdentResponse
+  , infoProcResponse
+  , infoRootResponse
   , invokeResponse
   , openApiResponse
   , readResponse
@@ -12,12 +14,14 @@ module PostgREST.Response
   , addRetryHint
   , isServiceUnavailable
   , optionalRollback
+  , traceHeaderMiddleware
   ) where
 
 import qualified Data.Aeson                as JSON
 import qualified Data.ByteString.Char8     as BS
 import qualified Data.ByteString.Lazy      as LBS
 import qualified Data.HashMap.Strict       as HM
+import qualified Data.List                 as L
 import           Data.Text.Read            (decimal)
 import qualified Network.HTTP.Types.Header as HTTP
 import qualified Network.HTTP.Types.Status as HTTP
@@ -30,10 +34,10 @@ import qualified PostgREST.RangeQuery       as RangeQuery
 import qualified PostgREST.Response.OpenAPI as OpenAPI
 
 import PostgREST.ApiRequest              (ApiRequest (..),
-                                          InvokeMethod (..),
-                                          Target (..))
+                                          InvokeMethod (..))
 import PostgREST.ApiRequest.Preferences  (PreferRepresentation (..),
                                           PreferTransaction (..),
+                                          Preferences (..),
                                           shouldCount,
                                           toAppliedHeader)
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
@@ -46,13 +50,12 @@ import PostgREST.Response.GucHeader      (GucHeader, unwrapGucHeader)
 import PostgREST.SchemaCache             (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
                                           Schema)
-import PostgREST.SchemaCache.Proc        (ProcDescription (..),
-                                          ProcVolatility (..),
-                                          ProcsMap)
+import PostgREST.SchemaCache.Routine     (FuncVolatility (..),
+                                          Routine (..), RoutineMap)
 import PostgREST.SchemaCache.Table       (Table (..), TablesMap)
 
-import qualified PostgREST.ApiRequest.Types as ApiRequestTypes
-import qualified PostgREST.SchemaCache.Proc as Proc
+import qualified PostgREST.ApiRequest.Types    as ApiRequestTypes
+import qualified PostgREST.SchemaCache.Routine as Routine
 
 import Protolude      hiding (Handler, toS)
 import Protolude.Conv (toS)
@@ -83,7 +86,7 @@ readResponse headersOnly identifier ctxApiRequest@ApiRequest{..} resultSet = cas
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
 createResponse :: QualifiedIdentifier -> MutateReadPlan -> ApiRequest -> ResultSet -> Wai.Response
-createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}, ..} resultSet = case resultSet of
   RSStandard{..} -> do
     let
       pkCols = case mrMutatePlan of { Insert{insPkCols} -> insPkCols; _ -> mempty;}
@@ -100,14 +103,15 @@ createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiReques
                     <> HTTP.renderSimpleQuery True rsLocation
                 )
           , Just . RangeQuery.contentRangeH 1 0 $
-              if shouldCount iPreferCount then Just rsQueryTotal else Nothing
+              if shouldCount preferCount then Just rsQueryTotal else Nothing
           , if null pkCols && isNothing (qsOnConflict iQueryParams) then
               Nothing
             else
-              toAppliedHeader <$> iPreferResolution
+              toAppliedHeader <$> preferResolution
+          , toAppliedHeader <$> preferMissing
           ]
 
-    if iPreferRepresentation == Full then
+    if preferRepresentation == Full then
       response HTTP.status201 (headers ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
     else
       response HTTP.status201 headers mempty
@@ -116,16 +120,16 @@ createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiReques
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
 updateResponse :: ApiRequest -> ResultSet -> Wai.Response
-updateResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+updateResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
   RSStandard{..} -> do
     let
       response = gucResponse rsGucStatus rsGucHeaders
       contentRangeHeader =
-        RangeQuery.contentRangeH 0 (rsQueryTotal - 1) $
-          if shouldCount iPreferCount then Just rsQueryTotal else Nothing
-      headers = [contentRangeHeader]
+        Just . RangeQuery.contentRangeH 0 (rsQueryTotal - 1) $
+          if shouldCount preferCount then Just rsQueryTotal else Nothing
+      headers = catMaybes [contentRangeHeader, toAppliedHeader <$> preferMissing]
 
-    if iPreferRepresentation == Full then
+    if preferRepresentation == Full then
         response HTTP.status200
           (headers ++ contentTypeHeaders ctxApiRequest)
           (LBS.fromStrict rsBody)
@@ -136,12 +140,12 @@ updateResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
 singleUpsertResponse :: ApiRequest -> ResultSet -> Wai.Response
-singleUpsertResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+singleUpsertResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
   RSStandard {..} -> do
     let
       response = gucResponse rsGucStatus rsGucHeaders
 
-    if iPreferRepresentation == Full then
+    if preferRepresentation == Full then
       response HTTP.status200 (contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
     else
       response HTTP.status204 [] mempty
@@ -150,16 +154,16 @@ singleUpsertResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
 deleteResponse :: ApiRequest -> ResultSet -> Wai.Response
-deleteResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+deleteResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
   RSStandard {..} -> do
     let
       response = gucResponse rsGucStatus rsGucHeaders
       contentRangeHeader =
         RangeQuery.contentRangeH 1 0 $
-          if shouldCount iPreferCount then Just rsQueryTotal else Nothing
+          if shouldCount preferCount then Just rsQueryTotal else Nothing
       headers = [contentRangeHeader]
 
-    if iPreferRepresentation == Full then
+    if preferRepresentation == Full then
         response HTTP.status200
           (headers ++ contentTypeHeaders ctxApiRequest)
           (LBS.fromStrict rsBody)
@@ -169,20 +173,12 @@ deleteResponse ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
-infoResponse :: Target -> SchemaCache -> Wai.Response
-infoResponse target sCache =
-  case target of
-    TargetIdent identifier ->
-      case HM.lookup identifier (dbTables sCache) of
-        Just tbl -> respondInfo $ allowH tbl
-        Nothing  -> Error.errorResponseFor $ Error.ApiRequestError ApiRequestTypes.NotFound
-    TargetProc pd _
-      | pdVolatility pd == Volatile -> respondInfo "OPTIONS,POST"
-      | otherwise                   -> respondInfo "OPTIONS,GET,HEAD,POST"
-    TargetDefaultSpec _             -> respondInfo "OPTIONS,GET,HEAD"
+infoIdentResponse :: QualifiedIdentifier -> SchemaCache -> Wai.Response
+infoIdentResponse identifier sCache =
+  case HM.lookup identifier (dbTables sCache) of
+    Just tbl -> respondInfo $ allowH tbl
+    Nothing  -> Error.errorResponseFor $ Error.ApiRequestError ApiRequestTypes.NotFound
   where
-    respondInfo allowHeader = Wai.responseLBS HTTP.status200 [allOrigins, (HTTP.hAllow, allowHeader)] mempty
-    allOrigins = ("Access-Control-Allow-Origin", "*")
     allowH table =
       let hasPK = not . null $ tablePKCols table in
       BS.intercalate "," $
@@ -192,7 +188,19 @@ infoResponse target sCache =
           ["PATCH" | tableUpdatable table] ++
           ["DELETE" | tableDeletable table]
 
-invokeResponse :: InvokeMethod -> ProcDescription -> ApiRequest -> ResultSet -> Wai.Response
+infoProcResponse :: Routine -> Wai.Response
+infoProcResponse proc | pdVolatility proc == Volatile = respondInfo "OPTIONS,POST"
+                      | otherwise                     = respondInfo "OPTIONS,GET,HEAD,POST"
+
+infoRootResponse :: Wai.Response
+infoRootResponse = respondInfo "OPTIONS,GET,HEAD"
+
+respondInfo :: ByteString -> Wai.Response
+respondInfo allowHeader =
+  let allOrigins = ("Access-Control-Allow-Origin", "*") in
+  Wai.responseLBS HTTP.status200 [allOrigins, (HTTP.hAllow, allowHeader)] mempty
+
+invokeResponse :: InvokeMethod -> Routine -> ApiRequest -> ResultSet -> Wai.Response
 invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
   RSStandard {..} -> do
     let
@@ -205,7 +213,7 @@ invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resu
         else LBS.fromStrict rsBody
       headers = [contentRange]
 
-    if Proc.procReturnsVoid proc then
+    if Routine.funcReturnsVoid proc then
         response HTTP.status204 headers mempty
       else
         response status
@@ -215,7 +223,7 @@ invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resu
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
-openApiResponse :: Bool -> Maybe (TablesMap, ProcsMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
+openApiResponse :: Bool -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
 openApiResponse headersOnly body conf sCache schema negotiatedByProfile =
   Wai.responseLBS HTTP.status200
     (MediaType.toContentType MTOpenAPI : maybeToList (profileHeader schema negotiatedByProfile))
@@ -263,14 +271,14 @@ isServiceUnavailable :: Wai.Response -> Bool
 isServiceUnavailable response = Wai.responseStatus response == HTTP.status503
 
 optionalRollback :: AppConfig -> ApiRequest -> ExceptT Error.Error IO Wai.Response -> ExceptT Error.Error IO Wai.Response
-optionalRollback AppConfig{..} ApiRequest{..} resp = do
+optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} resp = do
   newRes <- catchError resp $ return . Error.errorResponseFor
   return $ Wai.mapResponseHeaders preferenceApplied newRes
   where
     shouldCommit =
-      configDbTxAllowOverride && iPreferTransaction == Just Commit
+      configDbTxAllowOverride && preferTransaction == Just Commit
     shouldRollback =
-      configDbTxAllowOverride && iPreferTransaction == Just Rollback
+      configDbTxAllowOverride && preferTransaction == Just Rollback
     preferenceApplied
       | shouldCommit =
           addHeadersIfNotIncluded
@@ -286,3 +294,11 @@ addHeadersIfNotIncluded :: [HTTP.Header] -> [HTTP.Header] -> [HTTP.Header]
 addHeadersIfNotIncluded newHeaders initialHeaders =
   filter (\(nk, _) -> isNothing $ find (\(ik, _) -> ik == nk) initialHeaders) newHeaders ++
   initialHeaders
+
+traceHeaderMiddleware :: AppConfig -> Wai.Middleware
+traceHeaderMiddleware AppConfig{configServerTraceHeader} app req respond =
+  case configServerTraceHeader of
+    Nothing -> app req respond
+    Just hdr ->
+      let hdrVal = L.lookup hdr $ Wai.requestHeaders req in
+      app req (respond . Wai.mapResponseHeaders ([(hdr, fromMaybe mempty hdrVal)] ++))

src/PostgREST/Response/OpenAPI.hs

@@ -28,20 +28,21 @@ import PostgREST.Config                   (AppConfig (..), Proxy (..),
                                            isMalformedProxyUri, toURI)
 import PostgREST.SchemaCache              (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
-import PostgREST.SchemaCache.Proc         (ProcDescription (..),
-                                           ProcParam (..))
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                            Relationship (..),
                                            RelationshipsMap)
+import PostgREST.SchemaCache.Routine      (Routine (..),
+                                           RoutineParam (..))
 import PostgREST.SchemaCache.Table        (Column (..), Table (..),
-                                           TablesMap)
+                                           TablesMap,
+                                           tableColumnsList)
 import PostgREST.Version                  (docsVersion, prettyVersion)
 
 import PostgREST.MediaType
 
 import Protolude hiding (Proxy, get)
 
-encode :: AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [ProcDescription] -> Maybe Text -> LBS.ByteString
+encode :: AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
 encode conf sCache tables procs schemaDescription =
   JSON.encode $
     postgrestSpec
@@ -66,10 +67,22 @@ toSwaggerType "bigint"            = Just SwaggerInteger
 toSwaggerType "numeric"           = Just SwaggerNumber
 toSwaggerType "real"              = Just SwaggerNumber
 toSwaggerType "double precision"  = Just SwaggerNumber
-toSwaggerType "ARRAY"             = Just SwaggerArray
 toSwaggerType "json"              = Nothing
 toSwaggerType "jsonb"             = Nothing
-toSwaggerType _                   = Just SwaggerString
+toSwaggerType colType             = case T.takeEnd 2 colType of
+  "[]" -> Just SwaggerArray
+  _    -> Just SwaggerString
+
+typeFromArray :: Text -> Text
+typeFromArray = T.dropEnd 2
+
+toSwaggerTypeFromArray :: Text -> Maybe (SwaggerType t)
+toSwaggerTypeFromArray arrType = toSwaggerType $ typeFromArray arrType
+
+makePropertyItems :: Text -> Maybe (Referenced Schema)
+makePropertyItems arrType = case toSwaggerType arrType of
+  Just SwaggerArray -> Just $ Inline (mempty & type_ .~ toSwaggerTypeFromArray arrType)
+  _                 -> Nothing
 
 parseDefault :: Text -> Text -> Text
 parseDefault colType colDefault =
@@ -87,8 +100,8 @@ makeTableDef rels t =
       (tn, (mempty :: Schema)
         & description .~ tableDescription t
         & type_ ?~ SwaggerObject
-        & properties .~ fromList (makeProperty t rels <$> tableColumns t)
-        & required .~ fmap colName (filter (not . colNullable) $ tableColumns t))
+        & properties .~ fromList (makeProperty t rels <$> tableColumnsList t)
+        & required .~ fmap colName (filter (not . colNullable) $ tableColumnsList t))
 
 makeProperty :: Table -> RelationshipsMap -> Column -> (Text, Referenced Schema)
 makeProperty tbl rels col = (colName col, Inline s)
@@ -97,11 +110,14 @@ makeProperty tbl rels col = (colName col, Inline s)
     fk :: Maybe Text
     fk =
       let
+        searchedRels = fromMaybe mempty $ HM.lookup (QualifiedIdentifier (tableSchema tbl) (tableName tbl), tableSchema tbl) rels
+        -- Sorts the relationship list to get tables first
+        relsSortedByIsView = sortOn relFTableIsView [ r | r@Relationship{} <- searchedRels]
         -- Finds the relationship that has a single column foreign key
         rel = find (\case
           Relationship{relCardinality=(M2O _ relColumns)} -> [colName col] == (fst <$> relColumns)
           _                                               -> False
-          ) $ fromMaybe mempty $ HM.lookup (QualifiedIdentifier (tableSchema tbl) (tableName tbl), tableSchema tbl) rels
+          ) relsSortedByIsView
         fCol = (headMay . (\r -> snd <$> relColumns (relCardinality r)) =<< rel)
         fTbl = qiName . relForeignTable <$> rel
         fTblCol = (,) <$> fTbl <*> fCol
@@ -127,8 +143,9 @@ makeProperty tbl rels col = (colName col, Inline s)
         & format ?~ colType col
         & maxLength .~ (fromIntegral <$> colMaxLen col)
         & type_ .~ toSwaggerType (colType col)
+        & items .~ (SwaggerItemsObject <$> makePropertyItems (colType col))
 
-makeProcSchema :: ProcDescription -> Schema
+makeProcSchema :: Routine -> Schema
 makeProcSchema pd =
   (mempty :: Schema)
   & description .~ pdDescription pd
@@ -136,11 +153,12 @@ makeProcSchema pd =
   & properties .~ fromList (fmap makeProcProperty (pdParams pd))
   & required .~ fmap ppName (filter ppReq (pdParams pd))
 
-makeProcProperty :: ProcParam -> (Text, Referenced Schema)
-makeProcProperty (ProcParam n t _ _) = (n, Inline s)
+makeProcProperty :: RoutineParam -> (Text, Referenced Schema)
+makeProcProperty (RoutineParam n t _ _) = (n, Inline s)
   where
     s = (mempty :: Schema)
           & type_ .~ toSwaggerType t
+          & items .~ (SwaggerItemsObject <$> makePropertyItems t)
           & format ?~ t
 
 makePreferParam :: [Text] -> Param
@@ -152,10 +170,47 @@ makePreferParam ts =
   & schema .~ ParamOther ((mempty :: ParamOtherSchema)
     & in_ .~ ParamHeader
     & type_ ?~ SwaggerString
-    & enum_ .~ JSON.decode (JSON.encode ts))
+    & enum_ .~ JSON.decode (JSON.encode $ foldl (<>) [] (val <$> ts)))
+  where
+    val :: Text -> [Text]
+    val = \case
+      "count"      -> ["count=none"]
+      "params"     -> ["params=single-object"]
+      "return"     -> ["return=representation", "return=minimal", "return=none"]
+      "resolution" -> ["resolution=ignore-duplicates", "resolution=merge-duplicates"]
+      _            -> []
 
-makeProcParam :: ProcDescription -> [Referenced Param]
-makeProcParam pd =
+makeProcGetParam :: RoutineParam -> Referenced Param
+makeProcGetParam (RoutineParam n t r v) =
+  Inline $ (mempty :: Param)
+    & name .~ n
+    & required ?~ r
+    & schema .~ ParamOther fullSchema
+  where
+    fullSchema = if v then schemaMulti else schemaNotMulti
+    baseSchema = (mempty :: ParamOtherSchema)
+      & in_ .~ ParamQuery
+    schemaNotMulti = baseSchema
+      & format ?~ t
+      & type_ ?~ toParamType (toSwaggerType t)
+    schemaMulti = baseSchema
+      & type_ ?~ fromMaybe SwaggerString (toSwaggerType t)
+      & items ?~ SwaggerItemsPrimitive (Just CollectionMulti)
+        ((mempty :: ParamSchema x)
+          & type_ .~ toSwaggerTypeFromArray t
+          & format ?~ typeFromArray t)
+    toParamType paramType = case paramType of
+      -- Array uses {} in query params
+      Just SwaggerArray -> SwaggerString
+      -- Type must be specified in query params
+      Nothing           -> SwaggerString
+      _                 -> fromJust paramType
+
+makeProcGetParams :: [RoutineParam] -> [Referenced Param]
+makeProcGetParams = fmap makeProcGetParam
+
+makeProcPostParams :: Routine -> [Referenced Param]
+makeProcPostParams pd =
   [ Inline $ (mempty :: Param)
     & name     .~ "args"
     & required ?~ True
@@ -165,9 +220,11 @@ makeProcParam pd =
 
 makeParamDefs :: [Table] -> [(Text, Param)]
 makeParamDefs ti =
-  [ ("preferParams", makePreferParam ["params=single-object"])
-  , ("preferReturn", makePreferParam ["return=representation", "return=minimal", "return=none"])
-  , ("preferCount", makePreferParam ["count=none"])
+  -- TODO: create Prefer for each method (GET, PATCH, etc.)
+  [ ("preferParams", makePreferParam ["params"])
+  , ("preferReturn", makePreferParam ["return"])
+  , ("preferCount", makePreferParam ["count"])
+  , ("preferPost", makePreferParam ["return", "resolution"])
   , ("select", (mempty :: Param)
       & name        .~ "select"
       & description ?~ "Filtering Columns"
@@ -219,7 +276,7 @@ makeParamDefs ti =
         & in_ .~ ParamQuery
         & type_ ?~ SwaggerString))
   ]
-  <> concat [ makeObjectBody (tableName t) : makeRowFilters (tableName t) (tableColumns t)
+  <> concat [ makeObjectBody (tableName t) : makeRowFilters (tableName t) (tableColumnsList t)
             | t <- ti
             ]
 
@@ -267,7 +324,7 @@ makePathItem t = ("/" ++ T.unpack tn, p $ tableInsertable t || tableUpdatable t
         )
       )
     postOp = tOp
-      & parameters .~ fmap ref ["body." <> tn, "select", "preferReturn"]
+      & parameters .~ fmap ref ["body." <> tn, "select", "preferPost"]
       & at 201 ?~ "Created"
     patchOp = tOp
       & parameters .~ fmap ref (rs <> ["body." <> tn, "preferReturn"])
@@ -280,24 +337,29 @@ makePathItem t = ("/" ++ T.unpack tn, p $ tableInsertable t || tableUpdatable t
     p False = pr
     p True  = pw
     tn = tableName t
-    rs = [ T.intercalate "." ["rowFilter", tn, colName c ] | c <- tableColumns t ]
+    rs = [ T.intercalate "." ["rowFilter", tn, colName c ] | c <- tableColumnsList t ]
     ref = Ref . Reference
 
-makeProcPathItem :: ProcDescription -> (FilePath, PathItem)
+makeProcPathItem :: Routine -> (FilePath, PathItem)
 makeProcPathItem pd = ("/rpc/" ++ toS (pdName pd), pe)
   where
     -- Use first line of proc description as summary; rest as description (if present)
     -- We strip leading newlines from description so that users can include a blank line between summary and description
     (pSum, pDesc) = fmap fst &&& fmap (T.dropWhile (=='\n') . snd) $
                     T.breakOn "\n" <$> pdDescription pd
-    postOp = (mempty :: Operation)
+    procOp = (mempty :: Operation)
       & summary .~ pSum
       & description .~ mfilter (/="") pDesc
-      & parameters .~ makeProcParam pd
       & tags .~ Set.fromList ["(rpc) " <> pdName pd]
       & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON]
       & at 200 ?~ "OK"
-    pe = (mempty :: PathItem) & post ?~ postOp
+    getOp = procOp
+      & parameters .~ makeProcGetParams (pdParams pd)
+    postOp = procOp
+      & parameters .~ makeProcPostParams pd
+    pe = (mempty :: PathItem)
+      & get ?~ getOp
+      & post ?~ postOp
 
 makeRootPathItem :: (FilePath, PathItem)
 makeRootPathItem = ("/", p)
@@ -310,7 +372,7 @@ makeRootPathItem = ("/", p)
     pr = (mempty :: PathItem) & get ?~ getOp
     p = pr
 
-makePathItems :: [ProcDescription] -> [Table] -> InsOrdHashMap FilePath PathItem
+makePathItems :: [Routine] -> [Table] -> InsOrdHashMap FilePath PathItem
 makePathItems pds ti = fromList $ makeRootPathItem :
   fmap makePathItem ti ++ fmap makeProcPathItem pds
 
@@ -330,14 +392,14 @@ escapeHostName "*6" = "0.0.0.0"
 escapeHostName "!6" = "0.0.0.0"
 escapeHostName h    = h
 
-postgrestSpec :: RelationshipsMap -> [ProcDescription] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
+postgrestSpec :: RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
 postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
   & basePath ?~ T.unpack b
   & schemes ?~ [s']
   & info .~ ((mempty :: Info)
       & version .~ T.decodeUtf8 prettyVersion
-      & title .~ "PostgREST API"
-      & description ?~ d)
+      & title .~ fromMaybe "PostgREST API" dTitle
+      & description ?~ fromMaybe "This is a dynamic API generated by PostgREST" dDesc)
   & externalDocs ?~ ((mempty :: ExternalDocs)
     & description ?~ "PostgREST Documentation"
     & url .~ URL ("https://postgrest.org/en/" <> docsVersion <> "/api.html"))
@@ -352,8 +414,9 @@ postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
     where
       s' = if s == "http" then Http else Https
       h' = Just $ Host (T.unpack $ escapeHostName h) (Just (fromInteger p))
-      d = fromMaybe "This is a dynamic API generated by PostgREST" sd
       securityDefName = "JWT"
+      (dTitle, dDesc) = fmap fst &&& fmap (T.dropWhile (=='\n') . snd) $
+                    T.breakOn "\n" <$> sd
 
 pickProxy :: Maybe Text -> Maybe Proxy
 pickProxy proxy

src/PostgREST/SchemaCache.hs

@@ -22,17 +22,18 @@ module PostgREST.SchemaCache
   ( SchemaCache(..)
   , querySchemaCache
   , accessibleTables
-  , accessibleProcs
+  , accessibleFuncs
   , schemaDescription
   ) where
 
-import qualified Data.Aeson          as JSON
-import qualified Data.HashMap.Strict as HM
-import qualified Data.Set            as S
-import qualified Hasql.Decoders      as HD
-import qualified Hasql.Encoders      as HE
-import qualified Hasql.Statement     as SQL
-import qualified Hasql.Transaction   as SQL
+import qualified Data.Aeson                 as JSON
+import qualified Data.HashMap.Strict        as HM
+import qualified Data.HashMap.Strict.InsOrd as HMI
+import qualified Data.Set                   as S
+import qualified Hasql.Decoders             as HD
+import qualified Hasql.Encoders             as HE
+import qualified Hasql.Statement            as SQL
+import qualified Hasql.Transaction          as SQL
 
 import Contravariant.Extras          (contrazip2)
 import Text.InterpolatedString.Perl6 (q)
@@ -43,17 +44,16 @@ import PostgREST.Config.PgVersion         (PgVersion, pgVersion100,
 import PostgREST.SchemaCache.Identifiers  (AccessSet, FieldName,
                                            QualifiedIdentifier (..),
                                            Schema)
-import PostgREST.SchemaCache.Proc         (PgType (..),
-                                           ProcDescription (..),
-                                           ProcParam (..),
-                                           ProcVolatility (..),
-                                           ProcsMap, RetType (..))
 import PostgREST.SchemaCache.Relationship (Cardinality (..),
                                            Junction (..),
                                            Relationship (..),
                                            RelationshipsMap)
-import PostgREST.SchemaCache.Table        (Column (..), Table (..),
-                                           TablesMap)
+import PostgREST.SchemaCache.Routine      (FuncVolatility (..),
+                                           PgType (..), RetType (..),
+                                           Routine (..), RoutineMap,
+                                           RoutineParam (..))
+import PostgREST.SchemaCache.Table        (Column (..), ColumnMap,
+                                           Table (..), TablesMap)
 
 import Protolude
 
@@ -61,7 +61,7 @@ import Protolude
 data SchemaCache = SchemaCache
   { dbTables        :: TablesMap
   , dbRelationships :: RelationshipsMap
-  , dbProcs         :: ProcsMap
+  , dbRoutines      :: RoutineMap
   }
   deriving (Generic, JSON.ToJSON)
 
@@ -106,11 +106,11 @@ type SqlQuery = ByteString
 querySchemaCache :: [Schema] -> [Schema] -> Bool -> SQL.Transaction SchemaCache
 querySchemaCache schemas extraSearchPath prepared = do
   SQL.sql "set local schema ''" -- This voids the search path. The following queries need this for getting the fully qualified name(schema.name) of every db object
-  pgVer   <- SQL.statement mempty pgVersionStatement
+  pgVer   <- SQL.statement mempty $ pgVersionStatement prepared
   tabs    <- SQL.statement schemas $ allTables pgVer prepared
   keyDeps <- SQL.statement (schemas, extraSearchPath) $ allViewsKeyDependencies prepared
   m2oRels <- SQL.statement mempty $ allM2OandO2ORels pgVer prepared
-  procs   <- SQL.statement schemas $ allProcs pgVer prepared
+  funcs   <- SQL.statement schemas $ allFunctions pgVer prepared
   cRels   <- SQL.statement mempty $ allComputedRels prepared
 
   let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
@@ -119,7 +119,7 @@ querySchemaCache schemas extraSearchPath prepared = do
   return $ removeInternal schemas $ SchemaCache {
       dbTables = tabsWViewsPks
     , dbRelationships = getOverrideRelationshipsMap rels cRels
-    , dbProcs = procs
+    , dbRoutines = funcs
     }
 
 -- | overrides detected relationships with the computed relationships and gets the RelationshipsMap
@@ -149,7 +149,7 @@ removeInternal schemas dbStruct =
       dbTables        = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
     , dbRelationships = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
                         HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
-    , dbProcs         = dbProcs dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+    , dbRoutines      = dbRoutines dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
     }
   where
     hasInternalJunction ComputedRelationship{} = False
@@ -178,15 +178,20 @@ decodeTables =
     <*> column HD.bool
     <*> column HD.bool
     <*> arrayColumn HD.text
-    <*> compositeArrayColumn
-        (Column
+    <*> parseCols (compositeArrayColumn
+      (Column
         <$> compositeField HD.text
         <*> nullableCompositeField HD.text
         <*> compositeField HD.bool
         <*> compositeField HD.text
+        <*> compositeField HD.text
         <*> nullableCompositeField HD.int4
         <*> nullableCompositeField HD.text
-        <*> compositeFieldArray HD.text)
+        <*> compositeFieldArray HD.text))
+
+
+parseCols :: HD.Row [Column] -> HD.Row ColumnMap
+parseCols = fmap (HMI.fromList . map (\col@Column{colName} -> (colName, col)))
 
 decodeRels :: HD.Result [Relationship]
 decodeRels =
@@ -222,17 +227,17 @@ viewKeyDepFromRow (s1,t1,s2,v2,cons,consType,sCols) = ViewKeyDependency (Qualifi
            | consType == "f" = FKDep
            | otherwise       = FKDepRef -- f_ref, we build this type in the query
 
-decodeProcs :: HD.Result ProcsMap
-decodeProcs =
-  -- Duplicate rows for a function means they're overloaded, order these by least args according to ProcDescription Ord instance
-  map sort . HM.fromListWith (++) . map ((\(x,y) -> (x, [y])) . addKey) <$> HD.rowList procRow
+decodeFuncs :: HD.Result RoutineMap
+decodeFuncs =
+  -- Duplicate rows for a function means they're overloaded, order these by least args according to Routine Ord instance
+  map sort . HM.fromListWith (++) . map ((\(x,y) -> (x, [y])) . addKey) <$> HD.rowList funcRow
   where
-    procRow = ProcDescription
+    funcRow = Function
               <$> column HD.text
               <*> column HD.text
               <*> nullableColumn HD.text
               <*> compositeArrayColumn
-                  (ProcParam
+                  (RoutineParam
                   <$> compositeField HD.text
                   <*> compositeField HD.text
                   <*> compositeField HD.bool
@@ -245,38 +250,38 @@ decodeProcs =
                   <*> column HD.bool)
               <*> (parseVolatility <$> column HD.char)
               <*> column HD.bool
+              <*> nullableColumn HD.text
 
-    addKey :: ProcDescription -> (QualifiedIdentifier, ProcDescription)
+    addKey :: Routine -> (QualifiedIdentifier, Routine)
     addKey pd = (QualifiedIdentifier (pdSchema pd) (pdName pd), pd)
 
-    parseRetType :: Text -> Text -> Bool -> Bool -> Bool -> Maybe RetType
-    parseRetType schema name isSetOf isComposite isVoid
-      | isVoid    = Nothing
-      | isSetOf   = Just (SetOf pgType)
-      | otherwise = Just (Single pgType)
+    parseRetType :: Text -> Text -> Bool -> Bool -> Bool -> RetType
+    parseRetType schema name isSetOf isComposite isCompositeAlias
+      | isSetOf   = SetOf pgType
+      | otherwise = Single pgType
       where
         qi = QualifiedIdentifier schema name
         pgType
-          | isComposite = Composite qi
-          | otherwise   = Scalar
+          | isComposite = Composite qi isCompositeAlias
+          | otherwise   = Scalar qi
 
-    parseVolatility :: Char -> ProcVolatility
+    parseVolatility :: Char -> FuncVolatility
     parseVolatility v | v == 'i' = Immutable
                       | v == 's' = Stable
                       | otherwise = Volatile -- only 'v' can happen here
 
-allProcs :: PgVersion -> Bool -> SQL.Statement [Schema] ProcsMap
-allProcs pgVer = SQL.Statement sql (arrayParam HE.text) decodeProcs
+allFunctions :: PgVersion -> Bool -> SQL.Statement [Schema] RoutineMap
+allFunctions pgVer = SQL.Statement sql (arrayParam HE.text) decodeFuncs
   where
-    sql = procsSqlQuery pgVer <> " AND pn.nspname = ANY($1)"
+    sql = funcsSqlQuery pgVer <> " AND pn.nspname = ANY($1)"
 
-accessibleProcs :: PgVersion -> Bool -> SQL.Statement Schema ProcsMap
-accessibleProcs pgVer = SQL.Statement sql (param HE.text) decodeProcs
+accessibleFuncs :: PgVersion -> Bool -> SQL.Statement Schema RoutineMap
+accessibleFuncs pgVer = SQL.Statement sql (param HE.text) decodeFuncs
   where
-    sql = procsSqlQuery pgVer <> " AND pn.nspname = $1 AND has_function_privilege(p.oid, 'execute')"
+    sql = funcsSqlQuery pgVer <> " AND pn.nspname = $1 AND has_function_privilege(p.oid, 'execute')"
 
-procsSqlQuery :: PgVersion -> SqlQuery
-procsSqlQuery pgVer = [q|
+funcsSqlQuery :: PgVersion -> SqlQuery
+funcsSqlQuery pgVer = [q|
  -- Recursively get the base types of domains
   WITH
   base_types AS (
@@ -333,9 +338,10 @@ procsSqlQuery pgVer = [q|
      -- if any TABLE, INOUT or OUT arguments present, treat as composite
      or COALESCE(proargmodes::text[] && '{t,b,o}', false)
     ) AS rettype_is_composite,
-    ('void'::regtype = t.oid) AS rettype_is_void,
+    bt.oid <> bt.base as rettype_is_composite_alias,
     p.provolatile,
-    p.provariadic > 0 as hasvariadic
+    p.provariadic > 0 as hasvariadic,
+    lower((regexp_split_to_array((regexp_split_to_array(config, '='))[2], ','))[1]) AS transaction_isolation_level
   FROM pg_proc p
   LEFT JOIN arguments a ON a.oid = p.oid
   JOIN pg_namespace pn ON pn.oid = p.pronamespace
@@ -344,6 +350,7 @@ procsSqlQuery pgVer = [q|
   JOIN pg_namespace tn ON tn.oid = t.typnamespace
   LEFT JOIN pg_class comp ON comp.oid = t.typrelid
   LEFT JOIN pg_description as d ON d.objoid = p.oid
+  LEFT JOIN LATERAL unnest(proconfig) config ON config like 'default_transaction_isolation%'
   WHERE t.oid <> 'trigger'::regtype AND COALESCE(a.callable, true)
 |] <> (if pgVer >= pgVersion110 then "AND prokind = 'f'" else "AND NOT (proisagg OR proiswindow)")
 
@@ -399,7 +406,7 @@ test         | personnages_view     | test       | actors_view      | personnage
 -}
 addViewM2OAndO2ORels :: [ViewKeyDependency] -> [Relationship] -> [Relationship]
 addViewM2OAndO2ORels keyDeps rels =
-  rels ++ concat (viewRels <$> rels)
+  rels ++ concatMap viewRels rels
   where
     isM2O card = case card of {M2O _ _ -> True; _ -> False;}
     isO2O card = case card of {O2O _ _ -> True; _ -> False;}
@@ -449,7 +456,7 @@ addViewM2OAndO2ORels keyDeps rels =
         , keyDepColsTblVw <- expandKeyDepCols $ keyDepCols tblVw ]
       else []
     viewRels _ = []
-    expandKeyDepCols kdc = zip (fst <$> kdc) <$> sequenceA (snd <$> kdc)
+    expandKeyDepCols kdc = zip (fst <$> kdc) <$> traverse snd kdc
 
 addInverseRels :: [Relationship] -> [Relationship]
 addInverseRels rels =
@@ -485,7 +492,7 @@ addViewPrimaryKeys tabs keyDeps =
     -- * We don't have any logic that requires the client to name a PK column (compared to the column hints in embedding for FKs),
     --   so we don't need to know about the other references.
     -- * We need to choose a single reference for each column, otherwise we'd output too many columns in location headers etc.
-    takeFirstPK pkCols = catMaybes $ head . snd <$> pkCols
+    takeFirstPK = mapMaybe (head . snd)
 
 allTables :: PgVersion -> Bool -> SQL.Statement [Schema] TablesMap
 allTables pgVer =
@@ -507,22 +514,22 @@ tablesSqlQuery pgVer =
           c.relname::name AS table_name,
           a.attname::name AS column_name,
           d.description AS description,
-          pg_get_expr(ad.adbin, ad.adrelid)::text AS column_default,
+  |] <> columnDefault <>
+  [q|
           not (a.attnotnull OR t.typtype = 'd' AND t.typnotnull) AS is_nullable,
               CASE
                   WHEN t.typtype = 'd' THEN
                   CASE
-                      WHEN bt.typelem <> 0::oid AND bt.typlen = (-1) THEN 'ARRAY'::text
                       WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
                       ELSE format_type(a.atttypid, a.atttypmod)
                   END
                   ELSE
                   CASE
-                      WHEN t.typelem <> 0::oid AND t.typlen = (-1) THEN 'ARRAY'::text
                       WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
                       ELSE format_type(a.atttypid, a.atttypmod)
                   END
               END::text AS data_type,
+          t.oid AS data_type_id,
           information_schema._pg_char_max_length(
               information_schema._pg_truetypid(a.*, t.*),
               information_schema._pg_truetypmod(a.*, t.*)
@@ -558,6 +565,7 @@ tablesSqlQuery pgVer =
           info.description,
           info.is_nullable::boolean,
           info.data_type,
+          info.data_type_id::regtype::text,
           info.character_maximum_length,
           info.column_default,
           coalesce(enum_info.vals, '{}')) order by info.position) as columns
@@ -689,6 +697,16 @@ tablesSqlQuery pgVer =
   "ORDER BY table_schema, table_name"
   where
     relIsPartition = if pgVer >= pgVersion100 then " AND not c.relispartition " else mempty
+    -- detect default values on columns that have GENERATED .. AS IDENTITY
+    columnDefault =
+      if pgVer >= pgVersion100
+        then [q|
+        CASE
+           WHEN nullif(a.attidentity, '') is null
+             THEN pg_get_expr(ad.adbin, ad.adrelid)::text
+             ELSE format('nextval(%s)', quote_literal(pg_get_serial_sequence(a.attrelid::regclass::text, a.attname::text)))
+        END AS column_default,|]
+        else "pg_get_expr(ad.adbin, ad.adrelid)::text AS column_default,"
 
 
 -- | Gets many-to-one relationships and one-to-one(O2O) relationships, which are a refinement of the many-to-one's

src/PostgREST/SchemaCache/Proc.hs

@@ -1,91 +0,0 @@
-{-# LANGUAGE DeriveAnyClass #-}
-{-# LANGUAGE DeriveGeneric  #-}
-
-module PostgREST.SchemaCache.Proc
-  ( PgType(..)
-  , ProcDescription(..)
-  , ProcParam(..)
-  , ProcVolatility(..)
-  , ProcsMap
-  , RetType(..)
-  , procReturnsScalar
-  , procReturnsSingle
-  , procReturnsVoid
-  , procTableName
-  ) where
-
-import qualified Data.Aeson          as JSON
-import qualified Data.HashMap.Strict as HM
-
-import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
-                                          Schema, TableName)
-
-import Protolude
-
-data PgType
-  = Scalar
-  | Composite QualifiedIdentifier
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
-
-data RetType
-  = Single PgType
-  | SetOf PgType
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
-
-data ProcVolatility
-  = Volatile
-  | Stable
-  | Immutable
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
-
-data ProcDescription = ProcDescription
-  { pdSchema      :: Schema
-  , pdName        :: Text
-  , pdDescription :: Maybe Text
-  , pdParams      :: [ProcParam]
-  , pdReturnType  :: Maybe RetType
-  , pdVolatility  :: ProcVolatility
-  , pdHasVariadic :: Bool
-  }
-  deriving (Eq, Generic, JSON.ToJSON)
-
-data ProcParam = ProcParam
-  { ppName :: Text
-  , ppType :: Text
-  , ppReq  :: Bool
-  , ppVar  :: Bool
-  }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
-
--- Order by least number of params in the case of overloaded functions
-instance Ord ProcDescription where
-  ProcDescription schema1 name1 des1 prms1 rt1 vol1 hasVar1 `compare` ProcDescription schema2 name2 des2 prms2 rt2 vol2 hasVar2
-    | schema1 == schema2 && name1 == name2 && length prms1 < length prms2  = LT
-    | schema2 == schema2 && name1 == name2 && length prms1 > length prms2  = GT
-    | otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2)
-
--- | A map of all procs, all of which can be overloaded(one entry will have more than one ProcDescription).
--- | It uses a HashMap for a faster lookup.
-type ProcsMap = HM.HashMap QualifiedIdentifier [ProcDescription]
-
-procReturnsScalar :: ProcDescription -> Bool
-procReturnsScalar proc = case proc of
-  ProcDescription{pdReturnType = Just (Single Scalar)} -> True
-  ProcDescription{pdReturnType = Just (SetOf Scalar)}  -> True
-  _                                                    -> False
-
-procReturnsSingle :: ProcDescription -> Bool
-procReturnsSingle proc = case proc of
-  ProcDescription{pdReturnType = Just (Single _)} -> True
-  _                                               -> False
-
-procReturnsVoid :: ProcDescription -> Bool
-procReturnsVoid proc = case proc of
-  ProcDescription{pdReturnType = Nothing} -> True
-  _                                       -> False
-
-procTableName :: ProcDescription -> Maybe TableName
-procTableName proc = case pdReturnType proc of
-  Just (SetOf  (Composite qi)) -> Just $ qiName qi
-  Just (Single (Composite qi)) -> Just $ qiName qi
-  _                            -> Nothing

src/PostgREST/SchemaCache/Relationship.hs

@@ -6,6 +6,7 @@ module PostgREST.SchemaCache.Relationship
   , Relationship(..)
   , Junction(..)
   , RelationshipsMap
+  , relIsToOne
   ) where
 
 import qualified Data.Aeson          as JSON
@@ -55,10 +56,17 @@ data Junction = Junction
   { junTable       :: QualifiedIdentifier
   , junConstraint1 :: FKConstraint
   , junConstraint2 :: FKConstraint
-  , junColumns1    :: [(FieldName, FieldName)]
-  , junColumns2    :: [(FieldName, FieldName)]
+  , junColsSource  :: [(FieldName, FieldName)]
+  , junColsTarget  :: [(FieldName, FieldName)]
   }
   deriving (Eq, Ord, Generic, JSON.ToJSON)
 
 -- | Key based on the source table and the foreign table schema
 type RelationshipsMap = HM.HashMap (QualifiedIdentifier, Schema)  [Relationship]
+
+relIsToOne :: Relationship -> Bool
+relIsToOne rel = case rel of
+  Relationship{relCardinality=M2O _ _} -> True
+  Relationship{relCardinality=O2O _ _} -> True
+  ComputedRelationship{relToOne=True}  -> True
+  _                                    -> False

src/PostgREST/SchemaCache/Routine.hs

@@ -0,0 +1,104 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric  #-}
+
+module PostgREST.SchemaCache.Routine
+  ( PgType(..)
+  , Routine(..)
+  , RoutineParam(..)
+  , FuncVolatility(..)
+  , RoutineMap
+  , RetType(..)
+  , funcReturnsScalar
+  , funcReturnsSetOfScalar
+  , funcReturnsSingleComposite
+  , funcReturnsVoid
+  , funcTableName
+  , funcReturnsCompositeAlias
+  ) where
+
+import qualified Data.Aeson          as JSON
+import qualified Data.HashMap.Strict as HM
+
+import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
+                                          Schema, TableName)
+
+import Protolude
+
+data PgType
+  = Scalar QualifiedIdentifier
+  | Composite QualifiedIdentifier Bool -- True if the composite is a domain alias(used to work around a bug in pg 11 and 12, see QueryBuilder.hs)
+  deriving (Eq, Ord, Generic, JSON.ToJSON)
+
+data RetType
+  = Single PgType
+  | SetOf PgType
+  deriving (Eq, Ord, Generic, JSON.ToJSON)
+
+data FuncVolatility
+  = Volatile
+  | Stable
+  | Immutable
+  deriving (Eq, Ord, Generic, JSON.ToJSON)
+
+data Routine = Function
+  { pdSchema      :: Schema
+  , pdName        :: Text
+  , pdDescription :: Maybe Text
+  , pdParams      :: [RoutineParam]
+  , pdReturnType  :: RetType
+  , pdVolatility  :: FuncVolatility
+  , pdHasVariadic :: Bool
+  , pdIsoLvl      :: Maybe Text
+  }
+  deriving (Eq, Generic, JSON.ToJSON)
+
+data RoutineParam = RoutineParam
+  { ppName :: Text
+  , ppType :: Text
+  , ppReq  :: Bool
+  , ppVar  :: Bool
+  }
+  deriving (Eq, Ord, Generic, JSON.ToJSON)
+
+-- Order by least number of params in the case of overloaded functions
+instance Ord Routine where
+  Function schema1 name1 des1 prms1 rt1 vol1 hasVar1 iso1 `compare` Function schema2 name2 des2 prms2 rt2 vol2 hasVar2 iso2
+    | schema1 == schema2 && name1 == name2 && length prms1 < length prms2  = LT
+    | schema2 == schema2 && name1 == name2 && length prms1 > length prms2  = GT
+    | otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1, iso1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2, iso2)
+
+-- | A map of all procs, all of which can be overloaded(one entry will have more than one Routine).
+-- | It uses a HashMap for a faster lookup.
+type RoutineMap = HM.HashMap QualifiedIdentifier [Routine]
+
+funcReturnsScalar :: Routine -> Bool
+funcReturnsScalar proc = case proc of
+  Function{pdReturnType = Single (Scalar{})} -> True
+  _                                          -> False
+
+funcReturnsSetOfScalar :: Routine -> Bool
+funcReturnsSetOfScalar proc = case proc of
+  Function{pdReturnType = SetOf (Scalar{})} -> True
+  _                                         -> False
+
+funcReturnsCompositeAlias :: Routine -> Bool
+funcReturnsCompositeAlias proc = case proc of
+  Function{pdReturnType = Single (Composite _ True)} -> True
+  Function{pdReturnType = SetOf (Composite _ True)}  -> True
+  _                                                  -> False
+
+funcReturnsSingleComposite :: Routine -> Bool
+funcReturnsSingleComposite proc = case proc of
+  Function{pdReturnType = Single (Composite _ _)} -> True
+  _                                               -> False
+
+funcReturnsVoid :: Routine -> Bool
+funcReturnsVoid proc = case proc of
+  Function{pdReturnType = Single (Scalar (QualifiedIdentifier "pg_catalog" "void"))} -> True
+  _                                                                                  -> False
+
+funcTableName :: Routine -> Maybe TableName
+funcTableName proc = case pdReturnType proc of
+  SetOf  (Composite qi _) -> Just $ qiName qi
+  Single (Composite qi _) -> Just $ qiName qi
+  _                       -> Nothing

src/PostgREST/SchemaCache/Table.hs

@@ -1,14 +1,18 @@
-{-# LANGUAGE DeriveAnyClass #-}
-{-# LANGUAGE DeriveGeneric  #-}
+{-# LANGUAGE DeriveAnyClass    #-}
+{-# LANGUAGE DeriveGeneric     #-}
+{-# LANGUAGE FlexibleInstances #-}
 
 module PostgREST.SchemaCache.Table
   ( Column(..)
   , Table(..)
+  , tableColumnsList
   , TablesMap
+  , ColumnMap
   ) where
 
-import qualified Data.Aeson          as JSON
-import qualified Data.HashMap.Strict as HM
+import qualified Data.Aeson                 as JSON
+import qualified Data.HashMap.Strict        as HM
+import qualified Data.HashMap.Strict.InsOrd as HMI
 
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier (..),
@@ -28,9 +32,12 @@ data Table = Table
   , tableUpdatable   :: Bool
   , tableDeletable   :: Bool
   , tablePKCols      :: [FieldName]
-  , tableColumns     :: [Column]
+  , tableColumns     :: ColumnMap
   }
-  deriving (Show, Ord, Generic, JSON.ToJSON)
+  deriving (Show, Generic, JSON.ToJSON)
+
+tableColumnsList :: Table -> [Column]
+tableColumnsList = HMI.elems . tableColumns
 
 instance Eq Table where
   Table{tableSchema=s1,tableName=n1} == Table{tableSchema=s2,tableName=n2} = s1 == s2 && n1 == n2
@@ -40,6 +47,7 @@ data Column = Column
   , colDescription :: Maybe Text
   , colNullable    :: Bool
   , colType        :: Text
+  , colNominalType :: Text
   , colMaxLen      :: Maybe Int32
   , colDefault     :: Maybe Text
   , colEnum        :: [Text]
@@ -47,3 +55,4 @@ data Column = Column
   deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 type TablesMap = HM.HashMap QualifiedIdentifier Table
+type ColumnMap = HMI.InsOrdHashMap FieldName Column

src/PostgREST/Workers.hs

@@ -9,11 +9,8 @@ module PostgREST.Workers
   , runAdmin
   ) where
 
-import qualified Data.Aeson                 as JSON
 import qualified Data.ByteString            as BS
-import qualified Data.ByteString.Lazy       as LBS
 import qualified Data.Text                  as T
-import qualified Data.Text.Encoding         as T
 import qualified Hasql.Notifications        as SQL
 import qualified Hasql.Session              as SQL
 import qualified Hasql.Transaction.Sessions as SQL
@@ -30,10 +27,10 @@ import Network.Socket.ByteString
 
 import PostgREST.AppState         (AppState)
 import PostgREST.Config           (AppConfig (..), readAppConfig)
-import PostgREST.Config.Database  (queryDbSettings, queryPgVersion)
+import PostgREST.Config.Database  (queryDbSettings, queryPgVersion,
+                                   queryRoleSettings)
 import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
-import PostgREST.Error            (PgError (PgError), checkIsFatal,
-                                   errorPayload)
+import PostgREST.Error            (checkIsFatal)
 import PostgREST.SchemaCache      (querySchemaCache)
 
 import qualified PostgREST.AppState as AppState
@@ -128,12 +125,11 @@ establishConnection appState =
 
     getConnectionStatus :: IO ConnectionStatus
     getConnectionStatus = do
-      pgVersion <- AppState.usePool appState queryPgVersion
+      pgVersion <- AppState.usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
       case pgVersion of
         Left e -> do
-          let err = PgError False e
-          AppState.logWithZTime appState . T.decodeUtf8 . LBS.toStrict $ errorPayload err
-          case checkIsFatal err of
+          AppState.logPgrstError appState e
+          case checkIsFatal e of
             Just reason ->
               return $ FatalConnectionError reason
             Nothing ->
@@ -168,25 +164,20 @@ loadSchemaCache appState = do
       querySchemaCache (toList configDbSchemas) configDbExtraSearchPath configDbPreparedStatements
   case result of
     Left e -> do
-      let
-        err = PgError False e
-        putErr = AppState.logWithZTime appState . T.decodeUtf8 . LBS.toStrict $ errorPayload err
-      case checkIsFatal err of
+      case checkIsFatal e of
         Just hint -> do
           AppState.logWithZTime appState "A fatal error ocurred when loading the schema cache"
-          putErr
+          AppState.logPgrstError appState e
           AppState.logWithZTime appState hint
           return SCFatalFail
         Nothing -> do
           AppState.putSchemaCache appState Nothing
           AppState.logWithZTime appState "An error ocurred when loading the schema cache"
-          putErr
+          AppState.logPgrstError appState e
           return SCOnRetry
 
     Right sCache -> do
       AppState.putSchemaCache appState (Just sCache)
-      when (isJust configDbRootSpec) .
-        AppState.putJsonDbS appState . LBS.toStrict $ JSON.encode sCache
       AppState.logWithZTime appState "Schema cache loaded"
       return SCLoaded
 
@@ -230,16 +221,15 @@ listener appState = do
       listener appState
 
     handleNotification _ msg
-      | BS.null msg            = scLoader -- reload the schema cache
-      | msg == "reload schema" = scLoader -- reload the schema cache
-      | msg == "reload config" = reReadConfig False appState -- reload the config
+      | BS.null msg            = cacheReloader
+      | msg == "reload schema" = cacheReloader
+      | msg == "reload config" = reReadConfig False appState
       | otherwise              = pure () -- Do nothing if anything else than an empty message is sent
 
-    scLoader =
-      -- It's not necessary to check the loadSchemaCache success
-      -- here. If the connection drops, the thread will die and
-      -- proceed to recover.
-      void $ loadSchemaCache appState
+    cacheReloader =
+      -- reloads the schema cache + restarts pool connections
+      -- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
+      connectionWorker appState
 
 -- | Re-reads the config plus config options from the db
 reReadConfig :: Bool -> AppState -> IO ()
@@ -250,23 +240,31 @@ reReadConfig startingUp appState = do
       qDbSettings <- AppState.usePool appState $ queryDbSettings configDbPreparedStatements
       case qDbSettings of
         Left e -> do
-          let
-            err = PgError False e
-            putErr = AppState.logWithZTime appState . T.decodeUtf8 . LBS.toStrict $ errorPayload err
           AppState.logWithZTime appState
             "An error ocurred when trying to query database settings for the config parameters"
-          case checkIsFatal err of
+          case checkIsFatal e of
             Just hint -> do
-              putErr
+              AppState.logPgrstError appState e
               AppState.logWithZTime appState hint
               killThread (AppState.getMainThreadId appState)
             Nothing -> do
-              putErr
-          pure []
+              AppState.logPgrstError appState e
+          pure mempty
         Right x -> pure x
     else
       pure mempty
-  readAppConfig dbSettings configFilePath (Just configDbUri) >>= \case
+  roleSettings <-
+    if configDbConfig then do
+      rSettings <- AppState.usePool appState $ queryRoleSettings configDbPreparedStatements
+      case rSettings of
+        Left e -> do
+          AppState.logWithZTime appState "An error ocurred when trying to query the role settings"
+          AppState.logPgrstError appState e
+          pure mempty
+        Right x -> pure x
+    else
+      pure mempty
+  readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings >>= \case
     Left err   ->
       if startingUp then
         panic err -- die on invalid config if the program is starting up

stack.yaml

@@ -1,4 +1,4 @@
-resolver: lts-19.14 # 2022-07-01, GHC 9.0.2
+resolver: lts-20.6 # 2023-01-09, GHC 9.2.5
 
 nix:
   packages:
@@ -10,23 +10,7 @@ nix:
   pure: false
 
 extra-deps:
-  - HTTP-4000.3.16
-  - configurator-pg-0.2.6
-  - hashable-1.4.1.0
-  - hashtables-1.3
-  - hasql-1.6.1.1
-  - hasql-dynamic-statements-0.3.1.2
-  - hasql-implicits-0.1.0.5
-  - hasql-notifications-0.2.0.3
-  - hasql-pool-0.8.0.6
-  - hasql-transaction-1.0.1.2
-  - isomorphism-class-0.1.0.6
-  - lens-aeson-1.1.3
-  - optparse-applicative-0.16.1.0
-  - postgresql-binary-0.12.5
-  - protolude-0.3.2
-  - ptr-0.16.8.2
-  - text-builder-0.6.7
-  - text-builder-dev-0.3.3
   - git: https://github.com/PostgREST/postgresql-libpq.git
-    commit: 33ff97db570b5b432255f5f24a68db51453f6eb8
+    commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
+  - hasql-notifications-0.2.0.4
+  - hasql-pool-0.9

stack.yaml.lock

@@ -5,145 +5,33 @@
 
 packages:
 - completed:
-    hackage: HTTP-4000.3.16@sha256:6042643c15a0b43e522a6693f1e322f05000d519543a84149cb80aeffee34f71,5947
-    pantry-tree:
-      size: 1428
-      sha256: b73a7f6d21cf20bbf819e19039409c9010efb5000d2b72cdd8fd67a9027c14e8
-  original:
-    hackage: HTTP-4000.3.16
-- completed:
-    hackage: configurator-pg-0.2.6@sha256:cd9b06a458428e493a4d6def725af7ab1ab0fef678fbd871f9586fc7f9aa70be,2849
-    pantry-tree:
-      size: 2463
-      sha256: 97efe7a22afc93033bda5adcffdabc0f1c30dc32b2c3ba02114ce7cd74c942fd
-  original:
-    hackage: configurator-pg-0.2.6
-- completed:
-    hackage: hashable-1.4.1.0@sha256:50b2f002c68fe67730ee7a3cd8607486197dd99b084255005ad51ecd6970a41b,5019
-    pantry-tree:
-      size: 1248
-      sha256: 9af2f7a42674f7effcabbebc043f97057240783f1709338a77f58216f4a5f18c
-  original:
-    hackage: hashable-1.4.1.0
-- completed:
-    hackage: hashtables-1.3@sha256:ab21804fdafbbd8ad918b2911dabb729ae0ea891780fe66bf7804cbcd07edadf,10379
-    pantry-tree:
-      size: 2895
-      sha256: e71f113ad989dbc994e0fb52bcc219d62930de9afa8b3441bf7909e864481b33
-  original:
-    hackage: hashtables-1.3
-- completed:
-    hackage: hasql-1.6.1.1@sha256:948a2137308cc5354e4997bc3666753867124cd25db792424cb9614b1c1b44cf,6626
-    pantry-tree:
-      size: 2622
-      sha256: 28d21bf061522fc513f040e9c383b90532222b7258216cc094e07736add8be10
-  original:
-    hackage: hasql-1.6.1.1
-- completed:
-    hackage: hasql-dynamic-statements-0.3.1.2@sha256:417aa533c84f074e2fa16bb2c4d4231326aa512097dd1025d915388e56acd1eb,2675
-    pantry-tree:
-      size: 595
-      sha256: 91696d3f3e0ef3254772ae5a8e4e89be68285febb49b302ed83d85ac4037a417
-  original:
-    hackage: hasql-dynamic-statements-0.3.1.2
-- completed:
-    hackage: hasql-implicits-0.1.0.5@sha256:d16aacad6dc21428d72447d3ae8bcc03839a2f0aa1ec29c797ed9aca4609f9af,1361
-    pantry-tree:
-      size: 264
-      sha256: 0451b99a0a1d02db673d0c40acdf60d4e769e15852eed9e8dc05bffaf43efb70
-  original:
-    hackage: hasql-implicits-0.1.0.5
-- completed:
-    hackage: hasql-notifications-0.2.0.3@sha256:aca3f7ee847a8f0b7ef6f989dc48f4a094a06c1a34e92aa3c8bb230085966ea6,2027
-    pantry-tree:
-      size: 452
-      sha256: 999f0f2856a00d21f4498a8a58452bbefc4ea972fe2984fd234a68a5fe61d98b
-  original:
-    hackage: hasql-notifications-0.2.0.3
-- completed:
-    hackage: hasql-pool-0.8.0.6@sha256:b63bb83409bab5bc20ff24f5d62205e9b117701a0fc24531ddeac20ab8c2a42c,1818
-    pantry-tree:
-      size: 346
-      sha256: c4100946b7eae44375511e35a393abe2e1db0e5637c68cea8f53176b796bfd5b
-  original:
-    hackage: hasql-pool-0.8.0.6
-- completed:
-    hackage: hasql-transaction-1.0.1.2@sha256:297b158cd1f0727f9b0e175bd7d3741c1bcb725a8094956d0ee79b41aafdb30a,2890
-    pantry-tree:
-      size: 983
-      sha256: 3679e6d5c835cc17a8fa0c252b8221e282880044b7219aa1de2531bbd5c40691
-  original:
-    hackage: hasql-transaction-1.0.1.2
-- completed:
-    hackage: isomorphism-class-0.1.0.6@sha256:d93da31287359c761953b876354de28381f409c5c50e3241c572a443e50c553d,1703
-    pantry-tree:
-      size: 465
-      sha256: c97f922d1ae8f1a0db4c28fac9383d2716934879e95ff0b2b88ebb861d6fba14
-  original:
-    hackage: isomorphism-class-0.1.0.6
-- completed:
-    hackage: lens-aeson-1.1.3@sha256:52c8eaecd2d1c2a969c0762277c4a8ee72c339a686727d5785932e72ef9c3050,1764
-    pantry-tree:
-      size: 541
-      sha256: b31392b78f2a03111c805f4400007778eb93b49f998ab41dfbebaaf9b5526bad
-  original:
-    hackage: lens-aeson-1.1.3
-- completed:
-    hackage: optparse-applicative-0.16.1.0@sha256:418c22ed6a19124d457d96bc66bd22c93ac22fad0c7100fe4972bbb4ac989731,4982
-    pantry-tree:
-      size: 2979
-      sha256: dd092d843091c08691485d68a1908517079b1bc6f3d73928f37635a19dc27fc1
-  original:
-    hackage: optparse-applicative-0.16.1.0
-- completed:
-    hackage: postgresql-binary-0.12.5@sha256:de9da3cba9be541d6c75ae8da2858c33d83dc1b2e0c639b0b9781816b78a91f4,5594
-    pantry-tree:
-      size: 1619
-      sha256: b392337f91031a5b3407393e2f04dfe4e7a28019e88eae6a9370538b90e28c51
-  original:
-    hackage: postgresql-binary-0.12.5
-- completed:
-    hackage: protolude-0.3.2@sha256:2a38b3dad40d238ab644e234b692c8911423f9d3ed0e36b62287c4a698d92cd1,2240
-    pantry-tree:
-      size: 1594
-      sha256: a36d2912ac552d950ba4476de7d950b56b82dd28e48b9f4d0efee938f10bc525
-  original:
-    hackage: protolude-0.3.2
-- completed:
-    hackage: ptr-0.16.8.2@sha256:708ebb95117f2872d2c5a554eb6804cf1126e86abe793b2673f913f14e5eb1ac,3959
-    pantry-tree:
-      size: 1303
-      sha256: 557c438345de19f82bf01d676100da2a191ef06f624e7a4b90b09ac17cbb52a5
-  original:
-    hackage: ptr-0.16.8.2
-- completed:
-    hackage: text-builder-0.6.7@sha256:efbb3e06107e9c8d1cfe85c963938ca9f375a74379af03da3173be4ef5c37bcf,2364
-    pantry-tree:
-      size: 425
-      sha256: cd0ae197e6f9f3860a8ab71f5b87c4a8452ed1fce2fdfd35e36d68ded6e6648e
-  original:
-    hackage: text-builder-0.6.7
-- completed:
-    hackage: text-builder-dev-0.3.3@sha256:79ec422defcc2e5b34f94129c72b98d34b2efc1ed8bbd945ccb8f4f535a892c3,2784
-    pantry-tree:
-      size: 724
-      sha256: 8883631a132438e7892fcb13e89d6bbcdc0ac76c56fbea8df8d7aa482ce81f73
-  original:
-    hackage: text-builder-dev-0.3.3
-- completed:
+    commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
+    git: https://github.com/PostgREST/postgresql-libpq.git
     name: postgresql-libpq
-    version: 0.9.4.3
-    git: https://github.com/PostgREST/postgresql-libpq.git
     pantry-tree:
-      size: 1081
-      sha256: 0df271e48af32eb8292a45301af45e114110d54099ee73dbc609d39770e8175e
-    commit: 33ff97db570b5b432255f5f24a68db51453f6eb8
+      sha256: 074668b9669b9c49f3c522c8af5c608799a1965e203c463b188b2632995beac2
+      size: 1414
+    version: 0.9.4.3
   original:
+    commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
     git: https://github.com/PostgREST/postgresql-libpq.git
-    commit: 33ff97db570b5b432255f5f24a68db51453f6eb8
+- completed:
+    hackage: hasql-notifications-0.2.0.4@sha256:9a09fa9b97feadd9492c8bd8bc6b9cffe0513510102f08374b0c45ecd479ed67,2028
+    pantry-tree:
+      sha256: 56f9e240728e7a65711dde45fa2e2075b914e32cd370424aaa4572392378a60e
+      size: 452
+  original:
+    hackage: hasql-notifications-0.2.0.4
+- completed:
+    hackage: hasql-pool-0.9@sha256:db7a37f6b3a922c37adc3c7ced47a7c10786d1f171e47a735a6e812a587ba44c,2111
+    pantry-tree:
+      sha256: 49b1181d28c6f5317e794671c2dae155754b834bdcfa30f7e5dbad28e4cf0249
+      size: 346
+  original:
+    hackage: hasql-pool-0.9
 snapshots:
 - completed:
-    size: 618951
-    url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/19/14.yaml
-    sha256: 4c31d4ef975b0211078862566aedf3b82b6cea569fc2cde4c72a51e5a8d236ce
-  original: lts-19.14
+    sha256: 4905c93319aa94aa53da8f41d614d7bacdbfe6c63a8c6132d32e6e62f24a9af4
+    size: 649315
+    url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/20/6.yaml
+  original: lts-20.6

test/doc/Main.hs

@@ -15,4 +15,5 @@ main =
     , "src/PostgREST/Query/SqlFragment.hs"
     , "src/PostgREST/ApiRequest/Preferences.hs"
     , "src/PostgREST/ApiRequest/QueryParams.hs"
+    , "src/PostgREST/Error.hs"
     ]

test/io/configs/expected/aliases.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public"
 db-max-rows = 1000
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
 db-pre-request = "check_alias"
 db-prepared-statements = true
 db-root-spec = "open_alias"
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/expected/boolean-numeric.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
 db-pre-request = ""
 db-prepared-statements = false
 db-root-spec = ""
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/expected/boolean-string.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
 db-pre-request = ""
 db-prepared-statements = false
 db-root-spec = ""
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/expected/defaults.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
 db-pre-request = ""
 db-prepared-statements = true
 db-root-spec = ""
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/expected/no-defaults-with-db-other-authenticator.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public,extensions,other"
 db-max-rows = 100
 db-plan-enabled = true
 db-pool = 1
-db-pool-acquisition-timeout = 10
+db-pool-acquisition-timeout = 30
+db-pool-max-lifetime = 3600
 db-pre-request = "test.other_custom_headers"
 db-prepared-statements = false
 db-root-spec = "other_root"
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = "https://otherexample.org/api"
 raw-media-types = "application/vnd.pgrst.other-db-config"
 server-host = "0.0.0.0"
 server-port = 80
+server-trace-header = "traceparent"
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001

test/io/configs/expected/no-defaults-with-db.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public,extensions,private"
 db-max-rows = 1000
 db-plan-enabled = true
 db-pool = 1
-db-pool-acquisition-timeout = 10
+db-pool-acquisition-timeout = 30
+db-pool-max-lifetime = 3600
 db-pre-request = "test.custom_headers"
 db-prepared-statements = false
 db-root-spec = "root"
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = "https://example.org/api"
 raw-media-types = "application/vnd.pgrst.db-config"
 server-host = "0.0.0.0"
 server-port = 80
+server-trace-header = "CF-Ray"
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001

test/io/configs/expected/no-defaults.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public,test"
 db-max-rows = 1000
 db-plan-enabled = true
 db-pool = 1
-db-pool-acquisition-timeout = 10
+db-pool-acquisition-timeout = 30
+db-pool-max-lifetime = 3600
 db-pre-request = "please_run_fast"
 db-prepared-statements = false
 db-root-spec = "openapi_v3"
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = "https://postgrest.org"
 raw-media-types = "application/vnd.pgrst.config"
 server-host = "0.0.0.0"
 server-port = 80
+server-trace-header = "X-Request-Id"
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001

test/io/configs/expected/types.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false
 db-pool = 10
-db-pool-acquisition-timeout = ""
+db-pool-acquisition-timeout = 10
+db-pool-max-lifetime = 1800
 db-pre-request = ""
 db-prepared-statements = true
 db-root-spec = ""
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = ""
 raw-media-types = ""
 server-host = "!4"
 server-port = 3000
+server-trace-header = ""
 server-unix-socket = ""
 server-unix-socket-mode = "660"
 admin-server-port = ""

test/io/configs/no-defaults-env.yaml

@@ -7,7 +7,8 @@ PGRST_DB_EXTRA_SEARCH_PATH: public, test
 PGRST_DB_MAX_ROWS: 1000
 PGRST_DB_PLAN_ENABLED: true
 PGRST_DB_POOL: 1
-PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
+PGRST_DB_POOL_ACQUISITION_TIMEOUT: 30
+PGRST_DB_POOL_MAX_LIFETIME: 3600
 PGRST_DB_PREPARED_STATEMENTS: false
 PGRST_DB_PRE_REQUEST: please_run_fast
 PGRST_DB_ROOT_SPEC: openapi_v3
@@ -28,6 +29,7 @@ PGRST_OPENAPI_SERVER_PROXY_URI: 'https://postgrest.org'
 PGRST_RAW_MEDIA_TYPES: application/vnd.pgrst.config
 PGRST_SERVER_HOST: 0.0.0.0
 PGRST_SERVER_PORT: 80
+PGRST_SERVER_TRACE_HEADER: X-Request-Id
 PGRST_SERVER_UNIX_SOCKET: /tmp/pgrst_io_test.sock
 PGRST_SERVER_UNIX_SOCKET_MODE: 777
 PGRST_ADMIN_SERVER_PORT: 3001

test/io/configs/no-defaults.config

@@ -5,7 +5,8 @@ db-extra-search-path = "public, test"
 db-max-rows = 1000
 db-plan-enabled = true
 db-pool = 1
-db-pool-acquisition-timeout = 10
+db-pool-acquisition-timeout = 30
+db-pool-max-lifetime = 3600
 db-pre-request = "please_run_fast"
 db-prepared-statements = false
 db-root-spec = "openapi_v3"
@@ -25,6 +26,7 @@ openapi-server-proxy-uri = "https://postgrest.org"
 raw-media-types = "application/vnd.pgrst.config"
 server-host = "0.0.0.0"
 server-port = 80
+server-trace-header = "X-Request-Id"
 server-unix-socket = "/tmp/pgrst_io_test.sock"
 server-unix-socket-mode = "777"
 admin-server-port = 3001

test/io/db_config.sql

@@ -17,6 +17,7 @@ ALTER ROLE db_config_authenticator SET pgrst.db_pre_request = 'test.custom_heade
 ALTER ROLE db_config_authenticator SET pgrst.db_max_rows = '1000';
 ALTER ROLE db_config_authenticator SET pgrst.db_extra_search_path = 'public, extensions';
 ALTER ROLE db_config_authenticator SET pgrst.not_existing = 'should be ignored';
+ALTER ROLE db_config_authenticator SET pgrst.server_trace_header = 'CF-Ray';
 
 -- override with database specific setting
 ALTER ROLE db_config_authenticator IN DATABASE :DBNAME SET pgrst.jwt_secret = 'OVERRIDE=REALLY=REALLY=REALLY=REALLY=VERY=SAFE';
@@ -60,6 +61,7 @@ ALTER ROLE other_authenticator SET pgrst.db_max_rows = '100';
 ALTER ROLE other_authenticator SET pgrst.db_extra_search_path = 'public, extensions, other';
 ALTER ROLE other_authenticator SET pgrst.openapi_mode = 'disabled';
 ALTER ROLE other_authenticator SET pgrst.openapi_security_active = 'false';
+ALTER ROLE other_authenticator SET pgrst.server_trace_header = 'traceparent';
 
 -- authenticator used for tests that manipulate statement timeout
 CREATE ROLE timeout_authenticator LOGIN NOINHERIT;

test/io/fixtures.sql

@@ -8,7 +8,13 @@ ALTER ROLE :USER SET pgrst.db_anon_role = 'postgrest_test_anonymous';
 
 CREATE ROLE postgrest_test_author;
 
-GRANT postgrest_test_anonymous, postgrest_test_author TO :USER;
+CREATE ROLE postgrest_test_serializable;
+alter role postgrest_test_serializable set default_transaction_isolation = 'serializable';
+
+CREATE ROLE postgrest_test_repeatable_read;
+alter role postgrest_test_repeatable_read set default_transaction_isolation = 'REPEATABLE READ';
+
+GRANT postgrest_test_anonymous, postgrest_test_author, postgrest_test_serializable, postgrest_test_repeatable_read TO :USER;
 
 CREATE SCHEMA v1;
 GRANT USAGE ON SCHEMA v1 TO postgrest_test_anonymous;
@@ -86,3 +92,53 @@ $$ language sql;
 create or replace function hello() returns text as $$
   select 'hello';
 $$ language sql;
+
+create table cats(id uuid primary key, name text);
+grant all on cats to postgrest_test_anonymous;
+
+create function drop_change_cats() returns void
+language sql security definer
+as $$
+  drop table cats;
+  create table cats(id bigint primary key, name text);
+  grant all on table cats to postgrest_test_anonymous;
+  notify pgrst, 'reload schema';
+$$;
+
+alter role postgrest_test_anonymous set statement_timeout to '2s';
+alter role postgrest_test_author set statement_timeout to '10s';
+
+create function change_role_statement_timeout(timeout text) returns void as $_$
+begin
+  execute format($$
+    alter role current_user set statement_timeout = %L;
+  $$, timeout);
+end $_$ volatile language plpgsql ;
+
+create table items as select x as id from generate_series(1,5) x;
+
+create view items_w_isolation_level as
+select
+  id,
+  current_setting('transaction_isolation', true) as isolation_level
+from items;
+
+grant all on items_w_isolation_level to postgrest_test_anonymous, postgrest_test_repeatable_read, postgrest_test_serializable;
+
+create function default_isolation_level()
+returns text as $$
+  select current_setting('transaction_isolation', true);
+$$
+language sql;
+
+create function serializable_isolation_level()
+returns text as $$
+  select current_setting('transaction_isolation', true);
+$$
+language sql set default_transaction_isolation = 'serializable';
+
+create function repeatable_read_isolation_level()
+returns text as $$
+  select current_setting('transaction_isolation', true);
+$$
+language sql set default_transaction_isolation = 'REPEATABLE READ';

test/io/postgrest.py

@@ -42,6 +42,18 @@ class PostgrestProcess:
     process: object
     session: object
 
+    def read_stdout(self, nlines=1):
+        "Wait for line(s) on standard output."
+        output = []
+        for _ in range(10):
+            l = self.process.stdout.readline()
+            if l:
+                output.append(l.decode())
+                if len(output) >= nlines:
+                    break
+            time.sleep(0.1)
+        return output
+
 
 @contextlib.contextmanager
 def run(
@@ -50,6 +62,7 @@ def run(
     env=None,
     port=None,
     host=None,
+    wait_for_readiness=True,
     no_pool_connection_available=False,
 ):
     "Run PostgREST and yield an endpoint that is ready for connections."
@@ -88,7 +101,8 @@ def run(
             process.stdin.write(stdin or b"")
             process.stdin.close()
 
-            wait_until_ready(adminurl + "/ready")
+            if wait_for_readiness:
+                wait_until_ready(adminurl + "/ready")
 
             process.stdout.read()
 
@@ -137,6 +151,14 @@ def freeport():
         return s.getsockname()[1]
 
 
+def wait_until_exit(postgrest):
+    "Wait for PostgREST to exit, or times out"
+    try:
+        return postgrest.process.wait(timeout=1)
+    except subprocess.TimeoutExpired:
+        raise PostgrestTimedOut()
+
+
 def wait_until_ready(url):
     "Wait for the given HTTP endpoint to return a status of 200."
     session = requests_unixsocket.Session()

test/io/test_io.py

@@ -66,6 +66,15 @@ def test_read_secret_from_stdin_dbconfig(defaultenv):
         assert response.status_code == 200
 
 
+def test_fail_with_invalid_password(defaultenv):
+    "Connecting with an invalid password should fail without retries."
+    uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
+    env = {**defaultenv, "PGRST_DB_URI": uri}
+    with run(env=env, wait_for_readiness=False) as postgrest:
+        exitCode = wait_until_exit(postgrest)
+        assert exitCode == 1
+
+
 def test_connect_with_dburi(dburi, defaultenv):
     "Connecting with db-uri instead of LIPQ* environment variables should work."
     defaultenv_without_libpq = {
@@ -325,7 +334,7 @@ def test_db_schema_notify_reload(defaultenv):
             "/rpc/change_db_schema_and_full_reload", data={"schemas": "v1"}
         )
 
-        time.sleep(0.1)
+        time.sleep(0.2)
 
         response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
         assert response.text == '"\\"v1\\", \\"public\\""'
@@ -407,14 +416,8 @@ def test_invalid_role_claim_key_notify_reload(defaultenv):
     with run(env=env) as postgrest:
         postgrest.session.post("/rpc/invalid_role_claim_key_reload")
 
-        output = None
-        for _ in range(10):
-            output = postgrest.process.stdout.readline()
-            if output:
-                break
-            time.sleep(0.1)
-
-        assert "failed to parse role-claim-key value" in output.decode()
+        output = postgrest.read_stdout()
+        assert "failed to parse role-claim-key value" in output[0]
 
         response = postgrest.session.post("/rpc/reset_invalid_role_claim_key")
         assert response.status_code == 204
@@ -526,7 +529,6 @@ def test_pool_size(defaultenv, metapostgrest):
     }
 
     with run(env=env) as postgrest:
-
         start = time.time()
         threads = []
         for i in range(4):
@@ -563,6 +565,11 @@ def test_pool_acquisition_timeout(defaultenv, metapostgrest):
         data = response.json()
         assert data["message"] == "Timed out acquiring connection from connection pool."
 
+        # ensure the message appears on the logs as well
+        output = sorted(postgrest.read_stdout(nlines=2))
+        assert " 504 " in output[0]
+        assert "Timed out acquiring connection from connection pool." in output[1]
+
 
 def test_change_statement_timeout_held_connection(defaultenv, metapostgrest):
     "Statement timeout changes take effect immediately, even with a request outliving the reconfiguration"
@@ -653,7 +660,6 @@ def test_admin_ready_includes_schema_cache_state(defaultenv, metapostgrest):
     }
 
     with run(env=env) as postgrest:
-
         # make it impossible to load the schema cache, by setting statement timeout to 1ms
         set_statement_timeout(metapostgrest, role, 1)
 
@@ -709,7 +715,6 @@ def test_admin_works_with_host_special_values(specialhostvalue, defaultenv):
     "Should get a success from the admin live and ready endpoints when using special host values for the main app"
 
     with run(env=defaultenv, port=freeport(), host=specialhostvalue) as postgrest:
-
         response = postgrest.admin.get("/live")
         assert response.status_code == 200
 
@@ -765,7 +770,6 @@ def test_no_pool_connection_required_on_bad_http_logic(defaultenv):
     "no pool connection should be consumed for failing on invalid http logic"
 
     with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
-
         # not found nested route shouldn't require opening a connection
         response = postgrest.session.head("/path/notfound")
         assert response.status_code == 404
@@ -781,7 +785,6 @@ def test_no_pool_connection_required_on_options(defaultenv):
     "no pool connection should be consumed for OPTIONS requests"
 
     with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
-
         # OPTIONS on a table shouldn't require opening a connection
         response = postgrest.session.options("/projects")
         assert response.status_code == 200
@@ -801,7 +804,6 @@ def test_no_pool_connection_required_on_bad_jwt_claim(defaultenv):
     env = {**defaultenv, "PGRST_JWT_SECRET": SECRET}
 
     with run(env=env, no_pool_connection_available=True) as postgrest:
-
         # A JWT with an invalid signature shouldn't open a connection
         headers = jwtauthheader({"role": "postgrest_test_author"}, "Wrong Secret")
         response = postgrest.session.get("/projects", headers=headers)
@@ -812,12 +814,138 @@ def test_no_pool_connection_required_on_bad_embedding(defaultenv):
     "no pool connection should be consumed for failing to embed"
 
     with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
-
         # OPTIONS on a table shouldn't require opening a connection
         response = postgrest.session.get("/projects?select=*,unexistent(*)")
         assert response.status_code == 400
 
 
+def test_notify_reloading_catalog_cache(defaultenv):
+    "notify should reload the connection catalog cache"
+
+    with run(env=defaultenv) as postgrest:
+        # first the id col is an uuid
+        response = postgrest.session.get(
+            "/cats?id=eq.dea27321-f988-4a57-93e4-8eeb38f3cf1e"
+        )
+        assert response.status_code == 200
+
+        # change it to a bigint
+        response = postgrest.session.post("/rpc/drop_change_cats")
+        assert response.status_code == 204
+        time.sleep(0.1)
+
+        # next request should succeed with a bigint value
+        response = postgrest.session.get("/cats?id=eq.1")
+        assert response.status_code == 200
+
+
+def test_role_settings(defaultenv):
+    "statement_timeout should be set per role"
+
+    env = {
+        **defaultenv,
+        "PGRST_JWT_SECRET": SECRET,
+    }
+
+    with run(env=env) as postgrest:
+        # statement_timeout for postgrest_test_anonymous
+        response = postgrest.session.get("/rpc/get_guc_value?name=statement_timeout")
+        assert response.text == '"2s"'
+
+        # reload statement_timeout with NOTIFY
+        response = postgrest.session.post(
+            "/rpc/change_role_statement_timeout", data={"timeout": "5s"}
+        )
+        assert response.status_code == 204
+
+        response = postgrest.session.get("/rpc/reload_pgrst_config")
+        assert response.status_code == 204
+        time.sleep(0.1)
+
+        response = postgrest.session.get("/rpc/get_guc_value?name=statement_timeout")
+        assert response.text == '"5s"'
+
+        # statement_timeout for postgrest_test_author
+        headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
+        response = postgrest.session.get(
+            "/rpc/get_guc_value?name=statement_timeout", headers=headers
+        )
+        assert response.text == '"10s"'
+
+
+def test_isolation_level(defaultenv):
+    "isolation_level should be set per role and per function"
+
+    env = {
+        **defaultenv,
+        "PGRST_JWT_SECRET": SECRET,
+    }
+
+    with run(env=env) as postgrest:
+        # default isolation level for postgrest_test_anonymous
+        response = postgrest.session.get(
+            "/items_w_isolation_level?select=isolation_level&limit=1"
+        )
+        assert response.text == '[{"isolation_level":"read committed"}]'
+
+        # isolation level for postgrest_test_repeatable_read on GET
+        headers = jwtauthheader({"role": "postgrest_test_repeatable_read"}, SECRET)
+        response = postgrest.session.get(
+            "/items_w_isolation_level?select=isolation_level&limit=1", headers=headers
+        )
+        assert response.text == '[{"isolation_level":"repeatable read"}]'
+
+        # isolation level for postgrest_test_serializable on POST
+        headers = jwtauthheader({"role": "postgrest_test_serializable"}, SECRET)
+        headers["Prefer"] = "return=representation"
+        response = postgrest.session.post(
+            "/items_w_isolation_level?select=isolation_level",
+            json={"id": "666"},
+            headers=headers,
+        )
+        assert response.text == '[{"isolation_level":"serializable"}]'
+
+        # isolation level for postgrest_test_serializable on PATCH
+        headers = jwtauthheader({"role": "postgrest_test_serializable"}, SECRET)
+        headers["Prefer"] = "return=representation"
+        response = postgrest.session.patch(
+            "/items_w_isolation_level?select=isolation_level&id=eq.666",
+            json={"id": "666"},
+            headers=headers,
+        )
+        assert response.text == '[{"isolation_level":"serializable"}]'
+
+        # isolation level for postgrest_test_serializable on DELETE
+        headers = jwtauthheader({"role": "postgrest_test_serializable"}, SECRET)
+        headers["Prefer"] = "return=representation"
+        response = postgrest.session.delete(
+            "/items_w_isolation_level?select=isolation_level&id=eq.666", headers=headers
+        )
+        assert response.text == '[{"isolation_level":"serializable"}]'
+
+        # default isolation level for function
+        response = postgrest.session.get("/rpc/default_isolation_level")
+        assert response.text == '"read committed"'
+
+        # changes with role isolation level
+        headers = jwtauthheader({"role": "postgrest_test_repeatable_read"}, SECRET)
+        response = postgrest.session.get(
+            "/rpc/default_isolation_level", headers=headers
+        )
+        assert response.text == '"repeatable read"'
+
+        # isolation level can be set per function
+        response = postgrest.session.get("/rpc/serializable_isolation_level")
+        assert response.text == '"serializable"'
+        response = postgrest.session.get("/rpc/repeatable_read_isolation_level")
+        assert response.text == '"repeatable read"'
+
+        # isolation level for a function overrides the role isolation level
+        headers = jwtauthheader({"role": "postgrest_test_repeatable_read"}, SECRET)
+        response = postgrest.session.get("/rpc/serializable_isolation_level")
+        assert response.text == '"serializable"'
+
+
 # TODO: This test fails now because of https://github.com/PostgREST/postgrest/pull/2122
 # The stack size of 1K(-with-rtsopts=-K1K) is not enough and this fails with "stack overflow"
 # A stack size of 200K seems to be enough for succeess

test/pgbench/1567/new.sql

@@ -0,0 +1,15 @@
+INSERT INTO "test"."complex_items"("arr_data", "field-with_sep", "id", "name")
+SELECT pgrst_body."arr_data", pgrst_body."field-with_sep", pgrst_body."id", pgrst_body."name"
+FROM (
+  SELECT '[{"id": 4, "name": "Vier"}, {"id": 5, "name": "Funf", "arr_data": null}, {"id": 6, "name": "Sechs", "arr_data": [1, 2, 3], "field-with_sep": 6}]'::jsonb as json_data
+) pgrst_payload,
+LATERAL (
+  SELECT CASE WHEN jsonb_typeof(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE jsonb_build_array(pgrst_payload.json_data) END AS val
+) pgrst_uniform_json,
+LATERAL (
+  SELECT jsonb_agg(jsonb_build_object('field-with_sep', 1) || elem) AS vals from jsonb_array_elements(pgrst_uniform_json.val) elem
+) pgrst_json_defs,
+LATERAL (
+  SELECT * FROM jsonb_to_recordset (pgrst_json_defs.vals) AS _ ("arr_data" integer[], "field-with_sep" integer, "id" bigint, "name" text)
+) pgrst_body
+RETURNING "test"."complex_items".*;

test/pgbench/1567/old.sql

@@ -0,0 +1,12 @@
+INSERT INTO "test"."complex_items"("arr_data", "field-with_sep", "id", "name")
+SELECT pgrst_body."arr_data", pgrst_body."field-with_sep", pgrst_body."id", pgrst_body."name"
+FROM (
+  SELECT '[{"id": 4, "name": "Vier"}, {"id": 5, "name": "Funf", "arr_data": null}, {"id": 6, "name": "Sechs", "arr_data": [1, 2, 3], "field-with_sep": 6}]'::jsonb as json_data
+) pgrst_payload,
+LATERAL (
+  SELECT CASE WHEN jsonb_typeof(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE jsonb_build_array(pgrst_payload.json_data) END AS val
+) pgrst_uniform_json,
+LATERAL (
+  SELECT * FROM jsonb_to_recordset (pgrst_uniform_json.val) AS _ ("arr_data" integer[], "field-with_sep" integer, "id" bigint, "name" text)
+) pgrst_body
+RETURNING "test"."complex_items".*

test/pgbench/1652/new.sql

@@ -0,0 +1,20 @@
+WITH pgrst_source AS (
+  SELECT pgrst_call.*
+  FROM (
+    SELECT '{"id": 4}'::json as json_data
+  ) pgrst_payload,
+  LATERAL (
+    SELECT CASE WHEN json_typeof(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE json_build_array(pgrst_payload.json_data) END AS val
+  ) pgrst_uniform_json,
+  LATERAL (
+    SELECT * FROM json_to_recordset(pgrst_uniform_json.val) AS _("id" integer) LIMIT 1
+  ) pgrst_body,
+  LATERAL "test"."get_projects_below"("id" := pgrst_body.id) pgrst_call
+)
+SELECT
+  null::bigint AS total_result_set,
+  pg_catalog.count(_postgrest_t) AS page_total,
+  coalesce(json_agg(_postgrest_t), '[]')::character varying AS body,
+  nullif(current_setting('response.headers', true), '') AS response_headers,
+  nullif(current_setting('response.status', true), '') AS response_status
+FROM (SELECT "projects".* FROM "pgrst_source" AS "projects") _postgrest_t;

test/pgbench/1652/old.sql

@@ -0,0 +1,15 @@
+WITH pgrst_source AS (
+  WITH
+  pgrst_payload AS (SELECT '{"id": 4}'::json AS json_data),
+  pgrst_body AS ( SELECT CASE WHEN json_typeof(json_data) = 'array' THEN json_data ELSE json_build_array(json_data) END AS val FROM pgrst_payload),
+  pgrst_args AS ( SELECT * FROM json_to_recordset((SELECT val FROM pgrst_body)) AS _("id" integer) )
+  SELECT "get_projects_below".*
+  FROM "test"."get_projects_below"("id" := (SELECT "id" FROM pgrst_args LIMIT 1))
+)
+SELECT
+  null::bigint AS total_result_set,
+  pg_catalog.count(_postgrest_t) AS page_total,
+  coalesce(json_agg(_postgrest_t), '[]')::character varying AS body,
+  nullif(current_setting('response.headers', true), '') AS response_headers,
+  nullif(current_setting('response.status', true), '') AS response_status
+FROM (SELECT "projects".* FROM "pgrst_source" AS "projects") _postgrest_t;

test/pgbench/2677/new.sql

@@ -0,0 +1,12 @@
+INSERT INTO "test"."complex_items"("arr_data", "field-with_sep", "id", "name")
+SELECT pgrst_body."arr_data", pgrst_body."field-with_sep", pgrst_body."id", pgrst_body."name"
+FROM (
+  SELECT '[{"id": 4, "name": "Vier"}, {"id": 5, "name": "Funf", "arr_data": null}, {"id": 6, "name": "Sechs", "arr_data": [1, 2, 3], "field-with_sep": 6}]'::json as json_data
+) pgrst_payload,
+LATERAL (
+  SELECT CASE WHEN json_typeof(pgrst_payload.json_data) = 'array' THEN pgrst_payload.json_data ELSE json_build_array(pgrst_payload.json_data) END AS val
+) pgrst_uniform_json,
+LATERAL (
+  SELECT * FROM json_to_recordset (pgrst_uniform_json.val) AS _ ("arr_data" integer[], "field-with_sep" integer, "id" bigint, "name" text)
+) pgrst_body
+RETURNING "test"."complex_items".*

test/pgbench/2677/old.sql

@@ -0,0 +1,7 @@
+WITH
+pgrst_payload AS (SELECT '[{"id": 4, "name": "Vier"}, {"id": 5, "name": "Funf", "arr_data": null}, {"id": 6, "name": "Sechs", "arr_data": [1, 2, 3], "field-with_sep": 6}]'::json AS json_data),
+pgrst_body AS ( SELECT CASE WHEN json_typeof(json_data) = 'array' THEN json_data ELSE json_build_array(json_data) END AS val FROM pgrst_payload)
+INSERT INTO "test"."complex_items"("arr_data", "field-with_sep", "id", "name")
+SELECT "arr_data", "field-with_sep", "id", "name"
+FROM json_to_recordset ((SELECT val FROM pgrst_body)) AS _ ("arr_data" integer[], "field-with_sep" integer, "id" bigint, "name" text)
+RETURNING "test"."complex_items".*

test/pgbench/README.md

@@ -0,0 +1,13 @@
+## pgbench tests
+
+Can be used as:
+
+```
+postgrest-with-postgresql-15 -f test/pgbench/fixtures.sql pgbench -n -T 10 -f test/pgbench/1567/old.sql
+
+postgrest-with-postgresql-15 -f test/pgbench/fixtures.sql pgbench -n -T 10 -f test/pgbench/1567/new.sql
+```
+
+## Directory structure
+
+The directory name is the issue number on github.

test/pgbench/fixtures.sql

@@ -0,0 +1,7 @@
+\ir ../spec/fixtures/load.sql
+
+ALTER TABLE test.complex_items
+    DROP CONSTRAINT complex_items_pkey;
+
+ALTER TABLE test.complex_items
+    ALTER COLUMN "field-with_sep" DROP NOT NULL;

test/spec/Feature/ObservabilitySpec.hs

@@ -0,0 +1,34 @@
+module Feature.ObservabilitySpec where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+
+import Protolude
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "Observability" $ do
+    it "includes the server trace header on the response" $ do
+      request methodHead "/"
+          [ ("X-Request-Id", "1") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "X-Request-Id" <:> "1"] }
+
+      request methodHead "/projects"
+          [ ("X-Request-Id", "2") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "X-Request-Id" <:> "2"] }
+
+      request methodHead "/rpc/add_them?a=2&b=4"
+          [ ("X-Request-Id", "3") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "X-Request-Id" <:> "3"] }

test/spec/Feature/OpenApi/OpenApiSpec.hs

@@ -42,6 +42,20 @@ spec actualPgVersion = describe "OpenAPI" $ do
 
     liftIO $ docsUrl `shouldBe` Just (String ("https://postgrest.org/en/" <> docsVersion <> "/api.html"))
 
+  describe "schema" $ do
+
+    it "includes title and comments to schema" $ do
+      r <- simpleBody <$> get "/"
+
+      let childGetTitle = r ^? key "info" . key "title"
+      let childGetDescription = r ^? key "info" . key "description"
+
+      liftIO $ do
+
+        childGetTitle `shouldBe` Just "My API title"
+
+        childGetDescription `shouldBe` Just "My API description\nthat spans\nmultiple lines"
+
   describe "table" $ do
 
     it "includes paths to tables" $ do
@@ -91,7 +105,7 @@ spec actualPgVersion = describe "OpenAPI" $ do
             [
               { "$ref": "#/parameters/body.child_entities" },
               { "$ref": "#/parameters/select" },
-              { "$ref": "#/parameters/preferReturn" }
+              { "$ref": "#/parameters/preferPost" }
             ]
           |]
 
@@ -310,6 +324,23 @@ spec actualPgVersion = describe "OpenAPI" $ do
             }
           |]
 
+  describe "VIEW created for a TABLE with a O2M relationship" $ do
+
+    it "fk points to destination TABLE instead of the VIEW" $ do
+      r <- simpleBody <$> get "/"
+
+      let referralLink = r ^? key "definitions" . key "projects" . key "properties" . key "client_id"
+
+      liftIO $
+        referralLink `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "integer",
+              "type": "integer",
+              "description": "Note:\nThis is a Foreign Key to `clients.id`.<fk table='clients' column='id'/>"
+            }
+          |]
+
   describe "PostgreSQL to Swagger Type Mapping" $ do
 
     it "character varying to string" $ do
@@ -490,6 +521,117 @@ spec actualPgVersion = describe "OpenAPI" $ do
             }
           |]
 
+    it "array types to array" $ do
+      r <- simpleBody <$> get "/"
+
+      let text_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_text_arr"
+      let int_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_int_arr"
+      let bool_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_bool_arr"
+      let char_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_char_arr"
+      let varchar_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_varchar_arr"
+      let bigint_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_bigint_arr"
+      let numeric_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_numeric_arr"
+      let json_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_json_arr"
+      let jsonb_arr_types = r ^? key "definitions" . key "openapi_types" . key "properties" . key "a_jsonb_arr"
+
+      liftIO $ do
+
+        text_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "text[]",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          |]
+
+        int_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "integer[]",
+              "type": "array",
+              "items": {
+                "type": "integer"
+              }
+            }
+          |]
+
+        bool_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "boolean[]",
+              "type": "array",
+              "items": {
+                "type": "boolean"
+              }
+            }
+          |]
+
+        char_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "character[]",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          |]
+
+        varchar_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "character varying[]",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          |]
+
+        bigint_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "bigint[]",
+              "type": "array",
+              "items": {
+                "type": "integer"
+              }
+            }
+          |]
+
+        numeric_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "numeric[]",
+              "type": "array",
+              "items": {
+                "type": "number"
+              }
+            }
+          |]
+
+        json_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "json[]",
+              "type": "array",
+              "items": {}
+            }
+          |]
+
+        jsonb_arr_types `shouldBe` Just
+          [aesonQQ|
+            {
+              "format": "jsonb[]",
+              "type": "array",
+              "items": {}
+            }
+          |]
+
+
   describe "Detects default values" $ do
 
     it "text" $ do
@@ -566,10 +708,156 @@ spec actualPgVersion = describe "OpenAPI" $ do
 
   describe "RPC" $ do
 
-    it "includes function summary/description and body schema for arguments" $ do
+    it "includes function summary/description and query parameters for arguments in the get path item" $ do
       r <- simpleBody <$> get "/"
 
-      let method s = key "paths" . key "/rpc/varied_arguments" . key s
+      let method s = key "paths" . key "/rpc/varied_arguments_openapi" . key s
+          args = r ^? method "get" . key "parameters"
+          summary = r ^? method "get" . key "summary"
+          description = r ^? method "get" . key "description"
+
+      liftIO $ do
+
+        summary `shouldBe` Just "An RPC function"
+
+        description `shouldBe` Just "Just a test for RPC function arguments"
+
+        args `shouldBe` Just
+          [aesonQQ|
+            [
+              {
+                "format": "double precision",
+                "in": "query",
+                "name": "double",
+                "required": true,
+                "type": "number"
+              },
+              {
+                "format": "character varying",
+                "in": "query",
+                "name": "varchar",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "boolean",
+                "in": "query",
+                "name": "boolean",
+                "required": true,
+                "type": "boolean"
+              },
+              {
+                "format": "date",
+                "in": "query",
+                "name": "date",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "money",
+                "in": "query",
+                "name": "money",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "enum_menagerie_type",
+                "in": "query",
+                "name": "enum",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "text[]",
+                "in": "query",
+                "name": "text_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "integer[]",
+                "in": "query",
+                "name": "int_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "boolean[]",
+                "in": "query",
+                "name": "bool_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "character[]",
+                "in": "query",
+                "name": "char_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "character varying[]",
+                "in": "query",
+                "name": "varchar_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "bigint[]",
+                "in": "query",
+                "name": "bigint_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "numeric[]",
+                "in": "query",
+                "name": "numeric_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "json[]",
+                "in": "query",
+                "name": "json_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "jsonb[]",
+                "in": "query",
+                "name": "jsonb_arr",
+                "required": true,
+                "type": "string"
+              },
+              {
+                "format": "integer",
+                "in": "query",
+                "name": "integer",
+                "required": false,
+                "type": "integer"
+              },
+              {
+                "format": "json",
+                "in": "query",
+                "name": "json",
+                "required": false,
+                "type": "string"
+              },
+              {
+                "format": "jsonb",
+                "in": "query",
+                "name": "jsonb",
+                "required": false,
+                "type": "string"
+              }
+            ]
+          |]
+
+    it "includes function summary/description and body schema for arguments in the post path item" $ do
+      r <- simpleBody <$> get "/"
+
+      let method s = key "paths" . key "/rpc/varied_arguments_openapi" . key s
           args = r ^? method "post" . key "parameters" . nth 0 . key "schema"
           summary = r ^? method "post" . key "summary"
           description = r ^? method "post" . key "description"
@@ -590,7 +878,15 @@ spec actualPgVersion = describe "OpenAPI" $ do
                 "date",
                 "money",
                 "enum",
-                "arr"
+                "text_arr",
+                "int_arr",
+                "bool_arr",
+                "char_arr",
+                "varchar_arr",
+                "bigint_arr",
+                "numeric_arr",
+                "json_arr",
+                "jsonb_arr"
               ],
               "properties": {
                 "double": {
@@ -617,9 +913,64 @@ spec actualPgVersion = describe "OpenAPI" $ do
                   "format": "enum_menagerie_type",
                   "type": "string"
                 },
-                "arr": {
+                "text_arr": {
                   "format": "text[]",
-                  "type": "string"
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                "int_arr": {
+                  "format": "integer[]",
+                  "type": "array",
+                  "items": {
+                    "type": "integer"
+                  }
+                },
+                "bool_arr": {
+                  "format": "boolean[]",
+                  "type": "array",
+                  "items": {
+                    "type": "boolean"
+                  }
+                },
+                "char_arr": {
+                  "format": "character[]",
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                "varchar_arr": {
+                  "format": "character varying[]",
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                "bigint_arr": {
+                  "format": "bigint[]",
+                  "type": "array",
+                  "items": {
+                    "type": "integer"
+                  }
+                },
+                "numeric_arr": {
+                  "format": "numeric[]",
+                  "type": "array",
+                  "items": {
+                    "type": "number"
+                  }
+                },
+                "json_arr": {
+                  "format": "json[]",
+                  "type": "array",
+                  "items": {}
+                },
+                "jsonb_arr": {
+                  "format": "jsonb[]",
+                  "type": "array",
+                  "items": {}
                 },
                 "integer": {
                   "format": "integer",
@@ -668,6 +1019,26 @@ spec actualPgVersion = describe "OpenAPI" $ do
 
       liftIO $ params `shouldBe` Just [aesonQQ|["num", "str"]|]
 
+    it "uses a multi collection format when the function has a VARIADIC parameter" $ do
+      r <- simpleBody <$> get "/"
+      let param = r ^? key "paths" . key "/rpc/variadic_param"
+                     . key "get" . key "parameters" .  nth 0
+
+      liftIO $ param `shouldBe` Just
+        [aesonQQ|
+          {
+            "collectionFormat": "multi",
+            "in": "query",
+            "items": {
+              "format": "text",
+              "type": "string"
+            },
+            "name": "v",
+            "required": false,
+            "type": "array"
+          }
+        |]
+
   describe "Security" $
     it "does not include security or security definitions by default" $ do
       r <- simpleBody <$> get "/"

test/spec/Feature/OpenApi/RootSpec.hs

@@ -9,8 +9,6 @@ import Test.Hspec.Wai.JSON
 
 import Protolude hiding (get)
 
-import SpecHelper
-
 spec :: SpecWith ((), Application)
 spec =
   describe "root spec function" $ do
@@ -22,9 +20,3 @@ spec =
            "info": {"title": "PostgREST API", "description": "This is a dynamic API generated by PostgREST"}
           }|]
         { matchHeaders = ["Content-Type" <:> "application/openapi+json; charset=utf-8"] }
-
-    it "accepts application/json" $
-      request methodGet "/"
-        [("Accept", "application/json")] "" `shouldRespondWith`
-        200
-        { matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/AndOrParamsSpec.hs

@@ -92,6 +92,9 @@ spec actualPgVersion =
               {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" },
               {"text_search_vector": "'art':4 'spass':5 'unmog':7"}
             ]|] { matchHeaders = [matchContentTypeJson] }
+        it "can handle isdistinct" $
+          get "/entities?and=(id.gte.2,arr.isdistinct.{1,2})&select=id" `shouldRespondWith`
+            [json|[{ "id": 3 }, { "id": 4 }]|] { matchHeaders = [matchContentTypeJson] }
 
         when (actualPgVersion >= pgVersion112) $
           it "can handle wfts (websearch_to_tsquery)" $
@@ -138,6 +141,8 @@ spec actualPgVersion =
             [json|[{ "id": 3 }, { "id": 4 }]|] { matchHeaders = [matchContentTypeJson] }
           get "/ranges?range=adj.(3,10]&select=id" `shouldRespondWith`
             [json|[{ "id": 1 }]|] { matchHeaders = [matchContentTypeJson] }
+          get "/ranges?range=isdistinct.[1,3]&select=id" `shouldRespondWith`
+            [json|[{ "id": 2 }, { "id": 3 }, { "id": 4 }, {"id": 5}]|] { matchHeaders = [matchContentTypeJson] }
 
         it "can handle array operators" $ do
           get "/entities?arr=eq.{1,2,3}&select=id" `shouldRespondWith`
@@ -166,6 +171,8 @@ spec actualPgVersion =
             [json|[{ "id": 3 }]|] { matchHeaders = [matchContentTypeJson] }
           get "/entities?arr=ov.{2,3}&select=id" `shouldRespondWith`
             [json|[{ "id": 2 }, { "id": 3 }]|] { matchHeaders = [matchContentTypeJson] }
+          get "/entities?arr=isdistinct.{1,2}&select=id" `shouldRespondWith`
+            [json|[{ "id": 1 }, { "id": 3 }, { "id": 4 }]|] { matchHeaders = [matchContentTypeJson] }
 
         context "operators with not" $ do
           it "eq, cs, like can be negated" $
@@ -180,6 +187,9 @@ spec actualPgVersion =
           it "gt, lte, ilike can be negated" $
             get "/entities?and=(name.not.ilike.*ITY2,or(id.not.gt.4,id.not.lte.1))&select=id" `shouldRespondWith`
               [json|[{"id": 1}, {"id": 2}, {"id": 3}]|] { matchHeaders = [matchContentTypeJson] }
+          it "isdistinct can be negated" $
+            get "/entities?and=(id.not.eq.2,arr.not.isdistinct.{1,2,3})&select=id" `shouldRespondWith`
+              [json|[{"id": 3}]|] { matchHeaders = [matchContentTypeJson] }
 
       context "and/or params with quotes" $ do
         it "eq can have quotes" $
@@ -252,35 +262,3 @@ spec actualPgVersion =
 
     it "can query columns that begin with and/or reserved words" $
       get "/grandchild_entities?or=(and_starting_col.eq.smth, or_starting_col.eq.smth)" `shouldRespondWith` 200
-
-    it "fails when using IN without () and provides meaningful error message" $
-      get "/entities?or=(id.in.1,2,id.eq.3)" `shouldRespondWith`
-        [json|{
-          "details": "unexpected \"1\" expecting \"(\"",
-          "message": "\"failed to parse logic tree ((id.in.1,2,id.eq.3))\" (line 1, column 10)",
-          "code": "PGRST100",
-          "hint": null
-        }|] { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-
-    it "fails on malformed query params and provides meaningful error message" $ do
-      get "/entities?or=)(" `shouldRespondWith`
-        [json|{
-          "details": "unexpected \")\" expecting \"(\"",
-          "message": "\"failed to parse logic tree ()()\" (line 1, column 3)",
-          "code": "PGRST100",
-          "hint": null
-        }|] { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-      get "/entities?and=(ord(id.eq.1,id.eq.1),id.eq.2)" `shouldRespondWith`
-        [json|{
-          "details": "unexpected \"d\" expecting \"(\"",
-          "message": "\"failed to parse logic tree ((ord(id.eq.1,id.eq.1),id.eq.2))\" (line 1, column 7)",
-          "code": "PGRST100",
-          "hint": null
-        }|] { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-      get "/entities?or=(id.eq.1,not.xor(id.eq.2,id.eq.3))" `shouldRespondWith`
-        [json|{
-          "details": "unexpected \"x\" expecting logic operator (and, or)",
-          "message": "\"failed to parse logic tree ((id.eq.1,not.xor(id.eq.2,id.eq.3)))\" (line 1, column 16)",
-          "code": "PGRST100",
-          "hint": null
-        }|] { matchStatus = 400, matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/DeleteSpec.hs

@@ -154,7 +154,7 @@ spec =
       it "works with the limit and offset query params" $
         baseTable "limited_delete_items" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items?order=id&limit=1&offset=1" mempty
+        requestMutation methodDelete "/limited_delete_items?order=id&limit=1&offset=1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
@@ -164,7 +164,7 @@ spec =
       it "works with the limit query param plus a filter" $
         baseTable "limited_delete_items" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items?order=id&limit=1&id=gt.1" mempty
+        requestMutation methodDelete "/limited_delete_items?order=id&limit=1&id=gt.1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
@@ -200,7 +200,7 @@ spec =
       it "works with views with an explicit order by unique col" $
         baseTable "limited_delete_items_view" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items_view?order=id&limit=1&offset=1" mempty
+        requestMutation methodDelete "/limited_delete_items_view?order=id&limit=1&offset=1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
@@ -210,7 +210,7 @@ spec =
       it "works with views with an explicit order by composite pk" $
         baseTable "limited_delete_items_cpk_view" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items_cpk_view?order=id,name&limit=1&offset=1" mempty
+        requestMutation methodDelete "/limited_delete_items_cpk_view?order=id,name&limit=1&offset=1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
@@ -220,9 +220,53 @@ spec =
       it "works on a table without a pk by ordering by 'ctid'" $
         baseTable "limited_delete_items_no_pk" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/limited_delete_items_no_pk?order=ctid&limit=1&offset=1" mempty
+        requestMutation methodDelete "/limited_delete_items_no_pk?order=ctid&limit=1&offset=1" mempty mempty
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "item-1" }
         , { "id": 3, "name": "item-3" }
         ]|]
+
+      it "ignores the Range header" $ do
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items"
+          (rangeHdrs (ByteRangeFromTo 0 0)) mempty
+         `shouldMutateInto`
+         [json|[]|]
+
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items?id=gte.2"
+          (rangeHdrs (ByteRangeFromTo 0 0)) mempty
+         `shouldMutateInto`
+         [json|[ { "id": 1, "name": "item-1" } ]|]
+
+      it "ignores the Range header and does not do a limited delete" $
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items?order=id"
+          (rangeHdrs (ByteRangeFromTo 0 0)) mempty
+         `shouldMutateInto`
+         [json|[]|]
+
+      it "ignores the Range header and does not throw an invalid range error" $
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items?order=id&limit=1&offset=1"
+          (rangeHdrs (ByteRangeFromTo 0 0)) mempty
+         `shouldMutateInto`
+         [json|[
+           { "id": 1, "name": "item-1" }
+         , { "id": 3, "name": "item-3" }
+         ]|]
+
+      it "ignores the Range header but not the limit and offset params" $
+        baseTable "limited_delete_items" "id" tblDataBefore
+         `mutatesWith`
+         requestMutation methodDelete "/limited_delete_items?order=id&limit=2&offset=1"
+          (rangeHdrs (ByteRangeFromTo 1 1)) mempty
+         `shouldMutateInto`
+         [json|[
+           { "id": 1, "name": "item-1" }
+         ]|]

test/spec/Feature/Query/EmbedDisambiguationSpec.hs

@@ -68,10 +68,8 @@ spec =
           , matchHeaders = [matchContentTypeJson]
           }
 
-      it "errs when there are more than two fks on a junction table(currently impossible to disambiguate, only choice is to split the table)" $
+      it "errs when there are more than two fks on a junction table but it can be disambiguated with spread embeds" $ do
         -- We have 4 possibilities for doing the junction JOIN here.
-        -- This could be solved by specifying two additional fks, like whatev_projects!fk1!fk2(*)
-        -- If the need arises this capability can be added later without causing a breaking change
         get "/whatev_sites?select=*,whatev_projects(*)" `shouldRespondWith`
           [json|
             {
@@ -105,6 +103,23 @@ spec =
           { matchStatus  = 300
           , matchHeaders = [matchContentTypeJson]
           }
+        -- Each of those 4 possibilities can be done with spread embeds, by following the details in the error above
+        get "/whatev_sites?select=*,whatev_jobs!site_id_1(...whatev_projects!project_id_1(*))" `shouldRespondWith` [json|[]|]
+        get "/whatev_sites?select=*,whatev_jobs!site_id_1(...whatev_projects!project_id_2(*))" `shouldRespondWith` [json|[]|]
+        get "/whatev_sites?select=*,whatev_jobs!site_id_2(...whatev_projects!project_id_1(*))" `shouldRespondWith` [json|[]|]
+        get "/whatev_sites?select=*,whatev_jobs!site_id_2(...whatev_projects!project_id_2(*))" `shouldRespondWith` [json|[]|]
+
+      it "can disambiguate a recursive m2m with spread embeds" $ do
+        get "/posters?select=*,subscribers:subscriptions!subscribed(...posters!subscriber(*))&limit=1" `shouldRespondWith`
+          [json| [ {"id":1,"name":"Mark","subscribers":[{"id":3,"name":"Bill"}, {"id":4,"name":"Jeff"}]}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }
+        get "/posters?select=*,subscriptions!subscriber(...posters!subscribed(*))&limit=1" `shouldRespondWith`
+          [json| [{"id":1,"name":"Mark","subscriptions":[{"id":2,"name":"Elon"}]}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }
 
       it "errs on an ambiguous embed that has two one-to-one relationships" $
         get "/first?select=second(*)" `shouldRespondWith`
@@ -202,10 +217,10 @@ spec =
         it "fails if the fk is not known" $
           get "/message?select=id,sender:person!space(name)&id=lt.4" `shouldRespondWith`
             [json|{
-              "hint":"Verify that 'message' and 'person' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
+              "hint":null,
               "message":"Could not find a relationship between 'message' and 'person' in the schema cache",
               "code": "PGRST200",
-              "details": null}|]
+              "details":"Searched for a foreign key relationship between 'message' and 'person' using the hint 'space' in the schema 'test', but no matches were found."}|]
             { matchStatus = 400
             , matchHeaders = [matchContentTypeJson] }
 
@@ -492,10 +507,10 @@ spec =
       it "doesn't work if the junction is only internal" $
         get "/end_1?select=end_2(*)" `shouldRespondWith`
           [json|{
-            "hint":"Verify that 'end_1' and 'end_2' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
+            "hint": null,
             "message":"Could not find a relationship between 'end_1' and 'end_2' in the schema cache",
             "code":"PGRST200",
-            "details": null}|]
+            "details": "Searched for a foreign key relationship between 'end_1' and 'end_2' in the schema 'test', but no matches were found."}|]
           { matchStatus  = 400
           , matchHeaders = [matchContentTypeJson] }
       it "shouldn't try to embed if the private junction has an exposed homonym" $
@@ -503,10 +518,10 @@ spec =
         -- Ref: https://github.com/PostgREST/postgrest/issues/1587#issuecomment-734995669
         get "/schauspieler?select=filme(*)" `shouldRespondWith`
           [json|{
-            "hint":"Verify that 'schauspieler' and 'filme' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
+            "hint":null,
             "message":"Could not find a relationship between 'schauspieler' and 'filme' in the schema cache",
             "code":"PGRST200",
-            "details": null}|]
+            "details":"Searched for a foreign key relationship between 'schauspieler' and 'filme' in the schema 'test', but no matches were found."}|]
           { matchStatus  = 400
           , matchHeaders = [matchContentTypeJson] }
 

test/spec/Feature/Query/InsertSpec.hs

@@ -11,8 +11,9 @@ import Test.Hspec.Wai
 import Test.Hspec.Wai.JSON
 import Text.Heredoc
 
-import PostgREST.Config.PgVersion (PgVersion, pgVersion110,
-                                   pgVersion112, pgVersion130)
+import PostgREST.Config.PgVersion (PgVersion, pgVersion100,
+                                   pgVersion110, pgVersion112,
+                                   pgVersion130)
 
 import Protolude  hiding (get)
 import SpecHelper
@@ -420,20 +421,102 @@ spec actualPgVersion = do
           , matchHeaders = []
           }
 
+      it "disallows ?columns which don't exist" $
+        post "/articles?columns=helicopter"
+          [json|[
+            {"id": 204, "body": "yyy"},
+            {"id": 205, "body": "zzz"}]|]
+          `shouldRespondWith`
+          [json|{"code":"PGRST204","details":null,"hint":null,"message":"Column 'helicopter' of relation 'articles' does not exist"} |]
+          { matchStatus  = 400
+          , matchHeaders = []
+          }
+
+      it "returns missing table error even if also has invalid ?columns" $
+        post "/garlic?columns=helicopter"
+          [json|[
+            {"id": 204, "body": "yyy"},
+            {"id": 205, "body": "zzz"}]|]
+          `shouldRespondWith`
+          [json|{} |]
+          { matchStatus  = 404
+          , matchHeaders = []
+          }
+
       it "disallows array elements that are not json objects" $
         post "/articles?columns=id,body"
           [json|[
             {"id": 204, "body": "yyy"},
             333,
             "asdf",
-            {"id": 205, "body": "zzz"}]|] `shouldRespondWith`
-          [json|{
-              "code": "22023",
-              "details": null,
-              "hint": null,
-              "message": "argument of json_populate_recordset must be an array of objects"}|]
-          { matchStatus  = 400
-          , matchHeaders = []
+            {"id": 205, "body": "zzz"}]|] `shouldRespondWith` 400
+
+      context "apply defaults on missing values" $ do
+        -- inserting the array fails on pg 9.6, but the feature should work normally
+        when (actualPgVersion >= pgVersion100) $
+          it "inserts table default values(field-with_sep) when json keys are undefined" $
+            request methodPost "/complex_items?columns=id,name,field-with_sep,arr_data" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+                [json|[
+                  {"id": 4, "name": "Vier"},
+                  {"id": 5, "name": "Funf", "arr_data": null},
+                  {"id": 6, "name": "Sechs", "field-with_sep": 6, "arr_data": "{1,2,3}"}
+                ]|]
+              `shouldRespondWith`
+                [json|[
+                  {"id": 4, "name": "Vier", "field-with_sep": 1, "settings":null,"arr_data":null},
+                  {"id": 5, "name": "Funf", "field-with_sep": 1, "settings":null,"arr_data":null},
+                  {"id": 6, "name": "Sechs", "field-with_sep": 6, "settings":null,"arr_data":[1,2,3]}
+                ]|]
+                { matchStatus  = 201
+                , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+                }
+
+        it "inserts view default values(field-with_sep) when json keys are undefined" $
+          request methodPost "/complex_items_view?columns=id,name" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+              [json|[
+                {"id": 7, "name": "Sieben"},
+                {"id": 8}
+              ]|]
+            `shouldRespondWith`
+              [json|[
+                {"id": 7, "name": "Sieben", "field-with_sep": 1, "settings":null,"arr_data":null},
+                {"id": 8, "name": "Default", "field-with_sep": 1, "settings":null,"arr_data":null}
+              ]|]
+              { matchStatus  = 201
+              , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+              }
+
+        it "doesn't insert json duplicate keys(since it uses jsonb)" $
+          request methodPost "/tbl_w_json?columns=id,data" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+              [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
+            `shouldRespondWith`
+              [json| [ { "data": { "a": 2 }, "id": 3 } ] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+              }
+
+        when (actualPgVersion >= pgVersion100) $
+          it "inserts a default on a generated by default as identity column" $ do
+            request methodPost "/channels?columns=id,data,slug&select=data,slug" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+                [json| { "slug": "foo" } |]
+              `shouldRespondWith`
+                [json| [{"data":{"foo": "bar"},"slug":"foo"}] |] -- id 1 was inserted here, we don't get it for idempotence in the tests
+                { matchStatus  = 201
+                , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+                }
+
+    it "inserts json that has duplicate keys" $ do
+      request methodPost "/tbl_w_json" [("Prefer", "return=representation")]
+          [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
+        `shouldRespondWith`
+          [json| [ { "data": { "a": 1, "a": 2 }, "id": 3 } ] |]
+          { matchStatus  = 201
+          }
+      request methodPost "/tbl_w_json?columns=id,data" [("Prefer", "return=representation")]
+          [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
+        `shouldRespondWith`
+          [json| [ { "data": { "a": 1, "a": 2 }, "id": 3 } ] |]
+          { matchStatus  = 201
           }
 
     context "with unicode values" $ do

test/spec/Feature/Query/JsonOperatorSpec.hs

@@ -292,25 +292,11 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         [json| [{"data":[{"a": [1,2,3]}, {"b": [4,5]}]}] |]
         { matchHeaders = [matchContentTypeJson] }
 
-    it "should fail on badly formed negatives" $ do
-      get "/json_arr?select=data->>-78xy" `shouldRespondWith`
-        [json|
-          {"details": "unexpected 'x' expecting digit, \"->\", \"::\", \".\", \",\" or end of input",
-           "message": "\"failed to parse select parameter (data->>-78xy)\" (line 1, column 11)",
-           "code": "PGRST100",
-           "hint": null} |]
-        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-      get "/json_arr?select=data->>--34" `shouldRespondWith`
-        [json|
-          {"details": "unexpected \"-\" expecting digit",
-           "message": "\"failed to parse select parameter (data->>--34)\" (line 1, column 9)",
-           "code": "PGRST100",
-           "hint": null} |]
-        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
-      get "/json_arr?select=data->>-xy-4" `shouldRespondWith`
-        [json|
-          {"details":"unexpected \"x\" expecting digit",
-           "message":"\"failed to parse select parameter (data->>-xy-4)\" (line 1, column 9)",
-           "code": "PGRST100",
-           "hint": null} |]
-        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
+  it "gives a meaningful error on bad syntax" $
+    get "/json_arr?select=data->>--34" `shouldRespondWith`
+      [json|
+        {"details": "unexpected \"-\" expecting digit",
+         "message": "\"failed to parse select parameter (data->>--34)\" (line 1, column 9)",
+         "code": "PGRST100",
+         "hint": null} |]
+      { matchStatus = 400, matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/PgSafeUpdateSpec.hs

@@ -38,7 +38,7 @@ spec =
       it "allows full table update if a filter is present" $
         baseTable "safe_update_items" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodPatch "/safe_update_items?id=gt.0" [json| {"name": "updated-item"} |]
+        requestMutation methodPatch "/safe_update_items?id=gt.0" mempty [json| {"name": "updated-item"} |]
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "updated-item", "observation": null }
@@ -61,7 +61,7 @@ spec =
       it "allows full table delete if a filter is present" $
         baseTable "safe_delete_items" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/safe_delete_items?id=gt.0" mempty
+        requestMutation methodDelete "/safe_delete_items?id=gt.0" mempty mempty
         `shouldMutateInto`
         [json|[]|]
 
@@ -72,7 +72,7 @@ disabledSpec =
       it "works if no condition is present" $
         baseTable "unsafe_update_items" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodPatch "/unsafe_update_items" [json| {"name": "updated-item"} |]
+        requestMutation methodPatch "/unsafe_update_items" mempty [json| {"name": "updated-item"} |]
         `shouldMutateInto`
         [json|[
           { "id": 1, "name": "updated-item", "observation": null }
@@ -84,6 +84,6 @@ disabledSpec =
       it "works if no condition is present" $
         baseTable "unsafe_delete_items" "id" tblDataBefore
         `mutatesWith`
-        requestMutation methodDelete "/unsafe_delete_items" mempty
+        requestMutation methodDelete "/unsafe_delete_items" mempty mempty
         `shouldMutateInto`
         [json|[]|]

test/spec/Feature/Query/PlanSpec.hs

@@ -8,14 +8,16 @@ import Network.Wai.Test (SResponse (..))
 
 import           Data.Aeson.Lens
 import           Data.Aeson.QQ
+import qualified Data.ByteString      as BS
 import qualified Data.ByteString.Lazy as LBS
+import qualified Data.Text            as T
 import           Network.HTTP.Types
 import           Test.Hspec           hiding (pendingWith)
 import           Test.Hspec.Wai
 import           Test.Hspec.Wai.JSON
 
-import PostgREST.Config.PgVersion (PgVersion, pgVersion100,
-                                   pgVersion120, pgVersion130)
+import PostgREST.Config.PgVersion (PgVersion, pgVersion120,
+                                   pgVersion130)
 import Protolude                  hiding (get)
 import SpecHelper
 
@@ -26,7 +28,7 @@ spec actualPgVersion = do
       r <- request methodGet "/projects?id=in.(1,2,3)"
              (acceptHdrs "application/vnd.pgrst.plan+json") ""
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
+      let totalCost  = planCost r
           resHeaders = simpleHeaders r
           resStatus  = simpleStatus r
 
@@ -35,14 +37,14 @@ spec actualPgVersion = do
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe`
           if actualPgVersion > pgVersion120
-            then Just [aesonQQ|15.63|]
-            else Just [aesonQQ|15.69|]
+            then 15.63
+            else 15.69
 
     it "outputs the total cost for a single filter on a view" $ do
       r <- request methodGet "/projects_view?id=gt.2"
              (acceptHdrs "application/vnd.pgrst.plan+json") ""
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
+      let totalCost  = planCost r
           resHeaders = simpleHeaders r
           resStatus  = simpleStatus r
 
@@ -51,34 +53,20 @@ spec actualPgVersion = do
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe`
           if actualPgVersion > pgVersion120
-            then Just [aesonQQ|24.28|]
-            else Just [aesonQQ|32.28|]
+            then 24.28
+            else 32.27
 
     it "outputs blocks info when using the buffers option" $
       if actualPgVersion >= pgVersion130
         then do
           r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=buffers") ""
 
-          let blocks  = simpleBody r ^? nth 0 . key "Planning"
+          let resBody  = simpleBody r
               resHeaders = simpleHeaders r
 
           liftIO $ do
             resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=buffers; charset=utf-8")
-            blocks `shouldBe`
-              Just [aesonQQ|
-                {
-                  "Shared Hit Blocks": 0,
-                  "Shared Read Blocks": 0,
-                  "Shared Dirtied Blocks": 0,
-                  "Shared Written Blocks": 0,
-                  "Local Hit Blocks": 0,
-                  "Local Read Blocks": 0,
-                  "Local Dirtied Blocks": 0,
-                  "Local Written Blocks": 0,
-                  "Temp Read Blocks": 0,
-                  "Temp Written Blocks": 0
-                }
-              |]
+            resBody `shouldSatisfy` (\t -> T.isInfixOf "Shared Hit Blocks" (decodeUtf8 $ BS.toStrict t))
         else do
           -- analyze is required for buffers on pg < 13
           r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=analyze|buffers") ""
@@ -137,8 +125,8 @@ spec actualPgVersion = do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=verbose; charset=utf-8")
         aggCol `shouldBe`
           if actualPgVersion >= pgVersion120
-            then Just [aesonQQ| "(COALESCE(json_agg(ROW(projects.id, projects.name, projects.client_id)), '[]'::json))::character varying" |]
-            else Just [aesonQQ| "(COALESCE(json_agg(ROW(pgrst_source.id, pgrst_source.name, pgrst_source.client_id)), '[]'::json))::character varying" |]
+            then Just [aesonQQ| "COALESCE(json_agg(ROW(projects.id, projects.name, projects.client_id)), '[]'::json)" |]
+            else Just [aesonQQ| "COALESCE(json_agg(ROW(pgrst_source.id, pgrst_source.name, pgrst_source.client_id)), '[]'::json)" |]
 
     it "outputs the plan for application/vnd.pgrst.object " $ do
       r <- request methodGet "/projects_view" (acceptHdrs "application/vnd.pgrst.plan+json; for=\"application/vnd.pgrst.object\"; options=verbose") ""
@@ -150,71 +138,62 @@ spec actualPgVersion = do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/vnd.pgrst.object+json\"; options=verbose; charset=utf-8")
         aggCol `shouldBe`
           if actualPgVersion >= pgVersion120
-            then Just [aesonQQ| "COALESCE(((json_agg(ROW(projects.id, projects.name, projects.client_id)) -> 0))::text, 'null'::text)" |]
-            else Just [aesonQQ| "COALESCE(((json_agg(ROW(pgrst_source.id, pgrst_source.name, pgrst_source.client_id)) -> 0))::text, 'null'::text)" |]
+            then Just [aesonQQ| "COALESCE((json_agg(ROW(projects.id, projects.name, projects.client_id)) -> 0), 'null'::json)" |]
+            else Just [aesonQQ| "COALESCE((json_agg(ROW(pgrst_source.id, pgrst_source.name, pgrst_source.client_id)) -> 0), 'null'::json)" |]
 
   describe "writes plans" $ do
     it "outputs the total cost for an insert" $ do
       r <- request methodPost "/projects"
              (acceptHdrs "application/vnd.pgrst.plan+json") [json|{"id":100, "name": "Project 100"}|]
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
+      let totalCost  = planCost r
           resHeaders = simpleHeaders r
           resStatus  = simpleStatus r
 
       liftIO $ do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe`
-          if actualPgVersion > pgVersion120
-            then Just [aesonQQ|3.28|]
-            else Just [aesonQQ|3.33|]
+        totalCost `shouldBe` 3.27
 
     it "outputs the total cost for an update" $ do
       r <- request methodPatch "/projects?id=eq.3"
              (acceptHdrs "application/vnd.pgrst.plan+json") [json|{"name": "Patched Project"}|]
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
+      let totalCost  = planCost r
           resHeaders = simpleHeaders r
           resStatus  = simpleStatus r
 
       liftIO $ do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe`
-          if actualPgVersion > pgVersion120
-            then Just [aesonQQ|12.45|]
-            else Just [aesonQQ|12.5|]
+        totalCost `shouldBe` 12.45
 
     it "outputs the total cost for a delete" $ do
       r <- request methodDelete "/projects?id=in.(1,2,3)"
              (acceptHdrs "application/vnd.pgrst.plan+json") ""
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
+      let totalCost  = planCost r
           resHeaders = simpleHeaders r
           resStatus  = simpleStatus r
 
       liftIO $ do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe` Just [aesonQQ|15.68|]
+        totalCost `shouldBe` 15.68
 
     it "outputs the total cost for a single upsert" $ do
       r <- request methodPut "/tiobe_pls?name=eq.Go"
             (acceptHdrs "application/vnd.pgrst.plan+json")
             [json| [ { "name": "Go", "rank": 19 } ]|]
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
+      let totalCost  = planCost r
           resHeaders = simpleHeaders r
           resStatus  = simpleStatus r
 
       liftIO $ do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe`
-          if actualPgVersion >= pgVersion120
-            then Just [aesonQQ|1.3|]
-            else Just [aesonQQ|1.35|]
+        totalCost `shouldBe` 1.29
 
     it "outputs the plan for application/vnd.pgrst.object" $ do
       r <- request methodDelete "/projects?id=eq.6"
@@ -225,21 +204,21 @@ spec actualPgVersion = do
 
       liftIO $ do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/vnd.pgrst.object+json\"; options=verbose; charset=utf-8")
-        aggCol `shouldBe` Just [aesonQQ| "COALESCE(((json_agg(ROW(projects.id, projects.name, projects.client_id)) -> 0))::text, 'null'::text)" |]
+        aggCol `shouldBe` Just [aesonQQ| "COALESCE((json_agg(ROW(projects.id, projects.name, projects.client_id)) -> 0), 'null'::json)" |]
 
   describe "function plan" $ do
     it "outputs the total cost for a function call" $ do
       r <- request methodGet "/rpc/getallprojects?id=in.(1,2,3)"
              (acceptHdrs "application/vnd.pgrst.plan+json") ""
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
+      let totalCost  = planCost r
           resHeaders = simpleHeaders r
           resStatus  = simpleStatus r
 
       liftIO $ do
         resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
-        totalCost `shouldBe` Just [aesonQQ|68.57|]
+        totalCost `shouldBe` 68.56
 
     it "outputs the plan for text/xml" $ do
       r <- request methodGet "/rpc/return_scalar_xml"
@@ -282,33 +261,93 @@ spec actualPgVersion = do
   describe "resource embedding costs" $ do
     it "a one to many doesn't surpass a threshold" $ do
       r <- request methodGet "/clients?select=*,projects(*)&id=eq.1"
-             (acceptHdrs "application/vnd.pgrst.plan+json") ""
+             [planHdr] ""
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
-      liftIO $ totalCost `shouldBe`
-          if actualPgVersion > pgVersion120
-            then Just [aesonQQ|33.25|]
-            else Just [aesonQQ|33.27|]
+      liftIO $ planCost r `shouldSatisfy` (< 33.3)
 
     it "a many to one doesn't surpass a threshold" $ do
       r <- request methodGet "/projects?select=*,clients(*)&id=eq.1"
-             (acceptHdrs "application/vnd.pgrst.plan+json") ""
+             [planHdr] ""
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
-      liftIO $ totalCost `shouldBe`
-          if actualPgVersion > pgVersion120
-            then Just [aesonQQ|16.39|]
-            else Just [aesonQQ|16.41|]
+      liftIO $ planCost r `shouldSatisfy` (< 16.5)
 
     it "a many to many doesn't surpass a threshold" $ do
       r <- request methodGet "/users?select=*,tasks(*)&id=eq.1"
              (acceptHdrs "application/vnd.pgrst.plan+json") ""
 
-      let totalCost  = simpleBody r ^? nth 0 . key "Plan" . key "Total Cost"
-      liftIO $ totalCost `shouldBe`
-          if | actualPgVersion > pgVersion120 -> Just [aesonQQ|69.34|]
-             | actualPgVersion > pgVersion100 -> Just [aesonQQ|69.36|]
-             | otherwise                      -> Just [aesonQQ|70.81|]
+      liftIO $ planCost r `shouldSatisfy` (< 70.9)
+
+    context "!inner vs embed not null" $ do
+      it "on an o2m, an !inner has a similar cost to not.null" $ do
+        r1 <- request methodGet "/clients?select=*,projects!inner(*)&id=eq.1"
+               [planHdr] ""
+
+        liftIO $ planCost r1 `shouldSatisfy` (< 33.3)
+
+        r2 <- request methodGet "/clients?select=*,projects(*)&projects=not.is.null&id=eq.1"
+               [planHdr] ""
+
+        liftIO $ planCost r2 `shouldSatisfy` (< 33.3)
+
+      it "on an m2o, an !inner has a similar cost to not.null" $ do
+        r1 <- request methodGet "/projects?select=*,clients!inner(*)&id=eq.1"
+               [planHdr] ""
+
+        liftIO $ planCost r1 `shouldSatisfy` (< 16.42)
+
+        r2 <- request methodGet "/projects?select=*,clients(*)&clients=not.is.null&id=eq.1"
+               [planHdr] ""
+
+        liftIO $ planCost r2 `shouldSatisfy` (< 16.42)
+
+      it "on an m2m, an !inner has a similar cost to not.null" $ do
+        r1 <- request methodGet "/users?select=*,tasks!inner(*)&tasks.id=eq.1"
+               [planHdr] ""
+
+        liftIO $ planCost r1 `shouldSatisfy` (< 20876.14)
+
+        r2 <- request methodGet "/users?select=*,tasks(*)&tasks.id=eq.1&tasks=not.is.null"
+               [planHdr] ""
+
+        liftIO $ planCost r2 `shouldSatisfy` (< 20876.14)
+
+  describe "function call costs" $ do
+    it "should not exceed cost when calling setof composite proc" $ do
+      r <- request methodGet "/rpc/get_projects_below?id=3"
+             [planHdr] ""
+
+      liftIO $ planCost r `shouldSatisfy` (< 45.4)
+
+    it "should not exceed cost when calling setof composite proc with empty params" $ do
+      r <- request methodGet "/rpc/getallprojects"
+             [planHdr] ""
+
+      liftIO $ planCost r `shouldSatisfy` (< 71.0)
+
+    it "should not exceed cost when calling scalar proc" $ do
+      r <- request methodGet "/rpc/add_them?a=3&b=4"
+             [planHdr] ""
+
+      liftIO $ planCost r `shouldSatisfy` (< 1.18)
+
+    context "function inlining" $ do
+      it "should inline a zero argument function(the function won't appear in the plan tree)" $ do
+        r <- request methodGet "/rpc/getallusers?id=eq.1"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getallusers" (decodeUtf8 $ BS.toStrict t))
+
+      it "should inline a function with arguments(the function won't appear in the plan tree)" $ do
+        r <- request methodGet "/rpc/getitemrange?min=10&max=15"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getitemrange" (decodeUtf8 $ BS.toStrict t))
 
 disabledSpec :: SpecWith ((), Application)
 disabledSpec =

test/spec/Feature/Query/QuerySpec.hs

@@ -291,7 +291,7 @@ spec actualPgVersion = do
           {"hint":"Verify that 'non_existent_projects' is included in the 'select' query parameter.",
            "details":null,
            "code":"PGRST108",
-           "message":"Cannot apply filter because 'non_existent_projects' is not an embedded resource in this request"}|]
+           "message":"'non_existent_projects' is not an embedded resource in this request"}|]
         { matchStatus  = 400
         , matchHeaders = [matchContentTypeJson]
         }
@@ -300,7 +300,7 @@ spec actualPgVersion = do
           {"hint":"Verify that 'amiga_projectsss' is included in the 'select' query parameter.",
            "details":null,
            "code":"PGRST108",
-           "message":"Cannot apply filter because 'amiga_projectsss' is not an embedded resource in this request"}|]
+           "message":"'amiga_projectsss' is not an embedded resource in this request"}|]
         { matchStatus  = 400
         , matchHeaders = [matchContentTypeJson]
         }
@@ -309,7 +309,7 @@ spec actualPgVersion = do
           {"hint":"Verify that 'tasks2' is included in the 'select' query parameter.",
            "details":null,
            "code":"PGRST108",
-           "message":"Cannot apply filter because 'tasks2' is not an embedded resource in this request"}|]
+           "message":"'tasks2' is not an embedded resource in this request"}|]
         { matchStatus  = 400
         , matchHeaders = [matchContentTypeJson]
         }
@@ -324,14 +324,24 @@ spec actualPgVersion = do
         [json|[{"id":1},{"id":2}]|]
         { matchHeaders = [matchContentTypeJson] }
 
+    it "matches with IS DISTINCT FROM" $
+      get "/no_pk?select=a&a=isdistinct.2" `shouldRespondWith`
+        [json|[{"a":null},{"a":"1"}]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "matches with IS DISTINCT FROM using not operator" $
+      get "/no_pk?select=a&a=not.isdistinct.2" `shouldRespondWith`
+        [json|[{"a":"2"}]|]
+        { matchHeaders = [matchContentTypeJson] }
+
   describe "Shaping response with select parameter" $ do
     it "selectStar works in absense of parameter" $
       get "/complex_items?id=eq.3" `shouldRespondWith`
-        [json|[{"id":3,"name":"Three","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":1}]|]
+        [json|[{"id":3,"name":"Three","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":3}]|]
 
     it "dash `-` in column names is accepted" $
       get "/complex_items?id=eq.3&select=id,field-with_sep" `shouldRespondWith`
-        [json|[{"id":3,"field-with_sep":1}]|]
+        [json|[{"id":3,"field-with_sep":3}]|]
 
     it "one simple column" $
       get "/complex_items?select=id" `shouldRespondWith`
@@ -590,8 +600,8 @@ spec actualPgVersion = do
           it "cannot request partitions as children from a partitioned table" $
             get "/car_models?id=in.(1,2,4)&select=id,name,car_model_sales_202101(id)&order=id.asc" `shouldRespondWith`
               [json|
-                {"hint":"Verify that 'car_models' and 'car_model_sales_202101' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
-                 "details":null,
+                {"hint":"Perhaps you meant 'car_model_sales' instead of 'car_model_sales_202101'.",
+                 "details":"Searched for a foreign key relationship between 'car_models' and 'car_model_sales_202101' in the schema 'test', but no matches were found.",
                  "code":"PGRST200",
                  "message":"Could not find a relationship between 'car_models' and 'car_model_sales_202101' in the schema cache"} |]
               { matchStatus  = 400
@@ -601,8 +611,8 @@ spec actualPgVersion = do
           it "cannot request a partitioned table as parent from a partition" $
             get "/car_model_sales_202101?select=id,name,car_models(id,name)&order=id.asc" `shouldRespondWith`
               [json|
-                {"hint":"Verify that 'car_model_sales_202101' and 'car_models' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
-                 "details":null,
+                {"hint":"Perhaps you meant 'car_model_sales' instead of 'car_model_sales_202101'.",
+                 "details":"Searched for a foreign key relationship between 'car_model_sales_202101' and 'car_models' in the schema 'test', but no matches were found.",
                  "code":"PGRST200",
                  "message":"Could not find a relationship between 'car_model_sales_202101' and 'car_models' in the schema cache"} |]
               { matchStatus  = 400
@@ -612,8 +622,8 @@ spec actualPgVersion = do
           it "cannot request a partition as parent from a partitioned table" $
             get "/car_model_sales?id=in.(1,3,4)&select=id,name,car_models_default(id,name)&order=id.asc" `shouldRespondWith`
               [json|
-                {"hint":"Verify that 'car_model_sales' and 'car_models_default' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
-                 "details":null,
+                {"hint":"Perhaps you meant 'car_models' instead of 'car_models_default'.",
+                 "details":"Searched for a foreign key relationship between 'car_model_sales' and 'car_models_default' in the schema 'test', but no matches were found.",
                  "code":"PGRST200",
                  "message":"Could not find a relationship between 'car_model_sales' and 'car_models_default' in the schema cache"} |]
               { matchStatus  = 400
@@ -623,8 +633,8 @@ spec actualPgVersion = do
           it "cannot request partitioned tables as children from a partition" $
             get "/car_models_default?select=id,name,car_model_sales(id,name)&order=id.asc" `shouldRespondWith`
               [json|
-                {"hint":"Verify that 'car_models_default' and 'car_model_sales' exist in the schema 'test' and that there is a foreign key relationship between them. If a new relationship was created, try reloading the schema cache.",
-                 "details":null,
+                {"hint":"Perhaps you meant 'car_model_sales' instead of 'car_models_default'.",
+                 "details":"Searched for a foreign key relationship between 'car_models_default' and 'car_model_sales' in the schema 'test', but no matches were found.",
                  "code":"PGRST200",
                  "message":"Could not find a relationship between 'car_models_default' and 'car_model_sales' in the schema cache"} |]
               { matchStatus  = 400
@@ -921,50 +931,12 @@ spec actualPgVersion = do
       get "/projects?id=eq.1&select=id, name, clients(id, name)&clients.order=name.asc" `shouldRespondWith`
         [json|[{"id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"}}]|]
 
-    context "order syntax errors" $ do
-      it "gives meaningful error messages when asc/desc/nulls{first,last} are misspelled" $ do
-        get "/items?order=id.ac" `shouldRespondWith`
-          [json|{"details":"unexpected \"c\" expecting \"asc\", \"desc\", \"nullsfirst\" or \"nullslast\"","message":"\"failed to parse order (id.ac)\" (line 1, column 4)","code":"PGRST100","hint":null}|]
-          { matchStatus  = 400
-          , matchHeaders = [matchContentTypeJson]
-          }
-        get "/items?order=id.descc" `shouldRespondWith`
-          [json|{"details":"unexpected 'c' expecting delimiter (.), \",\" or end of input","message":"\"failed to parse order (id.descc)\" (line 1, column 8)","code":"PGRST100","hint":null}|]
-          { matchStatus  = 400
-          , matchHeaders = [matchContentTypeJson]
-          }
-        get "/items?order=id.nulsfist" `shouldRespondWith`
-          [json|{"details":"unexpected \"n\" expecting \"asc\", \"desc\", \"nullsfirst\" or \"nullslast\"","message":"\"failed to parse order (id.nulsfist)\" (line 1, column 4)","code":"PGRST100","hint":null}|]
-          { matchStatus  = 400
-          , matchHeaders = [matchContentTypeJson]
-          }
-        get "/items?order=id.nullslasttt" `shouldRespondWith`
-          [json|{"details":"unexpected 't' expecting \",\" or end of input","message":"\"failed to parse order (id.nullslasttt)\" (line 1, column 13)","code":"PGRST100","hint":null}|]
-          { matchStatus  = 400
-          , matchHeaders = [matchContentTypeJson]
-          }
-        get "/items?order=id.smth34" `shouldRespondWith`
-          [json|{"details":"unexpected \"s\" expecting \"asc\", \"desc\", \"nullsfirst\" or \"nullslast\"","message":"\"failed to parse order (id.smth34)\" (line 1, column 4)","code":"PGRST100","hint":null}|]
-          { matchStatus  = 400
-          , matchHeaders = [matchContentTypeJson]
-          }
-
-      it "gives meaningful error messages when nulls{first,last} are misspelled after asc/desc" $ do
-        get "/items?order=id.asc.nlsfst" `shouldRespondWith`
-          [json|{"details":"unexpected \"l\" expecting \"nullsfirst\" or \"nullslast\"","message":"\"failed to parse order (id.asc.nlsfst)\" (line 1, column 8)","code":"PGRST100","hint":null}|]
-          { matchStatus  = 400
-          , matchHeaders = [matchContentTypeJson]
-          }
-        get "/items?order=id.asc.nullslasttt" `shouldRespondWith`
-          [json|{"details":"unexpected 't' expecting \",\" or end of input","message":"\"failed to parse order (id.asc.nullslasttt)\" (line 1, column 17)","code":"PGRST100","hint":null}|]
-          { matchStatus  = 400
-          , matchHeaders = [matchContentTypeJson]
-          }
-        get "/items?order=id.asc.smth34" `shouldRespondWith`
-          [json|{"details":"unexpected \"s\" expecting \"nullsfirst\" or \"nullslast\"","message":"\"failed to parse order (id.asc.smth34)\" (line 1, column 8)","code":"PGRST100","hint":null}|]
-          { matchStatus  = 400
-          , matchHeaders = [matchContentTypeJson]
-          }
+    it "gives meaningful error message on bad syntax" $ do
+      get "/items?order=id.asc.nullslasttt" `shouldRespondWith`
+        [json|{"details":"unexpected 't' expecting \",\" or end of input","message":"\"failed to parse order (id.asc.nullslasttt)\" (line 1, column 17)","code":"PGRST100","hint":null}|]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
 
   describe "Accept headers" $ do
     it "should respond an unknown accept type with 415" $
@@ -1040,7 +1012,8 @@ spec actualPgVersion = do
         { matchHeaders = [matchContentTypeJson] }
 
     it "fails if an operator is not given" $
-      get "/ghostBusters?id=0" `shouldRespondWith` [json| {"details":"Failed to parse [(\"id\",\"0\")]","message":"Unexpected param or filter missing operator","code":"PGRST104","hint":null} |]
+      get "/ghostBusters?id=0" `shouldRespondWith`
+        [json| {"code":"PGRST100","details":"unexpected \"0\" expecting \"not\" or operator (eq, gt, ...)","hint":null,"message":"\"failed to parse filter (0)\" (line 1, column 1)"} |]
         { matchStatus  = 400
         , matchHeaders = [matchContentTypeJson]
         }
@@ -1262,3 +1235,96 @@ spec actualPgVersion = do
     liftIO $ do
       let respHeaders = simpleHeaders r
       respHeaders `shouldSatisfy` noProfileHeader
+
+  context "empty embed" $ do
+    it "works on a many-to-one relationship" $ do
+      get "/projects?select=id,name,clients()" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Windows 7"},
+          {"id":2,"name":"Windows 10"},
+          {"id":3,"name":"IOS"},
+          {"id":4,"name":"OSX"},
+          {"id":5,"name":"Orphan"}]|]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/projects?select=id,name,clients!inner()&clients.id=eq.2" `shouldRespondWith`
+        [json|[
+          {"id":3,"name":"IOS"},
+          {"id":4,"name":"OSX"}]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "works on a one-to-many relationship" $ do
+      get "/clients?select=id,name,projects()" `shouldRespondWith`
+        [json| [{"id":1,"name":"Microsoft"}, {"id":2,"name":"Apple"}]|]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/clients?select=id,name,projects!inner()&projects.name=eq.IOS" `shouldRespondWith`
+        [json|[{"id":2,"name":"Apple"}]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "works on a many-to-many relationship" $ do
+      get "/users?select=*,tasks!inner()" `shouldRespondWith`
+        [json| [{"id":1,"name":"Angela Martin"}, {"id":2,"name":"Michael Scott"}, {"id":3,"name":"Dwight Schrute"}]|]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/users?select=*,tasks!inner()&tasks.id=eq.3" `shouldRespondWith`
+        [json|[{"id":1,"name":"Angela Martin"}]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+  context "empty root select" $
+    it "gives all columns" $ do
+      get "/projects?select=" `shouldRespondWith`
+        [json|[
+          {"id":1,"name":"Windows 7","client_id":1},
+          {"id":2,"name":"Windows 10","client_id":1},
+          {"id":3,"name":"IOS","client_id":2},
+          {"id":4,"name":"OSX","client_id":2},
+          {"id":5,"name":"Orphan","client_id":null}]|]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/rpc/getallprojects?select=" `shouldRespondWith`
+        [json|[
+          {"id":1,"name":"Windows 7","client_id":1},
+          {"id":2,"name":"Windows 10","client_id":1},
+          {"id":3,"name":"IOS","client_id":2},
+          {"id":4,"name":"OSX","client_id":2},
+          {"id":5,"name":"Orphan","client_id":null}]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+  context "any/all quantifiers" $ do
+    it "works with the eq operator" $
+      get "/projects?id=eq(any).{3,4,5}" `shouldRespondWith`
+        [json|[
+          {"id":3,"name":"IOS","client_id":2},
+          {"id":4,"name":"OSX","client_id":2},
+          {"id":5,"name":"Orphan","client_id":null}
+        ]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "works with the gt/gte operator" $ do
+      get "/projects?id=gt(all).{4,3}" `shouldRespondWith`
+        [json|[{"id":5,"name":"Orphan","client_id":null}]|]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/projects?id=gte(all).{4,3}" `shouldRespondWith`
+        [json|[{"id":4,"name":"OSX","client_id":2}, {"id":5,"name":"Orphan","client_id":null}]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "works with the lt/lte operator" $ do
+      get "/projects?id=lt(all).{4,3}" `shouldRespondWith`
+        [json|[{"id":1,"name":"Windows 7","client_id":1}, {"id":2,"name":"Windows 10","client_id":1}]|]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/projects?id=lte(all).{4,3}" `shouldRespondWith`
+        [json|[{"id":1,"name":"Windows 7","client_id":1}, {"id":2,"name":"Windows 10","client_id":1}, {"id":3,"name":"IOS","client_id":2}]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "works with the like/ilike operator" $ do
+      get "/articles?body=like(any).{%plan%,%brain%}&select=id" `shouldRespondWith`
+        [json|[ {"id":1}, {"id":2} ]|]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/articles?body=ilike(all).{%plan%,%greatness%}&select=id" `shouldRespondWith`
+        [json|[ {"id":1} ]|]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "works with the match/imatch operator" $ do
+      get "/articles?body=match(any).{stop,thing}&select=id" `shouldRespondWith`
+        [json|[{"id":1}]|]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/articles?body=imatch(any).{stop,thing}&select=id" `shouldRespondWith`
+        [json|[{"id":1}, {"id":2}]|]
+        { matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/RangeSpec.hs

@@ -1,7 +1,5 @@
 module Feature.Query.RangeSpec where
 
-import qualified Data.ByteString.Lazy as BL
-
 import Network.Wai      (Application)
 import Network.Wai.Test (SResponse (simpleHeaders, simpleStatus))
 
@@ -13,36 +11,29 @@ import Test.Hspec.Wai.JSON
 import Protolude  hiding (get)
 import SpecHelper
 
-defaultRange :: BL.ByteString
-defaultRange = [json| { "min": 0, "max": 15 } |]
-
-emptyRange :: BL.ByteString
-emptyRange = [json| { "min": 2, "max": 2 } |]
-
 spec :: SpecWith ((), Application)
 spec = do
-  describe "POST /rpc/getitemrange" $ do
+  describe "GET /rpc/getitemrange" $ do
     context "without range headers" $ do
       context "with response under server size limit" $
         it "returns whole range with status 200" $
-           post "/rpc/getitemrange" defaultRange `shouldRespondWith` 200
+           get "/rpc/getitemrange?min=0&max=15" `shouldRespondWith` 200
 
       context "when I don't want the count" $ do
         it "returns range Content-Range with */* for empty range" $
-          request methodPost "/rpc/getitemrange" [] emptyRange
+          get "/rpc/getitemrange?min=2&max=2"
             `shouldRespondWith` [json| [] |] {matchHeaders = ["Content-Range" <:> "*/*"]}
 
         it "returns range Content-Range with range/*" $
-          post "/rpc/getitemrange?order=id"
-              defaultRange
+          get "/rpc/getitemrange?order=id&min=0&max=15"
             `shouldRespondWith`
               [json| [{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13},{"id":14},{"id":15}] |]
               { matchHeaders = ["Content-Range" <:> "0-14/*"] }
 
       context "of invalid range" $ do
         it "refuses a range with nonzero start when there are no items" $
-          request methodPost "/rpc/getitemrange?offset=1"
-                  [("Prefer", "count=exact")] emptyRange
+          request methodGet "/rpc/getitemrange?offset=1&min=2&max=2"
+                  [("Prefer", "count=exact")] mempty
             `shouldRespondWith`
               [json| {
                 "message":"Requested range not satisfiable",
@@ -55,8 +46,8 @@ spec = do
             }
 
         it "refuses a range requesting start past last item" $
-          request methodPost "/rpc/getitemrange?offset=100"
-                  [("Prefer", "count=exact")] defaultRange
+          request methodGet "/rpc/getitemrange?offset=100&min=0&max=15"
+                  [("Prefer", "count=exact")] mempty
             `shouldRespondWith`
               [json| {
                 "message":"Requested range not satisfiable",
@@ -71,37 +62,37 @@ spec = do
     context "with range headers" $ do
       context "of acceptable range" $ do
         it "succeeds with partial content" $ do
-          r <- request methodPost  "/rpc/getitemrange"
-                       (rangeHdrs $ ByteRangeFromTo 0 1) defaultRange
+          r <- request methodGet  "/rpc/getitemrange?min=0&max=15"
+                       (rangeHdrs $ ByteRangeFromTo 0 1) mempty
           liftIO $ do
             simpleHeaders r `shouldSatisfy`
               matchHeader "Content-Range" "0-1/*"
             simpleStatus r `shouldBe` ok200
 
         it "understands open-ended ranges" $
-          request methodPost "/rpc/getitemrange"
-                  (rangeHdrs $ ByteRangeFrom 0) defaultRange
+          request methodGet "/rpc/getitemrange?min=0&max=15"
+                  (rangeHdrs $ ByteRangeFrom 0) mempty
             `shouldRespondWith` 200
 
         it "returns an empty body when there are no results" $
-          request methodPost "/rpc/getitemrange"
-                  (rangeHdrs $ ByteRangeFromTo 0 1) emptyRange
+          request methodGet "/rpc/getitemrange?min=2&max=2"
+                  (rangeHdrs $ ByteRangeFromTo 0 1) mempty
             `shouldRespondWith` "[]"
             { matchStatus  = 200
             , matchHeaders = ["Content-Range" <:> "*/*"]
             }
 
         it "allows one-item requests" $ do
-          r <- request methodPost  "/rpc/getitemrange"
-                       (rangeHdrs $ ByteRangeFromTo 0 0) defaultRange
+          r <- request methodGet  "/rpc/getitemrange?min=0&max=15"
+                       (rangeHdrs $ ByteRangeFromTo 0 0) mempty
           liftIO $ do
             simpleHeaders r `shouldSatisfy`
               matchHeader "Content-Range" "0-0/*"
             simpleStatus r `shouldBe` ok200
 
         it "handles ranges beyond collection length via truncation" $ do
-          r <- request methodPost  "/rpc/getitemrange"
-                       (rangeHdrs $ ByteRangeFromTo 10 100) defaultRange
+          r <- request methodGet  "/rpc/getitemrange?min=0&max=15"
+                       (rangeHdrs $ ByteRangeFromTo 10 100) mempty
           liftIO $ do
             simpleHeaders r `shouldSatisfy`
               matchHeader "Content-Range" "10-14/*"
@@ -109,8 +100,8 @@ spec = do
 
       context "of invalid range" $ do
         it "fails with 416 for offside range" $
-          request methodPost  "/rpc/getitemrange"
-                  (rangeHdrs $ ByteRangeFromTo 1 0) emptyRange
+          request methodGet  "/rpc/getitemrange?min=2&max=2"
+                  (rangeHdrs $ ByteRangeFromTo 1 0) mempty
             `shouldRespondWith`
               [json| {
                 "message":"Requested range not satisfiable",
@@ -121,8 +112,8 @@ spec = do
             { matchStatus = 416 }
 
         it "refuses a range with nonzero start when there are no items" $
-          request methodPost "/rpc/getitemrange"
-                  (rangeHdrsWithCount $ ByteRangeFromTo 1 2) emptyRange
+          request methodGet "/rpc/getitemrange?min=2&max=2"
+                  (rangeHdrsWithCount $ ByteRangeFromTo 1 2) mempty
             `shouldRespondWith`
               [json| {
                 "message":"Requested range not satisfiable",
@@ -135,8 +126,8 @@ spec = do
             }
 
         it "refuses a range requesting start past last item" $
-          request methodPost "/rpc/getitemrange"
-                  (rangeHdrsWithCount $ ByteRangeFromTo 100 199) defaultRange
+          request methodGet "/rpc/getitemrange?min=0&max=15"
+                  (rangeHdrsWithCount $ ByteRangeFromTo 100 199) mempty
             `shouldRespondWith`
               [json| {
                 "message":"Requested range not satisfiable",

test/spec/Feature/Query/RelatedQueriesSpec.hs

@@ -0,0 +1,256 @@
+module Feature.Query.RelatedQueriesSpec where
+
+import Network.Wai (Application)
+
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import Protolude  hiding (get)
+import SpecHelper
+
+spec :: SpecWith ((), Application)
+spec = describe "related queries" $ do
+  context "related orders" $ do
+    it "works on a many-to-one relationship" $ do
+      get "/projects?select=id,clients(name)&order=clients(name).nullsfirst" `shouldRespondWith`
+        [json|[
+          {"id":5,"clients":null},
+          {"id":3,"clients":{"name":"Apple"}},
+          {"id":4,"clients":{"name":"Apple"}},
+          {"id":1,"clients":{"name":"Microsoft"}},
+          {"id":2,"clients":{"name":"Microsoft"}} ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/projects?select=id,client:clients(name)&order=client(name).asc" `shouldRespondWith`
+        [json|[
+          {"id":3,"client":{"name":"Apple"}},
+          {"id":4,"client":{"name":"Apple"}},
+          {"id":1,"client":{"name":"Microsoft"}},
+          {"id":2,"client":{"name":"Microsoft"}},
+          {"id":5,"client":null} ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/videogames?select=id,computed_designers(id)&order=computed_designers(id).desc" `shouldRespondWith`
+        [json|[
+          {"id":3,"computed_designers":{"id":2}},
+          {"id":4,"computed_designers":{"id":2}},
+          {"id":1,"computed_designers":{"id":1}},
+          {"id":2,"computed_designers":{"id":1}}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "works on a one-to-one relationship and jsonb column" $ do
+      get "/trash?select=id,trash_details(id,jsonb_col)&order=trash_details(jsonb_col->key).asc" `shouldRespondWith`
+        [json|[
+          {"id":2,"trash_details":{"id":2,"jsonb_col":{"key": 6}}},
+          {"id":3,"trash_details":{"id":3,"jsonb_col":{"key": 8}}},
+          {"id":1,"trash_details":{"id":1,"jsonb_col":{"key": 10}}}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/trash?select=id,trash_details(id,jsonb_col)&order=trash_details(jsonb_col->key).desc" `shouldRespondWith`
+        [json|[
+          {"id":1,"trash_details":{"id":1,"jsonb_col":{"key": 10}}},
+          {"id":3,"trash_details":{"id":3,"jsonb_col":{"key": 8}}},
+          {"id":2,"trash_details":{"id":2,"jsonb_col":{"key": 6}}}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "works on an embedded resource" $ do
+      get "/users?select=name,tasks(id,name,projects(id,name))&tasks.order=projects(id).desc&limit=1" `shouldRespondWith`
+        [json| [{
+          "name":"Angela Martin",
+          "tasks":[
+            {"id": 3, "name":"Design w10","projects":{"id":2,"name":"Windows 10"}},
+            {"id": 4, "name":"Code w10","projects":{"id":2,"name":"Windows 10"}},
+            {"id": 1, "name":"Design w7","projects":{"id":1,"name":"Windows 7"}},
+            {"id": 2, "name":"Code w7","projects":{"id":1,"name":"Windows 7"}}
+          ]
+        }]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/users?select=name,tasks(id,name,projects(id,name))&tasks.order=projects(id).desc,name&limit=1" `shouldRespondWith`
+        [json| [{
+          "name":"Angela Martin",
+          "tasks":[
+            {"id": 4, "name":"Code w10","projects":{"id":2,"name":"Windows 10"}},
+            {"id": 3, "name":"Design w10","projects":{"id":2,"name":"Windows 10"}},
+            {"id": 2, "name":"Code w7","projects":{"id":1,"name":"Windows 7"}},
+            {"id": 1, "name":"Design w7","projects":{"id":1,"name":"Windows 7"}}
+          ]
+        }]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/users?select=name,tasks(id,name,projects(id,name))&tasks.order=projects(id).asc&limit=1" `shouldRespondWith`
+        [json|[{
+          "name":"Angela Martin",
+          "tasks":[
+            {"id":1,"name":"Design w7","projects":{"id":1,"name":"Windows 7"}},
+            {"id":2,"name":"Code w7","projects":{"id":1,"name":"Windows 7"}},
+            {"id":3,"name":"Design w10","projects":{"id":2,"name":"Windows 10"}},
+            {"id":4,"name":"Code w10","projects":{"id":2,"name":"Windows 10"}}
+          ]
+        }]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "fails when is not a to-one relationship" $ do
+      get "/clients?select=*,projects(*)&order=projects(id)" `shouldRespondWith`
+        [json|{
+          "code":"PGRST118",
+          "details":"'clients' and 'projects' do not form a many-to-one or one-to-one relationship",
+          "hint":null,
+          "message":"A related order on 'projects' is not possible"
+        }|]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/clients?select=*,pros:projects(*)&order=pros(id)" `shouldRespondWith`
+        [json|{
+          "code":"PGRST118",
+          "details":"'clients' and 'pros' do not form a many-to-one or one-to-one relationship",
+          "hint":null,
+          "message":"A related order on 'pros' is not possible"
+        }|]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/designers?select=id,computed_videogames(id)&order=computed_videogames(id).desc" `shouldRespondWith`
+        [json|{
+          "code":"PGRST118",
+          "details":"'designers' and 'computed_videogames' do not form a many-to-one or one-to-one relationship",
+          "hint":null,
+          "message":"A related order on 'computed_videogames' is not possible"
+        }|]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "fails when the resource is not embedded" $
+      get "/projects?select=id,clients(name)&order=clientsx(name).nullsfirst" `shouldRespondWith`
+        [json|{
+          "code":"PGRST108",
+          "details":null,
+          "hint":"Verify that 'clientsx' is included in the 'select' query parameter.",
+          "message":"'clientsx' is not an embedded resource in this request"
+        }|]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+  context "related conditions through null operator on embed" $ do
+    it "works on a many-to-one relationship" $ do
+      get "/projects?select=name,clients()&clients=not.is.null" `shouldRespondWith`
+        [json|[
+          {"name":"Windows 7"},
+          {"name":"Windows 10"},
+          {"name":"IOS"},
+          {"name":"OSX"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/projects?select=name,clients()&clients=is.null" `shouldRespondWith`
+        [json|[{"name":"Orphan"}]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/projects?select=name,computed_clients()&computed_clients=is.null" `shouldRespondWith`
+        [json|[{"name":"Orphan"}]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "works on a one-to-many relationship" $ do
+      get "/entities?select=name,child_entities()&child_entities=not.is.null" `shouldRespondWith`
+        [json|[
+          {"name":"entity 1"},
+          {"name":"entity 2"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/entities?select=name,child_entities()&child_entities=is.null" `shouldRespondWith`
+        [json|[
+          {"name":"entity 3"},
+          {"name":null}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/entities?select=name,childs:child_entities()&childs=is.null" `shouldRespondWith`
+        [json|[
+          {"name":"entity 3"},
+          {"name":null}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "works on a many-to-many relationship" $ do
+      get "/users?select=name,tasks()&tasks.id=eq.1&tasks=not.is.null" `shouldRespondWith`
+        [json|[
+          {"name":"Angela Martin"},
+          {"name":"Dwight Schrute"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/users?select=name,tasks()&tasks.id=eq.1&tasks=is.null" `shouldRespondWith`
+        [json|[
+          {"name":"Michael Scott"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "works on nested embeds" $ do
+      get "/entities?select=name,child_entities(name,grandchild_entities())&child_entities.grandchild_entities=not.is.null&child_entities=not.is.null" `shouldRespondWith`
+        [json|[
+          {"name":"entity 1","child_entities":[{"name":"child entity 1"}, {"name":"child entity 2"}]}]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "can do an or across embeds" $
+      get "/client?select=*,clientinfo(),contact()&clientinfo.other=ilike.*main*&contact.name=ilike.*tabby*&or=(clientinfo.not.is.null,contact.not.is.null)" `shouldRespondWith`
+        [json|[
+          {"id":1,"name":"Walmart"},
+          {"id":2,"name":"Target"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "only works with is null or is not null operators" $
+      get "/projects?select=name,clients(*)&clients=eq.3" `shouldRespondWith`
+        [json|{
+          "code":"PGRST120",
+          "details":"Only is null or not is null filters are allowed on embedded resources",
+          "hint":null,
+          "message":"Bad operator on the 'clients' embedded resource"
+        }|]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "doesn't interfere filtering when embedding using the column name" $
+      get "/projects?select=name,client_id,client:client_id(name)&client_id=eq.2" `shouldRespondWith`
+        [json|[
+          {"name":"IOS","client_id":2,"client":{"name":"Apple"}},
+          {"name":"OSX","client_id":2,"client":{"name":"Apple"}}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }

test/spec/Feature/Query/RpcSpec.hs

@@ -24,50 +24,65 @@ spec :: PgVersion -> SpecWith ((), Application)
 spec actualPgVersion =
   describe "remote procedure call" $ do
     context "a proc that returns a set" $ do
-      it "returns paginated results" $ do
-        request methodPost "/rpc/getitemrange"
-                (rangeHdrs (ByteRangeFromTo 0 0))  [json| { "min": 2, "max": 4 } |]
-           `shouldRespondWith` [json| [{"id":3}] |]
-            { matchStatus = 200
-            , matchHeaders = ["Content-Range" <:> "0-0/*"]
-            }
-        request methodGet "/rpc/getitemrange?min=2&max=4"
-                (rangeHdrs (ByteRangeFromTo 0 0)) ""
-           `shouldRespondWith` [json| [{"id":3}] |]
-            { matchStatus = 200
-            , matchHeaders = ["Content-Range" <:> "0-0/*"]
-            }
-        request methodHead "/rpc/getitemrange?min=2&max=4"
-            (rangeHdrs (ByteRangeFromTo 0 0)) ""
-          `shouldRespondWith`
-            ""
-            { matchStatus = 200
-            , matchHeaders = [ matchContentTypeJson
-                             , "Content-Range" <:> "0-0/*" ]
-            }
+      context "returns paginated results" $ do
+        it "using the Range header" $
+          request methodGet "/rpc/getitemrange?min=2&max=4"
+                  (rangeHdrs (ByteRangeFromTo 1 1)) mempty
+             `shouldRespondWith` [json| [{"id":4}] |]
+              { matchStatus = 200
+              , matchHeaders = ["Content-Range" <:> "1-1/*"]
+              }
 
-      it "includes total count if requested" $ do
-        request methodPost "/rpc/getitemrange"
-                (rangeHdrsWithCount (ByteRangeFromTo 0 0))
-                [json| { "min": 2, "max": 4 } |]
-           `shouldRespondWith` [json| [{"id":3}] |]
-            { matchStatus = 206 -- it now knows the response is partial
-            , matchHeaders = ["Content-Range" <:> "0-0/2"]
-            }
-        request methodGet "/rpc/getitemrange?min=2&max=4"
-                (rangeHdrsWithCount (ByteRangeFromTo 0 0)) ""
-           `shouldRespondWith` [json| [{"id":3}] |]
-            { matchStatus = 206
-            , matchHeaders = ["Content-Range" <:> "0-0/2"]
-            }
-        request methodHead "/rpc/getitemrange?min=2&max=4"
-            (rangeHdrsWithCount (ByteRangeFromTo 0 0)) ""
-          `shouldRespondWith`
-            ""
-            { matchStatus = 206
-            , matchHeaders = [ matchContentTypeJson
-                             , "Content-Range" <:> "0-0/2" ]
-            }
+        it "using limit and offset" $ do
+          post "/rpc/getitemrange?limit=1&offset=1" [json| { "min": 2, "max": 4 } |]
+             `shouldRespondWith` [json| [{"id":4}] |]
+              { matchStatus = 200
+              , matchHeaders = ["Content-Range" <:> "1-1/*"]
+              }
+          get "/rpc/getitemrange?min=2&max=4&limit=1&offset=1"
+             `shouldRespondWith` [json| [{"id":4}] |]
+              { matchStatus = 200
+              , matchHeaders = ["Content-Range" <:> "1-1/*"]
+              }
+          request methodHead "/rpc/getitemrange?min=2&max=4&limit=1&offset=1" mempty mempty
+            `shouldRespondWith`
+              ""
+              { matchStatus = 200
+              , matchHeaders = [ matchContentTypeJson
+                               , "Content-Range" <:> "1-1/*" ]
+              }
+
+      context "includes total count if requested" $ do
+        it "using the Range header" $
+          request methodGet "/rpc/getitemrange?min=2&max=4"
+                  (rangeHdrsWithCount (ByteRangeFromTo 1 1)) ""
+             `shouldRespondWith` [json| [{"id":4}] |]
+              { matchStatus = 206 -- it now knows the response is partial
+              , matchHeaders = ["Content-Range" <:> "1-1/2"]
+              }
+
+        it "using limit and offset" $ do
+          request methodPost "/rpc/getitemrange?limit=1&offset=1"
+                  [("Prefer", "count=exact")]
+                  [json| { "min": 2, "max": 4 } |]
+             `shouldRespondWith` [json| [{"id":4}] |]
+              { matchStatus = 206 -- it now knows the response is partial
+              , matchHeaders = ["Content-Range" <:> "1-1/2"]
+              }
+          request methodGet "/rpc/getitemrange?min=2&max=4&limit=1&offset=1"
+                  [("Prefer", "count=exact")] mempty
+             `shouldRespondWith` [json| [{"id":4}] |]
+              { matchStatus = 206
+              , matchHeaders = ["Content-Range" <:> "1-1/2"]
+              }
+          request methodHead "/rpc/getitemrange?min=2&max=4&limit=1&offset=1"
+              [("Prefer", "count=exact")] mempty
+            `shouldRespondWith`
+              ""
+              { matchStatus = 206
+              , matchHeaders = [ matchContentTypeJson
+                               , "Content-Range" <:> "1-1/2" ]
+              }
 
       it "includes exact count if requested" $ do
         request methodHead "/rpc/getallprojects"
@@ -113,6 +128,58 @@ spec actualPgVersion =
             , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8"]
             }
 
+      context "ignores Range header when method is different than GET" $ do
+        it "without limit and offset" $ do
+          request methodPost "/rpc/getitemrange"
+                  (rangeHdrsWithCount (ByteRangeFromTo 1 1))
+                  [json| { "min": 2, "max": 4 } |]
+             `shouldRespondWith` [json| [{"id": 3}, {"id": 4}] |]
+              { matchStatus = 200
+              , matchHeaders = ["Content-Range" <:> "0-1/2"]
+              }
+          request methodHead "/rpc/getitemrange?min=2&max=4"
+              (rangeHdrsWithCount (ByteRangeFromTo 1 1)) ""
+            `shouldRespondWith`
+              ""
+              { matchStatus = 200
+              , matchHeaders = [ matchContentTypeJson
+                               , "Content-Range" <:> "0-1/2" ]
+              }
+
+        it "with limit and offset" $ do
+          request methodPost "/rpc/getitemrange?limit=2&offset=1"
+                  (rangeHdrsWithCount (ByteRangeFromTo 1 1))
+                  [json| { "min": 2, "max": 5 } |]
+             `shouldRespondWith` [json| [{"id": 4}, {"id": 5}] |]
+              { matchStatus = 206
+              , matchHeaders = ["Content-Range" <:> "1-2/3"]
+              }
+          request methodHead "/rpc/getitemrange?min=2&max=5&limit=2&offset=1"
+              (rangeHdrsWithCount (ByteRangeFromTo 1 1)) ""
+            `shouldRespondWith`
+              ""
+              { matchStatus = 206
+              , matchHeaders = [ matchContentTypeJson
+                               , "Content-Range" <:> "1-2/3" ]
+              }
+
+        it "does not throw an invalid range error" $ do
+          request methodPost "/rpc/getitemrange?limit=2&offset=1"
+                  (rangeHdrsWithCount (ByteRangeFromTo 0 0))
+                  [json| { "min": 2, "max": 5 } |]
+             `shouldRespondWith` [json| [{"id": 4}, {"id": 5}] |]
+              { matchStatus = 206
+              , matchHeaders = ["Content-Range" <:> "1-2/3"]
+              }
+          request methodHead "/rpc/getitemrange?min=2&max=5&limit=2&offset=1"
+              (rangeHdrsWithCount (ByteRangeFromTo 0 0)) ""
+            `shouldRespondWith`
+              ""
+              { matchStatus = 206
+              , matchHeaders = [ matchContentTypeJson
+                               , "Content-Range" <:> "1-2/3" ]
+              }
+
     context "unknown function" $ do
       it "returns 404" $
         post "/rpc/fakefunc" [json| {} |] `shouldRespondWith` 404
@@ -120,17 +187,39 @@ spec actualPgVersion =
       it "should fail with 404 on unknown proc name" $
         get "/rpc/fake" `shouldRespondWith` 404
 
+      it "should fail with 404 and hint the closest proc on unknown proc name" $
+        get "/rpc/sayhell" `shouldRespondWith`
+          [json| {
+            "hint":"Perhaps you meant to call the function test.sayhello",
+            "message":"Could not find the function test.sayhell without parameters in the schema cache",
+            "code":"PGRST202",
+            "details":"Searched for the function test.sayhell without parameters, but no matches were found in the schema cache."} |]
+          { matchStatus  = 404
+          , matchHeaders = [matchContentTypeJson]
+          }
+
       it "should fail with 404 on unknown proc args" $ do
         get "/rpc/sayhello" `shouldRespondWith` 404
         get "/rpc/sayhello?any_arg=value" `shouldRespondWith` 404
 
+      it "should fail with 404 and hint the closest args on unknown proc args" $
+        get "/rpc/sayhello?nam=Peter" `shouldRespondWith`
+          [json| {
+            "hint":"Perhaps you meant to call the function test.sayhello(name)",
+            "message":"Could not find the function test.sayhello(nam) in the schema cache",
+            "code":"PGRST202",
+            "details":"Searched for the function test.sayhello with parameter nam, but no matches were found in the schema cache."} |]
+          { matchStatus  = 404
+          , matchHeaders = [matchContentTypeJson]
+          }
+
       it "should not ignore unknown args and fail with 404" $
         get "/rpc/add_them?a=1&b=2&smthelse=blabla" `shouldRespondWith`
         [json| {
-          "hint":"If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-          "message":"Could not find the test.add_them(a, b, smthelse) function in the schema cache",
+          "hint":"Perhaps you meant to call the function test.add_them(a, b)",
+          "message":"Could not find the function test.add_them(a, b, smthelse) in the schema cache",
           "code":"PGRST202",
-          "details":null} |]
+          "details":"Searched for the function test.add_them with parameters a, b, smthelse, but no matches were found in the schema cache."} |]
         { matchStatus  = 404
         , matchHeaders = [matchContentTypeJson]
         }
@@ -141,10 +230,10 @@ spec actualPgVersion =
           [json|{}|]
         `shouldRespondWith`
           [json| {
-            "hint":"If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-            "message":"Could not find the test.sayhello function with a single json or jsonb parameter in the schema cache",
+            "hint":null,
+            "message":"Could not find the function test.sayhello in the schema cache",
             "code":"PGRST202",
-            "details":null} |]
+            "details":"Searched for the function test.sayhello with a single json/jsonb parameter, but no matches were found in the schema cache."} |]
         { matchStatus  = 404
         , matchHeaders = [matchContentTypeJson]
         }
@@ -152,19 +241,19 @@ spec actualPgVersion =
       it "should fail with 404 for overloaded functions with unknown args" $ do
         get "/rpc/overloaded?wrong_arg=value" `shouldRespondWith`
           [json| {
-            "hint":"If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-            "message":"Could not find the test.overloaded(wrong_arg) function in the schema cache",
+            "hint":null,
+            "message":"Could not find the function test.overloaded(wrong_arg) in the schema cache",
             "code":"PGRST202",
-            "details":null} |]
+            "details":"Searched for the function test.overloaded with parameter wrong_arg, but no matches were found in the schema cache."} |]
           { matchStatus  = 404
           , matchHeaders = [matchContentTypeJson]
           }
         get "/rpc/overloaded?a=1&b=2&wrong_arg=value" `shouldRespondWith`
           [json| {
-            "hint":"If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-            "message":"Could not find the test.overloaded(a, b, wrong_arg) function in the schema cache",
+            "hint":"Perhaps you meant to call the function test.overloaded(a, b, c)",
+            "message":"Could not find the function test.overloaded(a, b, wrong_arg) in the schema cache",
             "code":"PGRST202",
-            "details":null} |]
+            "details":"Searched for the function test.overloaded with parameters a, b, wrong_arg, but no matches were found in the schema cache."} |]
           { matchStatus  = 404
           , matchHeaders = [matchContentTypeJson]
           }
@@ -246,13 +335,17 @@ spec actualPgVersion =
           `shouldRespondWith`
             [json|{"id": 2, "articleStars": [{"userId": 3}]}|]
 
-      it "can embed an M2M relationship table" $
+      it "can embed an M2M relationship table" $ do
         get "/rpc/getallusers?select=name,tasks(name)&id=gt.1"
           `shouldRespondWith` [json|[
             {"name":"Michael Scott", "tasks":[{"name":"Design IOS"}, {"name":"Code IOS"}, {"name":"Design OSX"}]},
             {"name":"Dwight Schrute","tasks":[{"name":"Design w7"}, {"name":"Design IOS"}]}
           ]|]
           { matchHeaders = [matchContentTypeJson] }
+        -- https://github.com/PostgREST/postgrest/issues/2565
+        get "/rpc/get_yards?select=groups(*)"
+          `shouldRespondWith` [json|[]|]
+          { matchHeaders = [matchContentTypeJson] }
 
       it "can embed an M2M relationship table that has a parent relationship table" $
         get "/rpc/getallusers?select=name,tasks(name,project:projects(name))&id=gt.1"
@@ -836,42 +929,6 @@ spec actualPgVersion =
           `shouldRespondWith` "3"
           { matchHeaders = [matchContentTypeJson] }
 
-    context "bulk RPC with params=multiple-objects" $ do
-      it "works with a scalar function an returns a json array" $
-        request methodPost "/rpc/add_them" [("Prefer", "params=multiple-objects")]
-          [json|[
-            {"a": 1, "b": 2},
-            {"a": 4, "b": 6},
-            {"a": 100, "b": 200} ]|]
-          `shouldRespondWith`
-          [json|
-            [3, 10, 300]
-          |] { matchHeaders = [matchContentTypeJson] }
-
-      it "works with a scalar function an returns a json array when posting CSV" $
-        request methodPost "/rpc/add_them" [("Content-Type", "text/csv"), ("Prefer", "params=multiple-objects")]
-          "a,b\n1,2\n4,6\n100,200"
-          `shouldRespondWith`
-          [json|
-            [3, 10, 300]
-          |]
-          { matchStatus  = 200
-          , matchHeaders = [matchContentTypeJson]
-          }
-
-      it "works with a non-scalar result" $
-        request methodPost "/rpc/get_projects_below?select=id,name" [("Prefer", "params=multiple-objects")]
-          [json|[
-            {"id": 1},
-            {"id": 5} ]|]
-          `shouldRespondWith`
-          [json|
-            [{"id":1,"name":"Windows 7"},
-             {"id":2,"name":"Windows 10"},
-             {"id":3,"name":"IOS"},
-             {"id":4,"name":"OSX"}]
-          |] { matchHeaders = [matchContentTypeJson] }
-
     context "HTTP request env vars" $ do
       it "custom header is set" $
         request methodPost "/rpc/get_guc_value"
@@ -1247,10 +1304,10 @@ spec actualPgVersion =
               [json|{"x": 1, "y": 2}|]
             `shouldRespondWith`
               [json|{
-                "hint": "If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-                "message": "Could not find the test.unnamed_int_param(x, y) function or the test.unnamed_int_param function with a single unnamed json or jsonb parameter in the schema cache",
+                "hint": "Perhaps you meant to call the function test.unnamed_text_param",
+                "message": "Could not find the function test.unnamed_int_param(x, y) in the schema cache",
                 "code":"PGRST202",
-                "details":null
+                "details":"Searched for the function test.unnamed_int_param with parameters x, y or with a single unnamed json/jsonb parameter, but no matches were found in the schema cache."
               }|]
               { matchStatus  = 404
               , matchHeaders = [ matchContentTypeJson ]
@@ -1262,10 +1319,10 @@ spec actualPgVersion =
               [str|a simple text|]
             `shouldRespondWith`
               [json|{
-                "hint": "If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-                "message": "Could not find the test.unnamed_int_param function with a single unnamed text parameter in the schema cache",
+                "hint": null,
+                "message": "Could not find the function test.unnamed_int_param in the schema cache",
                 "code":"PGRST202",
-                "details":null
+                "details":"Searched for the function test.unnamed_int_param with a single unnamed text parameter, but no matches were found in the schema cache."
               }|]
               { matchStatus  = 404
               , matchHeaders = [ matchContentTypeJson ]
@@ -1277,10 +1334,10 @@ spec actualPgVersion =
               [str|a simple text|]
             `shouldRespondWith`
               [json|{
-                "hint": "If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-                "message": "Could not find the test.unnamed_int_param function with a single unnamed xml parameter in the schema cache",
+                "hint": null,
+                "message": "Could not find the function test.unnamed_int_param in the schema cache",
                 "code":"PGRST202",
-                "details":null
+                "details":"Searched for the function test.unnamed_int_param with a single unnamed xml parameter, but no matches were found in the schema cache."
               }|]
               { matchStatus  = 404
               , matchHeaders = [ matchContentTypeJson ]
@@ -1293,10 +1350,10 @@ spec actualPgVersion =
               file
           `shouldRespondWith`
             [json|{
-              "hint": "If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-              "message": "Could not find the test.unnamed_int_param function with a single unnamed bytea parameter in the schema cache",
+              "hint": null,
+              "message": "Could not find the function test.unnamed_int_param in the schema cache",
               "code":"PGRST202",
-              "details":null
+              "details":"Searched for the function test.unnamed_int_param with a single unnamed bytea parameter, but no matches were found in the schema cache."
             }|]
             { matchStatus  = 404
             , matchHeaders = [ matchContentTypeJson ]
@@ -1353,10 +1410,10 @@ spec actualPgVersion =
               "a,b\n1,2\n4,6\n100,200"
             `shouldRespondWith`
               [json| {
-                "hint":"If a new function was created in the database with this name and parameters, try reloading the schema cache.",
-                "message":"Could not find the test.overloaded_unnamed_param(a, b) function in the schema cache",
+                "hint":"Perhaps you meant to call the function test.overloaded_unnamed_param(x, y)",
+                "message":"Could not find the function test.overloaded_unnamed_param(a, b) in the schema cache",
                 "code":"PGRST202",
-                "details":null
+                "details":"Searched for the function test.overloaded_unnamed_param with parameters a, b, but no matches were found in the schema cache."
               }|]
               { matchStatus  = 404
               , matchHeaders = [matchContentTypeJson]

test/spec/Feature/Query/SpreadQueriesSpec.hs

@@ -0,0 +1,114 @@
+module Feature.Query.SpreadQueriesSpec where
+
+import Network.Wai (Application)
+
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import Protolude  hiding (get)
+import SpecHelper
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "spread embeds" $ do
+    it "works on a many-to-one relationship" $ do
+      get "/projects?select=id,...clients(client_name:name)" `shouldRespondWith`
+        [json|[
+          {"id":1,"client_name":"Microsoft"},
+          {"id":2,"client_name":"Microsoft"},
+          {"id":3,"client_name":"Apple"},
+          {"id":4,"client_name":"Apple"},
+          {"id":5,"client_name":null}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/grandchild_entities?select=name,...child_entities(parent_name:name,...entities(grandparent_name:name))&limit=3" `shouldRespondWith`
+        [json|[
+          {"name":"grandchild entity 1","parent_name":"child entity 1","grandparent_name":"entity 1"},
+          {"name":"grandchild entity 2","parent_name":"child entity 1","grandparent_name":"entity 1"},
+          {"name":"grandchild entity 3","parent_name":"child entity 2","grandparent_name":"entity 1"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/videogames?select=name,...computed_designers(designer_name:name)" `shouldRespondWith`
+        [json|[
+          {"name":"Civilization I","designer_name":"Sid Meier"},
+          {"name":"Civilization II","designer_name":"Sid Meier"},
+          {"name":"Final Fantasy I","designer_name":"Hironobu Sakaguchi"},
+          {"name":"Final Fantasy II","designer_name":"Hironobu Sakaguchi"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "works inside a normal embed" $
+      get "/grandchild_entities?select=name,child_entity:child_entities(name,...entities(parent_name:name))&limit=1" `shouldRespondWith`
+        [json|[
+          {"name":"grandchild entity 1","child_entity":{"name":"child entity 1","parent_name":"entity 1"}}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "works on a one-to-one relationship" $
+      get "/country?select=name,...capital(capital:name)" `shouldRespondWith`
+        [json|[
+          {"name":"Afghanistan","capital":"Kabul"},
+          {"name":"Algeria","capital":"Algiers"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "fails when is not a to-one relationship" $ do
+      get "/clients?select=*,...projects(*)" `shouldRespondWith`
+        [json|{
+          "code":"PGRST119",
+          "details":"'clients' and 'projects' do not form a many-to-one or one-to-one relationship",
+          "hint":null,
+          "message":"A spread operation on 'projects' is not possible"
+        }|]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/designers?select=*,...computed_videogames(*)" `shouldRespondWith`
+        [json|{
+          "code":"PGRST119",
+          "details":"'designers' and 'computed_videogames' do not form a many-to-one or one-to-one relationship",
+          "hint":null,
+          "message":"A spread operation on 'computed_videogames' is not possible"
+        }|]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "can include or exclude attributes of the junction on a m2m" $ do
+      get "/users?select=*,tasks:users_tasks(*,...tasks(*))&limit=1" `shouldRespondWith`
+        [json|[{
+          "id":1,"name":"Angela Martin",
+          "tasks": [
+            {"user_id":1,"task_id":1,"id":1,"name":"Design w7","project_id":1},
+            {"user_id":1,"task_id":2,"id":2,"name":"Code w7","project_id":1},
+            {"user_id":1,"task_id":3,"id":3,"name":"Design w10","project_id":2},
+            {"user_id":1,"task_id":4,"id":4,"name":"Code w10","project_id":2}
+          ]
+        }]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/users?select=*,tasks:users_tasks(...tasks(*))&limit=1" `shouldRespondWith`
+        [json|[{
+          "id":1,"name":"Angela Martin",
+          "tasks":[
+            {"id":1,"name":"Design w7","project_id":1},
+            {"id":2,"name":"Code w7","project_id":1},
+            {"id":3,"name":"Design w10","project_id":2},
+            {"id":4,"name":"Code w10","project_id":2}
+          ]
+        }]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }

test/spec/Feature/Query/UpdateSpec.hs

@@ -308,99 +308,256 @@ spec = do
         request methodPatch "/articles?id=eq.2001&columns=body" [("Prefer", "return=representation")]
           [json| {"body": "Some real content", "smth": "here", "other": "stuff", "fake_id": 13} |] `shouldRespondWith` 200
 
+      it "disallows ?columns which don't exist" $ do
+        request methodPatch "/articles?id=eq.1&columns=helicopter"
+          [("Prefer", "return=representation")]
+          [json|{"body": "yyy"}|]
+          `shouldRespondWith`
+          [json|{"code":"PGRST204","details":null,"hint":null,"message":"Column 'helicopter' of relation 'articles' does not exist"} |]
+          { matchStatus  = 400
+          , matchHeaders = []
+          }
+
+      it "returns missing table error even if also has invalid ?columns" $ do
+        request methodPatch "/garlic?columns=helicopter"
+          [("Prefer", "return=representation")]
+          [json|[
+            {"id": 204, "body": "yyy"},
+            {"id": 205, "body": "zzz"}]|]
+          `shouldRespondWith`
+          [json|{} |]
+          { matchStatus  = 404
+          , matchHeaders = []
+          }
+
+      context "apply defaults on missing values" $ do
+        it "updates table using default values(field-with_sep) when json keys are undefined" $ do
+          request methodPatch "/complex_items?id=eq.3&columns=name,field-with_sep"
+            [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+            [json|{"name": "Tres"}|]
+            `shouldRespondWith`
+            [json|[
+              {"id":3,"name":"Tres","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":1}
+            ]|]
+            { matchStatus  = 200
+            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            }
+
+        it "updates with limit/offset using table default values(field-with_sep) when json keys are undefined" $ do
+          request methodPatch "/complex_items?select=id,name&columns=name,field-with_sep&limit=1&offset=2&order=id"
+            [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+            [json|{"name": "Tres"}|]
+            `shouldRespondWith`
+            [json|[
+              {"id":3,"name":"Tres"}
+            ]|]
+            { matchStatus  = 200
+            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            }
+
+        it "updates table default values(field-with_sep) when json keys are undefined" $ do
+          request methodPatch "/complex_items?id=eq.3&columns=name,field-with_sep"
+            [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+            [json|{"name": "Tres"}|]
+            `shouldRespondWith`
+            [json|[
+              {"id":3,"name":"Tres","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":1}
+            ]|]
+            { matchStatus  = 200
+            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            }
+
+        it "updates view default values(field-with_sep) when json keys are undefined" $
+          request methodPatch "/complex_items_view?id=eq.3&columns=arr_data,name"
+            [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+            [json|
+              {"arr_data":null}
+            |]
+            `shouldRespondWith`
+            [json|[
+              {"id":3,"name":"Default","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":null,"field-with_sep":3}
+            ]|]
+            { matchStatus  = 200
+            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            }
+
   context "tables with self reference foreign keys" $ do
-    it "embeds children after update" $
-      request methodPatch "/web_content?id=eq.0&select=id,name,web_content(name)"
-              [("Prefer", "return=representation")]
-        [json|{"name": "tardis-patched"}|]
-        `shouldRespondWith`
-        [json|
-          [ { "id": 0, "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" }, { "name": "bar" } ]} ]
-        |]
-        { matchStatus  = 200,
-          matchHeaders = [matchContentTypeJson]
-        }
-
-    it "embeds parent, children and grandchildren after update" $
-      request methodPatch "/web_content?id=eq.0&select=id,name,web_content(name,web_content(name)),parent_content:p_web_id(name)"
-              [("Prefer", "return=representation")]
-        [json|{"name": "tardis-patched-2"}|]
-        `shouldRespondWith`
-        [json| [
-          {
-            "id": 0,
-            "name": "tardis-patched-2",
-            "parent_content": { "name": "wat" },
-            "web_content": [
-                { "name": "fezz", "web_content": [ { "name": "wut" } ] },
-                { "name": "foo",  "web_content": [] },
-                { "name": "bar",  "web_content": [] }
-            ]
+    context "embeds children after update" $ do
+      it "without filters" $
+        request methodPatch "/web_content?id=eq.0&select=id,name,web_content(name)"
+                [("Prefer", "return=representation")]
+          [json|{"name": "tardis-patched"}|]
+          `shouldRespondWith`
+          [json|
+            [ { "id": 0, "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" }, { "name": "bar" } ]} ]
+          |]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
           }
-        ] |]
-        { matchStatus  = 200,
-          matchHeaders = [matchContentTypeJson]
-        }
 
-    it "embeds children after update without explicitly including the id in the ?select" $
-      request methodPatch "/web_content?id=eq.0&select=name,web_content(name)"
-              [("Prefer", "return=representation")]
-        [json|{"name": "tardis-patched"}|]
-        `shouldRespondWith`
-        [json|
-          [ { "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" }, { "name": "bar" } ]} ]
-        |]
-        { matchStatus  = 200,
-          matchHeaders = [matchContentTypeJson]
-        }
+      it "with filters" $
+        request methodPatch "/web_content?id=eq.0&select=id,name,web_content(name)&web_content.name=like.f*"
+                [("Prefer", "return=representation")]
+          [json|{"name": "tardis-patched"}|]
+          `shouldRespondWith`
+          [json|
+            [ { "id": 0, "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" } ]} ]
+          |]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
+          }
 
-    it "embeds an M2M relationship plus parent after update" $
-      request methodPatch "/users?id=eq.1&select=name,tasks(name,project:projects(name))"
-              [("Prefer", "return=representation")]
-        [json|{"name": "Kevin Malone"}|]
-        `shouldRespondWith`
-        [json|[
-          {
-            "name": "Kevin Malone",
-            "tasks": [
-                { "name": "Design w7", "project": { "name": "Windows 7" } },
-                { "name": "Code w7", "project": { "name": "Windows 7" } },
-                { "name": "Design w10", "project": { "name": "Windows 10" } },
-                { "name": "Code w10", "project": { "name": "Windows 10" } }
-            ]
+    context "embeds parent, children and grandchildren after update" $ do
+      it "without filters" $
+        request methodPatch "/web_content?id=eq.0&select=id,name,web_content(name,web_content(name)),parent_content:p_web_id(name)"
+                [("Prefer", "return=representation")]
+          [json|{"name": "tardis-patched-2"}|]
+          `shouldRespondWith`
+          [json| [
+            {
+              "id": 0,
+              "name": "tardis-patched-2",
+              "parent_content": { "name": "wat" },
+              "web_content": [
+                  { "name": "fezz", "web_content": [ { "name": "wut" } ] },
+                  { "name": "foo",  "web_content": [] },
+                  { "name": "bar",  "web_content": [] }
+              ]
+            }
+          ] |]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
           }
-        ]|]
-        { matchStatus  = 200,
-          matchHeaders = [matchContentTypeJson]
-        }
 
-    it "embeds an O2O relationship after update" $ do
-      request methodPatch "/students?id=eq.1&select=name,students_info(address)"
-              [("Prefer", "return=representation")]
-        [json|{"name": "Johnny Doe"}|]
-        `shouldRespondWith`
-        [json|[
-          {
-            "name": "Johnny Doe",
-            "students_info":{"address":"Street 1"}
+      it "with filters" $
+        request methodPatch "/web_content?id=eq.0&select=id,name,web_content(name,web_content(name)),parent_content:p_web_id(name)&web_content.name=like.f*&web_content.web_content.id=eq.4&parent_content.name=neq.wat"
+                [("Prefer", "return=representation")]
+          [json|{"name": "tardis-patched-2"}|]
+          `shouldRespondWith`
+          [json| [
+            {
+              "id": 0,
+              "name": "tardis-patched-2",
+              "parent_content": null,
+              "web_content": [
+                  { "name": "fezz", "web_content": [ { "name": "wut" } ] },
+                  { "name": "foo",  "web_content": [] }
+              ]
+            }
+          ] |]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
           }
-        ]|]
-        { matchStatus  = 200,
-          matchHeaders = [matchContentTypeJson]
-        }
-      request methodPatch "/students_info?id=eq.1&select=address,students(name)"
-              [("Prefer", "return=representation")]
-        [json|{"address": "New Street 1"}|]
-        `shouldRespondWith`
-        [json|[
-          {
-            "address": "New Street 1",
-            "students":{"name": "John Doe"}
+
+    context "embeds children after update without explicitly including the id in the ?select" $ do
+      it "without filters" $
+        request methodPatch "/web_content?id=eq.0&select=name,web_content(name)"
+                [("Prefer", "return=representation")]
+          [json|{"name": "tardis-patched"}|]
+          `shouldRespondWith`
+          [json|
+            [ { "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" }, { "name": "bar" } ]} ]
+          |]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
+          }
+
+      it "with filters" $
+        request methodPatch "/web_content?id=eq.0&select=name,web_content(name)&web_content.name=like.b*"
+                [("Prefer", "return=representation")]
+          [json|{"name": "tardis-patched"}|]
+          `shouldRespondWith`
+          [json|
+            [ { "name": "tardis-patched", "web_content": [ { "name": "bar" } ]} ]
+          |]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
+          }
+
+  context "tables with foreign keys referencing other tables" $ do
+    context "embeds an M2M relationship plus parent after update" $ do
+      it "without filters" $
+        request methodPatch "/users?id=eq.1&select=name,tasks(name,project:projects(name))"
+                [("Prefer", "return=representation")]
+          [json|{"name": "Kevin Malone"}|]
+          `shouldRespondWith`
+          [json|[
+            {
+              "name": "Kevin Malone",
+              "tasks": [
+                  { "name": "Design w7", "project": { "name": "Windows 7" } },
+                  { "name": "Code w7", "project": { "name": "Windows 7" } },
+                  { "name": "Design w10", "project": { "name": "Windows 10" } },
+                  { "name": "Code w10", "project": { "name": "Windows 10" } }
+              ]
+            }
+          ]|]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
+          }
+
+      it "with filters" $
+        request methodPatch "/users?id=eq.1&select=name,tasks(name,project:projects(name))&tasks.name=ilike.code*&tasks.project.name=like.*10"
+                [("Prefer", "return=representation")]
+          [json|{"name": "Kevin Malone"}|]
+          `shouldRespondWith`
+          [json|[
+            {
+              "name": "Kevin Malone",
+              "tasks": [
+                  { "name": "Code w7", "project": null },
+                  { "name": "Code w10", "project": { "name": "Windows 10" } }
+              ]
+            }
+          ]|]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
+          }
+
+
+    context "embeds an O2O relationship after update" $ do
+      it "without filters" $ do
+        request methodPatch "/students?id=eq.1&select=name,students_info(address)"
+                [("Prefer", "return=representation")]
+          [json|{"name": "Johnny Doe"}|]
+          `shouldRespondWith`
+          [json|[
+            {
+              "name": "Johnny Doe",
+              "students_info":{"address":"Street 1"}
+            }
+          ]|]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
+          }
+        request methodPatch "/students_info?id=eq.1&select=address,students(name)"
+                [("Prefer", "return=representation")]
+          [json|{"address": "New Street 1"}|]
+          `shouldRespondWith`
+          [json|[
+            {
+              "address": "New Street 1",
+              "students":{"name": "John Doe"}
+            }
+          ]|]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
+          }
+
+      it "with filters" $ do
+        request methodPatch "/students?id=eq.1&select=name,students_info(address)&students_info.code=like.0002"
+                [("Prefer", "return=representation")]
+          [json|{"name": "Johnny Doe"}|]
+          `shouldRespondWith`
+          [json|[
+            {
+              "name": "Johnny Doe",
+              "students_info": null
+            }
+          ]|]
+          { matchStatus  = 200,
+            matchHeaders = [matchContentTypeJson]
           }
-        ]|]
-        { matchStatus  = 200,
-          matchHeaders = [matchContentTypeJson]
-        }
 
   context "table with limited privileges" $ do
     it "succeeds updating row and gives a 204 when using return=minimal" $
@@ -427,7 +584,7 @@ spec = do
     it "works with the limit query param" $
       baseTable "limited_update_items" "id" tblDataBefore
       `mutatesWith`
-      requestMutation methodPatch "/limited_update_items?order=id&limit=2"
+      requestMutation methodPatch "/limited_update_items?order=id&limit=2" mempty
         [json| {"name": "updated-item"} |]
       `shouldMutateInto`
       [json|[
@@ -439,7 +596,7 @@ spec = do
     it "works with the limit query param plus a filter" $
       baseTable "limited_update_items" "id" tblDataBefore
       `mutatesWith`
-      requestMutation methodPatch "/limited_update_items?order=id&limit=1&id=gt.2"
+      requestMutation methodPatch "/limited_update_items?order=id&limit=1&id=gt.2" mempty
         [json| {"name": "updated-item"} |]
       `shouldMutateInto`
       [json|[
@@ -451,7 +608,7 @@ spec = do
     it "works with the limit and offset query params" $
       baseTable "limited_update_items" "id" tblDataBefore
       `mutatesWith`
-      requestMutation methodPatch "/limited_update_items?order=id&limit=1&offset=1"
+      requestMutation methodPatch "/limited_update_items?order=id&limit=1&offset=1" mempty
         [json| {"name": "updated-item"} |]
       `shouldMutateInto`
       [json|[
@@ -489,7 +646,7 @@ spec = do
     it "works with views with an explicit order by unique col" $
       baseTable "limited_update_items_view" "id" tblDataBefore
       `mutatesWith`
-      requestMutation methodPatch "/limited_update_items_view?order=id&limit=1&offset=1"
+      requestMutation methodPatch "/limited_update_items_view?order=id&limit=1&offset=1" mempty
         [json| {"name": "updated-item"} |]
       `shouldMutateInto`
       [json|[
@@ -501,7 +658,7 @@ spec = do
     it "works with views with an explicit order by composite pk" $
       baseTable "limited_update_items_cpk_view" "id" tblDataBefore
       `mutatesWith`
-      requestMutation methodPatch "/limited_update_items_cpk_view?order=id,name&limit=1&offset=1"
+      requestMutation methodPatch "/limited_update_items_cpk_view?order=id,name&limit=1&offset=1" mempty
         [json| {"name": "updated-item"} |]
       `shouldMutateInto`
       [json|[
@@ -513,7 +670,7 @@ spec = do
     it "works on a table without a pk by ordering by 'ctid'" $
       baseTable "limited_update_items_no_pk" "id" tblDataBefore
       `mutatesWith`
-      requestMutation methodPatch "/limited_update_items_no_pk?order=ctid&limit=1"
+      requestMutation methodPatch "/limited_update_items_no_pk?order=ctid&limit=1" mempty
         [json| {"name": "updated-item"} |]
       `shouldMutateInto`
       [json|[
@@ -521,3 +678,67 @@ spec = do
       , { "id": 2, "name": "item-2" }
       , { "id": 3, "name": "item-3" }
       ]|]
+
+    it "ignores the Range header" $ do
+      baseTable "limited_update_items" "id" tblDataBefore
+       `mutatesWith`
+       requestMutation methodPatch "/limited_update_items"
+        (rangeHdrs (ByteRangeFromTo 0 0))
+        [json| {"name": "updated-item"} |]
+       `shouldMutateInto`
+       [json|[
+         { "id": 1, "name": "updated-item" }
+       , { "id": 2, "name": "updated-item" }
+       , { "id": 3, "name": "updated-item" }
+       ]|]
+
+      baseTable "limited_update_items" "id" tblDataBefore
+       `mutatesWith`
+       requestMutation methodPatch "/limited_update_items?id=gte.2"
+        (rangeHdrs (ByteRangeFromTo 0 0))
+        [json| {"name": "updated-item"} |]
+       `shouldMutateInto`
+       [json|[
+         { "id": 1, "name": "item-1" }
+       , { "id": 2, "name": "updated-item" }
+       , { "id": 3, "name": "updated-item" }
+       ]|]
+
+    it "ignores the Range header and does not do a limited update" $
+      baseTable "limited_update_items" "id" tblDataBefore
+       `mutatesWith`
+       requestMutation methodPatch "/limited_update_items?order=id"
+        (rangeHdrs (ByteRangeFromTo 0 0))
+        [json| {"name": "updated-item"} |]
+       `shouldMutateInto`
+       [json|[
+         { "id": 1, "name": "updated-item" }
+       , { "id": 2, "name": "updated-item" }
+       , { "id": 3, "name": "updated-item" }
+       ]|]
+
+    it "ignores the Range header and does not throw an invalid range error" $
+      baseTable "limited_update_items" "id" tblDataBefore
+      `mutatesWith`
+      requestMutation methodPatch "/limited_update_items?order=id&limit=1&offset=1"
+        (rangeHdrs (ByteRangeFromTo 0 0))
+        [json| {"name": "updated-item"} |]
+      `shouldMutateInto`
+      [json|[
+       { "id": 1, "name": "item-1" }
+      , { "id": 2, "name": "updated-item" }
+      , { "id": 3, "name": "item-3" }
+      ]|]
+
+    it "ignores the Range header but not the limit and offset params" $
+      baseTable "limited_update_items" "id" tblDataBefore
+       `mutatesWith`
+       requestMutation methodPatch "/limited_update_items?order=id&limit=2&offset=1"
+         (rangeHdrs (ByteRangeFromTo 1 1))
+         [json| {"name": "updated-item"} |]
+       `shouldMutateInto`
+       [json|[
+         { "id": 1, "name": "item-1" }
+       , { "id": 2, "name": "updated-item" }
+       , { "id": 3, "name": "updated-item" }
+       ]|]

test/spec/Feature/Query/UpsertSpec.hs

@@ -195,25 +195,18 @@ spec actualPgVersion =
 
     context "with PUT" $ do
       context "Restrictions" $ do
-        it "fails if Range is specified" $
-          request methodPut "/tiobe_pls?name=eq.Javascript" [("Range", "0-5")]
-            [json| [ { "name": "Javascript", "rank": 1 } ]|]
-            `shouldRespondWith`
-            [json|{"message":"Range header and limit/offset querystring parameters are not allowed for PUT","code":"PGRST114","details":null,"hint":null}|]
-            { matchStatus = 400 , matchHeaders = [matchContentTypeJson] }
-
         it "fails if limit is specified" $
           put "/tiobe_pls?name=eq.Javascript&limit=1"
             [json| [ { "name": "Javascript", "rank": 1 } ]|]
             `shouldRespondWith`
-            [json|{"message":"Range header and limit/offset querystring parameters are not allowed for PUT","code":"PGRST114","details":null,"hint":null}|]
+            [json|{"message":"limit/offset querystring parameters are not allowed for PUT","code":"PGRST114","details":null,"hint":null}|]
             { matchStatus = 400 , matchHeaders = [matchContentTypeJson] }
 
         it "fails if offset is specified" $
           put "/tiobe_pls?name=eq.Javascript&offset=1"
             [json| [ { "name": "Javascript", "rank": 1 } ]|]
             `shouldRespondWith`
-            [json|{"message":"Range header and limit/offset querystring parameters are not allowed for PUT","code":"PGRST114","details":null,"hint":null}|]
+            [json|{"message":"limit/offset querystring parameters are not allowed for PUT","code":"PGRST114","details":null,"hint":null}|]
             { matchStatus = 400 , matchHeaders = [matchContentTypeJson] }
 
         it "rejects every other filter than pk cols eq's" $ do
@@ -382,6 +375,18 @@ spec actualPgVersion =
             `shouldRespondWith`
               [json|[ { "id": 1 } ]|]
 
+        it "ignores the Range header" $ do
+          -- assert that the next request will indeed be an update
+          get "/tiobe_pls?name=eq.Java"
+            `shouldRespondWith`
+              [json|[ { "name": "Java", "rank": 1 } ]|]
+
+          request methodPut "/tiobe_pls?name=eq.Java"
+              [("Prefer", "return=representation"), ("Range", "1-1")]
+              [json| [ { "name": "Java", "rank": 5 } ]|]
+            `shouldRespondWith`
+              [json| [ { "name": "Java", "rank": 5 } ]|]
+
       -- TODO: move this to SingularSpec?
       it "works with return=representation and vnd.pgrst.object+json" $
         request methodPut "/tiobe_pls?name=eq.Ruby"

test/spec/Main.hs

@@ -1,6 +1,5 @@
 module Main where
 
-import qualified Data.Aeson                 as JSON
 import qualified Hasql.Pool                 as P
 import qualified Hasql.Transaction.Sessions as HT
 
@@ -10,11 +9,10 @@ import Data.List.NonEmpty (toList)
 import Test.Hspec
 
 import PostgREST.App             (postgrest)
-import PostgREST.Config          (AppConfig (..), LogLevel (..))
+import PostgREST.Config          (AppConfig (..))
 import PostgREST.Config.Database (queryPgVersion)
 import PostgREST.SchemaCache     (querySchemaCache)
 import Protolude                 hiding (toList, toS)
-import Protolude.Conv            (toS)
 import SpecHelper
 
 import qualified PostgREST.AppState as AppState
@@ -29,6 +27,7 @@ import qualified Feature.ConcurrentSpec
 import qualified Feature.CorsSpec
 import qualified Feature.ExtraSearchPathSpec
 import qualified Feature.LegacyGucsSpec
+import qualified Feature.ObservabilitySpec
 import qualified Feature.OpenApi.DisabledOpenApiSpec
 import qualified Feature.OpenApi.IgnorePrivOpenApiSpec
 import qualified Feature.OpenApi.OpenApiSpec
@@ -53,8 +52,10 @@ import qualified Feature.Query.QueryLimitedSpec
 import qualified Feature.Query.QuerySpec
 import qualified Feature.Query.RangeSpec
 import qualified Feature.Query.RawOutputTypesSpec
+import qualified Feature.Query.RelatedQueriesSpec
 import qualified Feature.Query.RpcSpec
 import qualified Feature.Query.SingularSpec
+import qualified Feature.Query.SpreadQueriesSpec
 import qualified Feature.Query.UnicodeSpec
 import qualified Feature.Query.UpdateSpec
 import qualified Feature.Query.UpsertSpec
@@ -64,9 +65,9 @@ import qualified Feature.RpcPreRequestGucsSpec
 
 main :: IO ()
 main = do
-  pool <- P.acquire 3 Nothing $ toUtf8 $ configDbUri testCfg
+  pool <- P.acquire 3 10 60 $ toUtf8 $ configDbUri testCfg
 
-  actualPgVersion <- either (panic . show) id <$> P.use pool queryPgVersion
+  actualPgVersion <- either (panic . show) id <$> P.use pool (queryPgVersion False)
 
   baseSchemaCache <-
     loadSchemaCache pool
@@ -79,9 +80,7 @@ main = do
       appState <- AppState.initWithPool pool config
       AppState.putPgVersion appState actualPgVersion
       AppState.putSchemaCache appState (Just baseSchemaCache)
-      when (isJust $ configDbRootSpec config) $
-        AppState.putJsonDbS appState $ toS $ JSON.encode baseSchemaCache
-      return ((), postgrest LogCrit appState $ pure ())
+      return ((), postgrest config appState $ pure ())
 
     -- For tests that run with a different SchemaCache(depends on configSchemas)
     appDbs config = do
@@ -92,9 +91,7 @@ main = do
       appState <- AppState.initWithPool pool config
       AppState.putPgVersion appState actualPgVersion
       AppState.putSchemaCache appState (Just customSchemaCache)
-      when (isJust $ configDbRootSpec config) $
-        AppState.putJsonDbS appState $ toS $ JSON.encode baseSchemaCache
-      return ((), postgrest LogCrit appState $ pure ())
+      return ((), postgrest config appState $ pure ())
 
   let withApp              = app testCfg
       maxRowsApp           = app testMaxRowsCfg
@@ -115,6 +112,7 @@ main = do
       testCfgLegacyGucsApp = app testCfgLegacyGucs
       planEnabledApp       = app testPlanEnabledCfg
       pgSafeUpdateApp      = app testPgSafeUpdateEnabledCfg
+      obsApp               = app testObservabilityCfg
 
       extraSearchPathApp   = appDbs testCfgExtraSearchPath
       unicodeApp           = appDbs testUnicodeCfg
@@ -149,6 +147,8 @@ main = do
         , ("Feature.Query.UpdateSpec"                    , Feature.Query.UpdateSpec.spec)
         , ("Feature.Query.UpsertSpec"                    , Feature.Query.UpsertSpec.spec actualPgVersion)
         , ("Feature.Query.ComputedRelsSpec"              , Feature.Query.ComputedRelsSpec.spec)
+        , ("Feature.Query.RelatedQueriesSpec"            , Feature.Query.RelatedQueriesSpec.spec)
+        , ("Feature.Query.SpreadQueriesSpec"             , Feature.Query.SpreadQueriesSpec.spec)
         ]
 
   hspec $ do
@@ -243,6 +243,10 @@ main = do
     parallel $ before pgSafeUpdateApp $
       describe "Feature.Query.PgSafeUpdateSpec.spec" Feature.Query.PgSafeUpdateSpec.spec
 
+    -- this test runs with server-trace-header set
+    parallel $ before obsApp $
+      describe "Feature.ObservabilitySpec.spec" Feature.ObservabilitySpec.spec
+
     -- Note: the rollback tests can not run in parallel, because they test persistance and
     -- this results in race conditions
 

test/spec/QueryCost.hs

@@ -1,86 +0,0 @@
--- TODO Can be replaced now by obtaining the EXPLAIN plan and adding the cost tests on PlanSpec.hs
-module Main where
-
-import           Control.Lens                      ((^?))
-import qualified Data.Aeson.Lens                   as L
-import qualified Hasql.Decoders                    as HD
-import qualified Hasql.DynamicStatements.Snippet   as H
-import qualified Hasql.DynamicStatements.Statement as H
-import qualified Hasql.Pool                        as P
-import qualified Hasql.Statement                   as H
-import qualified Hasql.Transaction                 as HT
-import qualified Hasql.Transaction.Sessions        as HT
-import           Text.Heredoc
-
-import Protolude hiding (get, toS)
-
-import PostgREST.Plan.CallPlan
-import PostgREST.Query.QueryBuilder (callPlanToQuery)
-
-import PostgREST.SchemaCache.Identifiers
-import PostgREST.SchemaCache.Proc
-
-import Test.Hspec
-
-main :: IO ()
-main = do
-  pool <- P.acquire 3 Nothing "postgresql://"
-
-  hspec $ describe "QueryCost" $
-    context "call proc query" $ do
-      it "should not exceed cost when calling setof composite proc" $ do
-        cost <- exec pool $
-          callPlanToQuery (FunctionCall (QualifiedIdentifier "test" "get_projects_below")
-          (KeyParams [ProcParam "id" "int" True False])
-          (Just [str| {"id": 3} |]) False False [])
-        liftIO $
-          cost `shouldSatisfy` (< Just 40)
-
-      it "should not exceed cost when calling setof composite proc with empty params" $ do
-        cost <- exec pool $
-          callPlanToQuery (FunctionCall (QualifiedIdentifier "test" "getallprojects") (KeyParams []) Nothing False False [])
-        liftIO $
-          cost `shouldSatisfy` (< Just 30)
-
-      it "should not exceed cost when calling scalar proc" $ do
-        cost <- exec pool $
-          callPlanToQuery (FunctionCall (QualifiedIdentifier "test" "add_them")
-          (KeyParams [ProcParam "a" "int" True False, ProcParam "b" "int" True False])
-          (Just [str| {"a": 3, "b": 4} |]) True False [])
-        liftIO $
-          cost `shouldSatisfy` (< Just 10)
-
-      context "params=multiple-objects" $ do
-        it "should not exceed cost when calling setof composite proc" $ do
-          cost <- exec pool $
-            callPlanToQuery (FunctionCall (QualifiedIdentifier "test" "get_projects_below")
-            (KeyParams [ProcParam "id" "int" True False])
-            (Just [str| [{"id": 1}, {"id": 4}] |]) False True [])
-          liftIO $ do
-            -- lower bound needed for now to make sure that cost is not Nothing
-            cost `shouldSatisfy` (> Just 2000)
-            cost `shouldSatisfy` (< Just 2100)
-
-        it "should not exceed cost when calling scalar proc" $ do
-          cost <- exec pool $
-            callPlanToQuery (FunctionCall (QualifiedIdentifier "test" "add_them")
-            (KeyParams [ProcParam "a" "int" True False, ProcParam "b" "int" True False])
-            (Just [str| [{"a": 3, "b": 4}, {"a": 1, "b": 2}, {"a": 8, "b": 7}] |]) True False [])
-          liftIO $
-            cost `shouldSatisfy` (< Just 10)
-
-
-exec :: P.Pool -> H.Snippet -> IO (Maybe Int64)
-exec pool query =
-  join . rightToMaybe <$>
-  P.use pool (HT.transaction HT.ReadCommitted HT.Read $ HT.statement mempty $ explainCost query)
-
-explainCost :: H.Snippet -> H.Statement () (Maybe Int64)
-explainCost query =
-  H.dynamicallyParameterized snippet decodeExplain False
- where
-   snippet = "EXPLAIN (FORMAT JSON) " <> query
-   decodeExplain :: HD.Result (Maybe Int64)
-   decodeExplain =
-     let row = HD.singleRow $ HD.column $ HD.nonNullable HD.bytea in
-     (^? L.nth 0 . L.key "Plan" .  L.key "Total Cost" . L._Integral) <$> row

test/spec/SpecHelper.hs

@@ -1,13 +1,16 @@
 module SpecHelper where
 
+import           Control.Lens           ((^?))
+import           Data.Aeson.Lens
 import qualified Data.ByteString.Base64 as B64 (decodeLenient)
 import qualified Data.ByteString.Char8  as BS
 import qualified Data.ByteString.Lazy   as BL
 import qualified Data.Map.Strict        as M
+import           Data.Scientific        (toRealFloat)
 import qualified Data.Set               as S
 
 import Data.Aeson           (Value (..), decode, encode)
-import Data.CaseInsensitive (CI (..), original)
+import Data.CaseInsensitive (CI (..), mk, original)
 import Data.List            (lookup)
 import Data.List.NonEmpty   (fromList)
 import Network.Wai.Test     (SResponse (simpleBody, simpleHeaders, simpleStatus))
@@ -79,7 +82,8 @@ baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
   , configDbMaxRows             = Nothing
   , configDbPlanEnabled         = False
   , configDbPoolSize            = 10
-  , configDbPoolAcquisitionTimeout = Nothing
+  , configDbPoolAcquisitionTimeout = 10
+  , configDbPoolMaxLifetime     = 1800
   , configDbPreRequest          = Just $ QualifiedIdentifier "test" "switch_role"
   , configDbPreparedStatements  = True
   , configDbRootSpec            = Nothing
@@ -100,11 +104,13 @@ baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
   , configRawMediaTypes         = []
   , configServerHost            = "localhost"
   , configServerPort            = 3000
+  , configServerTraceHeader     = Nothing
   , configServerUnixSocket      = Nothing
   , configServerUnixSocketMode  = 432
   , configDbTxAllowOverride     = True
   , configDbTxRollbackAll       = True
   , configAdminServerPort       = Nothing
+  , configRoleSettings          = mempty
   }
 
 testCfg :: AppConfig
@@ -200,6 +206,9 @@ testCfgLegacyGucs = baseCfg { configDbUseLegacyGucs = False }
 testPgSafeUpdateEnabledCfg :: AppConfig
 testPgSafeUpdateEnabledCfg = baseCfg { configDbPreRequest = Just $ QualifiedIdentifier "test" "load_safeupdate" }
 
+testObservabilityCfg :: AppConfig
+testObservabilityCfg = baseCfg { configServerTraceHeader = Just $ mk "X-Request-Id" }
+
 analyzeTable :: Text -> IO ()
 analyzeTable tableName =
   void $ readProcess "psql" ["--set", "ON_ERROR_STOP=1", "-a", "-c", toS $ "ANALYZE test.\"" <> tableName <> "\""] []
@@ -213,6 +222,9 @@ rangeHdrsWithCount r = ("Prefer", "count=exact") : rangeHdrs r
 acceptHdrs :: BS.ByteString -> [Header]
 acceptHdrs mime = [(hAccept, mime)]
 
+planHdr :: Header
+planHdr = (hAccept, "application/vnd.pgrst.plan+json")
+
 rangeUnit :: Header
 rangeUnit = ("Range-Unit" :: CI BS.ByteString, "items")
 
@@ -270,9 +282,19 @@ baseTable :: ByteString -> ByteString -> Value -> BaseTable
 baseTable = BaseTable
 
 -- | The mutation (update/delete) that will be applied to the base table
-requestMutation :: Method -> ByteString -> BL.ByteString -> WaiExpectation ()
-requestMutation method path body =
-  request method path [("Prefer", "tx=commit")] body `shouldRespondWith` 204
+requestMutation :: Method -> ByteString -> [Header] -> BL.ByteString -> WaiExpectation ()
+requestMutation method path headers body =
+  request method path (("Prefer", "tx=commit") : headers) body `shouldRespondWith` 204
 
 data BaseTable = BaseTable ByteString ByteString Value
 data MutationCheck = MutationCheck BaseTable (WaiExpectation ())
+
+planCost :: SResponse -> Float
+planCost resp =
+  let res = simpleBody resp ^? nth 0 . key "Plan" . key "Total Cost" in
+  -- big value in case parsing fails
+  fromMaybe 1000000000.0 $ unbox =<< res
+  where
+    unbox :: Value -> Maybe Float
+    unbox (Number n) = Just $ toRealFloat n
+    unbox _          = Nothing

test/spec/TestTypes.hs

@@ -1,37 +0,0 @@
-module TestTypes (
-  IncPK(..)
-, CompoundPK(..)
-) where
-
-import           Data.Aeson ((.:))
-import qualified Data.Aeson as JSON
-
-import Protolude
-
-data IncPK = IncPK {
-  incId          :: Int
-, incNullableStr :: Maybe Text
-, incStr         :: Text
-, incInsert      :: Text
-} deriving (Eq, Show)
-
-instance JSON.FromJSON IncPK where
-  parseJSON (JSON.Object r) = IncPK <$>
-    r .: "id" <*>
-    r .: "nullable_string" <*>
-    r .: "non_nullable_string" <*>
-    r .: "inserted_at"
-  parseJSON _ = mzero
-
-data CompoundPK = CompoundPK {
-  compoundK1    :: Int
-, compoundK2    :: Text
-, compoundExtra :: Maybe Int
-} deriving (Eq, Show)
-
-instance JSON.FromJSON CompoundPK where
-  parseJSON (JSON.Object r) = CompoundPK <$>
-    r .: "k1" <*>
-    r .: "k2" <*>
-    r .: "extra"
-  parseJSON _ = mzero

test/spec/fixtures/data.sql

@@ -168,7 +168,7 @@ INSERT INTO touched_files VALUES
 TRUNCATE TABLE complex_items CASCADE;
 INSERT INTO complex_items VALUES (1, 'One', '{"foo":{"int":1,"bar":"baz"}}', '{1}');
 INSERT INTO complex_items VALUES (2, 'Two', '{"foo":{"int":1,"bar":"baz"}}', '{1,2}');
-INSERT INTO complex_items VALUES (3, 'Three', '{"foo":{"int":1,"bar":"baz"}}', '{1,2,3}');
+INSERT INTO complex_items VALUES (3, 'Three', '{"foo":{"int":1,"bar":"baz"}}', '{1,2,3}', 3);
 
 
 --
@@ -379,6 +379,7 @@ INSERT INTO ranges VALUES (1, '[1,3]');
 INSERT INTO ranges VALUES (2, '[3,6]');
 INSERT INTO ranges VALUES (3, '[6,9]');
 INSERT INTO ranges VALUES (4, '[9,12]');
+INSERT INTO ranges VALUES (5, null);
 
 TRUNCATE TABLE being CASCADE;
 INSERT INTO being VALUES (1), (2), (3), (4);
@@ -826,3 +827,15 @@ INSERT INTO country(id, name) VALUES (1, 'Afghanistan'), (2, 'Algeria');
 
 TRUNCATE TABLE capital CASCADE;
 INSERT INTO capital(id, name, country_id) VALUES (1, 'Kabul', 1), (2, 'Algiers', 2);
+
+TRUNCATE TABLE trash CASCADE;
+INSERT INTO trash(id) VALUES (1), (2), (3);
+
+TRUNCATE TABLE trash_details CASCADE;
+INSERT INTO trash_details(id,jsonb_col) VALUES (1,'{"key": 10}'), (2,'{"key": 6}'), (3,'{"key": 8}');
+
+TRUNCATE TABLE posters CASCADE;
+INSERT INTO posters(id,name) VALUES (1,'Mark'), (2,'Elon'), (3,'Bill'), (4,'Jeff');
+
+TRUNCATE TABLE subscriptions CASCADE;
+INSERT INTO subscriptions(subscriber,subscribed) VALUES (3,1), (4,1), (1,2);

test/spec/fixtures/privileges.sql

@@ -41,6 +41,14 @@ GRANT USAGE ON SEQUENCE
     , leak_id_seq
 TO postgrest_test_anonymous;
 
+DO $do$
+BEGIN
+  IF current_setting('server_version_num')::INT >= 100000 THEN
+    GRANT USAGE ON SEQUENCE channels_id_seq TO postgrest_test_anonymous;
+  END IF;
+END
+$do$;
+
 -- Privileges for non anonymous users
 GRANT USAGE ON SCHEMA test TO postgrest_test_author;
 GRANT ALL ON TABLE authors_only TO postgrest_test_author;