bump version to 11.2.0

This allows using doctests on the Response.hs module

.github/actions/setup-nix/action.yaml

@@ -11,7 +11,7 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: cachix/install-nix-action@v18
+    - uses: cachix/install-nix-action@v22
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12

.github/dependabot.yml

@@ -1,6 +1,11 @@
 version: 2
 updates:
-- package-ecosystem: github-actions
-  directory: /
-  schedule:
-    interval: weekly
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+
+  - package-ecosystem: github-actions
+    directory: /.github/actions/setup-nix
+    schedule:
+      interval: weekly

.github/scripts/arm/build.sh

@@ -4,13 +4,15 @@
 
 [ -z "$1" ] && { echo "Missing 1st argument: PostgREST github commit SHA"; exit 1; }
 [ -z "$2" ] && { echo "Missing 2nd argument: Build environment directory name"; exit 1; }
+[ -z "$3" ] && { echo "Missing 3rd argument: GHC version"; exit 1; }
 
 PGRST_GITHUB_COMMIT="$1"
 SCRIPT_DIR="$2"
 
 DOCKER_BUILD_DIR="$SCRIPT_DIR/docker-env"
-CABAL_VERSION="3.6.0.0"
-GHC_VERSION="9.2.4"
+# latest is a shortcut documented on https://www.haskell.org/ghcup/guide/#tags-and-shortcuts
+CABAL_VERSION="latest"
+GHC_VERSION="$3"
 
 install_packages() {
   sudo apt-get update -y

.github/scripts/arm/docker-env/Dockerfile

@@ -13,4 +13,6 @@ EXPOSE 3000
 
 USER 1000
 
-CMD postgrest
+# Use the array form to avoid running the command using bash, which does not handle `SIGTERM` properly. 
+# See https://docs.docker.com/compose/faq/#why-do-my-services-take-10-seconds-to-recreate-or-stop 
+CMD ["postgrest"]

.github/workflows/ci.yaml

@@ -46,7 +46,7 @@ jobs:
       - name: Run coverage (IO tests and Spec tests against PostgreSQL 15)
         run: postgrest-coverage
       - name: Upload coverage to codecov
-        uses: codecov/codecov-action@v3.1.1
+        uses: codecov/codecov-action@v3.1.4
         with:
           files: ./coverage/codecov.json
 
@@ -216,7 +216,7 @@ jobs:
   Build-Cabal:
     strategy:
       matrix:
-        ghc: ['9.2.4']
+        ghc: ['9.0.2', '9.2.4']
       fail-fast: false
     name: Build Linux (Cabal, GHC ${{ matrix.ghc }})
     runs-on: ubuntu-latest
@@ -247,13 +247,18 @@ jobs:
         run: cabal build --enable-tests --enable-benchmarks all
 
   Build-Cabal-Arm:
-    name: Build aarch64 (Cabal)
+    strategy:
+      matrix:
+        ghc: ['9.2.4']
+      fail-fast: false
+    name: Build aarch64 (Cabal, GHC ${{ matrix.ghc }})
     if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/rel-') }}
     runs-on: ubuntu-latest
     outputs:
       remotepath: ${{ steps.Remote-Dir.outputs.remotepath }}
     env:
       GITHUB_COMMIT: ${{ github.sha }}
+      GHC_VERSION: ${{ matrix.ghc }}
     steps:
       - uses: actions/checkout@v3
       - id: Remote-Dir
@@ -280,8 +285,8 @@ jobs:
           fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
           command_timeout: 120m
           script_stop: true
-          envs: GITHUB_COMMIT,REMOTE_DIR
-          script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR"
+          envs: GITHUB_COMMIT,REMOTE_DIR,GHC_VERSION
+          script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR" "GHC_VERSION"
       - name: Download binaries from remote server
         uses: nicklasfrahm/scp-action@main
         with:

.github/workflows/report.yaml

@@ -22,7 +22,7 @@ jobs:
           name: loadtest.md
           path: artifacts
       - name: Upload to GitHub Checks
-        uses: LouisBrunner/checks-action@v1.6.0
+        uses: LouisBrunner/checks-action@v1.6.2
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sha: ${{ github.event.workflow_run.head_sha }}

ARCHITECTURE.md

@@ -55,12 +55,6 @@ It builds the OpenAPI response using the schema cache.
 
 This module provides functions to deal with JWT authorization.
 
-### Workers.hs
-
-This spawns threads which are used to execute concurrent jobs.
-
-Jobs include connection recovery, a listener for the PostgreSQL LISTEN command, and an admin server.
-
 ### SchemaCache.hs
 
 This queries the PostgreSQL system catalogs and caches the metadata into a SchemaCache type,
@@ -68,3 +62,7 @@ This queries the PostgreSQL system catalogs and caches the metadata into a Schem
 ### AppState.hs
 
 The state of the App which is kept across requests.
+
+This spawns threads which are used to execute concurrent jobs.
+
+Jobs include connection recover and a listener for the PostgreSQL LISTEN command.

CHANGELOG.md

@@ -5,6 +5,78 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+## [11.2.0] - 2023-08-10
+
+### Added
+
+ - #2523, Data representations - @aljungberg
+   + Allows for flexible API output formatting and input parsing on a per-column type basis using regular SQL functions configured in the database
+   + Enables greater flexibility in the form and shape of your APIs, both for output and input, making PostgREST a more versatile general-purpose API server
+   + Examples include base64 encode/decode your binary data (like a `bytea` column containing an image), choose whether to present a timestamp column as seconds since the Unix epoch or as an ISO 8601 string, or represent fixed precision decimals as strings, not doubles, to preserve precision
+   + ...and accept the same in `POST/PUT/PATCH` by configuring the reverse transformation(s)
+   + Other use-cases include custom representation of enums, arrays, nested objects, CSS hex colour strings, gzip compressed fields, metric to imperial conversions, and much more
+   + Works when using the `select` parameter to select only a subset of columns, embedding through complex joins, renaming fields, with views and computed columns
+   + Works when filtering on a formatted column without extra indexes by parsing to the canonical representation
+   + Works for data `RETURNING` operations, such as requesting the full body in a POST/PUT/PATCH with `Prefer: return=representation`
+   + Works for batch updates and inserts
+   + Completely optional, define the functions in the database and they will be used automatically everywhere
+   + Data representations preserve the ability to write to the original column and require no extra storage or complex triggers (compared to using `GENERATED ALWAYS` columns)
+   + Note: data representations require Postgres 10 (Postgres 11 if using `IN` predicates); data representations are not implemented for RPC
+ - #2647, Allow to verify the PostgREST version in SQL: `select distinct application_name from pg_stat_activity`. - @laurenceisla
+ - #2856, Add the `--version` CLI option that prints the version information - @laurenceisla
+ - #1655, Improve `details` field of the singular error response - @taimoorzaeem
+ - #740, Add `Preference-Applied` in response for `Prefer: return=representation/headers-only/minimal` - @taimoorzaeem
+ - #1601, Add optional `nulls=stripped` parameter for mediatypes `application/vnd.pgrst.array+json` and `application/vnd.pgrst.object+json` - @taimoorzaeem
+
+### Fixed
+
+ - #2821, Fix OPTIONS not accepting all available media types - @steve-chavez
+ - #2834, Fix compilation on Ubuntu by being compatible with GHC 9.0.2 - @steve-chavez
+ - #2840, Fix `Prefer: missing=default` with DOMAIN default values - @steve-chavez
+ - #2849, Fix HEAD unnecessarily executing aggregates - @steve-chavez
+ - #2594, Fix unused index on jsonb/jsonb arrow filter and order (``/bets?data->>contractId=eq.1`` and ``/bets?order=data->>contractId``) - @steve-chavez
+ - #2861, Fix character and bit columns with fixed length not inserting/updating properly - @laurenceisla
+   + Fixes the error "value too long for type character(1)" when the char length of the column was bigger than one.
+ - #2862, Fix null filtering on embedded resource when using a column name equal to the relation name - @steve-chavez
+ - #1586, Fix function parameters of type character and bit not ignoring length - @laurenceisla
+   + Fixes the error "value too long for type character(1)" when the char length of the parameter was bigger than one.
+ - #2881, Fix error when a function returns `RECORD` or `SET OF RECORD` - @laurenceisla
+ - #2896, Fix applying superuser settings for impersonated role - @steve-chavez
+
+### Deprecated
+
+ - #2863, Deprecate resource embedding target disambiguation - @steve-chavez
+   + The `/table?select=*,other!fk(*)` must be used to disambiguate
+   + The server aids in choosing the `!fk` by sending a `hint` on the error whenever an ambiguous request happens.
+
+## [11.1.0] - 2023-06-07
+
+### Added
+
+ - #2786, Limit idle postgresql connection lifetime - @robx
+   + New option `db-pool-max-idletime` (default 30s).
+   + This is equivalent to the old option `db-pool-timeout` of PostgREST 10.0.0.
+   + A config alias for `db-pool-timeout` is included.
+ - #2703, Add pre-config function - @steve-chavez
+    + New config option `db-pre-config`(empty by default)
+    + Allows using the in-database configuration without SUPERUSER
+ - #2781, When `db-channel-enabled` is false, start automatic connection recovery on a new request when pool connections are closed with `pg_terminate_backend` - @steve-chavez
+    + Mitigates the lack of LISTEN/NOTIFY for schema cache reloading on read replicas.
+
+### Fixed
+
+ - #2791, Fix dropping schema cache reload notifications  - @steve-chavez
+ - #2801, Stop retrying connection when "no password supplied" - @steve-chavez
+
+## [11.0.1] - 2023-04-27
+
+### Fixed
+
+ - #2762, Fixes "permission denied for schema" error during schema cache load - @steve-chavez
+ - #2756, Fix bad error message on generated columns when using `Prefer: missing=default` - @steve-chavez
+ - #1139, Allow a 30 second skew for JWT validation - @steve-chavez
+   + It used to be 1 second, which was too strict
+
 ## [11.0.0] - 2023-04-16
 
 ### Added

nix/README.md

@@ -5,24 +5,14 @@ for developing, testing and building PostgREST.
 
 ## Getting started with Nix
 
-You'll need to [get Nix](https://nixos.org/download.html). The installer will
-create your Nix store in the `/nix/` directory, where all build artifacts and
-their dependencies will be stored. It will also link the Nix executables like
-`nix-env`, `nix-build` and `nix-shell` into your PATH. Nix will manage all
-other PostgREST dependencies from here on out. To clean up older build
-artifacts from the `/nix/store`, you can run `nix-collect-garbage`.
-
-If you are on a system that does not support nix, for example Windows, you can
-run the nix development environment in a docker container. Inside the `nix/`
-directory run `docker-compose run --rm nix` to start the docker container. This
-will set up the binary cache and launch `nix-shell` automatically.
+You'll need to [get Nix](https://nixos.org/download.html). Follow the recommended installation for your operating system from the official download website.
 
 ## Building PostgREST
 
 To build PostgREST from your local checkout of the repository, run:
 
 ```bash
-nix-build --attr postgrestPackage
+$ nix-build --attr postgrestPackage
 
 ```
 
@@ -39,10 +29,10 @@ We recommend that you use the PostgREST binary cache on
 
 ```bash
 # Install cachix:
-nix-env -iA cachix -f https://cachix.org/api/v1/install
+$ nix-env -iA cachix -f https://cachix.org/api/v1/install
 
 # Set cachix up to use the PostgREST binary cache:
-cachix use postgrest
+$ cachix use postgrest
 
 ```
 
@@ -56,7 +46,7 @@ following command will put you into a new shell that has GHC and Cabal on the
 PATH:
 
 ```bash
-nix-shell
+$ nix-shell
 
 ```
 
@@ -146,10 +136,10 @@ Note: Once inside nix-shell, the utilities work from any directory inside
 the PostgREST repo. Paths are resolved relative to the repo root:
 
 ```bash
-$ cd src
+[nix-shell]$ cd src
 # Even though the current directory is ./src, the config path must still start
 # from the repo root:
-$ postgrest-run test/io/configs/simple.conf
+[nix-shell]$ postgrest-run test/io/configs/simple.conf
 ```
 
 ## Testing
@@ -177,21 +167,21 @@ run with `postgrest-test-io`. The test runner under the hood is
 
 ```bash
 # Filter the tests to run by name, including all that contain 'config':
-postgrest-test-io -k config
+[nix-shell]$ postgrest-test-io -k config
 
 # Run tests in parallel using xdist, specifying the number of processes:
-postgrest-test-io -n auto
-postgrest-test-io -n 8
+[nix-shell]$ postgrest-test-io -n auto
+[nix-shell]$ postgrest-test-io -n 8
 ```
 
 The memory tests check that we don't surpass a memory threshold for big request bodies.
 
 ```bash
 # Build the dependencies needed for the memory test
-nix-shell --arg memory true
+$ nix-shell --arg memory true
 
 # Run the memory test
-postgrest-test-memory
+[nix-shell]$ postgrest-test-memory
 ```
 
 The loadtests ensure that performance doesn't drop on a change. Underlyingly they use
@@ -199,25 +189,25 @@ The loadtests ensure that performance doesn't drop on a change. Underlyingly the
 
 ```bash
 # Run the loadtests on the latest commit(HEAD)
-postgrest-loadtest
+[nix-shell]$ postgrest-loadtest
 
 # You can loadtest comparing to a different branch
-postgrest-loadtest-against master
+[nix-shell]$ postgrest-loadtest-against master
 
 # You can simulate latency client/postgrest and postgrest/database
-PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest
+[nix-shell]$ PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest
 
 # You can build postgrest directly with cabal for faster iteration
-PGRST_BUILD_CABAL=1 postgrest-loadtest
+[nix-shell]$ PGRST_BUILD_CABAL=1 postgrest-loadtest
 
 # Produce a markdown report to be used on CI
-postgrest-loadtest-report
+[nix-shell]$ postgrest-loadtest-report
 ```
 
 doctests for some of our modules are also available:
 
 ```bash
-postgrest-test-doctest
+[nix-shell]$ postgrest-test-doctest
 ```
 
 ## Code coverage
@@ -226,11 +216,11 @@ Code coverage is available under the `postgrest-coverage` command. This will pro
 
 ```bash
 # Will run all the tests and produce a coverage dir
-postgrest-coverage
+[nix-shell]$ postgrest-coverage
 
 # Visualize the output
-cd coverage
-python -mSimpleHTTPServer 8080
+[nix-shell]$ cd coverage
+[nix-shell]$ python -mSimpleHTTPServer 8080
 ```
 
 ## Linting and styling code

nix/overlays/haskell-packages.nix

@@ -43,8 +43,8 @@ let
         (prev.callHackageDirect
           {
             pkg = "hasql-notifications";
-            ver = "0.2.0.4";
-            sha256 = "sha256-fm1xiDyvDkb5WLOJ73/s8wrWEW23XFS7luAv2brfr8I=";
+            ver = "0.2.0.5";
+            sha256 = "sha256-KV36zs/RTgJh/oBBPZaNiQshusmakWnjYTcdoNa4JLA=";
           }
           { });
 
@@ -52,8 +52,8 @@ let
         (prev.callHackageDirect
           {
             pkg = "hasql-pool";
-            ver = "0.9";
-            sha256 = "sha256-5UshbbaBVY8eJ/9VagNVVxonRwMcd7UmGqDc35pJNFY=";
+            ver = "0.10";
+            sha256 = "sha256-kHzoqtNV9BFWnn1h560JRqMooQRwxokVKgDRBexamNI=";
           }
           { });
     } // extraOverrides final prev;

nix/tools/cabalTools.nix

@@ -38,11 +38,21 @@ let
       {
         name = "postgrest-run";
         docs = "Run PostgREST after building it interactively with cabal-install";
-        args = [ "ARG_LEFTOVERS([PostgREST arguments])" ];
+        args =
+          [
+            "ARG_USE_ENV([PGRST_DB_ANON_ROLE], [postgrest_test_anonymous], [PostgREST anonymous role])"
+            "ARG_USE_ENV([PGRST_DB_POOL], [1], [PostgREST pool size])"
+            "ARG_USE_ENV([PGRST_DB_POOL_ACQUISITION_TIMEOUT], [1], [PostgREST pool size])"
+            "ARG_LEFTOVERS([PostgREST arguments])"
+          ];
         inRootDir = true;
         withEnv = postgrest.env;
       }
       ''
+        export PGRST_DB_ANON_ROLE
+        export PGRST_DB_POOL
+        export PGRST_DB_POOL_ACQUISITION_TIMEOUT
+
         exec ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
           postgrest "''${_arg_leftovers[@]}"
       '';

nix/tools/withTools.nix

@@ -15,11 +15,14 @@
 let
   withTmpDb =
     { name, postgresql }:
-    let commandName = "postgrest-with-${name}"; in
+    let
+      commandName = "postgrest-with-${name}";
+      superuserRole = "postgres";
+    in
     checkedShellScript
       {
         name = commandName;
-        docs = "Run the given command in a temporary database with ${name}";
+        docs = "Run the given command in a temporary database with ${name}. If you wish to mutate the database, login with the '${superuserRole}' role.";
         args =
           [
             "ARG_OPTIONAL_SINGLE([fixtures], [f], [SQL file to load fixtures from], [test/spec/fixtures/load.sql])"
@@ -29,6 +32,7 @@ let
             "ARG_USE_ENV([PGDATABASE], [postgres], [PG database name])"
             "ARG_USE_ENV([PGRST_DB_SCHEMAS], [test], [Schema to expose])"
             "ARG_USE_ENV([PGTZ], [utc], [Timezone to use])"
+            "ARG_USE_ENV([PGOPTIONS], [-c search_path=public,test], [PG options to use])"
           ];
         positionalCompletion = "_command";
         inRootDir = true;
@@ -58,6 +62,7 @@ let
         export PGDATABASE
         export PGRST_DB_SCHEMAS
         export PGTZ
+        export PGOPTIONS
 
         HBA_FILE="$tmpdir/pg_hba.conf"
         echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
@@ -66,7 +71,8 @@ let
         log "Initializing database cluster..."
         # We try to make the database cluster as independent as possible from the host
         # by specifying the timezone, locale and encoding.
-        PGTZ=UTC initdb --no-locale --encoding=UTF8 --nosync -U "$PGUSER" --auth=trust \
+        # initdb -U creates a superuser(man initdb)
+        PGTZ=UTC initdb --no-locale --encoding=UTF8 --nosync -U "${superuserRole}" --auth=trust \
           >> "$setuplog"
 
         log "Starting the database cluster..."
@@ -82,12 +88,15 @@ let
         }
         trap stop EXIT
 
-        log "Loading fixtures..."
-        psql -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
+        log "Creating a minimally privileged $PGUSER connection role..."
+        createuser "$PGUSER" -U "${superuserRole}" --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login
+
+        log "Loading fixtures under the ${superuserRole} role..."
+        psql -U "${superuserRole}" -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
 
         log "Done. Running command..."
 
-        echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U $PGUSER"
+        echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U ${superuserRole}"
         echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log"
 
         ("$_arg_command" "''${_arg_leftovers[@]}")

postgrest.cabal

@@ -1,5 +1,5 @@
 name:               postgrest
-version:            11.0.0
+version:            11.2.0
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                     for tables, views, and functions, supporting all HTTP methods that security
@@ -34,7 +34,8 @@ library
   default-extensions: OverloadedStrings
                       NoImplicitPrelude
   hs-source-dirs:     src
-  exposed-modules:    PostgREST.App
+  exposed-modules:    PostgREST.Admin
+                      PostgREST.App
                       PostgREST.AppState
                       PostgREST.Auth
                       PostgREST.CLI
@@ -48,6 +49,7 @@ library
                       PostgREST.SchemaCache.Identifiers
                       PostgREST.SchemaCache.Routine
                       PostgREST.SchemaCache.Relationship
+                      PostgREST.SchemaCache.Representations
                       PostgREST.SchemaCache.Table
                       PostgREST.Error
                       PostgREST.Logger
@@ -70,7 +72,6 @@ library
                       PostgREST.Response.OpenAPI
                       PostgREST.Response.GucHeader
                       PostgREST.Version
-                      PostgREST.Workers
   other-modules:      Paths_postgrest
   build-depends:      base                      >= 4.9 && < 4.17
                     , HTTP                      >= 4000.3.7 && < 4000.5
@@ -86,12 +87,13 @@ library
                     , contravariant-extras      >= 0.3.3 && < 0.4
                     , cookie                    >= 0.4.2 && < 0.5
                     , either                    >= 4.4.1 && < 5.1
+                    , extra                     >= 1.7.0 && < 2.0
                     , fuzzyset                  >= 0.2.3
                     , gitrev                    >= 1.2 && < 1.4
                     , hasql                     >= 1.6.1.1 && < 1.7
                     , hasql-dynamic-statements  >= 0.3.1 && < 0.4
                     , hasql-notifications       >= 0.1 && < 0.3
-                    , hasql-pool                >= 0.9 && < 0.10
+                    , hasql-pool                >= 0.10 && < 0.11
                     , hasql-transaction         >= 1.0.1 && < 1.1
                     , heredoc                   >= 0.2 && < 0.3
                     , http-types                >= 0.12.2 && < 0.13
@@ -187,6 +189,7 @@ test-suite spec
                       Feature.CorsSpec
                       Feature.ExtraSearchPathSpec
                       Feature.LegacyGucsSpec
+                      Feature.NoSuperuserSpec
                       Feature.ObservabilitySpec
                       Feature.OpenApi.DisabledOpenApiSpec
                       Feature.OpenApi.IgnorePrivOpenApiSpec
@@ -215,6 +218,7 @@ test-suite spec
                       Feature.Query.RelatedQueriesSpec
                       Feature.Query.RpcSpec
                       Feature.Query.SingularSpec
+                      Feature.Query.NullsStrip
                       Feature.Query.SpreadQueriesSpec
                       Feature.Query.UnicodeSpec
                       Feature.Query.UpdateSpec
@@ -231,7 +235,7 @@ test-suite spec
                     , bytestring        >= 0.10.8 && < 0.12
                     , case-insensitive  >= 1.2 && < 1.3
                     , containers        >= 0.5.7 && < 0.7
-                    , hasql-pool        >= 0.9 && < 0.10
+                    , hasql-pool        >= 0.10 && < 0.11
                     , hasql-transaction >= 1.0.1 && < 1.1
                     , heredoc           >= 0.2 && < 0.3
                     , hspec             >= 2.3 && < 2.10

src/PostgREST/Admin.hs

@@ -0,0 +1,87 @@
+{-# LANGUAGE NamedFieldPuns  #-}
+{-# LANGUAGE RecordWildCards #-}
+
+module PostgREST.Admin
+  ( runAdmin
+  ) where
+
+import qualified Data.Text                 as T
+import qualified Hasql.Session             as SQL
+import qualified Network.HTTP.Types.Status as HTTP
+import qualified Network.Wai               as Wai
+import qualified Network.Wai.Handler.Warp  as Warp
+
+import Control.Monad.Extra (whenJust)
+
+import Network.Socket
+import Network.Socket.ByteString
+
+import PostgREST.AppState (AppState)
+import PostgREST.Config   (AppConfig (..))
+
+import qualified PostgREST.AppState as AppState
+
+import Protolude
+
+runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
+runAdmin conf@AppConfig{configAdminServerPort} appState settings =
+  whenJust configAdminServerPort $ \adminPort -> do
+    AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
+    void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
+  where
+    adminApp = admin appState conf
+
+-- | PostgREST admin application
+admin :: AppState.AppState -> AppConfig -> Wai.Application
+admin appState appConfig req respond  = do
+  isMainAppReachable  <- any isRight <$> reachMainApp appConfig
+  isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
+  isConnectionUp      <-
+    if configDbChannelEnabled appConfig
+      then AppState.getIsListenerOn appState
+      else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
+
+  case Wai.pathInfo req of
+    ["ready"] ->
+      respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
+    ["live"] ->
+      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
+    _ ->
+      respond $ Wai.responseLBS HTTP.status404 [] mempty
+
+-- Try to connect to the main app socket
+-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
+-- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
+-- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
+reachMainApp :: AppConfig -> IO [Either IOException ()]
+reachMainApp AppConfig{..} =
+  case configServerUnixSocket of
+    Just path ->  do
+      sock <- socket AF_UNIX Stream 0
+      (:[]) <$> try (do
+        connect sock $ SockAddrUnix path
+        withSocketsDo $ bracket (pure sock) close sendEmpty)
+    Nothing -> do
+      let
+        host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
+             | otherwise                                             = Just configServerHost
+        filterAddrs xs =
+          case configServerHost of
+              "*4" -> ipv4Addrs xs ++ ipv6Addrs xs
+              "!4" -> ipv4Addrs xs
+              "*6" -> ipv6Addrs xs ++ ipv4Addrs xs
+              "!6" -> ipv6Addrs xs
+              _    -> xs
+        ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
+        ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
+
+      addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
+      tryAddr `traverse` filterAddrs addrs
+  where
+    sendEmpty sock = void $ send sock mempty
+    tryAddr :: AddrInfo -> IO (Either IOException ())
+    tryAddr addr = do
+      sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
+      try $ do
+        connect sock $ addrAddress addr
+        withSocketsDo $ bracket (pure sock) close sendEmpty

src/PostgREST/ApiRequest.hs

@@ -51,8 +51,7 @@ import PostgREST.ApiRequest.Types        (ApiRequestError (..),
                                           RangeError (..))
 import PostgREST.Config                  (AppConfig (..),
                                           OpenAPIMode (..))
-import PostgREST.MediaType               (MTPlanAttrs (..),
-                                          MTPlanFormat (..),
+import PostgREST.MediaType               (MTPlanFormat (..),
                                           MediaType (..))
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           convertToLimitZeroRange,
@@ -207,7 +206,7 @@ getAction PathInfo{pathIsProc, pathIsDefSpec} method =
 
 getMediaTypes :: AppConfig -> RequestHeaders -> Action -> PathInfo -> Either ApiRequestError (MediaType, MediaType)
 getMediaTypes conf hdrs action path = do
-   acceptMediaType <- findAcceptMediaType conf action path accepts
+   acceptMediaType <- negotiateContent conf action path accepts
    pure (acceptMediaType, contentMediaType)
   where
     accepts = maybe [MTAny] (map MediaType.decodeMediaType . parseHttpAccept) $ lookupHeader "accept"
@@ -299,19 +298,6 @@ getPayload reqBody contentMediaType QueryParams{qsColumns} action PathInfo{pathI
       ActionInvoke InvPost        -> qsColumns
       _                           -> Nothing
 
-{-|
-  Find the best match from a list of media types accepted by the
-  client in order of decreasing preference and a list of types
-  producible by the server.  If there is no match but the client
-  accepts */* then return the top server pick.
--}
-mutuallyAgreeable :: [MediaType] -> [MediaType] -> Maybe MediaType
-mutuallyAgreeable sProduces cAccepts =
-  let exact = listToMaybe $ L.intersect cAccepts sProduces in
-  if isNothing exact && MTAny `elem` cAccepts
-     then listToMaybe sProduces
-     else exact
-
 type CsvData = V.Vector (M.Map Text LBS.ByteString)
 
 {-|
@@ -361,28 +347,33 @@ payloadAttributes raw json =
   where
     emptyPJArray = ProcessedJSON (JSON.encode emptyArray) S.empty
 
-findAcceptMediaType :: AppConfig -> Action -> PathInfo -> [MediaType] -> Either ApiRequestError MediaType
-findAcceptMediaType conf action path accepts =
-  case mutuallyAgreeable (requestMediaTypes conf action path) accepts of
-    Just ct ->
-      Right ct
-    Nothing ->
-      Left . MediaTypeError $ map MediaType.toMime accepts
 
-requestMediaTypes :: AppConfig -> Action -> PathInfo -> [MediaType]
-requestMediaTypes conf action path =
+-- | Do content negotiation. i.e. choose a media type based on the intersection of accepted/produced media types.
+negotiateContent :: AppConfig -> Action -> PathInfo -> [MediaType] -> Either ApiRequestError MediaType
+negotiateContent conf action path accepts =
+  case firstAcceptedPick of
+    Just MTAny -> Right MTApplicationJSON -- by default(for */*) we respond with json
+    Just mt    -> Right mt
+    Nothing    -> Left . MediaTypeError $ map MediaType.toMime accepts
+  where
+    -- if there are multiple accepted media types, pick the first
+    firstAcceptedPick = listToMaybe $ L.intersect accepts $ producedMediaTypes conf action path
+
+producedMediaTypes :: AppConfig -> Action -> PathInfo -> [MediaType]
+producedMediaTypes conf action path =
   case action of
     ActionRead _    -> defaultMediaTypes ++ rawMediaTypes
     ActionInvoke _  -> invokeMediaTypes
-    ActionInspect _ -> [MTOpenAPI, MTApplicationJSON]
-    ActionInfo      -> [MTTextCSV]
-    _               -> defaultMediaTypes
+    ActionInfo      -> defaultMediaTypes
+    ActionMutate _  -> defaultMediaTypes
+    ActionInspect _ -> inspectMediaTypes
   where
+    inspectMediaTypes = [MTOpenAPI, MTApplicationJSON, MTArrayJSONStrip, MTAny]
     invokeMediaTypes =
       defaultMediaTypes
         ++ rawMediaTypes
         ++ [MTOpenAPI | pathIsRootSpec path]
     defaultMediaTypes =
-      [MTApplicationJSON, MTSingularJSON, MTGeoJSON, MTTextCSV] ++
-      [MTPlan $ MTPlanAttrs Nothing PlanJSON mempty | configDbPlanEnabled conf]
+      [MTApplicationJSON, MTArrayJSONStrip, MTSingularJSON True, MTSingularJSON False, MTGeoJSON, MTTextCSV] ++
+      [MTPlan MTApplicationJSON PlanText mempty | configDbPlanEnabled conf] ++ [MTAny]
     rawMediaTypes = configRawMediaTypes conf `union` [MTOctetStream, MTTextPlain, MTTextXML]

src/PostgREST/ApiRequest/Preferences.hs

@@ -41,7 +41,7 @@ import Protolude
 data Preferences
   = Preferences
     { preferResolution     :: Maybe PreferResolution
-    , preferRepresentation :: PreferRepresentation
+    , preferRepresentation :: Maybe PreferRepresentation
     , preferParameters     :: Maybe PreferParameters
     , preferCount          :: Maybe PreferCount
     , preferTransaction    :: Maybe PreferTransaction
@@ -56,7 +56,7 @@ data Preferences
 -- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates, count=exact")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
---     , preferRepresentation = None
+--     , preferRepresentation = Nothing
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
@@ -68,7 +68,7 @@ data Preferences
 -- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
---     , preferRepresentation = None
+--     , preferRepresentation = Nothing
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
@@ -100,7 +100,7 @@ data Preferences
 -- >>> pPrint $ fromHeaders [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default")]
 -- Preferences
 --     { preferResolution = Nothing
---     , preferRepresentation = Full
+--     , preferRepresentation = Just Full
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Just Commit
@@ -110,12 +110,12 @@ data Preferences
 fromHeaders :: [HTTP.Header] -> Preferences
 fromHeaders headers =
   Preferences
-    { preferResolution = parsePrefs [MergeDuplicates, IgnoreDuplicates]
-    , preferRepresentation = fromMaybe None $ parsePrefs [Full, None, HeadersOnly]
-    , preferParameters = parsePrefs [SingleObject]
-    , preferCount = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
-    , preferTransaction = parsePrefs [Commit, Rollback]
-    , preferMissing = parsePrefs [ApplyDefaults, ApplyNulls]
+    { preferResolution     = parsePrefs [MergeDuplicates, IgnoreDuplicates]
+    , preferRepresentation = parsePrefs [Full, None, HeadersOnly]
+    , preferParameters     = parsePrefs [SingleObject]
+    , preferCount          = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
+    , preferTransaction    = parsePrefs [Commit, Rollback]
+    , preferMissing        = parsePrefs [ApplyDefaults, ApplyNulls]
     }
   where
     prefHeaders = filter ((==) HTTP.hPrefer . fst) headers
@@ -168,6 +168,8 @@ data PreferRepresentation
   | None        -- ^ Return nothing from the mutated data.
   deriving Eq
 
+instance ToAppliedHeader PreferRepresentation
+
 instance ToHeaderValue PreferRepresentation where
   toHeaderValue Full        = "return=representation"
   toHeaderValue None        = "return=minimal"

src/PostgREST/ApiRequest/QueryParams.hs

@@ -60,29 +60,6 @@ import PostgREST.ApiRequest.Types (EmbedParam (..), EmbedPath, Field,
 
 import Protolude hiding (try)
 
-
--- $setup
--- Setup for doctests
--- >>> import Text.Pretty.Simple (pPrint)
--- >>> deriving instance Show QPError
--- >>> deriving instance Show TrileanVal
--- >>> deriving instance Show FtsOperator
--- >>> deriving instance Show QuantOperator
--- >>> deriving instance Show SimpleOperator
--- >>> deriving instance Show OpQuantifier
--- >>> deriving instance Show Operation
--- >>> deriving instance Show OpExpr
--- >>> deriving instance Show JsonOperand
--- >>> deriving instance Show JsonOperation
--- >>> deriving instance Show Filter
--- >>> deriving instance Show JoinType
--- >>> deriving instance Show SelectItem
--- >>> deriving instance Show OrderDirection
--- >>> deriving instance Show OrderNulls
--- >>> deriving instance Show OrderTerm
--- >>> deriving instance Show LogicOperator
--- >>> deriving instance Show LogicTree
-
 data QueryParams =
   QueryParams
     { qsCanonical      :: ByteString

src/PostgREST/ApiRequest/Types.hs

@@ -62,7 +62,7 @@ data SelectItem
     , selHint     :: Maybe Hint
     , selJoinType :: Maybe JoinType
     }
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data ApiRequestError
   = AmbiguousRelBetween Text Text [Relationship]
@@ -86,12 +86,15 @@ data ApiRequestError
   | UnacceptableSchema [Text]
   | UnsupportedMethod ByteString
   | ColumnNotFound Text Text
+  deriving Show
 
 data QPError = QPError Text Text
+  deriving Show
 data RangeError
   = NegativeLimit
   | LowerGTUpper
   | OutOfBounds Text Text
+  deriving Show
 
 type NodeName = Text
 type Depth = Integer
@@ -108,17 +111,17 @@ data OrderTerm
     , otDirection :: Maybe OrderDirection
     , otNullOrder :: Maybe OrderNulls
     }
-  deriving Eq
+  deriving (Eq, Show)
 
 data OrderDirection
   = OrderAsc
   | OrderDesc
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data OrderNulls
   = OrderNullsFirst
   | OrderNullsLast
-  deriving (Eq)
+  deriving (Eq, Show)
 
 type Field = (FieldName, JsonPath)
 type Cast = Text
@@ -135,7 +138,7 @@ data EmbedParam
 data JoinType
   = JTInner
   | JTLeft
-  deriving Eq
+  deriving (Eq, Show)
 
 -- | Path of the embedded levels, e.g "clients.projects.name=eq.." gives Path
 -- ["clients", "projects"]
@@ -149,7 +152,7 @@ type JsonPath = [JsonOperation]
 data JsonOperation
   = JArrow { jOp :: JsonOperand }
   | J2Arrow { jOp :: JsonOperand }
-  deriving (Eq, Ord)
+  deriving (Eq, Show, Ord)
 
 -- | Represents the key(`->'key'`) or index(`->'1`::int`), the index is Text
 -- because we reuse our escaping functons and let pg do the casting with
@@ -157,7 +160,7 @@ data JsonOperation
 data JsonOperand
   = JKey { jVal :: Text }
   | JIdx { jVal :: Text }
-  deriving (Eq, Ord)
+  deriving (Eq, Show, Ord)
 
 -- | Boolean logic expression tree e.g. "and(name.eq.N,or(id.eq.1,id.eq.2))" is:
 --
@@ -169,28 +172,27 @@ data JsonOperand
 data LogicTree
   = Expr Bool LogicOperator [LogicTree]
   | Stmnt Filter
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data LogicOperator
   = And
   | Or
-  deriving Eq
+  deriving (Eq, Show)
 
 data Filter
   = Filter
   { field  :: Field
   , opExpr :: OpExpr
   }
-  | FilterNullEmbed Bool FieldName
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data OpExpr
   = OpExpr Bool Operation
   | NoOpExpr Text
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data OpQuantifier = QuantAny | QuantAll
-  deriving Eq
+  deriving (Eq, Show)
 
 data Operation
   = Op SimpleOperator SingleVal
@@ -199,7 +201,7 @@ data Operation
   | Is TrileanVal
   | IsDistinctFrom SingleVal
   | Fts FtsOperator (Maybe Language) SingleVal
-  deriving (Eq)
+  deriving (Eq, Show)
 
 type Language = Text
 
@@ -215,7 +217,7 @@ data TrileanVal
   | TriFalse
   | TriNull
   | TriUnknown
-  deriving Eq
+  deriving (Eq, Show)
 
 -- Operators that are quantifiable, i.e. they can be used with the any/all modifiers
 data QuantOperator
@@ -228,7 +230,7 @@ data QuantOperator
   | OpILike
   | OpMatch
   | OpIMatch
-  deriving Eq
+  deriving (Eq, Show)
 
 data SimpleOperator
   = OpNotEqual
@@ -240,7 +242,7 @@ data SimpleOperator
   | OpNotExtendsRight
   | OpNotExtendsLeft
   | OpAdjacent
-  deriving Eq
+  deriving (Eq, Show)
 
 --
 -- | Operators for full text search operators
@@ -249,4 +251,4 @@ data FtsOperator
   | FilterFtsPlain
   | FilterFtsPhrase
   | FilterFtsWebsearch
-  deriving Eq
+  deriving (Eq, Show)

src/PostgREST/App.hs

@@ -9,7 +9,6 @@ Some of its functionality includes:
 - Producing HTTP Headers according to RFCs.
 - Content Negotiation
 -}
-{-# LANGUAGE LambdaCase      #-}
 {-# LANGUAGE RecordWildCards #-}
 module PostgREST.App
   ( SignalHandlerInstaller
@@ -20,7 +19,7 @@ module PostgREST.App
 
 
 import Control.Monad.Except     (liftEither)
-import Data.Either.Combinators  (mapLeft, whenLeft)
+import Data.Either.Combinators  (mapLeft)
 import Data.Maybe               (fromJust)
 import Data.String              (IsString (..))
 import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
@@ -28,11 +27,12 @@ import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
 import System.Posix.Types       (FileMode)
 
 import qualified Data.HashMap.Strict        as HM
-import qualified Hasql.Pool                 as SQL
+import qualified Data.Text.Encoding         as T
 import qualified Hasql.Transaction.Sessions as SQL
 import qualified Network.Wai                as Wai
 import qualified Network.Wai.Handler.Warp   as Warp
 
+import qualified PostgREST.Admin            as Admin
 import qualified PostgREST.ApiRequest       as ApiRequest
 import qualified PostgREST.ApiRequest.Types as ApiRequestTypes
 import qualified PostgREST.AppState         as AppState
@@ -43,7 +43,6 @@ import qualified PostgREST.Logger           as Logger
 import qualified PostgREST.Plan             as Plan
 import qualified PostgREST.Query            as Query
 import qualified PostgREST.Response         as Response
-import qualified PostgREST.Workers          as Workers
 
 import PostgREST.ApiRequest          (Action (..), ApiRequest (..),
                                       Mutation (..), Target (..))
@@ -55,7 +54,7 @@ import PostgREST.Error               (Error)
 import PostgREST.Query               (DbHandler)
 import PostgREST.SchemaCache         (SchemaCache (..))
 import PostgREST.SchemaCache.Routine (Routine (..))
-import PostgREST.Version             (prettyVersion)
+import PostgREST.Version             (docsVersion, prettyVersion)
 
 import Protolude hiding (Handler)
 
@@ -68,14 +67,14 @@ type SocketRunner = Warp.Settings -> Wai.Application -> FileMode -> FilePath ->
 run :: SignalHandlerInstaller -> Maybe SocketRunner -> AppState -> IO ()
 run installHandlers maybeRunWithSocket appState = do
   conf@AppConfig{..} <- AppState.getConfig appState
-  Workers.connectionWorker appState -- Loads the initial SchemaCache
+  AppState.connectionWorker appState -- Loads the initial SchemaCache
   installHandlers appState
   -- reload schema cache + config on NOTIFY
-  Workers.runListener conf appState
+  AppState.runListener conf appState
 
-  Workers.runAdmin conf appState $ serverSettings conf
+  Admin.runAdmin conf appState $ serverSettings conf
 
-  let app = postgrest conf appState (Workers.connectionWorker appState)
+  let app = postgrest conf appState (AppState.connectionWorker appState)
 
   case configServerUnixSocket of
     Just socket ->
@@ -154,27 +153,17 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@
   Response.optionalRollback conf apiRequest $
     handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache
 
-runDbHandler :: AppState.AppState -> Maybe Text -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
+runDbHandler :: AppState.AppState -> SQL.IsolationLevel -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
 runDbHandler appState isoLvl mode authenticated prepared handler = do
   dbResp <- lift $ do
     let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction
-    res <- AppState.usePool appState . transaction (toIsolationLevel isoLvl) mode $ runExceptT handler
-    whenLeft res (\case
-      SQL.AcquisitionTimeoutUsageError -> AppState.debounceLogAcquisitionTimeout appState -- this can happen rapidly for many requests, so we debounce
-      _ -> pure ())
-    return res
+    AppState.usePool appState . transaction isoLvl mode $ runExceptT handler
 
   resp <-
     liftEither . mapLeft Error.PgErr $
       mapLeft (Error.PgError authenticated) dbResp
 
   liftEither resp
-  where
-    toIsolationLevel = \case
-      Nothing                -> SQL.ReadCommitted
-      Just "repeatable read" -> SQL.RepeatableRead
-      Just "serializable"    -> SQL.Serializable
-      _                      -> SQL.ReadCommitted
 
 handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> Bool -> Bool -> PgVersion -> ApiRequest -> SchemaCache -> Handler IO Wai.Response
 handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@ApiRequest{..} sCache =
@@ -206,12 +195,12 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A
 
     (ActionInvoke invMethod, TargetProc identifier _) -> do
       cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq invMethod
-      resultSet <- runQuery (roleIsoLvl <|> pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
+      resultSet <- runQuery (fromMaybe roleIsoLvl $ pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
       return $ Response.invokeResponse invMethod (Plan.crProc cPlan) apiReq resultSet
 
     (ActionInspect headersOnly, TargetDefaultSpec tSchema) -> do
       oaiResult <- runQuery roleIsoLvl Plan.inspectPlanTxMode $ Query.openApiQuery sCache pgVer conf tSchema
-      return $ Response.openApiResponse headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
+      return $ Response.openApiResponse (T.decodeUtf8 prettyVersion, docsVersion) headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
 
     (ActionInfo, TargetIdent identifier) ->
       return $ Response.infoIdentResponse identifier sCache
@@ -229,7 +218,7 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A
       throwError $ Error.ApiRequestError ApiRequestTypes.NotFound
   where
     roleSettings = fromMaybe mempty (HM.lookup authRole $ configRoleSettings conf)
-    roleIsoLvl = decodeUtf8 <$> HM.lookup "default_transaction_isolation" roleSettings
+    roleIsoLvl = HM.findWithDefault SQL.ReadCommitted authRole $ configRoleIsoLvl conf
     runQuery isoLvl mode query =
       runDbHandler appState isoLvl mode authenticated prepared $ do
         Query.setPgLocals conf authClaims authRole (HM.toList roleSettings) apiReq pgVer

src/PostgREST/AppState.hs

@@ -1,9 +1,10 @@
+{-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 
 module PostgREST.AppState
   ( AppState
   , destroy
-  , flushPool
   , getConfig
   , getSchemaCache
   , getIsListenerOn
@@ -11,40 +12,52 @@ module PostgREST.AppState
   , getPgVersion
   , getRetryNextIn
   , getTime
-  , getWorkerSem
   , init
   , initWithPool
   , logWithZTime
-  , logPgrstError
-  , putConfig
   , putSchemaCache
-  , putIsListenerOn
   , putPgVersion
-  , putRetryNextIn
-  , signalListener
   , usePool
-  , waitListener
-  , debounceLogAcquisitionTimeout
+  , loadSchemaCache
+  , reReadConfig
+  , connectionWorker
+  , runListener
   ) where
 
-import qualified Data.ByteString.Lazy as LBS
-import qualified Data.Text.Encoding   as T
-import qualified Hasql.Pool           as SQL
-import qualified Hasql.Session        as SQL
-import qualified PostgREST.Error      as Error
+import qualified Data.ByteString.Char8      as BS
+import qualified Data.ByteString.Lazy       as LBS
+import           Data.Either.Combinators    (whenLeft)
+import qualified Data.Text.Encoding         as T
+import           Hasql.Connection           (acquire)
+import qualified Hasql.Notifications        as SQL
+import qualified Hasql.Pool                 as SQL
+import qualified Hasql.Session              as SQL
+import qualified Hasql.Transaction.Sessions as SQL
+import qualified PostgREST.Error            as Error
+import           PostgREST.Version          (prettyVersion)
 
 import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
                            updateAction)
 import Control.Debounce
+import Control.Retry      (RetryStatus, capDelay, exponentialBackoff,
+                           retrying, rsPreviousDelay)
 import Data.IORef         (IORef, atomicWriteIORef, newIORef,
                            readIORef)
 import Data.Time          (ZonedTime, defaultTimeLocale, formatTime,
                            getZonedTime)
 import Data.Time.Clock    (UTCTime, getCurrentTime)
 
-import PostgREST.Config           (AppConfig (..))
-import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
-import PostgREST.SchemaCache      (SchemaCache)
+import PostgREST.Config                  (AppConfig (..),
+                                          addFallbackAppName,
+                                          readAppConfig)
+import PostgREST.Config.Database         (queryDbSettings,
+                                          queryPgVersion,
+                                          queryRoleSettings)
+import PostgREST.Config.PgVersion        (PgVersion (..),
+                                          minimumPgVersion)
+import PostgREST.SchemaCache             (SchemaCache,
+                                          querySchemaCache)
+import PostgREST.SchemaCache.Identifiers (dumpQi)
 
 import Protolude
 
@@ -56,8 +69,8 @@ data AppState = AppState
   , statePgVersion                :: IORef PgVersion
   -- | No schema cache at the start. Will be filled in by the connectionWorker
   , stateSchemaCache              :: IORef (Maybe SchemaCache)
-  -- | Binary semaphore to make sure just one connectionWorker can run at a time
-  , stateWorkerSem                :: MVar ()
+  -- | starts the connection worker with a debounce
+  , debouncedConnectionWorker     :: IO ()
   -- | Binary semaphore used to sync the listener(NOTIFY reload) with the connectionWorker.
   , stateListener                 :: MVar ()
   -- | State of the LISTEN channel, used for the admin server checks
@@ -86,7 +99,7 @@ initWithPool pool conf = do
   appState <- AppState pool
     <$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step
     <*> newIORef Nothing
-    <*> newEmptyMVar
+    <*> pure (pure ())
     <*> newEmptyMVar
     <*> newIORef False
     <*> newIORef conf
@@ -96,7 +109,8 @@ initWithPool pool conf = do
     <*> newIORef 0
     <*> pure (pure ())
 
-  deb <-
+
+  debLogTimeout <-
     let oneSecond = 1000000 in
     mkDebounce defaultDebounceSettings
        { debounceAction = logPgrstError appState SQL.AcquisitionTimeoutUsageError
@@ -104,7 +118,15 @@ initWithPool pool conf = do
        , debounceEdge = leadingEdge -- logs at the start and the end
        }
 
-  return appState { debounceLogAcquisitionTimeout = deb }
+  debWorker <-
+    let decisecond = 100000 in
+    mkDebounce defaultDebounceSettings
+       { debounceAction = internalConnectionWorker appState
+       , debounceFreq = decisecond
+       , debounceEdge = leadingEdge -- runs the worker at the start and the end
+       }
+
+  return appState { debounceLogAcquisitionTimeout = debLogTimeout, debouncedConnectionWorker = debWorker }
 
 destroy :: AppState -> IO ()
 destroy = destroyPool
@@ -115,11 +137,17 @@ initPool AppConfig{..} =
     configDbPoolSize
     (fromIntegral configDbPoolAcquisitionTimeout)
     (fromIntegral configDbPoolMaxLifetime)
-    (toUtf8 configDbUri)
+    (fromIntegral configDbPoolMaxIdletime)
+    (toUtf8 $ addFallbackAppName prettyVersion configDbUri)
 
 -- | Run an action with a database connection.
 usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a)
-usePool AppState{..} = SQL.use statePool
+usePool AppState{..} x = do
+  res <- SQL.use statePool x
+  whenLeft res (\case
+    SQL.AcquisitionTimeoutUsageError -> debounceLogAcquisitionTimeout -- this can happen rapidly for many requests, so we debounce
+    _                                -> pure ())
+  return res
 
 -- | Flush the connection pool so that any future use of the pool will
 -- use connections freshly established after this call.
@@ -142,8 +170,8 @@ getSchemaCache = readIORef . stateSchemaCache
 putSchemaCache :: AppState -> Maybe SchemaCache -> IO ()
 putSchemaCache appState = atomicWriteIORef (stateSchemaCache appState)
 
-getWorkerSem :: AppState -> MVar ()
-getWorkerSem = stateWorkerSem
+connectionWorker :: AppState -> IO ()
+connectionWorker = debouncedConnectionWorker
 
 getRetryNextIn :: AppState -> IO Int
 getRetryNextIn = readIORef . stateRetryNextIn
@@ -188,3 +216,263 @@ getIsListenerOn = readIORef . stateIsListenerOn
 
 putIsListenerOn :: AppState -> Bool -> IO ()
 putIsListenerOn = atomicWriteIORef . stateIsListenerOn
+
+-- | Schema cache status
+data SCacheStatus
+  = SCLoaded
+  | SCOnRetry
+  | SCFatalFail
+
+-- | Load the SchemaCache by using a connection from the pool.
+loadSchemaCache :: AppState -> IO SCacheStatus
+loadSchemaCache appState = do
+  conf@AppConfig{..} <- getConfig appState
+  result <-
+    let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
+    usePool appState . transaction SQL.ReadCommitted SQL.Read $
+      querySchemaCache conf
+  case result of
+    Left e -> do
+      case checkIsFatal e of
+        Just hint -> do
+          logWithZTime appState "A fatal error ocurred when loading the schema cache"
+          logPgrstError appState e
+          logWithZTime appState hint
+          return SCFatalFail
+        Nothing -> do
+          putSchemaCache appState Nothing
+          logWithZTime appState "An error ocurred when loading the schema cache"
+          logPgrstError appState e
+          return SCOnRetry
+
+    Right sCache -> do
+      putSchemaCache appState (Just sCache)
+      logWithZTime appState "Schema cache loaded"
+      return SCLoaded
+
+-- | Current database connection status data ConnectionStatus
+data ConnectionStatus
+  = NotConnected
+  | Connected PgVersion
+  | FatalConnectionError Text
+  deriving (Eq)
+
+-- | The purpose of this worker is to obtain a healthy connection to pg and an
+-- up-to-date schema cache(SchemaCache).  This method is meant to be called
+-- multiple times by the same thread, but does nothing if the previous
+-- invocation has not terminated. In all cases this method does not halt the
+-- calling thread, the work is performed in a separate thread.
+--
+-- Background thread that does the following :
+--  1. Tries to connect to pg server and will keep trying until success.
+--  2. Checks if the pg version is supported and if it's not it kills the main
+--     program.
+--  3. Obtains the sCache. If this fails, it goes back to 1.
+internalConnectionWorker :: AppState -> IO ()
+internalConnectionWorker appState = work
+  where
+    work = do
+      AppConfig{..} <- getConfig appState
+      logWithZTime appState $ "Starting PostgREST " <> T.decodeUtf8 prettyVersion <> "..."
+      logWithZTime appState "Attempting to connect to the database..."
+      connected <- establishConnection appState
+      case connected of
+        FatalConnectionError reason ->
+          -- Fatal error when connecting
+          logWithZTime appState reason >> killThread (getMainThreadId appState)
+        NotConnected ->
+          -- Unreachable because establishConnection will keep trying to connect
+          return ()
+        Connected actualPgVersion -> do
+          -- Procede with initialization
+          putPgVersion appState actualPgVersion
+          when configDbChannelEnabled $
+            signalListener appState
+          logWithZTime appState "Connection successful"
+          -- this could be fail because the connection drops, but the loadSchemaCache will pick the error and retry again
+          -- We cannot retry after it fails immediately, because db-pre-config could have user errors. We just log the error and continue.
+          when configDbConfig $ reReadConfig False appState
+          scStatus <- loadSchemaCache appState
+          case scStatus of
+            SCLoaded ->
+              -- do nothing and proceed if the load was successful
+              return ()
+            SCOnRetry ->
+              -- retry reloading the schema cache
+              work
+            SCFatalFail ->
+              -- die if our schema cache query has an error
+              killThread $ getMainThreadId appState
+
+-- | Repeatedly flush the pool, and check if a connection from the
+-- pool allows access to the PostgreSQL database.
+--
+-- Releasing the pool is key for rapid recovery. Otherwise, the pool
+-- timeout would have to be reached for new healthy connections to be acquired.
+-- Which might not happen if the server is busy with requests. No idle
+-- connection, no pool timeout.
+--
+-- The connection tries are capped, but if the connection times out no error is
+-- thrown, just 'False' is returned.
+establishConnection :: AppState -> IO ConnectionStatus
+establishConnection appState =
+  retrying retrySettings shouldRetry $
+    const $ flushPool appState >> getConnectionStatus
+  where
+    retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
+    delayMicroseconds = 32000000 -- 32 seconds
+    backoffMicroseconds = 1000000 -- 1 second
+
+    getConnectionStatus :: IO ConnectionStatus
+    getConnectionStatus = do
+      pgVersion <- usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
+      case pgVersion of
+        Left e -> do
+          logPgrstError appState e
+          case checkIsFatal e of
+            Just reason ->
+              return $ FatalConnectionError reason
+            Nothing ->
+              return NotConnected
+        Right version ->
+          if version < minimumPgVersion then
+            return . FatalConnectionError $
+              "Cannot run in this PostgreSQL version, PostgREST needs at least "
+              <> pgvName minimumPgVersion
+          else
+            return . Connected  $ version
+
+    shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
+    shouldRetry rs isConnSucc = do
+      let
+        delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
+        itShould = NotConnected == isConnSucc
+      when itShould . logWithZTime appState $
+        "Attempting to reconnect to the database in "
+        <> (show delay::Text)
+        <> " seconds..."
+      when itShould $ putRetryNextIn appState delay
+      return itShould
+
+-- | Re-reads the config plus config options from the db
+reReadConfig :: Bool -> AppState -> IO ()
+reReadConfig startingUp appState = do
+  AppConfig{..} <- getConfig appState
+  dbSettings <-
+    if configDbConfig then do
+      qDbSettings <- usePool appState $ queryDbSettings (dumpQi <$> configDbPreConfig) configDbPreparedStatements
+      case qDbSettings of
+        Left e -> do
+          logWithZTime appState
+            "An error ocurred when trying to query database settings for the config parameters"
+          case checkIsFatal e of
+            Just hint -> do
+              logPgrstError appState e
+              logWithZTime appState hint
+              killThread (getMainThreadId appState)
+            Nothing -> do
+              logPgrstError appState e
+          pure mempty
+        Right x -> pure x
+    else
+      pure mempty
+  (roleSettings, roleIsolationLvl) <-
+    if configDbConfig then do
+      rSettings <- usePool appState $ queryRoleSettings configDbPreparedStatements
+      case rSettings of
+        Left e -> do
+          logWithZTime appState "An error ocurred when trying to query the role settings"
+          logPgrstError appState e
+          pure (mempty, mempty)
+        Right x -> pure x
+    else
+      pure mempty
+  readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings roleIsolationLvl >>= \case
+    Left err   ->
+      if startingUp then
+        panic err -- die on invalid config if the program is starting up
+      else
+        logWithZTime appState $ "Failed reloading config: " <> err
+    Right newConf -> do
+      putConfig appState newConf
+      if startingUp then
+        pass
+      else
+        logWithZTime appState "Config reloaded"
+
+
+runListener :: AppConfig -> AppState -> IO ()
+runListener AppConfig{configDbChannelEnabled} appState =
+  when configDbChannelEnabled $ listener appState
+
+-- | Starts a dedicated pg connection to LISTEN for notifications.  When a
+-- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
+-- cache.  It uses the connectionWorker in case the LISTEN connection dies.
+listener :: AppState -> IO ()
+listener appState = do
+  AppConfig{..} <- getConfig appState
+  let dbChannel = toS configDbChannel
+
+  -- The listener has to wait for a signal from the connectionWorker.
+  -- This is because when the connection to the db is lost, the listener also
+  -- tries to recover the connection, but not with the same pace as the connectionWorker.
+  -- Not waiting makes stderr quickly fill with connection retries messages from the listener.
+  waitListener appState
+
+  -- forkFinally allows to detect if the thread dies
+  void . flip forkFinally (handleFinally dbChannel) $ do
+    dbOrError <- acquire $ toUtf8 (addFallbackAppName prettyVersion configDbUri)
+    case dbOrError of
+      Right db -> do
+        logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
+        putIsListenerOn appState True
+        SQL.listen db $ SQL.toPgIdentifier dbChannel
+        SQL.waitForNotifications handleNotification db
+      _ ->
+        die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
+  where
+    handleFinally dbChannel _ = do
+      -- if the thread dies, we try to recover
+      logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
+      putIsListenerOn appState False
+      -- assume the pool connection was also lost, call the connection worker
+      connectionWorker appState
+      -- retry the listener
+      listener appState
+
+    handleNotification _ msg
+      | BS.null msg            = cacheReloader
+      | msg == "reload schema" = cacheReloader
+      | msg == "reload config" = reReadConfig False appState
+      | otherwise              = pure () -- Do nothing if anything else than an empty message is sent
+
+    cacheReloader =
+      -- reloads the schema cache + restarts pool connections
+      -- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
+      connectionWorker appState
+
+checkIsFatal :: SQL.UsageError -> Maybe Text
+checkIsFatal (SQL.ConnectionUsageError e)
+  | isAuthFailureMessage = Just $ toS failureMessage
+  | otherwise = Nothing
+  where isAuthFailureMessage =
+          ("FATAL:  password authentication failed" `isInfixOf` failureMessage) ||
+          ("no password supplied" `isInfixOf` failureMessage)
+        failureMessage = BS.unpack $ fromMaybe mempty e
+checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
+  = case serverError of
+      -- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
+      SQL.ServerError "42601" _ _ _ _
+        -> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
+      -- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
+      -- This would mean that a connection pooler in transaction mode is being used
+      -- while prepared statements are enabled in the PostgREST configuration,
+      -- both of which are incompatible with each other.
+      SQL.ServerError "42P05" _ _ _ _
+        -> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
+      -- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
+      -- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
+      SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
+        -> Just "Hint: Connection poolers in statement mode are not supported."
+      _ -> Nothing
+checkIsFatal _ = Nothing

src/PostgREST/Auth.hs

@@ -63,7 +63,7 @@ parseToken AppConfig{..} token time = do
   liftEither . mapLeft jwtClaimsError $ JSON.toJSON <$> eitherClaims
   where
     validation =
-      JWT.defaultJWTValidationSettings audienceCheck & set JWT.allowedSkew 1
+      JWT.defaultJWTValidationSettings audienceCheck & set JWT.allowedSkew 30
 
     audienceCheck :: JWT.StringOrURI -> Bool
     audienceCheck = maybe (const True) (==) configJwtAudience

src/PostgREST/CLI.hs

@@ -21,7 +21,6 @@ import PostgREST.AppState    (AppState)
 import PostgREST.Config      (AppConfig (..))
 import PostgREST.SchemaCache (querySchemaCache)
 import PostgREST.Version     (prettyVersion)
-import PostgREST.Workers     (reReadConfig)
 
 import qualified PostgREST.App      as App
 import qualified PostgREST.AppState as AppState
@@ -33,7 +32,7 @@ import Protolude hiding (hPutStrLn)
 main :: App.SignalHandlerInstaller -> Maybe App.SocketRunner -> CLI -> IO ()
 main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
   conf@AppConfig{..} <-
-    either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty
+    either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty mempty
 
   -- Per https://github.com/PostgREST/postgrest/issues/268, we want to
   -- explicitly close the connections to PostgreSQL on shutdown.
@@ -43,7 +42,7 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
     AppState.destroy
     (\appState -> case cliCommand of
       CmdDumpConfig -> do
-        when configDbConfig $ reReadConfig True appState
+        when configDbConfig $ AppState.reReadConfig True appState
         putStr . Config.toText =<< AppState.getConfig appState
       CmdDumpSchema -> putStrLn =<< dumpSchema appState
       CmdRun -> App.run installSignalHandlers runAppWithSocket appState)
@@ -51,15 +50,12 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
 -- | Dump SchemaCache schema to JSON
 dumpSchema :: AppState -> IO LBS.ByteString
 dumpSchema appState = do
-  AppConfig{..} <- AppState.getConfig appState
+  conf@AppConfig{..} <- AppState.getConfig appState
   result <-
     let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
     AppState.usePool appState $
       transaction SQL.ReadCommitted SQL.Read $
-        querySchemaCache
-          (toList configDbSchemas)
-          configDbExtraSearchPath
-          configDbPreparedStatements
+        querySchemaCache conf
   case result of
     Left e -> do
       hPutStrLn stderr $ "An error ocurred when loading the schema cache:\n" <> show e
@@ -84,7 +80,7 @@ readCLIShowHelp =
   where
     prefs = O.prefs $ O.showHelpOnError <> O.showHelpOnEmpty
     opts = O.info parser $ O.fullDesc <> progDesc
-    parser = O.helper <*> exampleParser <*> cliParser
+    parser = O.helper <*> versionFlag <*> exampleParser <*> cliParser
 
     progDesc =
       O.progDesc $
@@ -92,6 +88,12 @@ readCLIShowHelp =
         <> BS.unpack prettyVersion
         <> " / create a REST API to an existing Postgres database"
 
+    versionFlag =
+      O.infoOption ("PostgREST " <> BS.unpack prettyVersion) $
+        O.long "version"
+        <> O.short 'v'
+        <> O.help "Show the version information"
+
     exampleParser =
       O.infoOption exampleConfigFile $
         O.long "example"
@@ -136,6 +138,9 @@ exampleConfigFile =
       |## Enable in-database configuration
       |db-config = true
       |
+      |## Function for in-database configuration
+      |## db-pre-config = "postgrest.pre_config"
+      |
       |## Extra schemas to add to the search_path of every request
       |db-extra-search-path = "public"
       |
@@ -154,6 +159,9 @@ exampleConfigFile =
       |## Time in seconds after which to recycle pool connections
       |# db-pool-max-lifetime = 1800
       |
+      |## Time in seconds after which to recycle unused pool connections
+      |# db-pool-max-idletime = 30
+      |
       |## Stored proc to exec immediately after auth
       |# db-pre-request = "stored_proc_name"
       |

src/PostgREST/Config.hs

@@ -24,6 +24,7 @@ module PostgREST.Config
   , readPGRSTEnvironment
   , toURI
   , parseSecret
+  , addFallbackAppName
   ) where
 
 import qualified Crypto.JOSE.Types      as JOSE
@@ -47,11 +48,15 @@ import Data.List               (lookup)
 import Data.List.NonEmpty      (fromList, toList)
 import Data.Maybe              (fromJust)
 import Data.Scientific         (floatingOrInteger)
+import Network.URI             (escapeURIString,
+                                isUnescapedInURIComponent, parseURI,
+                                uriQuery)
 import Numeric                 (readOct, showOct)
 import System.Environment      (getEnvironment)
 import System.Posix.Types      (FileMode)
 
-import PostgREST.Config.Database         (RoleSettings)
+import PostgREST.Config.Database         (RoleIsolationLvl,
+                                          RoleSettings)
 import PostgREST.Config.JSPath           (JSPath, JSPathExp (..),
                                           dumpJSPath, pRoleClaimKey)
 import PostgREST.Config.Proxy            (Proxy (..),
@@ -74,11 +79,13 @@ data AppConfig = AppConfig
   , configDbPoolSize               :: Int
   , configDbPoolAcquisitionTimeout :: Int
   , configDbPoolMaxLifetime        :: Int
+  , configDbPoolMaxIdletime        :: Int
   , configDbPreRequest             :: Maybe QualifiedIdentifier
   , configDbPreparedStatements     :: Bool
   , configDbRootSpec               :: Maybe QualifiedIdentifier
   , configDbSchemas                :: NonEmpty Text
   , configDbConfig                 :: Bool
+  , configDbPreConfig              :: Maybe QualifiedIdentifier
   , configDbTxAllowOverride        :: Bool
   , configDbTxRollbackAll          :: Bool
   , configDbUri                    :: Text
@@ -101,6 +108,8 @@ data AppConfig = AppConfig
   , configServerUnixSocketMode     :: FileMode
   , configAdminServerPort          :: Maybe Int
   , configRoleSettings             :: RoleSettings
+  , configRoleIsoLvl               :: RoleIsolationLvl
+  , configInternalSCSleep          :: Maybe Int32
   }
 
 data LogLevel = LogCrit | LogError | LogWarn | LogInfo
@@ -137,11 +146,13 @@ toText conf =
       ,("db-pool",                       show . configDbPoolSize)
       ,("db-pool-acquisition-timeout",   show . configDbPoolAcquisitionTimeout)
       ,("db-pool-max-lifetime",          show . configDbPoolMaxLifetime)
+      ,("db-pool-max-idletime",          show . configDbPoolMaxIdletime)
       ,("db-pre-request",            q . maybe mempty dumpQi . configDbPreRequest)
       ,("db-prepared-statements",        T.toLower . show . configDbPreparedStatements)
       ,("db-root-spec",              q . maybe mempty dumpQi . configDbRootSpec)
       ,("db-schemas",                q . T.intercalate "," . toList . configDbSchemas)
       ,("db-config",                     T.toLower . show . configDbConfig)
+      ,("db-pre-config",             q . maybe mempty dumpQi . configDbPreConfig)
       ,("db-tx-end",                 q . showTxEnd)
       ,("db-uri",                    q . configDbUri)
       ,("db-use-legacy-gucs",            T.toLower . show . configDbUseLegacyGucs)
@@ -193,13 +204,13 @@ instance JustIfMaybe a (Maybe a) where
 
 -- | Reads and parses the config and overrides its parameters from env vars,
 -- files or db settings.
-readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> IO (Either Text AppConfig)
-readAppConfig dbSettings optPath prevDbUri roleSettings = do
+readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> RoleIsolationLvl -> IO (Either Text AppConfig)
+readAppConfig dbSettings optPath prevDbUri roleSettings roleIsolationLvl = do
   env <- readPGRSTEnvironment
   -- if no filename provided, start with an empty map to read config from environment
   conf <- maybe (return $ Right M.empty) loadConfig optPath
 
-  case C.runParser (parser optPath env dbSettings roleSettings) =<< mapLeft show conf of
+  case C.runParser (parser optPath env dbSettings roleSettings roleIsolationLvl) =<< mapLeft show conf of
     Left err ->
       return . Left $ "Error in config " <> err
     Right parsedConfig ->
@@ -214,8 +225,8 @@ readAppConfig dbSettings optPath prevDbUri roleSettings = do
       decodeJWKS <$>
         (decodeSecret =<< readSecretFile =<< readDbUriFile prevDbUri parsedConfig)
 
-parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> C.Parser C.Config AppConfig
-parser optPath env dbSettings roleSettings =
+parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> RoleIsolationLvl -> C.Parser C.Config AppConfig
+parser optPath env dbSettings roleSettings roleIsolationLvl =
   AppConfig
     <$> parseAppSettings "app.settings"
     <*> (fmap encodeUtf8 <$> optString "db-anon-role")
@@ -228,6 +239,8 @@ parser optPath env dbSettings roleSettings =
     <*> (fromMaybe 10 <$> optInt "db-pool")
     <*> (fromMaybe 10 <$> optInt "db-pool-acquisition-timeout")
     <*> (fromMaybe 1800 <$> optInt "db-pool-max-lifetime")
+    <*> (fromMaybe 30 <$> optWithAlias (optInt "db-pool-timeout")
+                                       (optInt "db-pool-max-idletime"))
     <*> (fmap toQi <$> optWithAlias (optString "db-pre-request")
                                     (optString "pre-request"))
     <*> (fromMaybe True <$> optBool "db-prepared-statements")
@@ -236,6 +249,7 @@ parser optPath env dbSettings roleSettings =
     <*> (fromList . maybe ["public"] splitOnCommas <$> optWithAlias (optValue "db-schemas")
                                                                     (optValue "db-schema"))
     <*> (fromMaybe True <$> optBool "db-config")
+    <*> (fmap toQi <$> optString "db-pre-config")
     <*> parseTxEnd "db-tx-end" snd
     <*> parseTxEnd "db-tx-end" fst
     <*> (fromMaybe "postgresql://" <$> optString "db-uri")
@@ -260,6 +274,8 @@ parser optPath env dbSettings roleSettings =
     <*> parseSocketFileMode "server-unix-socket-mode"
     <*> optInt "admin-server-port"
     <*> pure roleSettings
+    <*> pure roleIsolationLvl
+    <*> optInt "internal-schema-cache-sleep"
   where
     parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
     parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
@@ -354,21 +370,14 @@ parser optPath env dbSettings roleSettings =
                                (C.Key -> C.Parser C.Value a -> C.Parser C.Config b) ->
                                C.Key -> (C.Value -> a) -> C.Parser C.Config b
     overrideFromDbOrEnvironment necessity key coercion =
-      case reloadableDbSetting <|> M.lookup envVarName env of
+      case dbConf <|> M.lookup envVarName env of
         Just dbOrEnvVal -> pure $ justIfMaybe $ coercion $ C.String dbOrEnvVal
-        Nothing  -> necessity key (coercion <$> C.value)
+        Nothing         -> necessity key (coercion <$> C.value)
       where
         dashToUnderscore '-' = '_'
         dashToUnderscore c   = c
         envVarName = "PGRST_" <> (toUpper . dashToUnderscore <$> toS key)
-        reloadableDbSetting =
-          let dbSettingName = T.pack $ dashToUnderscore <$> toS key in
-          if dbSettingName `notElem` [
-            "server_host", "server_port", "server_unix_socket", "server_unix_socket_mode", "admin_server_port", "log_level",
-            "db_uri", "db_channel_enabled", "db_channel", "db_pool", "db_pool_acquisition_timeout",
-            "db_pool_max_lifetime", "db_config"]
-          then lookup dbSettingName dbSettings
-          else Nothing
+        dbConf = lookup (T.pack $ dashToUnderscore <$> toS key) dbSettings
 
     coerceText :: C.Value -> Text
     coerceText (C.String s) = s
@@ -455,3 +464,41 @@ type Environment = M.Map [Char] Text
 readPGRSTEnvironment :: IO Environment
 readPGRSTEnvironment =
   M.map T.pack . M.fromList . filter (isPrefixOf "PGRST_" . fst) <$> getEnvironment
+
+-- | Adds a `fallback_application_name` value to the connection string. This allows querying the PostgREST version on pg_stat_activity.
+--
+-- >>> let ver = "11.1.0 (5a04ec7)"::ByteString
+-- >>> let strangeVer = "11'1&0@#$%,.:\"[]{}?+^()=asdfqwer"::ByteString
+--
+-- >>> addFallbackAppName ver "postgres://user:pass@host:5432/postgres"
+-- "postgres://user:pass@host:5432/postgres?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgres://user:pass@host:5432/postgres?"
+-- "postgres://user:pass@host:5432/postgres?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgres:///postgres?host=server&port=5432"
+-- "postgres:///postgres?host=server&port=5432&fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "host=localhost port=5432 dbname=postgres"
+-- "host=localhost port=5432 dbname=postgres fallback_application_name='PostgREST 11.1.0 (5a04ec7)'"
+--
+-- >>> addFallbackAppName ver "postgresql://"
+-- "postgresql://?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName strangeVer "host=localhost port=5432 dbname=postgres"
+-- "host=localhost port=5432 dbname=postgres fallback_application_name='PostgREST 11\\'1&0@#$%,.:\"[]{}?+^()=asdfqwer'"
+--
+-- >>> addFallbackAppName strangeVer "postgres:///postgres?host=server&port=5432"
+-- "postgres:///postgres?host=server&port=5432&fallback_application_name=PostgREST%2011%271%260%40%23%24%25%2C.%3A%22%5B%5D%7B%7D%3F%2B%5E%28%29%3Dasdfqwer"
+addFallbackAppName :: ByteString -> Text -> Text
+addFallbackAppName version dbUri = dbUri <>
+  case uriQuery <$> parseURI (toS dbUri) of
+    Nothing  -> " " <> keyValFmt -- Assume key/value connection string if the uri is not valid
+    Just ""  -> "?" <> uriFmt
+    Just "?" -> uriFmt
+    _        -> "&" <> uriFmt
+  where
+    uriFmt = pKeyWord <> toS (escapeURIString isUnescapedInURIComponent $ toS pgrstVer)
+    keyValFmt = pKeyWord <> "'" <> T.replace "'" "\\'" pgrstVer <> "'"
+    pKeyWord = "fallback_application_name="
+    pgrstVer = "PostgREST " <> T.decodeUtf8 version

src/PostgREST/Config/Database.hs

@@ -6,6 +6,8 @@ module PostgREST.Config.Database
   , queryRoleSettings
   , queryPgVersion
   , RoleSettings
+  , RoleIsolationLvl
+  , toIsolationLevel
   ) where
 
 import Control.Arrow ((***))
@@ -21,11 +23,47 @@ import qualified Hasql.Statement            as SQL
 import qualified Hasql.Transaction          as SQL
 import qualified Hasql.Transaction.Sessions as SQL
 
-import Text.InterpolatedString.Perl6 (q)
+import Text.InterpolatedString.Perl6 (q, qc)
 
 import Protolude
 
-type RoleSettings = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
+type RoleSettings     = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
+type RoleIsolationLvl = HM.HashMap ByteString SQL.IsolationLevel
+
+toIsolationLevel :: (Eq a, IsString a) => a -> SQL.IsolationLevel
+toIsolationLevel a = case a of
+  "repeatable read" -> SQL.RepeatableRead
+  "serializable"    -> SQL.Serializable
+  _                 -> SQL.ReadCommitted
+
+prefix :: Text
+prefix = "pgrst."
+
+-- | In-db settings names
+dbSettingsNames :: [Text]
+dbSettingsNames =
+  (prefix <>) <$>
+  ["db_anon_role"
+  ,"db_pre_config"
+  ,"db_extra_search_path"
+  ,"db_max_rows"
+  ,"db_plan_enabled"
+  ,"db_pre_request"
+  ,"db_prepared_statements"
+  ,"db_root_spec"
+  ,"db_schemas"
+  ,"db_tx_end"
+  ,"db_use_legacy_gucs"
+  ,"jwt_aud"
+  ,"jwt_role_claim_key"
+  ,"jwt_secret"
+  ,"jwt_secret_is_base64"
+  ,"openapi_mode"
+  ,"openapi_security_active"
+  ,"openapi_server_proxy_uri"
+  ,"raw_media_types"
+  ,"server_trace_header"
+  ]
 
 queryPgVersion :: Bool -> Session PgVersion
 queryPgVersion prepared = statement mempty $ pgVersionStatement prepared
@@ -36,46 +74,62 @@ pgVersionStatement = SQL.Statement sql HE.noParams versionRow
     sql = "SELECT current_setting('server_version_num')::integer, current_setting('server_version')"
     versionRow = HD.singleRow $ PgVersion <$> column HD.int4 <*> column HD.text
 
-queryDbSettings :: Bool -> Session [(Text, Text)]
-queryDbSettings prepared =
+-- | Query the in-database configuration. The settings have the following priorities:
+--
+-- 1. Role + with database-specific settings:
+--    ALTER ROLE authenticator IN DATABASE postgres SET <prefix>jwt_aud = 'val';
+-- 2. Role + with settings:
+--    ALTER ROLE authenticator SET <prefix>jwt_aud = 'overridden';
+-- 3. pre-config function:
+--    CREATE FUNCTION pre_config() .. PERFORM set_config(<prefix>jwt_aud, 'pre_config_aud'..)
+--
+-- The example above will result in <prefix>jwt_aud = 'val'
+-- A setting on the database only will have no effect: ALTER DATABASE postgres SET <prefix>jwt_aud = 'xx'
+queryDbSettings :: Maybe Text -> Bool -> Session [(Text, Text)]
+queryDbSettings preConfFunc prepared =
   let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
-  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ dbSettingsStatement prepared
-
--- | Get db settings from the connection role. Global settings will be overridden by database specific settings.
-dbSettingsStatement :: Bool -> SQL.Statement () [(Text, Text)]
-dbSettingsStatement = SQL.Statement sql HE.noParams decodeSettings
+  transaction SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared
   where
-    sql = [q|
+    sql = [qc|
       WITH
-      role_setting (database, setting) AS (
-        SELECT setdatabase,
-               unnest(setconfig)
-          FROM pg_catalog.pg_db_role_setting
-         WHERE setrole = CURRENT_USER::regrole::oid
-           AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
+      role_setting AS (
+        SELECT setdatabase as database,
+               unnest(setconfig) as setting
+        FROM pg_catalog.pg_db_role_setting
+        WHERE setrole = CURRENT_USER::regrole::oid
+          AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
       ),
-      kv_settings (database, k, v) AS (
+      kv_settings AS (
         SELECT database,
-               substr(setting, 1, strpos(setting, '=') - 1),
-               substr(setting, strpos(setting, '=') + 1)
-          FROM role_setting
-         WHERE setting LIKE 'pgrst.%'
+               substr(setting, 1, strpos(setting, '=') - 1) as k,
+               substr(setting, strpos(setting, '=') + 1) as v
+        FROM role_setting
+        {preConfigF}
       )
       SELECT DISTINCT ON (key)
-             replace(k, 'pgrst.', '') AS key,
+             replace(k, '{prefix}', '') AS key,
              v AS value
-        FROM kv_settings
-       ORDER BY key, database DESC;
+      FROM kv_settings
+      WHERE k = ANY($1) AND v IS NOT NULL
+      ORDER BY key, database DESC NULLS LAST;
     |]
+    preConfigF = case preConfFunc of
+      Nothing   -> mempty
+      Just func -> [qc|
+          UNION
+          SELECT
+            null as database,
+            x as k,
+            current_setting(x, true) as v
+          FROM unnest($1) x
+          JOIN {func}() _ ON TRUE
+      |]::Text
     decodeSettings = HD.rowList $ (,) <$> column HD.text <*> column HD.text
 
-queryRoleSettings :: Bool -> Session RoleSettings
+queryRoleSettings :: Bool -> Session (RoleSettings, RoleIsolationLvl)
 queryRoleSettings prepared =
   let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
-  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ roleSettingsStatement prepared
-
-roleSettingsStatement :: Bool -> SQL.Statement () RoleSettings
-roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
+  transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared
   where
     sql = [q|
       with
@@ -91,18 +145,41 @@ roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
           substr(setting, 1, strpos(setting, '=') - 1) as key,
           lower(substr(setting, strpos(setting, '=') + 1)) as value
         FROM role_setting
+      ),
+      iso_setting AS (
+        SELECT rolname, value
+        FROM kv_settings
+        WHERE key = 'default_transaction_isolation'
       )
-      select rolname, array_agg(row(key, value))
-      from kv_settings
-      group by rolname;
+      select
+        kv.rolname,
+        i.value as iso_lvl,
+        array_agg(row(kv.key, kv.value)) filter (where key <> 'default_transation_isolation') as role_settings
+      from kv_settings kv
+      join pg_settings ps on ps.name = kv.key and ps.context = 'user'
+      left join iso_setting i on i.rolname = kv.rolname
+      group by kv.rolname, i.value;
     |]
-    decodeRoleSettings = HM.fromList . map (bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>))) <$> HD.rowList aRow
-    aRow :: HD.Row (Text, [(Text, Text)])
-    aRow = (,) <$> column HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
+
+    processRows :: [(Text, Maybe Text, [(Text, Text)])] -> (RoleSettings, RoleIsolationLvl)
+    processRows rs =
+      let
+        rowsWRoleSettings = [ (x, z) | (x, _, z) <- rs ]
+        rowsWIsolation    = [ (x, y) | (x, Just y, _) <- rs ]
+      in
+      ( HM.fromList $ bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>)) <$> rowsWRoleSettings
+      , HM.fromList $ (encodeUtf8 *** toIsolationLevel) <$> rowsWIsolation
+      )
+
+    rows :: HD.Result [(Text, Maybe Text, [(Text, Text)])]
+    rows = HD.rowList $ (,,) <$> column HD.text <*> nullableColumn HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
 
 column :: HD.Value a -> HD.Row a
 column = HD.column . HD.nonNullable
 
+nullableColumn :: HD.Value a -> HD.Row (Maybe a)
+nullableColumn = HD.column . HD.nullable
+
 compositeField :: HD.Value a -> HD.Composite a
 compositeField = HD.field . HD.nonNullable
 
@@ -111,3 +188,9 @@ compositeArrayColumn = arrayColumn . HD.composite
 
 arrayColumn :: HD.Value a -> HD.Row [a]
 arrayColumn = column . HD.listArray . HD.nonNullable
+
+param :: HE.Value a -> HE.Params a
+param = HE.param . HE.nonNullable
+
+arrayParam :: HE.Value a -> HE.Params [a]
+arrayParam = param . HE.foldableArray . HE.nonNullable

src/PostgREST/Error.hs

@@ -11,7 +11,6 @@ module PostgREST.Error
   , PgError(..)
   , Error(..)
   , errorPayload
-  , checkIsFatal
   , singularityError
   ) where
 
@@ -429,6 +428,7 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
         '5':'3':_ -> HTTP.status503 -- insufficient resources
         '5':'4':_ -> HTTP.status413 -- too complex
         '5':'5':_ -> HTTP.status500 -- obj not on prereq state
+        '5':'7':'P':'0':'1':_ -> HTTP.status503 -- terminating connection due to administrator command
         '5':'7':_ -> HTTP.status500 -- operator intervention
         '5':'8':_ -> HTTP.status500 -- system error
         'F':'0':_ -> HTTP.status500 -- conf file error
@@ -446,29 +446,6 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
 
     _                       -> HTTP.status500
 
-checkIsFatal :: SQL.UsageError -> Maybe Text
-checkIsFatal (SQL.ConnectionUsageError e)
-  | isAuthFailureMessage = Just $ toS failureMessage
-  | otherwise = Nothing
-  where isAuthFailureMessage = "FATAL:  password authentication failed" `isInfixOf` failureMessage
-        failureMessage = BS.unpack $ fromMaybe mempty e
-checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
-  = case serverError of
-      -- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
-      SQL.ServerError "42601" _ _ _ _
-        -> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
-      -- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
-      -- This would mean that a connection pooler in transaction mode is being used
-      -- while prepared statements are enabled in the PostgREST configuration,
-      -- both of which are incompatible with each other.
-      SQL.ServerError "42P05" _ _ _ _
-        -> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
-      -- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
-      -- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
-      SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
-        -> Just "Hint: Connection poolers in statement mode are not supported."
-      _ -> Nothing
-checkIsFatal _ = Nothing
 
 
 data Error
@@ -501,7 +478,7 @@ instance PgrstError Error where
   headers (JwtTokenInvalid m)    = [MediaType.toContentType MTApplicationJSON, invalidTokenHeader m]
   headers JwtTokenRequired       = [MediaType.toContentType MTApplicationJSON, requiredTokenHeader]
   headers (PgErr err)            = headers err
-  headers SingularityError{}     = [MediaType.toContentType MTSingularJSON]
+  headers SingularityError{}     = [MediaType.toContentType (MTSingularJSON False)]
   headers _                      = [MediaType.toContentType MTApplicationJSON]
 
 instance JSON.ToJSON Error where
@@ -553,7 +530,7 @@ instance JSON.ToJSON Error where
   toJSON (SingularityError n) = JSON.object [
     "code"    .= ApiRequestErrorCode16,
     "message" .= ("JSON object requested, multiple (or no) rows returned" :: Text),
-    "details" .= T.unwords ["Results contain", show n, "rows,", T.decodeUtf8 (MediaType.toMime MTSingularJSON), "requires 1 row"],
+    "details" .= T.unwords ["The result contains", show n, "rows"],
     "hint"    .= JSON.Null]
 
   toJSON (PgErr err) = JSON.toJSON err

src/PostgREST/MediaType.hs

@@ -4,16 +4,13 @@ module PostgREST.MediaType
   ( MediaType(..)
   , MTPlanOption (..)
   , MTPlanFormat (..)
-  , MTPlanAttrs(..)
   , toContentType
   , toMime
   , decodeMediaType
-  , getMediaType
   ) where
 
 import qualified Data.ByteString          as BS
 import qualified Data.ByteString.Internal as BS (c2w)
-import           Data.Maybe               (fromJust)
 
 import Network.HTTP.Types.Header (Header, hContentType)
 
@@ -22,7 +19,8 @@ import Protolude
 -- | Enumeration of currently supported media types
 data MediaType
   = MTApplicationJSON
-  | MTSingularJSON
+  | MTArrayJSONStrip
+  | MTSingularJSON Bool
   | MTGeoJSON
   | MTTextCSV
   | MTTextPlain
@@ -32,18 +30,32 @@ data MediaType
   | MTOctetStream
   | MTAny
   | MTOther ByteString
-  | MTPlan MTPlanAttrs
-  deriving Eq
-
-data MTPlanAttrs = MTPlanAttrs (Maybe MediaType) MTPlanFormat [MTPlanOption]
-instance Eq MTPlanAttrs where
-  MTPlanAttrs {} == MTPlanAttrs {} = True -- we don't care about the attributes when comparing two MTPlan media types
+  -- TODO MTPlan should only have its options as [Text]. Its ResultAggregate should have the typed attributes.
+  | MTPlan MediaType MTPlanFormat [MTPlanOption]
+  deriving Show
+instance Eq MediaType where
+  MTApplicationJSON    == MTApplicationJSON = True
+  MTArrayJSONStrip     == MTArrayJSONStrip  = True
+  MTSingularJSON x     == MTSingularJSON y  = x == y
+  MTGeoJSON            == MTGeoJSON         = True
+  MTTextCSV            == MTTextCSV         = True
+  MTTextPlain          == MTTextPlain       = True
+  MTTextXML            == MTTextXML         = True
+  MTOpenAPI            == MTOpenAPI         = True
+  MTUrlEncoded         == MTUrlEncoded      = True
+  MTOctetStream        == MTOctetStream     = True
+  MTAny                == MTAny             = True
+  MTOther x            == MTOther y         = x == y
+  MTPlan{}             == MTPlan{}          = True
+  _                    == _                 = False
 
 data MTPlanOption
   = PlanAnalyze | PlanVerbose | PlanSettings | PlanBuffers | PlanWAL
+  deriving (Eq, Show)
 
 data MTPlanFormat
   = PlanJSON | PlanText
+  deriving (Eq, Show)
 
 -- | Convert MediaType to a Content-Type HTTP Header
 toContentType :: MediaType -> Header
@@ -56,20 +68,22 @@ toContentType ct = (hContentType, toMime ct <> charset)
 
 -- | Convert from MediaType to a ByteString representing the mime type
 toMime :: MediaType -> ByteString
-toMime MTApplicationJSON = "application/json"
-toMime MTGeoJSON         = "application/geo+json"
-toMime MTTextCSV         = "text/csv"
-toMime MTTextPlain       = "text/plain"
-toMime MTTextXML         = "text/xml"
-toMime MTOpenAPI         = "application/openapi+json"
-toMime MTSingularJSON    = "application/vnd.pgrst.object+json"
-toMime MTUrlEncoded      = "application/x-www-form-urlencoded"
-toMime MTOctetStream     = "application/octet-stream"
-toMime MTAny             = "*/*"
-toMime (MTOther ct)      = ct
-toMime (MTPlan (MTPlanAttrs mt fmt opts)) =
+toMime MTApplicationJSON      = "application/json"
+toMime MTArrayJSONStrip       = "application/vnd.pgrst.array+json;nulls=stripped"
+toMime MTGeoJSON              = "application/geo+json"
+toMime MTTextCSV              = "text/csv"
+toMime MTTextPlain            = "text/plain"
+toMime MTTextXML              = "text/xml"
+toMime MTOpenAPI              = "application/openapi+json"
+toMime (MTSingularJSON True)  = "application/vnd.pgrst.object+json;nulls=stripped"
+toMime (MTSingularJSON False) = "application/vnd.pgrst.object+json"
+toMime MTUrlEncoded           = "application/x-www-form-urlencoded"
+toMime MTOctetStream          = "application/octet-stream"
+toMime MTAny                  = "*/*"
+toMime (MTOther ct)           = ct
+toMime (MTPlan mt fmt opts)   =
   "application/vnd.pgrst.plan+" <> toMimePlanFormat fmt <>
-  (if isNothing mt then mempty else "; for=\"" <> toMime (fromJust mt) <> "\"") <>
+  ("; for=\"" <> toMime mt <> "\"") <>
   (if null opts then mempty else "; options=" <> BS.intercalate "|" (toMimePlanOption <$> opts))
 
 toMimePlanOption :: MTPlanOption -> ByteString
@@ -83,42 +97,72 @@ toMimePlanFormat :: MTPlanFormat -> ByteString
 toMimePlanFormat PlanJSON = "json"
 toMimePlanFormat PlanText = "text"
 
--- | Convert from ByteString to MediaType. Warning: discards MIME parameters
+-- | Convert from ByteString to MediaType.
+--
+-- >>> decodeMediaType "application/json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan;"
+-- MTPlan MTApplicationJSON PlanText []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan;for=\"application/json\""
+-- MTPlan MTApplicationJSON PlanText []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan+json;for=\"text/csv\""
+-- MTPlan MTTextCSV PlanJSON []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json;nulls=stripped"
+-- MTArrayJSONStrip
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json;nulls=stripped"
+-- MTSingularJSON True
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json"
+-- MTSingularJSON False
+
 decodeMediaType :: BS.ByteString -> MediaType
 decodeMediaType mt =
   case BS.split (BS.c2w ';') mt of
-    "application/json":_                   -> MTApplicationJSON
-    "application/geo+json":_               -> MTGeoJSON
-    "text/csv":_                           -> MTTextCSV
-    "text/plain":_                         -> MTTextPlain
-    "text/xml":_                           -> MTTextXML
-    "application/openapi+json":_           -> MTOpenAPI
-    "application/vnd.pgrst.object+json":_  -> MTSingularJSON
-    "application/vnd.pgrst.object":_       -> MTSingularJSON
-    "application/x-www-form-urlencoded":_  -> MTUrlEncoded
-    "application/octet-stream":_           -> MTOctetStream
-    "application/vnd.pgrst.plan":rest      -> getPlan PlanText rest
-    "application/vnd.pgrst.plan+text":rest -> getPlan PlanText rest
-    "application/vnd.pgrst.plan+json":rest -> getPlan PlanJSON rest
-    "*/*":_                                -> MTAny
-    other:_                                -> MTOther other
-    _                                      -> MTAny
+    "application/json":_                     -> MTApplicationJSON
+    "application/geo+json":_                 -> MTGeoJSON
+    "text/csv":_                             -> MTTextCSV
+    "text/plain":_                           -> MTTextPlain
+    "text/xml":_                             -> MTTextXML
+    "application/openapi+json":_             -> MTOpenAPI
+    "application/x-www-form-urlencoded":_    -> MTUrlEncoded
+    "application/octet-stream":_             -> MTOctetStream
+    "application/vnd.pgrst.plan":rest        -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+text":rest   -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+json":rest   -> getPlan PlanJSON rest
+    "application/vnd.pgrst.object+json":rest -> checkSingularNullStrip rest
+    "application/vnd.pgrst.object":rest      -> checkSingularNullStrip rest
+    "application/vnd.pgrst.array+json":rest  -> checkArrayNullStrip rest
+    "*/*":_                                  -> MTAny
+    other:_                                  -> MTOther other
+    _                                        -> MTAny
   where
-    getPlan fmt rest =
-     let
-       opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)
-       inOpts str   = str `elem` opts
-       mtFor        = decodeMediaType . dropAround (== BS.c2w '"') <$> (BS.stripPrefix "for=" =<< find (BS.isPrefixOf "for=") rest)
-       dropAround p = BS.dropWhile p . BS.dropWhileEnd p in
-     MTPlan $ MTPlanAttrs mtFor fmt $
-      [PlanAnalyze  | inOpts "analyze" ] ++
-      [PlanVerbose  | inOpts "verbose" ] ++
-      [PlanSettings | inOpts "settings"] ++
-      [PlanBuffers  | inOpts "buffers" ] ++
-      [PlanWAL      | inOpts "wal"     ]
+    checkArrayNullStrip ["nulls=stripped"] = MTArrayJSONStrip
+    checkArrayNullStrip  _                 = MTApplicationJSON
 
-getMediaType :: MediaType -> MediaType
-getMediaType mt = case mt of
-  MTPlan (MTPlanAttrs (Just mType) _ _) -> mType
-  MTPlan (MTPlanAttrs Nothing _ _)      -> MTApplicationJSON
-  other                                 -> other
+    checkSingularNullStrip ["nulls=stripped"] = MTSingularJSON True
+    checkSingularNullStrip  _                 = MTSingularJSON False
+
+    getPlan fmt rest =
+      let
+        opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)
+        inOpts str   = str `elem` opts
+        dropAround p = BS.dropWhile p . BS.dropWhileEnd p
+        mtFor        = fromMaybe MTApplicationJSON $ do
+          foundFor    <- find (BS.isPrefixOf "for=") rest
+          strippedFor <- BS.stripPrefix "for=" foundFor
+          pure . decodeMediaType $ dropAround (== BS.c2w '"') strippedFor
+      in
+      MTPlan mtFor fmt $
+        [PlanAnalyze  | inOpts "analyze" ] ++
+        [PlanVerbose  | inOpts "verbose" ] ++
+        [PlanSettings | inOpts "settings"] ++
+        [PlanBuffers  | inOpts "buffers" ] ++
+        [PlanWAL      | inOpts "wal"     ]

src/PostgREST/Plan.hs

@@ -13,7 +13,6 @@ resource.
 {-# LANGUAGE DuplicateRecordFields #-}
 {-# LANGUAGE LambdaCase            #-}
 {-# LANGUAGE NamedFieldPuns        #-}
-{-# LANGUAGE OverloadedRecordDot   #-}
 {-# LANGUAGE RecordWildCards       #-}
 
 module PostgREST.Plan
@@ -26,9 +25,9 @@ module PostgREST.Plan
   , inspectPlanTxMode
   ) where
 
-
 import qualified Data.ByteString.Lazy          as LBS
 import qualified Data.HashMap.Strict           as HM
+import qualified Data.HashMap.Strict.InsOrd    as HMI
 import qualified Data.List                     as L
 import qualified Data.Set                      as S
 import qualified PostgREST.SchemaCache.Routine as Routine
@@ -37,35 +36,40 @@ import Data.Either.Combinators (mapLeft, mapRight)
 import Data.List               (delete)
 import Data.Tree               (Tree (..))
 
-import PostgREST.ApiRequest               (Action (..),
-                                           ApiRequest (..),
-                                           InvokeMethod (..),
-                                           Mutation (..),
-                                           Payload (..))
-import PostgREST.Config                   (AppConfig (..))
-import PostgREST.Error                    (Error (..))
-import PostgREST.MediaType                (MTPlanAttrs (..),
-                                           MediaType (..))
-import PostgREST.Query.SqlFragment        (sourceCTEName)
-import PostgREST.RangeQuery               (NonnegRange, allRange,
-                                           convertToLimitZeroRange,
-                                           restrictRange)
-import PostgREST.SchemaCache              (SchemaCache (..))
-import PostgREST.SchemaCache.Identifiers  (FieldName,
-                                           QualifiedIdentifier (..),
-                                           Schema)
-import PostgREST.SchemaCache.Relationship (Cardinality (..),
-                                           Junction (..),
-                                           Relationship (..),
-                                           RelationshipsMap,
-                                           relIsToOne)
-import PostgREST.SchemaCache.Routine      (Routine (..), RoutineMap,
-                                           RoutineParam (..),
-                                           funcReturnsCompositeAlias,
-                                           funcReturnsScalar,
-                                           funcReturnsSetOfScalar)
-import PostgREST.SchemaCache.Table        (Table (tableName),
-                                           tablePKCols)
+import PostgREST.ApiRequest                  (Action (..),
+                                              ApiRequest (..),
+                                              InvokeMethod (..),
+                                              Mutation (..),
+                                              Payload (..))
+import PostgREST.Config                      (AppConfig (..))
+import PostgREST.Error                       (Error (..))
+import PostgREST.MediaType                   (MediaType (..))
+import PostgREST.Query.SqlFragment           (sourceCTEName)
+import PostgREST.RangeQuery                  (NonnegRange, allRange,
+                                              convertToLimitZeroRange,
+                                              restrictRange)
+import PostgREST.SchemaCache                 (SchemaCache (..))
+import PostgREST.SchemaCache.Identifiers     (FieldName,
+                                              QualifiedIdentifier (..),
+                                              Schema)
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
+                                              Junction (..),
+                                              Relationship (..),
+                                              RelationshipsMap,
+                                              relIsToOne)
+import PostgREST.SchemaCache.Representations (DataRepresentation (..),
+                                              RepresentationsMap)
+import PostgREST.SchemaCache.Routine         (ResultAggregate (..),
+                                              Routine (..),
+                                              RoutineMap,
+                                              RoutineParam (..),
+                                              funcReturnsCompositeAlias,
+                                              funcReturnsScalar,
+                                              funcReturnsSetOfScalar)
+import PostgREST.SchemaCache.Table           (Column (..), Table (..),
+                                              TablesMap,
+                                              tableColumnsList,
+                                              tablePKCols)
 
 import PostgREST.ApiRequest.Preferences
 import PostgREST.ApiRequest.Types
@@ -79,16 +83,21 @@ import qualified PostgREST.ApiRequest.QueryParams as QueryParams
 
 import Protolude hiding (from)
 
+-- $setup
+-- Setup for doctests
+-- >>> import Data.Ranged.Ranges (fullRange)
+
 data WrappedReadPlan = WrappedReadPlan {
   wrReadPlan :: ReadPlanTree
 , wrTxMode   :: SQL.Mode
-, wrBinField :: Maybe FieldName
+, wrResAgg   :: ResultAggregate
 }
 
 data MutateReadPlan = MutateReadPlan {
   mrReadPlan   :: ReadPlanTree
 , mrMutatePlan :: MutatePlan
 , mrTxMode     :: SQL.Mode
+, mrResAgg     :: ResultAggregate
 }
 
 data CallReadPlan = CallReadPlan {
@@ -96,20 +105,21 @@ data CallReadPlan = CallReadPlan {
 , crCallPlan :: CallPlan
 , crTxMode   :: SQL.Mode
 , crProc     :: Routine
-, crBinField :: Maybe FieldName
+, crResAgg   :: ResultAggregate
 }
 
 wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error WrappedReadPlan
 wrappedReadPlan  identifier conf sCache apiRequest = do
   rPlan <- readPlan identifier conf sCache apiRequest
   binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) Nothing rPlan
-  return $ WrappedReadPlan rPlan SQL.Read binField
+  return $ WrappedReadPlan rPlan SQL.Read $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
 
 mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error MutateReadPlan
 mutateReadPlan  mutation apiRequest identifier conf sCache = do
   rPlan <- readPlan identifier conf sCache apiRequest
+  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) Nothing rPlan
   mPlan <- mutatePlan mutation identifier apiRequest sCache rPlan
-  return $ MutateReadPlan rPlan mPlan SQL.Write
+  return $ MutateReadPlan rPlan mPlan SQL.Write $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
 
 callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CallReadPlan
 callReadPlan identifier conf sCache apiRequest invMethod = do
@@ -134,7 +144,7 @@ callReadPlan identifier conf sCache apiRequest invMethod = do
           (InvPost, Routine.Volatile)  -> SQL.Write
       cPlan = callPlan proc apiRequest paramKeys args rPlan
   binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) (Just proc) rPlan
-  return $ CallReadPlan rPlan cPlan txMode proc binField
+  return $ CallReadPlan rPlan cPlan txMode proc $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
   where
     Preferences{..} = iPreferences apiRequest
     qsParams' = QueryParams.qsParams (iQueryParams apiRequest)
@@ -199,26 +209,96 @@ findProc qi argumentsKeys paramsAsSingleObject allProcs contentMediaType isInvPo
 inspectPlanTxMode :: SQL.Mode
 inspectPlanTxMode = SQL.Read
 
+-- | During planning we need to resolve Field -> CoercibleField (finding the context specific target type and map function).
+-- | ResolverContext facilitates this without the need to pass around a laundry list of parameters.
+data ResolverContext = ResolverContext
+  { tables          :: TablesMap
+  , representations :: RepresentationsMap
+  , qi              :: QualifiedIdentifier  -- ^ The table we're currently attending; changes as we recurse into joins etc.
+  , outputType      :: Text                 -- ^ The output type for the response payload; e.g. "csv", "json", "binary".
+  }
+
+resolveColumnField :: Column -> CoercibleField
+resolveColumnField col = CoercibleField (colName col) mempty False (colNominalType col) Nothing (colDefault col)
+
+resolveTableFieldName :: Table -> FieldName -> CoercibleField
+resolveTableFieldName table fieldName =
+  fromMaybe (unknownField fieldName []) $ HMI.lookup fieldName (tableColumns table) >>=
+    Just . resolveColumnField
+
+resolveTableField :: Table -> Field -> CoercibleField
+resolveTableField table (fieldName, []) = resolveTableFieldName table fieldName
+resolveTableField table (fieldName, jp) =
+  case resolveTableFieldName table fieldName of
+    -- types that are already json/jsonb don't need to be converted with `to_jsonb` for using arrow operators `data->attr`
+    -- this prevents indexes not applying https://github.com/PostgREST/postgrest/issues/2594
+    cf@CoercibleField{cfIRType="json"}  -> cf{cfJsonPath=jp}
+    cf@CoercibleField{cfIRType="jsonb"} -> cf{cfJsonPath=jp}
+    -- other types will get converted `to_jsonb(col)->attr`
+    cf                                  -> cf{cfJsonPath=jp, cfToJson=True}
+
+-- | Resolve a type within the context based on the given field name and JSON path. Although there are situations where failure to resolve a field is considered an error (see `resolveOrError`), there are also situations where we allow it (RPC calls). If it should be an error and `resolveOrError` doesn't fit, ensure to check the `cfIRType` isn't empty.
+resolveTypeOrUnknown :: ResolverContext -> Field -> CoercibleField
+resolveTypeOrUnknown ResolverContext{..} field@(fn, jp) =
+  fromMaybe (unknownField fn jp) $ HM.lookup qi tables >>=
+    Just . flip resolveTableField field
+
+-- | Install any pre-defined data representation from source to target to coerce this reference.
+--
+-- Note that we change the IR type here. This might seem unintuitive. The short of it is that for a CoercibleField without a transformer, input type == output type. A transformer maps from a -> b, so by definition the input type will be a and the output type b after. And cfIRType is the *input* type.
+--
+-- It might feel odd that once a transformer is added we 'forget' the target type (because now a /= b). You might also note there's no obvious way to stack transforms (even if there was a stack, you erased what type you're working with so it's awkward). Alas as satisfying as it would be to engineer a layered mapping system with full type information, we just don't need it.
+withTransformer :: ResolverContext -> Text -> Text -> CoercibleField -> CoercibleField
+withTransformer ResolverContext{representations} sourceType targetType field =
+  fromMaybe field $ HM.lookup (sourceType, targetType) representations >>=
+    (\fieldRepresentation -> Just field{cfIRType=sourceType, cfTransform=Just (drFunction fieldRepresentation)})
+
+-- | Map the intermediate representation type to the output type, if available.
+withOutputFormat :: ResolverContext -> CoercibleField -> CoercibleField
+withOutputFormat ctx@ResolverContext{outputType} field@CoercibleField{cfIRType} = withTransformer ctx cfIRType outputType field
+
+-- | Map text into the intermediate representation type, if available.
+withTextParse :: ResolverContext -> CoercibleField -> CoercibleField
+withTextParse ctx field@CoercibleField{cfIRType} = withTransformer ctx "text" cfIRType field
+
+-- | Map json into the intermediate representation type, if available.
+withJsonParse :: ResolverContext -> CoercibleField -> CoercibleField
+withJsonParse ctx field@CoercibleField{cfIRType} = withTransformer ctx "json" cfIRType field
+
+-- | Map the intermediate representation type to the output type defined by the resolver context (normally json), if available.
+resolveOutputField :: ResolverContext -> Field -> CoercibleField
+resolveOutputField ctx field = withOutputFormat ctx $ resolveTypeOrUnknown ctx field
+
+-- | Map the query string format of a value (text) into the intermediate representation type, if available.
+resolveQueryInputField :: ResolverContext -> Field -> CoercibleField
+resolveQueryInputField ctx field = withTextParse ctx $ resolveTypeOrUnknown ctx field
+
 -- | Builds the ReadPlan tree on a number of stages.
 -- | Adds filters, order, limits on its respective nodes.
 -- | Adds joins conditions obtained from resource embedding.
 readPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error ReadPlanTree
-readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows} SchemaCache{dbRelationships} apiRequest  =
-  mapLeft ApiRequestError $
-  treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
-  addNullEmbedFilters =<<
-  validateSpreadEmbeds =<<
-  addRelatedOrders =<<
-  addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
-  addLogicTrees apiRequest =<<
-  addRanges apiRequest =<<
-  addOrders apiRequest =<<
-  addFilters apiRequest (initReadRequest qi $ QueryParams.qsSelect $ iQueryParams apiRequest)
+readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows} SchemaCache{dbTables, dbRelationships, dbRepresentations} apiRequest  =
+  let
+    -- JSON output format hardcoded for now. In the future we might want to support other output mappings such as CSV.
+    ctx = ResolverContext dbTables dbRepresentations qi "json"
+  in
+    mapLeft ApiRequestError $
+    treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
+    addNullEmbedFilters =<<
+    validateSpreadEmbeds =<<
+    addRelatedOrders =<<
+    addDataRepresentationAliases =<<
+    expandStarsForDataRepresentations ctx =<<
+    addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
+    addLogicTrees ctx apiRequest =<<
+    addRanges apiRequest =<<
+    addOrders ctx apiRequest =<<
+    addFilters ctx apiRequest (initReadRequest ctx $ QueryParams.qsSelect $ iQueryParams apiRequest)
 
 -- Build the initial read plan tree
-initReadRequest :: QualifiedIdentifier -> [Tree SelectItem] -> ReadPlanTree
-initReadRequest qi@QualifiedIdentifier{..} =
-  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi, relName=qiName, depth=rootDepth} []
+initReadRequest :: ResolverContext -> [Tree SelectItem] -> ReadPlanTree
+initReadRequest ctx@ResolverContext{qi=QualifiedIdentifier{..}} =
+  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi ctx, relName=qiName, depth=rootDepth} []
   where
     rootDepth = 0
     defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing False rootDepth
@@ -237,7 +317,49 @@ initReadRequest qi@QualifiedIdentifier{..} =
             (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth, relIsSpread=True} [])
             fldForest:rForest
         SelectField{..} ->
-          Node q{select=(selField, selCast, selAlias):select q} rForest
+          Node q{select=(resolveOutputField ctx{qi=from q} selField, selCast, selAlias):select q} rForest
+
+-- | Preserve the original field name if data representation is used to coerce the value.
+addDataRepresentationAliases :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addDataRepresentationAliases rPlanTree = Right $ fmap (\rPlan@ReadPlan{select=sel} -> rPlan{select=map aliasSelectItem sel}) rPlanTree
+  where
+    aliasSelectItem :: (CoercibleField, Maybe Cast, Maybe Alias) -> (CoercibleField, Maybe Cast, Maybe Alias)
+    -- If there already is an alias, don't overwrite it.
+    aliasSelectItem (fld@(CoercibleField{cfName=fieldName, cfTransform=(Just _)}), Nothing, Nothing) = (fld, Nothing, Just fieldName)
+    aliasSelectItem fld = fld
+
+knownColumnsInContext :: ResolverContext -> [Column]
+knownColumnsInContext ResolverContext{..} =
+  fromMaybe [] $ HM.lookup qi tables >>=
+  Just . tableColumnsList
+
+-- | Expand "select *" into explicit field names of the table, if necessary to apply data representations.
+expandStarsForDataRepresentations :: ResolverContext -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+expandStarsForDataRepresentations ctx@ResolverContext{qi} rPlanTree = Right $ fmap expandStars rPlanTree
+  where
+    expandStars :: ReadPlan -> ReadPlan
+    -- When the schema is "" and the table is the source CTE, we assume the true source table is given in the from
+    -- alias and belongs to the request schema. See the bit in `addRels` with `newFrom = ...`.
+    expandStars rPlan@ReadPlan{from=(QualifiedIdentifier "" "pgrst_source"), fromAlias=(Just tblAlias)} =
+      expandStarsForTable ctx{qi=qi{qiName=tblAlias}} rPlan
+    expandStars rPlan@ReadPlan{from=fromTable} =
+      expandStarsForTable ctx{qi=fromTable} rPlan
+
+expandStarsForTable :: ResolverContext -> ReadPlan -> ReadPlan
+expandStarsForTable ctx@ResolverContext{representations, outputType} rplan@ReadPlan{select=selectItems} =
+  -- If we have a '*' select AND the target table has at least one data representation, expand.
+  if ("*" `elem` map (\(field, _, _) -> cfName field) selectItems) && any hasOutputRep knownColumns
+    then rplan{select=concatMap (expandStarSelectItem knownColumns) selectItems}
+    else rplan
+  where
+    knownColumns = knownColumnsInContext ctx
+
+    hasOutputRep :: Column -> Bool
+    hasOutputRep col = HM.member (colNominalType col, outputType) representations
+
+    expandStarSelectItem :: [Column] -> (CoercibleField, Maybe Cast, Maybe Alias) -> [(CoercibleField, Maybe Cast, Maybe Alias)]
+    expandStarSelectItem columns (CoercibleField{cfName="*", cfJsonPath=[]}, b, c) = map (\col -> (withOutputFormat ctx $ resolveColumnField col, b, c)) columns
+    expandStarSelectItem _ selectItem = [selectItem]
 
 -- | Enforces the `max-rows` config on the result
 treeRestrictRange :: Maybe Integer -> Action -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -273,7 +395,7 @@ addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,re
       Node <$> newReadPlan <*> (updateForest . hush $ Node <$> newReadPlan <*> pure forest)
     Nothing -> -- root case
       let
-        newFrom  = QualifiedIdentifier mempty $ decodeUtf8 sourceCTEName
+        newFrom  = QualifiedIdentifier mempty sourceCTEName
         newAlias = Just (qiName $ from rPlan)
         newReadPlan = case action of
           -- the CTE for mutations/rpc is used as WITH sourceCTEName .. SELECT .. FROM sourceCTEName as alias,
@@ -363,7 +485,9 @@ findRel schema allRels origin target hint =
             target == qiName relForeignTable && isO2M relCardinality
             && matchFKRefSingleCol hnt relCardinality -- auditor
         else case hint of
+          -- DEPRECATED(remove after 2 major releases since v11.1.0): remove target
           -- target = table / view / constraint / column-from-origin (constraint/column-from-origin can only come from tables https://github.com/PostgREST/postgrest/issues/2277)
+          -- DEPRECATED(remove after 2 major releases since v11.1.0): remove hint as table/view/columns and only leave it as constraint
           -- hint   = table / view / constraint / column-from-origin / column-from-target (hint can take table / view values to aid in finding the junction in an m2m relationship)
           Nothing ->
               -- /projects?select=clients(*)
@@ -392,8 +516,8 @@ findRel schema allRels origin target hint =
             )
       ) $ fromMaybe mempty $ HM.lookup (QualifiedIdentifier schema origin, schema) allRels
 
-addFilters :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addFilters ApiRequest{..} rReq =
+addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addFilters ctx ApiRequest{..} rReq =
   foldr addFilterToNode (Right rReq) flts
   where
     QueryParams.QueryParams{..} = iQueryParams
@@ -405,10 +529,10 @@ addFilters ApiRequest{..} rReq =
 
     addFilterToNode :: (EmbedPath, Filter) -> Either ApiRequestError ReadPlanTree ->  Either ApiRequestError ReadPlanTree
     addFilterToNode =
-      updateNode (\flt (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest flt lf}  f)
+      updateNode (\flt (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest (resolveFilter ctx{qi=fromTable} flt) lf}  f)
 
-addOrders :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addOrders ApiRequest{..} rReq =
+addOrders :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addOrders ctx ApiRequest{..} rReq =
   case iAction of
     ActionMutate _ -> Right rReq
     _              -> foldr addOrderToNode (Right rReq) qsOrder
@@ -416,48 +540,115 @@ addOrders ApiRequest{..} rReq =
     QueryParams.QueryParams{..} = iQueryParams
 
     addOrderToNode :: (EmbedPath, [OrderTerm]) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
-    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=o} f)
+    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=resolveOrder ctx <$> o} f)
+
+resolveOrder :: ResolverContext -> OrderTerm -> CoercibleOrderTerm
+resolveOrder _ (OrderRelationTerm a b c d) = CoercibleOrderRelationTerm a b c d
+resolveOrder ctx (OrderTerm fld dir nulls) = CoercibleOrderTerm (resolveTypeOrUnknown ctx fld) dir nulls
 
 -- Validates that the related resource on the order is an embedded resource,
 -- e.g. if `clients` is inside the `select` in /projects?order=clients(id)&select=*,clients(*),
 -- and if it's a to-one relationship, it adds the right alias to the OrderRelationTerm so the generated query can succeed.
--- TODO might be clearer if there's an additional intermediate type
 addRelatedOrders :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
-  newOrder <- getRelOrder `traverse` order
+  newOrder <- newRelOrder `traverse` order
   Node rp{order=newOrder} <$> addRelatedOrders `traverse` forest
   where
-    getRelOrder ot@OrderTerm{}                   = Right ot
-    getRelOrder ot@OrderRelationTerm{otRelation} =
-      let foundRP = rootLabel <$> find (\(Node ReadPlan{relName, relAlias} _) -> otRelation == fromMaybe relName relAlias) forest in
+    newRelOrder cot@CoercibleOrderTerm{}                   = Right cot
+    newRelOrder cot@CoercibleOrderRelationTerm{coRelation} =
+      let foundRP = rootLabel <$> find (\(Node ReadPlan{relName, relAlias} _) -> coRelation == fromMaybe relName relAlias) forest in
       case foundRP of
         Just ReadPlan{relName,relAlias,relAggAlias,relToParent} ->
           let isToOne = relIsToOne <$> relToParent
               name    = fromMaybe relName relAlias in
           if isToOne == Just True
-            then Right $ ot{otRelation=relAggAlias}
+            then Right $ cot{coRelation=relAggAlias}
             else Left $ RelatedOrderNotToOne (qiName from) name
         Nothing ->
-          Left $ NotEmbedded otRelation
+          Left $ NotEmbedded coRelation
 
--- Searches for null filters on embeds, e.g. `clients` on /projects?select=*,clients()&clients=not.is.null.
--- If these are found, it changes the filter to use the internal aggregate name(`projects_clients_1`) so the filter can succeed.
--- It fails if operators other than is.null or not.is.null are used.
+-- | Searches for null filters on embeds, e.g. `projects=not.is.null` on `GET /clients?select=*,projects(*)&projects=not.is.null`
+--
+-- (It doesn't err but uses an Either ApiRequestError type so it can combine with the other functions that modify the read plan tree)
+--
+-- Setup:
+--
+-- >>> let nullOp    = OpExpr True (Is TriNull)
+-- >>> let nonNullOp = OpExpr False (Is TriNull)
+-- >>> let notEqOp   = OpExpr True (Op OpNotEqual "val")
+-- >>> :{
+-- -- this represents the `projects(*)` part on `/clients?select=*,projects(*)`
+-- let
+-- subForestPlan =
+--   [
+--     Node {
+--       rootLabel = ReadPlan {
+--         select = [], -- there will be fields at this stage but we just omit them for brevity
+--         from = QualifiedIdentifier {qiSchema = "test", qiName = "projects"},
+--         fromAlias = Just "projects_1", where_ = [], order = [], range_ = fullRange,
+--         relName = "projects",
+--         relToParent = Nothing,
+--         relJoinConds = [],
+--         relAlias = Nothing, relAggAlias = "clients_projects_1", relHint = Nothing, relJoinType = Nothing, relIsSpread = False, depth = 1
+--       },
+--       subForest = []
+--     }
+--   ]
+-- :}
+--
+-- >>> :{
+-- -- this represents the full URL `/clients?select=*,projects(*)&projects=not.is.null`, if subForst takes the above subForestPlan and nullOp
+-- let
+-- readPlanTree op subForst =
+--   Node {
+--     rootLabel = ReadPlan {
+--       select = [], -- there will be fields at this stage but we just omit them for brevity
+--       from = QualifiedIdentifier { qiSchema = "test", qiName = "clients"},
+--       fromAlias = Nothing,
+--       where_ = [
+--         CoercibleStmnt (
+--           CoercibleFilter {
+--            field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson=False, cfIRType = "", cfTransform = Nothing, cfDefault = Nothing},
+--            opExpr = op
+--           }
+--         )
+--       ],
+--       order = [], range_ = fullRange, relName = "clients", relToParent = Nothing, relJoinConds = [], relAlias = Nothing, relAggAlias = "", relHint = Nothing,
+--       relJoinType = Nothing, relIsSpread = False, depth = 0
+--     },
+--     subForest = subForst
+--   }
+-- :}
+--
+-- Don't do anything to the filter if there's no embedding (a subtree) on projects. Assume it's a normal filter.
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp [])
+-- Right [CoercibleStmnt (CoercibleFilter {field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson = False, cfIRType = "", cfTransform = Nothing, cfDefault = Nothing}, opExpr = OpExpr True (Is TriNull)})]
+--
+-- If there's an embedding on projects, then change the filter to use the internal aggregate name (`clients_projects_1`) so the filter can succeed later.
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp subForestPlan)
+-- Right [CoercibleStmnt (CoercibleFilterNullEmbed True "clients_projects_1")]
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nonNullOp subForestPlan)
+-- Right [CoercibleStmnt (CoercibleFilterNullEmbed False "clients_projects_1")]
 addNullEmbedFilters :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addNullEmbedFilters (Node rp@ReadPlan{where_=oldLogic} forest) = do
-  let readPlans = rootLabel <$> forest
-  newLogic <- getFilters readPlans `traverse` oldLogic
+addNullEmbedFilters (Node rp@ReadPlan{where_=curLogic} forest) = do
+  let forestReadPlans = rootLabel <$> forest
+  newLogic <- newNullFilters forestReadPlans `traverse` curLogic
   Node rp{ReadPlan.where_= newLogic} <$> (addNullEmbedFilters `traverse` forest)
   where
-    getFilters :: [ReadPlan] -> LogicTree -> Either ApiRequestError LogicTree
-    getFilters rPlans (Expr b lOp trees) = Expr b lOp <$> (getFilters rPlans `traverse` trees)
-    getFilters rPlans flt@(Stmnt (Filter (fld, []) opExpr)) =
-      let foundRP = find (\ReadPlan{relName, relAlias} -> fld == fromMaybe relName relAlias) rPlans in
-      case (foundRP, opExpr) of
-        (Just ReadPlan{relAggAlias}, OpExpr b (Is TriNull)) -> Right $ Stmnt $ FilterNullEmbed b relAggAlias
-        (Just ReadPlan{relName}, _)                         -> Left $ UnacceptableFilter relName
-        _                                                   -> Right flt
-    getFilters _ flt@(Stmnt _)        = Right flt
+    newNullFilters :: [ReadPlan] -> CoercibleLogicTree -> Either ApiRequestError CoercibleLogicTree
+    newNullFilters rPlans = \case
+      (CoercibleExpr b lOp trees) ->
+        CoercibleExpr b lOp <$> (newNullFilters rPlans `traverse` trees)
+      flt@(CoercibleStmnt (CoercibleFilter (CoercibleField fld [] _ _ _ _) opExpr)) ->
+        let foundRP = find (\ReadPlan{relName, relAlias} -> fld == fromMaybe relName relAlias) rPlans in
+        case (foundRP, opExpr) of
+          (Just ReadPlan{relAggAlias}, OpExpr b (Is TriNull)) -> Right $ CoercibleStmnt $ CoercibleFilterNullEmbed b relAggAlias
+          _                                                   -> Right flt
+      flt@(CoercibleStmnt _) ->
+        Right flt
 
 addRanges :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRanges ApiRequest{..} rReq =
@@ -471,14 +662,21 @@ addRanges ApiRequest{..} rReq =
     addRangeToNode :: (EmbedPath, NonnegRange) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
     addRangeToNode = updateNode (\r (Node q f) -> Node q{range_=r} f)
 
-addLogicTrees :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addLogicTrees ApiRequest{..} rReq =
+addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addLogicTrees ctx ApiRequest{..} rReq =
   foldr addLogicTreeToNode (Right rReq) qsLogic
   where
     QueryParams.QueryParams{..} = iQueryParams
 
     addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
-    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=t:lf} f)
+    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)
+
+resolveLogicTree :: ResolverContext -> LogicTree -> CoercibleLogicTree
+resolveLogicTree ctx (Stmnt flt) = CoercibleStmnt $ resolveFilter ctx flt
+resolveLogicTree ctx (Expr b op lts) = CoercibleExpr b op (map (resolveLogicTree ctx) lts)
+
+resolveFilter :: ResolverContext -> Filter -> CoercibleFilter
+resolveFilter ctx (Filter fld opExpr) = CoercibleFilter{field=resolveQueryInputField ctx fld, opExpr=opExpr}
 
 -- Validates that spread embeds are only done on to-one relationships
 validateSpreadEmbeds :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -504,10 +702,10 @@ updateNode f (targetNodeName:remainingPath, a) (Right (Node rootNode forest)) =
     findNode = find (\(Node ReadPlan{relName, relAlias} _) -> relName == targetNodeName || relAlias == Just targetNodeName) forest
 
 mutatePlan :: Mutation -> QualifiedIdentifier -> ApiRequest -> SchemaCache -> ReadPlanTree -> Either Error MutatePlan
-mutatePlan mutation qi ApiRequest{iPreferences=preferences, ..} sCache readReq = mapLeft ApiRequestError $
+mutatePlan mutation qi ApiRequest{iPreferences=Preferences{..}, ..} SchemaCache{dbTables, dbRepresentations} readReq = mapLeft ApiRequestError $
   case mutation of
     MutationCreate ->
-      mapRight (\typedColumns -> Insert qi typedColumns body ((,) <$> preferences.preferResolution <*> Just confCols) [] returnings pkCols applyDefaults) typedColumnsOrError
+      mapRight (\typedColumns -> Insert qi typedColumns body ((,) <$> preferResolution <*> Just confCols) [] returnings pkCols applyDefaults) typedColumnsOrError
     MutationUpdate ->
       mapRight (\typedColumns -> Update qi typedColumns body combinedLogic iTopLevelRange rootOrder returnings applyDefaults) typedColumnsOrError
     MutationSingleUpsert ->
@@ -522,27 +720,28 @@ mutatePlan mutation qi ApiRequest{iPreferences=preferences, ..} sCache readReq =
           Left InvalidFilters
     MutationDelete -> Right $ Delete qi combinedLogic iTopLevelRange rootOrder returnings
   where
+    ctx = ResolverContext dbTables dbRepresentations qi "json"
     confCols = fromMaybe pkCols qsOnConflict
     QueryParams.QueryParams{..} = iQueryParams
     returnings =
-      if preferences.preferRepresentation == None
+      if preferRepresentation == Just None || isNothing preferRepresentation
         then []
         else inferColsEmbedNeeds readReq pkCols
-    pkCols = maybe mempty tablePKCols $ HM.lookup qi $ dbTables sCache
-    logic = map snd qsLogic
-    rootOrder = maybe [] snd $ find (\(x, _) -> null x) qsOrder
-    combinedLogic = foldr addFilterToLogicForest logic qsFiltersRoot
+    tbl = HM.lookup qi dbTables
+    pkCols = maybe mempty tablePKCols tbl
+    logic = map (resolveLogicTree ctx . snd) qsLogic
+    rootOrder = resolveOrder ctx <$> maybe [] snd (find (\(x, _) -> null x) qsOrder)
+    combinedLogic = foldr (addFilterToLogicForest . resolveFilter ctx) logic qsFiltersRoot
     body = payRaw <$> iPayload -- the body is assumed to be json at this stage(ApiRequest validates)
-    tbl = HM.lookup qi $ dbTables sCache
-    typedColumnsOrError = resolveOrError tbl `traverse` S.toList iColumns
-    applyDefaults = preferences.preferMissing == Just ApplyDefaults
+    applyDefaults = preferMissing == Just ApplyDefaults
+    typedColumnsOrError = resolveOrError ctx tbl `traverse` S.toList iColumns
 
-resolveOrError :: Maybe Table -> FieldName -> Either ApiRequestError TypedField
-resolveOrError Nothing _ = Left NotFound
-resolveOrError (Just table) field =
-  case resolveTableField table field of
-    Nothing         -> Left $ ColumnNotFound (tableName table) field
-    Just typedField -> Right typedField
+resolveOrError :: ResolverContext -> Maybe Table -> FieldName -> Either ApiRequestError CoercibleField
+resolveOrError _ Nothing _ = Left NotFound
+resolveOrError ctx (Just table) field =
+  case resolveTableFieldName table field of
+    CoercibleField{cfIRType=""} -> Left $ ColumnNotFound (tableName table) field
+    cf                          -> Right $ withJsonParse ctx cf
 
 callPlan :: Routine -> ApiRequest -> S.Set FieldName -> LBS.ByteString -> ReadPlanTree -> CallPlan
 callPlan proc ApiRequest{iPreferences=Preferences{..}} paramKeys args readReq = FunctionCall {
@@ -571,7 +770,7 @@ inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols
   | "*" `elem` fldNames = ["*"]
   | otherwise           = returnings
   where
-    fldNames = (\((fld, _), _, _) -> fld) <$> select
+    fldNames = cfName . (\(f, _, _) -> f) <$> select
     -- Without fkCols, when a mutatePlan to
     -- /projects?select=name,clients(name) occurs, the RETURNING SQL part would
     -- be `RETURNING name`(see QueryBuilder).  This would make the embedding
@@ -610,8 +809,8 @@ inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols
 
 -- Traditional filters(e.g. id=eq.1) are added as root nodes of the LogicTree
 -- they are later concatenated with AND in the QueryBuilder
-addFilterToLogicForest :: Filter -> [LogicTree] -> [LogicTree]
-addFilterToLogicForest flt lf = Stmnt flt : lf
+addFilterToLogicForest :: CoercibleFilter -> [CoercibleLogicTree] -> [CoercibleLogicTree]
+addFilterToLogicForest flt lf = CoercibleStmnt flt : lf
 
 -- | If raw(binary) output is requested, check that MediaType is one of the
 -- admitted rawMediaTypes and that`?select=...` contains only one field other
@@ -634,12 +833,43 @@ binaryField AppConfig{configRawMediaTypes} acceptMediaType proc rpTree
   where
     isRawMediaType = acceptMediaType `elem` configRawMediaTypes `L.union` [MTOctetStream, MTTextPlain, MTTextXML] || isRawPlan acceptMediaType
     isRawPlan mt = case mt of
-      MTPlan (MTPlanAttrs (Just MTOctetStream) _ _) -> True
-      MTPlan (MTPlanAttrs (Just MTTextPlain) _ _)   -> True
-      MTPlan (MTPlanAttrs (Just MTTextXML) _ _)     -> True
-      _                                             -> False
+      MTPlan MTOctetStream _ _ -> True
+      MTPlan MTTextPlain _ _   -> True
+      MTPlan MTTextXML _ _     -> True
+      _                        -> False
 
     fstFieldName :: ReadPlanTree -> Maybe FieldName
-    fstFieldName (Node ReadPlan{select=(("*", []), _, _):_} [])  = Nothing
-    fstFieldName (Node ReadPlan{select=[((fld, []), _, _)]} []) = Just fld
+    fstFieldName (Node ReadPlan{select=(CoercibleField{cfName="*", cfJsonPath=[]}, _, _):_} [])  = Nothing
+    fstFieldName (Node ReadPlan{select=[(CoercibleField{cfName=fld, cfJsonPath=[]}, _, _)]} [])  = Just fld
     fstFieldName _                                               = Nothing
+
+
+mediaToAggregate :: MediaType -> Maybe FieldName -> ApiRequest -> ResultAggregate
+mediaToAggregate mt binField apiReq@ApiRequest{iAction=act, iPreferences=Preferences{preferRepresentation=rep}} =
+  if noAgg then NoAgg
+  else case mt of
+    MTApplicationJSON     -> BuiltinAggJson
+    MTSingularJSON strip  -> BuiltinAggSingleJson strip
+    MTArrayJSONStrip      -> BuiltinAggArrayJsonStrip
+    MTGeoJSON             -> BuiltinAggGeoJson
+    MTTextCSV             -> BuiltinAggCsv
+    MTAny                 -> BuiltinAggJson
+    MTOpenAPI             -> BuiltinAggJson
+    MTUrlEncoded          -> NoAgg -- TODO: unreachable since a previous step (producedMediaTypes) whitelists the media types that can become aggregates.
+
+    -- binary types
+    MTTextPlain           -> BuiltinAggBinary binField
+    MTTextXML             -> BuiltinAggXml binField
+    MTOctetStream         -> BuiltinAggBinary binField
+    MTOther _             -> BuiltinAggBinary binField
+
+    -- Doing `Accept: application/vnd.pgrst.plan; for="application/vnd.pgrst.plan"` doesn't make sense, so we just empty the body.
+    -- TODO: fail instead to be more strict
+    MTPlan (MTPlan{}) _ _ -> NoAgg
+    MTPlan media      _ _ -> mediaToAggregate media binField apiReq
+  where
+    noAgg = case act of
+      ActionMutate _         -> rep == Just HeadersOnly || rep == Just None || isNothing rep
+      ActionRead _isHead     -> _isHead -- no need for an aggregate on HEAD https://github.com/PostgREST/postgrest/issues/2849
+      ActionInvoke invMethod -> invMethod == InvHead
+      _                      -> False

src/PostgREST/Plan/MutatePlan.hs

@@ -6,8 +6,9 @@ where
 import qualified Data.ByteString.Lazy as LBS
 
 import PostgREST.ApiRequest.Preferences  (PreferResolution)
-import PostgREST.ApiRequest.Types        (LogicTree, OrderTerm)
-import PostgREST.Plan.Types              (TypedField)
+import PostgREST.Plan.Types              (CoercibleField,
+                                          CoercibleLogicTree,
+                                          CoercibleOrderTerm)
 import PostgREST.RangeQuery              (NonnegRange)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier)
@@ -18,28 +19,28 @@ import Protolude
 data MutatePlan
   = Insert
       { in_        :: QualifiedIdentifier
-      , insCols    :: [TypedField]
+      , insCols    :: [CoercibleField]
       , insBody    :: Maybe LBS.ByteString
       , onConflict :: Maybe (PreferResolution, [FieldName])
-      , where_     :: [LogicTree]
+      , where_     :: [CoercibleLogicTree]
       , returning  :: [FieldName]
       , insPkCols  :: [FieldName]
       , applyDefs  :: Bool
       }
   | Update
       { in_       :: QualifiedIdentifier
-      , updCols   :: [TypedField]
+      , updCols   :: [CoercibleField]
       , updBody   :: Maybe LBS.ByteString
-      , where_    :: [LogicTree]
+      , where_    :: [CoercibleLogicTree]
       , mutRange  :: NonnegRange
-      , mutOrder  :: [OrderTerm]
+      , mutOrder  :: [CoercibleOrderTerm]
       , returning :: [FieldName]
       , applyDefs :: Bool
       }
   | Delete
       { in_       :: QualifiedIdentifier
-      , where_    :: [LogicTree]
+      , where_    :: [CoercibleLogicTree]
       , mutRange  :: NonnegRange
-      , mutOrder  :: [OrderTerm]
+      , mutOrder  :: [CoercibleOrderTerm]
       , returning :: [FieldName]
       }

src/PostgREST/Plan/ReadPlan.hs

@@ -6,9 +6,11 @@ module PostgREST.Plan.ReadPlan
 
 import Data.Tree (Tree (..))
 
-import PostgREST.ApiRequest.Types         (Alias, Cast, Depth, Field,
-                                           Hint, JoinType, LogicTree,
-                                           NodeName, OrderTerm)
+import PostgREST.ApiRequest.Types         (Alias, Cast, Depth, Hint,
+                                           JoinType, NodeName)
+import PostgREST.Plan.Types               (CoercibleField (..),
+                                           CoercibleLogicTree,
+                                           CoercibleOrderTerm)
 import PostgREST.RangeQuery               (NonnegRange)
 import PostgREST.SchemaCache.Identifiers  (FieldName,
                                            QualifiedIdentifier)
@@ -23,14 +25,14 @@ data JoinCondition =
   JoinCondition
     (QualifiedIdentifier, FieldName)
     (QualifiedIdentifier, FieldName)
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data ReadPlan = ReadPlan
-  { select       :: [(Field, Maybe Cast, Maybe Alias)]
+  { select       :: [(CoercibleField, Maybe Cast, Maybe Alias)]
   , from         :: QualifiedIdentifier
   , fromAlias    :: Maybe Alias
-  , where_       :: [LogicTree]
-  , order        :: [OrderTerm]
+  , where_       :: [CoercibleLogicTree]
+  , order        :: [CoercibleOrderTerm]
   , range_       :: NonnegRange
   , relName      :: NodeName
   , relToParent  :: Maybe Relationship
@@ -43,4 +45,4 @@ data ReadPlan = ReadPlan
   , depth        :: Depth
   -- ^ used for aliasing
   }
-  deriving (Eq)
+  deriving (Eq, Show)

src/PostgREST/Plan/Types.hs

@@ -1,24 +1,67 @@
 module PostgREST.Plan.Types
-  ( TypedField(..)
-  , resolveTableField
+  ( CoercibleField(..)
+  , unknownField
+  , CoercibleLogicTree(..)
+  , CoercibleFilter(..)
+  , TransformerProc
+  , CoercibleOrderTerm(..)
   ) where
 
-import qualified Data.HashMap.Strict.InsOrd as HMI
+import PostgREST.ApiRequest.Types (Field, JsonPath, LogicOperator,
+                                   OpExpr, OrderDirection, OrderNulls)
 
 import PostgREST.SchemaCache.Identifiers (FieldName)
-import PostgREST.SchemaCache.Table       (Column (..), Table (..))
 
 import Protolude
 
--- | A TypedField is a field with sufficient information to be read from JSON with `json_to_recordset`.
-data TypedField = TypedField
-   { tfName    :: FieldName
-   , tfIRType  :: Text -- ^ The initial type of the field, before any casting.
-   , tfDefault :: Maybe Text
-   } deriving (Eq)
+type TransformerProc = Text
 
-resolveTableField :: Table -> FieldName -> Maybe TypedField
-resolveTableField table fieldName =
-  case HMI.lookup fieldName (tableColumns table) of
-    Just column -> Just $ TypedField (colName column) (colNominalType column) (colDefault column)
-    Nothing     -> Nothing
+-- | A CoercibleField pairs the name of a query element with any type coercion information we need for some specific use case.
+-- |
+-- | As suggested by the name, it's often a reference to a field in a table but really it can be any nameable element (function parameter, calculation with an alias, etc) with a knowable type.
+-- |
+-- | In the simplest case, it allows us to parse JSON payloads with `json_to_recordset`, for which we need to know both the name and the type of each thing we'd like to extract. At a higher level, CoercibleField generalises to reflect that any value we work with in a query may need type specific handling.
+-- |
+-- | CoercibleField is the foundation for the Data Representations feature. This feature allow user-definable mappings between database types so that the same data can be presented or interpreted in various ways as needed. Sometimes the way Postgres coerces data implicitly isn't right for the job. Different mappings might be appropriate for different situations: parsing a filter from a query string requires one function (text -> field type) while parsing a payload from JSON takes another (json -> field type). And the reverse, outputting a field as JSON, requires yet a third (field type -> json). CoercibleField is that "job specific" reference to an element paired with the type we desire for that particular purpose and the function we'll use to get there, if any.
+-- |
+-- | In the planning phase, we "resolve" generic named elements into these specialised CoercibleFields. Again this is context specific: two different CoercibleFields both representing the exact same table column in the database, even in the same query, might have two different target types and mapping functions. For example, one might represent a column in a filter, and another the very same column in an output role to be sent in the response body.
+-- |
+-- | The type value is allowed to be the empty string. The analog here is soft type checking in programming languages: sometimes we don't need a variable to have a specified type and things will work anyhow. So the empty type variant is valid when we don't know and *don't need to know* about the specific type in some context. Note that this variation should not be used if it guarantees failure: in that case you should instead raise an error at the planning stage and bail out. For example, we can't parse JSON with `json_to_recordset` without knowing the types of each recipient field, and so error out. Using the empty string for the type would be incorrect and futile. On the other hand we use the empty type for RPC calls since type resolution isn't implemented for RPC, but it's fine because the query still works with Postgres' implicit coercion. In the future, hopefully we will support data representations across the board and then the empty type may be permanently retired.
+data CoercibleField = CoercibleField
+  { cfName      :: FieldName
+  , cfJsonPath  :: JsonPath
+  , cfToJson    :: Bool
+  , cfIRType    :: Text                  -- ^ The native Postgres type of the field, the intermediate (IR) type before mapping.
+  , cfTransform :: Maybe TransformerProc -- ^ The optional mapping from irType -> targetType.
+  , cfDefault   :: Maybe Text
+  } deriving (Eq, Show)
+
+unknownField :: FieldName -> JsonPath -> CoercibleField
+unknownField name path = CoercibleField name path False "" Nothing Nothing
+
+-- | Like an API request LogicTree, but with coercible field information.
+data CoercibleLogicTree
+  = CoercibleExpr Bool LogicOperator [CoercibleLogicTree]
+  | CoercibleStmnt CoercibleFilter
+  deriving (Eq, Show)
+
+data CoercibleFilter = CoercibleFilter
+  { field  :: CoercibleField
+  , opExpr :: OpExpr
+  }
+  | CoercibleFilterNullEmbed Bool FieldName
+  deriving (Eq, Show)
+
+data CoercibleOrderTerm
+  = CoercibleOrderTerm
+    { coField     :: CoercibleField
+    , coDirection :: Maybe OrderDirection
+    , coNullOrder :: Maybe OrderNulls
+    }
+  | CoercibleOrderRelationTerm
+    { coRelation  :: FieldName
+    , coRelTerm   :: Field
+    , coDirection :: Maybe OrderDirection
+    , coNullOrder :: Maybe OrderNulls
+    }
+  deriving (Eq, Show)

src/PostgREST/Query.hs

@@ -24,16 +24,13 @@ import qualified Data.Text.Encoding                as T
 import qualified Hasql.Decoders                    as HD
 import qualified Hasql.DynamicStatements.Snippet   as SQL (Snippet)
 import qualified Hasql.DynamicStatements.Statement as SQL
-import qualified Hasql.Encoders                    as HE
-import qualified Hasql.Statement                   as SQL
 import qualified Hasql.Transaction                 as SQL
 
-import qualified PostgREST.Error               as Error
-import qualified PostgREST.Query.QueryBuilder  as QueryBuilder
-import qualified PostgREST.Query.Statements    as Statements
-import qualified PostgREST.RangeQuery          as RangeQuery
-import qualified PostgREST.SchemaCache         as SchemaCache
-import qualified PostgREST.SchemaCache.Routine as Routine
+import qualified PostgREST.Error              as Error
+import qualified PostgREST.Query.QueryBuilder as QueryBuilder
+import qualified PostgREST.Query.Statements   as Statements
+import qualified PostgREST.RangeQuery         as RangeQuery
+import qualified PostgREST.SchemaCache        as SchemaCache
 
 import Data.Scientific (FPFormat (..), formatScientific, isInteger)
 
@@ -52,8 +49,8 @@ import PostgREST.Plan                    (CallReadPlan (..),
                                           MutateReadPlan (..),
                                           WrappedReadPlan (..))
 import PostgREST.Plan.MutatePlan         (MutatePlan (..))
-import PostgREST.Query.SqlFragment       (fromQi, intercalateSnippet,
-                                          pgFmtIdentList,
+import PostgREST.Query.SqlFragment       (escapeIdentList, fromQi,
+                                          intercalateSnippet,
                                           setConfigLocal,
                                           setConfigLocalJson)
 import PostgREST.Query.Statements        (ResultSet (..))
@@ -68,7 +65,7 @@ import Protolude hiding (Handler)
 type DbHandler = ExceptT Error SQL.Transaction
 
 readQuery :: WrappedReadPlan -> AppConfig -> ApiRequest -> DbHandler ResultSet
-readQuery WrappedReadPlan{wrReadPlan, wrBinField} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} = do
+readQuery WrappedReadPlan{wrReadPlan, wrResAgg} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} = do
   let countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan
   resultSet <-
      lift . SQL.statement mempty $
@@ -82,7 +79,7 @@ readQuery WrappedReadPlan{wrReadPlan, wrBinField} conf@AppConfig{..} apiReq@ApiR
         )
         (shouldCount preferCount)
         iAcceptMediaType
-        wrBinField
+        wrResAgg
         configDbPreparedStatements
   failNotSingular iAcceptMediaType resultSet
   optionalRollback conf apiReq
@@ -153,19 +150,17 @@ deleteQuery mrPlan apiReq@ApiRequest{..} conf = do
   pure resultSet
 
 invokeQuery :: Routine -> CallReadPlan -> ApiRequest -> AppConfig -> PgVersion -> DbHandler ResultSet
-invokeQuery proc CallReadPlan{crReadPlan, crCallPlan, crBinField} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} conf@AppConfig{..} pgVer = do
+invokeQuery rout CallReadPlan{crReadPlan, crCallPlan, crResAgg} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} conf@AppConfig{..} pgVer = do
   resultSet <-
     lift . SQL.statement mempty $
       Statements.prepareCall
-        (Routine.funcReturnsScalar proc)
-        (Routine.funcReturnsSingleComposite proc)
-        (Routine.funcReturnsSetOfScalar proc)
+        rout
         (QueryBuilder.callPlanToQuery crCallPlan pgVer)
         (QueryBuilder.readPlanToQuery crReadPlan)
         (QueryBuilder.readPlanToCountQuery crReadPlan)
         (shouldCount preferCount)
         iAcceptMediaType
-        crBinField
+        crResAgg
         configDbPreparedStatements
 
   optionalRollback conf apiReq
@@ -190,7 +185,7 @@ openApiQuery sCache pgVer AppConfig{..} tSchema =
       pure Nothing
 
 writeQuery :: MutateReadPlan -> ApiRequest -> AppConfig  -> DbHandler ResultSet
-writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferences=Preferences{..}} conf =
+writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan, mrResAgg} apiReq@ApiRequest{iPreferences=Preferences{..}} conf =
   let
     (isInsert, pkCols) = case mrMutatePlan of {Insert{insPkCols} -> (True, insPkCols); _ -> (False, mempty);}
   in
@@ -200,6 +195,7 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferenc
       (QueryBuilder.mutatePlanToQuery mrMutatePlan)
       isInsert
       (iAcceptMediaType apiReq)
+      mrResAgg
       preferRepresentation
       pkCols
       (configDbPreparedStatements conf)
@@ -210,7 +206,7 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferenc
 failNotSingular :: MediaType -> ResultSet -> DbHandler ()
 failNotSingular _ RSPlan{} = pure ()
 failNotSingular mediaType RSStandard{rsQueryTotal=queryTotal} =
-  when (mediaType == MTSingularJSON && queryTotal /= 1) $ do
+  when (elem mediaType [MTSingularJSON True,MTSingularJSON False]  && queryTotal /= 1) $ do
     lift SQL.condemn
     throwError $ Error.singularityError queryTotal
 
@@ -257,7 +253,7 @@ setPgLocals AppConfig{..} claims role roleSettings req actualPgVersion = lift $
     roleSettingsSql = setConfigLocal mempty <$> roleSettings
     appSettingsSql = setConfigLocal mempty <$> (join bimap toUtf8 <$> configAppSettings)
     searchPathSql =
-      let schemas = pgFmtIdentList (iSchema req : configDbExtraSearchPath) in
+      let schemas = escapeIdentList (iSchema req : configDbExtraSearchPath) in
       setConfigLocal mempty ("search_path", schemas)
     usesLegacyGucs = configDbUseLegacyGucs && actualPgVersion < pgVersion140
 
@@ -272,8 +268,7 @@ setPgLocals AppConfig{..} claims role roleSettings req actualPgVersion = lift $
 runPreReq :: AppConfig -> DbHandler ()
 runPreReq conf = lift $ traverse_ (SQL.statement mempty . stmt) (configDbPreRequest conf)
   where
-    stmt req = SQL.Statement
+    stmt req = SQL.dynamicallyParameterized
       ("select " <> fromQi req <> "()")
-      HE.noParams
       HD.noResult
       (configDbPreparedStatements conf)

src/PostgREST/Query/QueryBuilder.hs

@@ -55,7 +55,7 @@ readPlanToQuery (Node ReadPlan{select,from=mainQi,fromAlias,where_=logicForest,o
   where
     fromFrag = fromF relToParent mainQi fromAlias
     qi = getQualifiedIdentifier relToParent mainQi fromAlias
-    defSelect = [(("*", []), Nothing, Nothing)] -- gets all the columns in case of an empty select, ignoring/obtaining these columns is done at the aggregation stage
+    defSelect = [(unknownField "*" [], Nothing, Nothing)] -- gets all the columns in case of an empty select, ignoring/obtaining these columns is done at the aggregation stage
     (selects, joins) = foldr getSelectsJoins ([],[]) forest
 
 getSelectsJoins :: ReadPlanTree -> ([SQL.Snippet], [SQL.Snippet]) -> ([SQL.Snippet], [SQL.Snippet])
@@ -66,45 +66,43 @@ getSelectsJoins rr@(Node ReadPlan{select, relName, relToParent=Just rel, relAggA
     aliasOrName = pgFmtIdent $ fromMaybe relName relAlias
     aggAlias = pgFmtIdent relAggAlias
     correlatedSubquery sub al cond =
-      (if relJoinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> SQL.sql al <> " ON " <> cond
+      (if relJoinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> al <> " ON " <> cond
     (sel, joi) = if relIsToOne rel
       then
         ( if relIsSpread
-            then SQL.sql aggAlias <> ".*"
-            else SQL.sql ("row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName)
+            then aggAlias <> ".*"
+            else "row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName
         , correlatedSubquery subquery aggAlias "TRUE")
       else
-        ( SQL.sql $ "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> aliasOrName
+        ( "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> aliasOrName
         , correlatedSubquery (
-            "SELECT json_agg(" <> SQL.sql aggAlias <> ") AS " <> SQL.sql aggAlias <>
-            "FROM (" <> subquery <> " ) AS " <> SQL.sql aggAlias
-          ) aggAlias $ if relJoinType == Just JTInner then SQL.sql aggAlias <> " IS NOT NULL" else "TRUE")
+            "SELECT json_agg(" <> aggAlias <> ") AS " <> aggAlias <>
+            "FROM (" <> subquery <> " ) AS " <> aggAlias
+          ) aggAlias $ if relJoinType == Just JTInner then aggAlias <> " IS NOT NULL" else "TRUE")
   in
   (if null select && null forest then selects else sel:selects, joi:joins)
 
 mutatePlanToQuery :: MutatePlan -> SQL.Snippet
 mutatePlanToQuery (Insert mainQi iCols body onConflct putConditions returnings _ applyDefaults) =
-  "INSERT INTO " <> SQL.sql (fromQi mainQi) <> SQL.sql (if null iCols then " " else "(" <> cols <> ") ") <>
+  "INSERT INTO " <> fromQi mainQi <> (if null iCols then " " else "(" <> cols <> ") ") <>
   fromJsonBodyF body iCols True False applyDefaults <>
   -- Only used for PUT
   (if null putConditions then mempty else "WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree (QualifiedIdentifier mempty "pgrst_body") <$> putConditions)) <>
-  SQL.sql (BS.unwords [
-    maybe mempty (\(oncDo, oncCols) ->
-      if null oncCols then
-        mempty
-      else
-        " ON CONFLICT(" <> BS.intercalate ", " (pgFmtIdent <$> oncCols) <> ") " <> case oncDo of
-        IgnoreDuplicates ->
-          "DO NOTHING"
-        MergeDuplicates  ->
-          if null iCols
-             then "DO NOTHING"
-             else "DO UPDATE SET " <> BS.intercalate ", " ((pgFmtIdent . tfName) <> const " = EXCLUDED." <> (pgFmtIdent . tfName) <$> iCols)
-      ) onConflct,
-    returningF mainQi returnings
-    ])
+  maybe mempty (\(oncDo, oncCols) ->
+    if null oncCols then
+      mempty
+    else
+      " ON CONFLICT(" <> intercalateSnippet ", " (pgFmtIdent <$> oncCols) <> ") " <> case oncDo of
+      IgnoreDuplicates ->
+        "DO NOTHING"
+      MergeDuplicates  ->
+        if null iCols
+           then "DO NOTHING"
+           else "DO UPDATE SET " <> intercalateSnippet ", " ((pgFmtIdent . cfName) <> const " = EXCLUDED." <> (pgFmtIdent . cfName) <$> iCols)
+    ) onConflct <> " " <>
+  returningF mainQi returnings
   where
-    cols = BS.intercalate ", " $ pgFmtIdent . tfName <$> iCols
+    cols = intercalateSnippet ", " $ pgFmtIdent . cfName <$> iCols
 
 -- An update without a limit is always filtered with a WHERE
 mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings applyDefaults)
@@ -112,82 +110,83 @@ mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings a
     -- if there are no columns we cannot do UPDATE table SET {empty}, it'd be invalid syntax
     -- selecting an empty resultset from mainQi gives us the column names to prevent errors when using &select=
     -- the select has to be based on "returnings" to make computed overloaded functions not throw
-    SQL.sql $ "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"
+    "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"
 
   | range == allRange =
-    "UPDATE " <> mainTbl <> " SET " <> SQL.sql nonRangeCols <> " " <>
+    "UPDATE " <> mainTbl <> " SET " <> nonRangeCols <> " " <>
     fromJsonBodyF body uCols False False applyDefaults <>
     whereLogic <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings
 
   | otherwise =
     "WITH " <>
     "pgrst_update_body AS (" <> fromJsonBodyF body uCols True True applyDefaults <> "), " <>
     "pgrst_affected_rows AS (" <>
-      "SELECT " <> SQL.sql rangeIdF <> " FROM " <> mainTbl <>
+      "SELECT " <> rangeIdF <> " FROM " <> mainTbl <>
       whereLogic <> " " <>
       orderF mainQi ordts <> " " <>
       limitOffsetF range <>
     ") " <>
-    "UPDATE " <> mainTbl <> " SET " <> SQL.sql rangeCols <>
+    "UPDATE " <> mainTbl <> " SET " <> rangeCols <>
     "FROM pgrst_affected_rows " <>
-    "WHERE " <> SQL.sql whereRangeIdF <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    "WHERE " <> whereRangeIdF <> " " <>
+    returningF mainQi returnings
 
   where
     whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
-    mainTbl = SQL.sql (fromQi mainQi)
-    emptyBodyReturnedColumns = if null returnings then "NULL" else BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
-    nonRangeCols = BS.intercalate ", " (pgFmtIdent . tfName <> const " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_body") . tfName <$> uCols)
-    rangeCols = BS.intercalate ", " ((\col -> pgFmtIdent (tfName col) <> " = (SELECT " <> pgFmtIdent (tfName col) <> " FROM pgrst_update_body) ") <$> uCols)
-    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
+    mainTbl = fromQi mainQi
+    emptyBodyReturnedColumns = if null returnings then "NULL" else intercalateSnippet ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
+    nonRangeCols = intercalateSnippet ", " (pgFmtIdent . cfName <> const " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_body") . cfName <$> uCols)
+    rangeCols = intercalateSnippet ", " ((\col -> pgFmtIdent (cfName col) <> " = (SELECT " <> pgFmtIdent (cfName col) <> " FROM pgrst_update_body) ") <$> uCols)
+    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (cfName . coField <$> ordts)
 
 mutatePlanToQuery (Delete mainQi logicForest range ordts returnings)
   | range == allRange =
-    "DELETE FROM " <> SQL.sql (fromQi mainQi) <> " " <>
+    "DELETE FROM " <> fromQi mainQi <> " " <>
     whereLogic <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings
 
   | otherwise =
     "WITH " <>
     "pgrst_affected_rows AS (" <>
-      "SELECT " <> SQL.sql rangeIdF <> " FROM " <> SQL.sql (fromQi mainQi) <>
+      "SELECT " <> rangeIdF <> " FROM " <> fromQi mainQi <>
        whereLogic <> " " <>
       orderF mainQi ordts <> " " <>
       limitOffsetF range <>
     ") " <>
-    "DELETE FROM " <> SQL.sql (fromQi mainQi) <> " " <>
+    "DELETE FROM " <> fromQi mainQi <> " " <>
     "USING pgrst_affected_rows " <>
-    "WHERE " <> SQL.sql whereRangeIdF <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    "WHERE " <> whereRangeIdF <> " " <>
+    returningF mainQi returnings
 
   where
     whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
-    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
+    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (cfName . coField <$> ordts)
 
 callPlanToQuery :: CallPlan -> PgVersion -> SQL.Snippet
 callPlanToQuery (FunctionCall qi params args returnsScalar returnsSetOfScalar returnsCompositeAlias returnings) pgVer =
-  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call AS pgrst_scalar " else returnedColumns) <> " " <>
+  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call.pgrst_scalar" else returnedColumns) <> " " <>
   fromCall
   where
     fromCall = case params of
       OnePosParam prm -> "FROM " <> callIt (singleParameter args $ encodeUtf8 $ ppType prm)
       KeyParams []    -> "FROM " <> callIt mempty
-      KeyParams prms  -> fromJsonBodyF args ((\p -> TypedField (ppName p) (ppType p) Nothing) <$> prms) False True False <> ", " <>
+      KeyParams prms  -> fromJsonBodyF args ((\p -> CoercibleField (ppName p) mempty False (ppTypeMaxLength p) Nothing Nothing) <$> prms) False True False <> ", " <>
                          "LATERAL " <> callIt (fmtParams prms)
 
     callIt :: SQL.Snippet -> SQL.Snippet
-    callIt argument | pgVer < pgVersion130 && pgVer >= pgVersion110 && returnsCompositeAlias = "(SELECT (" <> SQL.sql (fromQi qi) <> "(" <> argument <> ")).*) pgrst_call"
-                    | otherwise                                                              = SQL.sql (fromQi qi) <> "(" <> argument <> ") pgrst_call"
+    callIt argument | pgVer < pgVersion130 && pgVer >= pgVersion110 && returnsCompositeAlias = "(SELECT (" <> fromQi qi <> "(" <> argument <> ")).*) pgrst_call"
+                    | returnsScalar || returnsSetOfScalar                                    = "(SELECT " <> fromQi qi <> "(" <> argument <> ") pgrst_scalar) pgrst_call"
+                    | otherwise                                                              = fromQi qi <> "(" <> argument <> ") pgrst_call"
 
     fmtParams :: [RoutineParam] -> SQL.Snippet
-    fmtParams prms = SQL.sql $ BS.intercalate ", "
+    fmtParams prms = intercalateSnippet ", "
       ((\a -> (if ppVar a then "VARIADIC " else mempty) <> pgFmtIdent (ppName a) <> " := pgrst_body." <> pgFmtIdent (ppName a)) <$> prms)
 
     returnedColumns :: SQL.Snippet
     returnedColumns
       | null returnings = "*"
-      | otherwise       = SQL.sql $ BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty "pgrst_call") <$> returnings)
+      | otherwise       = intercalateSnippet ", " (pgFmtColumn (QualifiedIdentifier mempty "pgrst_call") <$> returnings)
 
 -- | SQL query meant for COUNTing the root node of the Tree.
 -- It only takes WHERE into account and doesn't include LIMIT/OFFSET because it would reduce the COUNT.
@@ -229,7 +228,7 @@ getQualifiedIdentifier rel mainQi tblAlias = case rel of
 
 -- FROM clause plus implicit joins
 fromF :: Maybe Relationship -> QualifiedIdentifier -> Maybe Alias -> SQL.Snippet
-fromF rel mainQi tblAlias = SQL.sql $ "FROM " <>
+fromF rel mainQi tblAlias = "FROM " <>
   (case rel of
     Just ComputedRelationship{relFunction,relTable} -> fromQi relFunction <> "(" <> pgFmtIdent (qiName relTable) <> ")"
     _                                               -> fromQi mainQi) <>

src/PostgREST/Query/SqlFragment.hs

@@ -4,18 +4,10 @@
 {-|
 Module      : PostgREST.Query.SqlFragment
 Description : Helper functions for PostgREST.QueryBuilder.
-
-Any function that outputs a SqlFragment should be in this module.
 -}
 module PostgREST.Query.SqlFragment
   ( noLocationF
-  , SqlFragment
-  , asBinaryF
-  , asCsvF
-  , asGeoJsonF
-  , asJsonF
-  , asJsonSingleF
-  , asXmlF
+  , aggF
   , countF
   , fromQi
   , limitOffsetF
@@ -24,7 +16,6 @@ module PostgREST.Query.SqlFragment
   , orderF
   , pgFmtColumn
   , pgFmtIdent
-  , pgFmtIdentList
   , pgFmtJoinCondition
   , pgFmtLogicTree
   , pgFmtOrderTerm
@@ -34,12 +25,15 @@ module PostgREST.Query.SqlFragment
   , responseStatusF
   , returningF
   , singleParameter
+  , sourceCTE
   , sourceCTEName
   , unknownEncoder
   , intercalateSnippet
   , explainF
   , setConfigLocal
   , setConfigLocalJson
+  , escapeIdent
+  , escapeIdentList
   ) where
 
 import qualified Data.Aeson                      as JSON
@@ -56,44 +50,50 @@ import Control.Arrow ((***))
 import Data.Foldable                 (foldr1)
 import Text.InterpolatedString.Perl6 (qc)
 
-import PostgREST.ApiRequest.Types        (Alias, Cast, Field,
-                                          Filter (..),
+import PostgREST.ApiRequest.Types        (Alias, Cast,
                                           FtsOperator (..),
                                           JsonOperand (..),
                                           JsonOperation (..),
                                           JsonPath,
                                           LogicOperator (..),
-                                          LogicTree (..), OpExpr (..),
+                                          OpExpr (..),
                                           OpQuantifier (..),
                                           Operation (..),
                                           OrderDirection (..),
                                           OrderNulls (..),
-                                          OrderTerm (..),
                                           QuantOperator (..),
                                           SimpleOperator (..),
                                           TrileanVal (..))
 import PostgREST.MediaType               (MTPlanFormat (..),
                                           MTPlanOption (..))
 import PostgREST.Plan.ReadPlan           (JoinCondition (..))
-import PostgREST.Plan.Types              (TypedField (..))
+import PostgREST.Plan.Types              (CoercibleField (..),
+                                          CoercibleFilter (..),
+                                          CoercibleLogicTree (..),
+                                          CoercibleOrderTerm (..),
+                                          unknownField)
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           rangeLimit, rangeOffset)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier (..))
+import PostgREST.SchemaCache.Routine     (ResultAggregate (..),
+                                          Routine (..),
+                                          funcReturnsScalar,
+                                          funcReturnsSetOfScalar,
+                                          funcReturnsSingleComposite)
 
 import Protolude hiding (cast)
 
-
--- | A part of a SQL query that cannot be executed independently
-type SqlFragment = ByteString
-
-noLocationF :: SqlFragment
-noLocationF = "array[]::text[]"
-
-sourceCTEName :: SqlFragment
+sourceCTEName :: Text
 sourceCTEName = "pgrst_source"
 
-simpleOperator :: SimpleOperator -> SqlFragment
+sourceCTE :: SQL.Snippet
+sourceCTE = "pgrst_source"
+
+noLocationF :: SQL.Snippet
+noLocationF = "array[]::text[]"
+
+simpleOperator :: SimpleOperator -> SQL.Snippet
 simpleOperator = \case
   OpNotEqual        -> "<>"
   OpContains        -> "@>"
@@ -105,7 +105,7 @@ simpleOperator = \case
   OpNotExtendsLeft  -> "&>"
   OpAdjacent        -> "-|-"
 
-quantOperator :: QuantOperator -> SqlFragment
+quantOperator :: QuantOperator -> SQL.Snippet
 quantOperator = \case
   OpEqual            -> "="
   OpGreaterThanEqual -> ">="
@@ -117,7 +117,7 @@ quantOperator = \case
   OpMatch            -> "~"
   OpIMatch           -> "~*"
 
-ftsOperator :: FtsOperator -> SqlFragment
+ftsOperator :: FtsOperator -> SQL.Snippet
 ftsOperator = \case
   FilterFts          -> "@@ to_tsquery"
   FilterFtsPlain     -> "@@ plainto_tsquery"
@@ -145,8 +145,11 @@ pgBuildArrayLiteral vals =
  "{" <> T.intercalate "," (escaped <$> vals) <> "}"
 
 -- TODO: refactor by following https://github.com/PostgREST/postgrest/pull/1631#issuecomment-711070833
-pgFmtIdent :: Text -> SqlFragment
-pgFmtIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
+pgFmtIdent :: Text -> SQL.Snippet
+pgFmtIdent x = SQL.sql $ escapeIdent x
+
+escapeIdent :: Text -> ByteString
+escapeIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
 
 -- Only use it if the input comes from the database itself, like on `jsonb_build_object('column_from_a_table', val)..`
 pgFmtLit :: Text -> Text
@@ -164,12 +167,12 @@ trimNullChars = T.takeWhile (/= '\x0')
 -- |
 -- Format a list of identifiers and separate them by commas.
 --
--- >>> pgFmtIdentList ["schema_1", "schema_2", "SPECIAL \"@/\\#~_-"]
+-- >>> escapeIdentList ["schema_1", "schema_2", "SPECIAL \"@/\\#~_-"]
 -- "\"schema_1\", \"schema_2\", \"SPECIAL \"\"@/\\#~_-\""
-pgFmtIdentList :: [Text] -> SqlFragment
-pgFmtIdentList schemas = BS.intercalate ", " $ pgFmtIdent <$> schemas
+escapeIdentList :: [Text] -> ByteString
+escapeIdentList schemas = BS.intercalate ", " $ escapeIdent <$> schemas
 
-asCsvF :: SqlFragment
+asCsvF :: SQL.Snippet
 asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
   where
     asCsvHeaderF =
@@ -177,34 +180,49 @@ asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
       "  FROM (" <>
       "    SELECT json_object_keys(r)::text as k" <>
       "    FROM ( " <>
-      "      SELECT row_to_json(hh) as r from " <> sourceCTEName <> " as hh limit 1" <>
+      "      SELECT row_to_json(hh) as r from " <> sourceCTE <> " as hh limit 1" <>
       "    ) s" <>
       "  ) a" <>
       ")"
     asCsvBodyF = "coalesce(string_agg(substring(_postgrest_t::text, 2, length(_postgrest_t::text) - 2), '\n'), '')"
 
-asJsonSingleF :: Bool -> SqlFragment
-asJsonSingleF returnsScalar
-  | returnsScalar = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
-  | otherwise     = "coalesce(json_agg(_postgrest_t)->0, 'null')"
+addNullsToSnip :: Bool -> SQL.Snippet -> SQL.Snippet
+addNullsToSnip strip snip =
+  if strip then "json_strip_nulls(" <> snip <> ")" else  snip
 
-asJsonF :: Bool -> Bool -> Bool -> SqlFragment
-asJsonF returnsScalar returnsSetOfScalar returnsSingleComposite
-  | returnsSingleComposite              = "coalesce(json_agg(_postgrest_t)->0, 'null')"
-  | returnsScalar                       = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
-  | returnsSetOfScalar                  = "coalesce(json_agg(_postgrest_t.pgrst_scalar), '[]')"
-  | otherwise                           = "coalesce(json_agg(_postgrest_t), '[]')"
+asJsonSingleF :: Maybe Routine -> Bool -> SQL.Snippet
+asJsonSingleF rout strip
+  | returnsScalar = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0"  <> ", 'null')"
+  | otherwise     = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0"  <> ", 'null')"
+  where
+    returnsScalar = maybe False funcReturnsScalar rout
 
-asXmlF :: FieldName -> SqlFragment
-asXmlF fieldName = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
+asJsonF :: Maybe Routine -> Bool -> SQL.Snippet
+asJsonF rout strip
+  | returnsSingleComposite    = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0" <> ", 'null')"
+  | returnsScalar             = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0" <> ", 'null')"
+  | returnsSetOfScalar        = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)" <> ", '[]')"
+  | otherwise                 = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)" <> ", '[]')"
+  where
+    (returnsSingleComposite, returnsScalar, returnsSetOfScalar) = case rout of
+      Just r  -> (funcReturnsSingleComposite r, funcReturnsScalar r, funcReturnsSetOfScalar r)
+      Nothing -> (False, False, False)
 
-asGeoJsonF ::  SqlFragment
+
+asXmlF :: Maybe FieldName -> SQL.Snippet
+asXmlF (Just fieldName) = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
+-- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
+asXmlF Nothing          = "coalesce(xmlagg(_postgrest_t), '')"
+
+asGeoJsonF ::  SQL.Snippet
 asGeoJsonF = "json_build_object('type', 'FeatureCollection', 'features', coalesce(json_agg(ST_AsGeoJSON(_postgrest_t)::json), '[]'))"
 
-asBinaryF :: FieldName -> SqlFragment
-asBinaryF fieldName = "coalesce(string_agg(_postgrest_t." <> pgFmtIdent fieldName <> ", ''), '')"
+asBinaryF :: Maybe FieldName -> SQL.Snippet
+asBinaryF (Just fieldName) = "coalesce(string_agg(_postgrest_t." <> pgFmtIdent fieldName <> ", ''), '')"
+-- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
+asBinaryF Nothing          = "coalesce(string_agg(_postgrest_t, ''), '')"
 
-locationF :: [Text] -> SqlFragment
+locationF :: [Text] -> SQL.Snippet
 locationF pKeys = [qc|(
   WITH data AS (SELECT row_to_json(_) AS row FROM {sourceCTEName} AS _ LIMIT 1)
   SELECT array_agg(json_data.key || '=' || coalesce('eq.' || json_data.value, 'is.null'))
@@ -214,34 +232,45 @@ locationF pKeys = [qc|(
   where
     fmtPKeys = T.intercalate "','" pKeys
 
-fromQi :: QualifiedIdentifier -> SqlFragment
+fromQi :: QualifiedIdentifier -> SQL.Snippet
 fromQi t = (if T.null s then mempty else pgFmtIdent s <> ".") <> pgFmtIdent n
   where
     n = qiName t
     s = qiSchema t
 
-pgFmtColumn :: QualifiedIdentifier -> Text -> SqlFragment
+pgFmtColumn :: QualifiedIdentifier -> Text -> SQL.Snippet
 pgFmtColumn table "*" = fromQi table <> ".*"
 pgFmtColumn table c   = fromQi table <> "." <> pgFmtIdent c
 
-pgFmtField :: QualifiedIdentifier -> Field -> SQL.Snippet
-pgFmtField table (c, []) = SQL.sql (pgFmtColumn table c)
--- Using to_jsonb instead of to_json to avoid missing operator errors when filtering:
--- "operator does not exist: json = unknown"
-pgFmtField table (c, jp) = SQL.sql ("to_jsonb(" <> pgFmtColumn table c <> ")") <> pgFmtJsonPath jp
+pgFmtCallUnary :: Text -> SQL.Snippet -> SQL.Snippet
+pgFmtCallUnary f x = SQL.sql (encodeUtf8 f) <> "(" <> x <> ")"
 
-pgFmtSelectItem :: QualifiedIdentifier -> (Field, Maybe Cast, Maybe Alias) -> SQL.Snippet
-pgFmtSelectItem table (f@(fName, jp), Nothing, alias) = pgFmtField table f <> SQL.sql (pgFmtAs fName jp alias)
+pgFmtField :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
+pgFmtField table CoercibleField{cfName=fn, cfJsonPath=[]}                                = pgFmtColumn table fn
+pgFmtField table CoercibleField{cfName=fn, cfToJson=doToJson, cfJsonPath=jp} | doToJson  = "to_jsonb(" <> pgFmtColumn table fn <> ")" <> pgFmtJsonPath jp
+                                                                             | otherwise = pgFmtColumn table fn <> pgFmtJsonPath jp
+
+-- Select the value of a named element from a table, applying its optional coercion mapping if any.
+pgFmtTableCoerce :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
+pgFmtTableCoerce table fld@(CoercibleField{cfTransform=(Just formatterProc)}) = pgFmtCallUnary formatterProc (pgFmtField table fld)
+pgFmtTableCoerce table f = pgFmtField table f
+
+-- | Like the previous but now we just have a name so no namespace or JSON paths.
+pgFmtCoerceNamed :: CoercibleField -> SQL.Snippet
+pgFmtCoerceNamed CoercibleField{cfName=fn, cfTransform=(Just formatterProc)} = pgFmtCallUnary formatterProc (pgFmtIdent fn) <> " AS " <> pgFmtIdent fn
+pgFmtCoerceNamed CoercibleField{cfName=fn} = pgFmtIdent fn
+
+pgFmtSelectItem :: QualifiedIdentifier -> (CoercibleField, Maybe Cast, Maybe Alias) -> SQL.Snippet
+pgFmtSelectItem table (fld, Nothing, alias) = pgFmtTableCoerce table fld <> pgFmtAs (cfName fld) (cfJsonPath fld) alias
 -- Ideally we'd quote the cast with "pgFmtIdent cast". However, that would invalidate common casts such as "int", "bigint", etc.
 -- Try doing: `select 1::"bigint"` - it'll err, using "int8" will work though. There's some parser magic that pg does that's invalidated when quoting.
 -- Not quoting should be fine, we validate the input on Parsers.
-pgFmtSelectItem table (f@(fName, jp), Just cast, alias) = "CAST (" <> pgFmtField table f <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )" <> SQL.sql (pgFmtAs fName jp alias)
+pgFmtSelectItem table (fld, Just cast, alias) = "CAST (" <> pgFmtTableCoerce table fld <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )" <> pgFmtAs (cfName fld) (cfJsonPath fld) alias
 
 -- TODO: At this stage there shouldn't be a Maybe since ApiRequest should ensure that an INSERT/UPDATE has a body
-fromJsonBodyF :: Maybe LBS.ByteString -> [TypedField] -> Bool -> Bool -> Bool -> SQL.Snippet
+fromJsonBodyF :: Maybe LBS.ByteString -> [CoercibleField] -> Bool -> Bool -> Bool -> SQL.Snippet
 fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
-  SQL.sql
-  (if includeSelect then "SELECT " <> parsedCols <> " " else mempty) <>
+  (if includeSelect then "SELECT " <> namedCols <> " " else mempty) <>
   "FROM (SELECT " <> jsonPlaceHolder <> " AS json_data) pgrst_payload, " <>
   -- convert a json object into a json array, this way we can use json_to_recordset for all json payloads
   -- Otherwise we'd have to use json_to_record for json objects and json_to_recordset for json arrays
@@ -250,22 +279,23 @@ fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
   (if includeDefaults
     then "LATERAL (SELECT jsonb_agg(jsonb_build_object(" <> defsJsonb <> ") || elem) AS val from jsonb_array_elements(pgrst_uniform_json.val) elem) pgrst_json_defs, "
     else mempty) <>
-  "LATERAL (SELECT * FROM " <>
+  "LATERAL (SELECT " <> parsedCols <> " FROM " <>
     (if null fields
       -- When we are inserting no columns (e.g. using default values), we can't use our ordinary `json_to_recordset`
       -- because it can't extract records with no columns (there's no valid syntax for the `AS (colName colType,...)`
       -- part). But we still need to ensure as many rows are created as there are array elements.
       then SQL.sql $ jsonArrayElementsF <> "(" <> finalBodyF <> ") _ "
-      else SQL.sql $ jsonToRecordsetF <> "(" <> finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
+      else jsonToRecordsetF <> "(" <> SQL.sql finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
     ) <>
   ") pgrst_body "
   where
-    parsedCols = BS.intercalate ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . tfName <$> fields
-    typedCols = BS.intercalate ", " $ pgFmtIdent . tfName <> const " " <> encodeUtf8 . tfIRType <$> fields
+    namedCols = intercalateSnippet ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . cfName <$> fields
+    parsedCols = intercalateSnippet ", " $ pgFmtCoerceNamed <$> fields
+    typedCols = intercalateSnippet ", " $ pgFmtIdent . cfName <> const " " <> SQL.sql . encodeUtf8 . cfIRType <$> fields
     defsJsonb = SQL.sql $ BS.intercalate "," fieldsWDefaults
     fieldsWDefaults = mapMaybe (\case
-        TypedField{tfName=nam, tfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
-        TypedField{tfDefault=Nothing} -> Nothing
+        CoercibleField{cfName=nam, cfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
+        CoercibleField{cfDefault=Nothing} -> Nothing
       ) fields
     (finalBodyF, jsonTypeofF, jsonBuildArrayF, jsonArrayElementsF, jsonToRecordsetF) =
       if includeDefaults
@@ -273,16 +303,16 @@ fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
         else ("pgrst_uniform_json.val", "json_typeof", "json_build_array", "json_array_elements", "json_to_recordset")
     jsonPlaceHolder = SQL.encoderAndParam (HE.nullable $ if includeDefaults then HE.jsonbLazyBytes else HE.jsonLazyBytes) body
 
-pgFmtOrderTerm :: QualifiedIdentifier -> OrderTerm -> SQL.Snippet
+pgFmtOrderTerm :: QualifiedIdentifier -> CoercibleOrderTerm -> SQL.Snippet
 pgFmtOrderTerm qi ot =
   fmtOTerm ot <> " " <>
   SQL.sql (BS.unwords [
-    maybe mempty direction $ otDirection ot,
-    maybe mempty nullOrder $ otNullOrder ot])
+    maybe mempty direction $ coDirection ot,
+    maybe mempty nullOrder $ coNullOrder ot])
   where
     fmtOTerm = \case
-      OrderTerm{otTerm}                        -> pgFmtField qi otTerm
-      OrderRelationTerm{otRelation, otRelTerm} -> pgFmtField (QualifiedIdentifier mempty otRelation) otRelTerm
+      CoercibleOrderTerm{coField=cof}                            -> pgFmtField qi cof
+      CoercibleOrderRelationTerm{coRelation, coRelTerm=(fn, jp)} -> pgFmtField (QualifiedIdentifier mempty coRelation) (unknownField fn jp)
 
     direction OrderAsc  = "ASC"
     direction OrderDesc = "DESC"
@@ -290,17 +320,31 @@ pgFmtOrderTerm qi ot =
     nullOrder OrderNullsFirst = "NULLS FIRST"
     nullOrder OrderNullsLast  = "NULLS LAST"
 
+-- | Interpret a literal in the way the planner indicated through the CoercibleField.
+pgFmtUnknownLiteralForField :: SQL.Snippet -> CoercibleField -> SQL.Snippet
+pgFmtUnknownLiteralForField value CoercibleField{cfTransform=(Just parserProc)} = pgFmtCallUnary parserProc value
+-- But when no transform is requested, we just use the literal as-is.
+pgFmtUnknownLiteralForField value _ = value
 
-pgFmtFilter :: QualifiedIdentifier -> Filter -> SQL.Snippet
-pgFmtFilter _ (FilterNullEmbed hasNot fld) = SQL.sql (pgFmtIdent fld) <> " IS " <> (if hasNot then "NOT" else mempty) <> " NULL"
-pgFmtFilter _ (Filter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
-pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
-   Op op val -> " " <> SQL.sql (simpleOperator op) <> " " <> unknownLiteral val
+-- | Array version of the above, used by ANY().
+pgFmtArrayLiteralForField :: [Text] -> CoercibleField -> SQL.Snippet
+-- When a transformation is requested, we need to apply the transformation to each element of the array. This could be done by just making a query with `parser(value)` for each value, but may lead to huge query lengths. Imagine `data_representations.color_from_text('...'::text)` for repeated for a hundred values. Instead we use `unnest()` to unpack a standard array literal and then apply the transformation to each element, like a map.
+-- Note the literals will be treated as text since in every case when we use ANY() the parameters are textual (coming from a query string). We want to rely on the `text->domain` parser to do the right thing.
+pgFmtArrayLiteralForField values CoercibleField{cfTransform=(Just parserProc)} = SQL.sql "(SELECT " <> pgFmtCallUnary parserProc (SQL.sql "unnest(" <> unknownLiteral (pgBuildArrayLiteral values) <> "::text[])") <> ")"
+-- When no transformation is requested, we don't need a subquery.
+pgFmtArrayLiteralForField values _ = unknownLiteral (pgBuildArrayLiteral values)
 
-   OpQuant op quant val -> " " <> SQL.sql (quantOperator op) <> " " <> case op of
+
+pgFmtFilter :: QualifiedIdentifier -> CoercibleFilter -> SQL.Snippet
+pgFmtFilter _ (CoercibleFilterNullEmbed hasNot fld) = pgFmtIdent fld <> " IS " <> (if hasNot then "NOT" else mempty) <> " NULL"
+pgFmtFilter _ (CoercibleFilter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
+pgFmtFilter table (CoercibleFilter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
+   Op op val  -> " " <> simpleOperator op <> " " <> pgFmtUnknownLiteralForField (unknownLiteral val) fld
+
+   OpQuant op quant val -> " " <> quantOperator op <> " " <> case op of
      OpLike  -> fmtQuant quant $ unknownLiteral (T.map star val)
      OpILike -> fmtQuant quant $ unknownLiteral (T.map star val)
-     _       -> fmtQuant quant $ unknownLiteral val
+     _       -> fmtQuant quant $ pgFmtUnknownLiteralForField (unknownLiteral val) fld
 
    -- IS cannot be prepared. `PREPARE boolplan AS SELECT * FROM projects where id IS $1` will give a syntax error.
    -- The above can be fixed by using `PREPARE boolplan AS SELECT * FROM projects where id IS NOT DISTINCT FROM $1;`
@@ -319,9 +363,9 @@ pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField
    -- + Can invalidate prepared statements: multiple parameters on an IN($1, $2, $3) will lead to using different prepared statements and not take advantage of caching.
    In vals -> " " <> case vals of
       [""] -> "= ANY('{}') "
-      _    -> "= ANY (" <> unknownLiteral (pgBuildArrayLiteral vals) <> ") "
+      _    -> "= ANY (" <> pgFmtArrayLiteralForField vals fld <> ") "
 
-   Fts op lang val -> " " <> SQL.sql (ftsOperator op) <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
+   Fts op lang val -> " " <> ftsOperator op <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
  where
    ftsLang = maybe mempty (\l -> unknownLiteral l <> ", ")
    notOp = if hasNot then "NOT" else mempty
@@ -333,16 +377,16 @@ pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField
 
 pgFmtJoinCondition :: JoinCondition -> SQL.Snippet
 pgFmtJoinCondition (JoinCondition (qi1, col1) (qi2, col2)) =
-  SQL.sql $ pgFmtColumn qi1 col1 <> " = " <> pgFmtColumn qi2 col2
+  pgFmtColumn qi1 col1 <> " = " <> pgFmtColumn qi2 col2
 
-pgFmtLogicTree :: QualifiedIdentifier -> LogicTree -> SQL.Snippet
-pgFmtLogicTree qi (Expr hasNot op forest) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTree qi <$> forest) <> ")"
+pgFmtLogicTree :: QualifiedIdentifier -> CoercibleLogicTree -> SQL.Snippet
+pgFmtLogicTree qi (CoercibleExpr hasNot op forest) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTree qi <$> forest) <> ")"
   where
     notOp =  if hasNot then "NOT" else mempty
 
     opSql And = " AND "
     opSql Or  = " OR "
-pgFmtLogicTree qi (Stmnt flt) = pgFmtFilter qi flt
+pgFmtLogicTree qi (CoercibleStmnt flt) = pgFmtFilter qi flt
 
 pgFmtJsonPath :: JsonPath -> SQL.Snippet
 pgFmtJsonPath = \case
@@ -353,7 +397,7 @@ pgFmtJsonPath = \case
     pgFmtJsonOperand (JKey k) = unknownLiteral k
     pgFmtJsonOperand (JIdx i) = unknownLiteral i <> "::int"
 
-pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SqlFragment
+pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SQL.Snippet
 pgFmtAs _ [] Nothing = mempty
 pgFmtAs fName jp Nothing = case jOp <$> lastMay jp of
   Just (JKey key) -> " AS " <> pgFmtIdent key
@@ -365,7 +409,7 @@ pgFmtAs fName jp Nothing = case jOp <$> lastMay jp of
   Nothing -> mempty
 pgFmtAs _ _ (Just alias) = " AS " <> pgFmtIdent alias
 
-countF :: SQL.Snippet -> Bool -> (SQL.Snippet, SqlFragment)
+countF :: SQL.Snippet -> Bool -> (SQL.Snippet, SQL.Snippet)
 countF countQuery shouldCount =
   if shouldCount
     then (
@@ -375,11 +419,11 @@ countF countQuery shouldCount =
         mempty
       , "null::bigint")
 
-returningF :: QualifiedIdentifier -> [FieldName] -> SqlFragment
+returningF :: QualifiedIdentifier -> [FieldName] -> SQL.Snippet
 returningF qi returnings =
   if null returnings
     then "RETURNING 1" -- For mutation cases where there's no ?select, we return 1 to know how many rows were modified
-    else "RETURNING " <> BS.intercalate ", " (pgFmtColumn qi <$> returnings)
+    else "RETURNING " <> intercalateSnippet ", " (pgFmtColumn qi <$> returnings)
 
 limitOffsetF :: NonnegRange -> SQL.Snippet
 limitOffsetF range =
@@ -388,25 +432,25 @@ limitOffsetF range =
     limit = maybe "ALL" (\l -> unknownEncoder (BS.pack $ show l)) $ rangeLimit range
     offset = unknownEncoder (BS.pack . show $ rangeOffset range)
 
-responseHeadersF :: SqlFragment
+responseHeadersF :: SQL.Snippet
 responseHeadersF = currentSettingF "response.headers"
 
-responseStatusF :: SqlFragment
+responseStatusF :: SQL.Snippet
 responseStatusF = currentSettingF "response.status"
 
-currentSettingF :: SqlFragment -> SqlFragment
+currentSettingF :: SQL.Snippet -> SQL.Snippet
 currentSettingF setting =
   -- nullif is used because of https://gist.github.com/steve-chavez/8d7033ea5655096903f3b52f8ed09a15
   "nullif(current_setting('" <> setting <> "', true), '')"
 
-mutRangeF :: QualifiedIdentifier -> [FieldName] -> (SqlFragment, SqlFragment)
+mutRangeF :: QualifiedIdentifier -> [FieldName] -> (SQL.Snippet, SQL.Snippet)
 mutRangeF mainQi rangeId =
   (
-    BS.intercalate " AND " $ (\col -> pgFmtColumn mainQi col <> " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_affected_rows") col) <$> rangeId
-  , BS.intercalate ", " (pgFmtColumn mainQi <$> rangeId)
+    intercalateSnippet " AND " $ (\col -> pgFmtColumn mainQi col <> " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_affected_rows") col) <$> rangeId
+  , intercalateSnippet ", " (pgFmtColumn mainQi <$> rangeId)
   )
 
-orderF :: QualifiedIdentifier -> [OrderTerm] -> SQL.Snippet
+orderF :: QualifiedIdentifier -> [CoercibleOrderTerm] -> SQL.Snippet
 orderF _ []    = mempty
 orderF qi ordts = "ORDER BY " <> intercalateSnippet ", " (pgFmtOrderTerm qi <$> ordts)
 
@@ -434,8 +478,8 @@ explainF fmt opts snip =
     fmtPlanOpt PlanBuffers  = "BUFFERS"
     fmtPlanOpt PlanWAL      = "WAL"
 
-    fmtPlanFmt PlanJSON = "FORMAT JSON"
     fmtPlanFmt PlanText = "FORMAT TEXT"
+    fmtPlanFmt PlanJSON = "FORMAT JSON"
 
 -- | Do a pg set_config(setting, value, true) call. This is equivalent to a SET LOCAL.
 setConfigLocal :: ByteString -> (ByteString, ByteString) -> SQL.Snippet
@@ -451,3 +495,14 @@ setConfigLocalJson prefix keyVals = [setConfigLocal mempty (prefix, gucJsonVal k
     gucJsonVal = LBS.toStrict . JSON.encode . HM.fromList . arrayByteStringToText
     arrayByteStringToText :: [(ByteString, ByteString)] -> [(Text,Text)]
     arrayByteStringToText keyVal = (T.decodeUtf8 *** T.decodeUtf8) <$> keyVal
+
+aggF :: Maybe Routine -> ResultAggregate -> SQL.Snippet
+aggF rout = \case
+  BuiltinAggJson             -> asJsonF rout False
+  BuiltinAggArrayJsonStrip   -> asJsonF rout True
+  BuiltinAggSingleJson strip -> asJsonSingleF rout strip
+  BuiltinAggGeoJson          -> asGeoJsonF
+  BuiltinAggCsv              -> asCsvF
+  BuiltinAggXml    bField    -> asXmlF    bField
+  BuiltinAggBinary bField    -> asBinaryF bField
+  NoAgg                      -> "''::text"

src/PostgREST/Query/Statements.hs

@@ -23,15 +23,13 @@ import qualified Hasql.DynamicStatements.Statement as SQL
 import qualified Hasql.Statement                   as SQL
 
 import Control.Lens ((^?))
-import Data.Maybe   (fromJust)
 
 import PostgREST.ApiRequest.Preferences
-import PostgREST.MediaType               (MTPlanAttrs (..),
-                                          MTPlanFormat (..),
-                                          MediaType (..),
-                                          getMediaType)
+import PostgREST.MediaType              (MTPlanFormat (..),
+                                         MediaType (..))
 import PostgREST.Query.SqlFragment
-import PostgREST.SchemaCache.Identifiers (FieldName)
+import PostgREST.SchemaCache.Routine    (ResultAggregate (..),
+                                         Routine)
 
 import Protolude
 
@@ -55,109 +53,82 @@ data ResultSet
   | RSPlan BS.ByteString -- ^ the plan of the query
 
 
-prepareWrite :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType ->
-                PreferRepresentation -> [Text] -> Bool -> SQL.Statement () ResultSet
-prepareWrite selectQuery mutateQuery isInsert mt rep pKeys =
+prepareWrite :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> ResultAggregate ->
+                Maybe PreferRepresentation -> [Text] -> Bool -> SQL.Statement () ResultSet
+prepareWrite selectQuery mutateQuery isInsert mt rAgg rep pKeys =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
  where
   snippet =
-    "WITH " <> SQL.sql sourceCTEName <> " AS (" <> mutateQuery <> ") " <>
-    SQL.sql (
+    "WITH " <> sourceCTE <> " AS (" <> mutateQuery <> ") " <>
     "SELECT " <>
       "'' AS total_result_set, " <>
       "pg_catalog.count(_postgrest_t) AS page_total, " <>
       locF <> " AS header, " <>
-      bodyF <> " AS body, " <>
+      aggF Nothing rAgg <> " AS body, " <>
       responseHeadersF <> " AS response_headers, " <>
-      responseStatusF  <> " AS response_status "
-    ) <>
+      responseStatusF  <> " AS response_status " <>
     "FROM (" <> selectF <> ") _postgrest_t"
 
   locF =
-    if isInsert && rep == HeadersOnly
-      then BS.unwords [
-        "CASE WHEN pg_catalog.count(_postgrest_t) = 1",
-          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ")",
-          "ELSE " <> noLocationF,
-        "END"]
+    if isInsert && rep == Just HeadersOnly
+      then
+        "CASE WHEN pg_catalog.count(_postgrest_t) = 1 " <>
+          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ") " <>
+          "ELSE " <> noLocationF <> " " <>
+        "END"
       else noLocationF
 
-  bodyF
-    | rep /= Full                       = "''"
-    | getMediaType mt == MTTextCSV      = asCsvF
-    | getMediaType mt == MTGeoJSON      = asGeoJsonF
-    | getMediaType mt == MTSingularJSON = asJsonSingleF False
-    | otherwise                         = asJsonF False False False
-
   selectF
     -- prevent using any of the column names in ?select= when no response is returned from the CTE
-    | rep /= Full = SQL.sql ("SELECT * FROM " <> sourceCTEName)
-    | otherwise   = selectQuery
+    | rAgg == NoAgg = "SELECT * FROM " <> sourceCTE
+    | otherwise     = selectQuery
 
   decodeIt :: HD.Result ResultSet
   decodeIt = case mt of
     MTPlan{} -> planRow
     _        -> fromMaybe (RSStandard Nothing 0 mempty mempty Nothing Nothing) <$> HD.rowMaybe (standardRow False)
 
-prepareRead :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> Maybe FieldName -> Bool -> SQL.Statement () ResultSet
-prepareRead selectQuery countQuery countTotal mt binaryField =
+prepareRead :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> ResultAggregate -> Bool -> SQL.Statement () ResultSet
+prepareRead selectQuery countQuery countTotal mt rAgg =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
  where
   snippet =
-    "WITH " <>
-    SQL.sql sourceCTEName <> " AS ( " <> selectQuery <> " ) " <>
+    "WITH " <> sourceCTE <> " AS ( " <> selectQuery <> " ) " <>
     countCTEF <> " " <>
-    SQL.sql ("SELECT " <>
+    "SELECT " <>
       countResultF <> " AS total_result_set, " <>
       "pg_catalog.count(_postgrest_t) AS page_total, " <>
-      bodyF <> " AS body, " <>
+      aggF Nothing rAgg <> " AS body, " <>
       responseHeadersF <> " AS response_headers, " <>
       responseStatusF <> " AS response_status " <>
-    "FROM ( SELECT * FROM " <> sourceCTEName <> " ) _postgrest_t")
+    "FROM ( SELECT * FROM " <> sourceCTE <> " ) _postgrest_t"
 
   (countCTEF, countResultF) = countF countQuery countTotal
 
-  bodyF
-    | getMediaType mt == MTTextCSV                       = asCsvF
-    | getMediaType mt == MTSingularJSON                  = asJsonSingleF False
-    | getMediaType mt == MTGeoJSON                       = asGeoJsonF
-    | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
-    | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-    | otherwise                                          = asJsonF False False False
-
   decodeIt :: HD.Result ResultSet
   decodeIt = case mt of
     MTPlan{} -> planRow
     _        -> HD.singleRow $ standardRow True
 
-prepareCall :: Bool -> Bool -> Bool -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
-               MediaType -> Maybe FieldName -> Bool ->
+prepareCall :: Routine -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
+               MediaType -> ResultAggregate -> Bool ->
                SQL.Statement () ResultSet
-prepareCall returnsScalar returnsSingleComposite returnsSetOfScalar callProcQuery selectQuery countQuery countTotal mt binaryField =
+prepareCall rout callProcQuery selectQuery countQuery countTotal mt rAgg =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
   where
     snippet =
-      "WITH " <> SQL.sql sourceCTEName <> " AS (" <> callProcQuery <> ") " <>
+      "WITH " <> sourceCTE <> " AS (" <> callProcQuery <> ") " <>
       countCTEF <>
-      SQL.sql (
       "SELECT " <>
         countResultF <> " AS total_result_set, " <>
         "pg_catalog.count(_postgrest_t) AS page_total, " <>
-        bodyF <> " AS body, " <>
+        aggF (Just rout) rAgg <> " AS body, " <>
         responseHeadersF <> " AS response_headers, " <>
-        responseStatusF <> " AS response_status ") <>
+        responseStatusF <> " AS response_status " <>
       "FROM (" <> selectQuery <> ") _postgrest_t"
 
     (countCTEF, countResultF) = countF countQuery countTotal
 
-    bodyF
-     | getMediaType mt == MTSingularJSON                  = asJsonSingleF returnsScalar
-     | getMediaType mt == MTTextCSV                       = asCsvF
-     | getMediaType mt == MTGeoJSON                       = asGeoJsonF
-     | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
-     | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-     | otherwise                                          = asJsonF returnsScalar returnsSetOfScalar returnsSingleComposite
-
     decodeIt :: HD.Result ResultSet
     decodeIt = case mt of
       MTPlan{} -> planRow
@@ -187,8 +158,8 @@ standardRow noLocation =
 
 mtSnippet :: MediaType -> SQL.Snippet -> SQL.Snippet
 mtSnippet mediaType snippet = case mediaType of
-  MTPlan (MTPlanAttrs _ fmt opts) -> explainF fmt opts snippet
-  _                               -> snippet
+  MTPlan _ fmt opts -> explainF fmt opts snippet
+  _                 -> snippet
 
 -- | We use rowList because when doing EXPLAIN (FORMAT TEXT), the result comes as many rows. FORMAT JSON comes as one.
 planRow :: HD.Result ResultSet

src/PostgREST/Response.hs

@@ -1,3 +1,7 @@
+{- |
+   Module      : PostgREST.Response
+   Description : Generate HTTP Response
+-}
 {-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 module PostgREST.Response
@@ -14,6 +18,8 @@ module PostgREST.Response
   , addRetryHint
   , isServiceUnavailable
   , optionalRollback
+  , concatPrefAppsHeaders
+  , addPrefToHeaders
   , traceHeaderMiddleware
   ) where
 
@@ -60,6 +66,7 @@ import qualified PostgREST.SchemaCache.Routine as Routine
 import Protolude      hiding (Handler, toS)
 import Protolude.Conv (toS)
 
+
 readResponse :: Bool -> QualifiedIdentifier -> ApiRequest -> ResultSet -> Wai.Response
 readResponse headersOnly identifier ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
   RSStandard{..} -> do
@@ -111,14 +118,17 @@ createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiReques
           , toAppliedHeader <$> preferMissing
           ]
 
-    if preferRepresentation == Full then
-      response HTTP.status201 (headers ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
-    else
-      response HTTP.status201 headers mempty
+    case preferRepresentation of
+      Just Full -> response HTTP.status201 (addPrefToHeaders headers Full ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
+      Just None -> response HTTP.status201 (addPrefToHeaders headers None) mempty
+      Just HeadersOnly -> response HTTP.status201 (addPrefToHeaders headers HeadersOnly) mempty
+      Nothing -> response HTTP.status201 headers mempty
+
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
+
 updateResponse :: ApiRequest -> ResultSet -> Wai.Response
 updateResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
   RSStandard{..} -> do
@@ -129,12 +139,11 @@ updateResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet
           if shouldCount preferCount then Just rsQueryTotal else Nothing
       headers = catMaybes [contentRangeHeader, toAppliedHeader <$> preferMissing]
 
-    if preferRepresentation == Full then
-        response HTTP.status200
-          (headers ++ contentTypeHeaders ctxApiRequest)
+    case preferRepresentation of
+        Just Full -> response HTTP.status200 (addPrefToHeaders headers Full ++ contentTypeHeaders ctxApiRequest)
           (LBS.fromStrict rsBody)
-      else
-        response HTTP.status204 headers mempty
+        Just None -> response HTTP.status204 (addPrefToHeaders headers None) mempty
+        _ -> response HTTP.status204 headers mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
@@ -145,10 +154,10 @@ singleUpsertResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resu
     let
       response = gucResponse rsGucStatus rsGucHeaders
 
-    if preferRepresentation == Full then
-      response HTTP.status200 (contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
-    else
-      response HTTP.status204 [] mempty
+    case preferRepresentation of
+      Just Full -> response HTTP.status200 (contentTypeHeaders ctxApiRequest ++ [toAppliedHeader Full]) (LBS.fromStrict rsBody)
+      Just None -> response HTTP.status204 [toAppliedHeader None] mempty
+      _ -> response HTTP.status204 [] mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
@@ -163,12 +172,11 @@ deleteResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet
           if shouldCount preferCount then Just rsQueryTotal else Nothing
       headers = [contentRangeHeader]
 
-    if preferRepresentation == Full then
-        response HTTP.status200
-          (headers ++ contentTypeHeaders ctxApiRequest)
+    case preferRepresentation of
+        Just Full -> response HTTP.status200 (addPrefToHeaders headers Full ++ contentTypeHeaders ctxApiRequest)
           (LBS.fromStrict rsBody)
-      else
-        response HTTP.status204 headers mempty
+        Just None -> response HTTP.status204 (addPrefToHeaders headers None) mempty
+        _ -> response HTTP.status204 headers mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
@@ -223,11 +231,11 @@ invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resu
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
-openApiResponse :: Bool -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
-openApiResponse headersOnly body conf sCache schema negotiatedByProfile =
+openApiResponse :: (Text, Text) -> Bool -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
+openApiResponse versions headersOnly body conf sCache schema negotiatedByProfile =
   Wai.responseLBS HTTP.status200
     (MediaType.toContentType MTOpenAPI : maybeToList (profileHeader schema negotiatedByProfile))
-    (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode conf sCache x y z) body)
+    (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache x y z) body)
 
 -- | Response with headers and status overridden from GUCs.
 gucResponse
@@ -292,9 +300,40 @@ optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} resp = d
 -- | Add headers not already included to allow the user to override them instead of duplicating them
 addHeadersIfNotIncluded :: [HTTP.Header] -> [HTTP.Header] -> [HTTP.Header]
 addHeadersIfNotIncluded newHeaders initialHeaders =
-  filter (\(nk, _) -> isNothing $ find (\(ik, _) -> ik == nk) initialHeaders) newHeaders ++
+  filter (\(nk, nv) -> isNothing $ find (\(ik, iv) -> ik == nk && nv == iv) initialHeaders) newHeaders ++
   initialHeaders
 
+-- |  Filters out multiple Preference-Applied Headers from the list and concatenate them into a single Preference-Applied header:
+--
+-- >>> :{
+--  concatPrefAppsHeaders
+--     [("Content-Type","application/json")
+--     , ("Preference-Applied","tx=commit")
+--     , ("Preference-Applied","return=minimal")]
+-- :}
+-- [("Content-Type","application/json"),("Preference-Applied","tx=commit, return=minimal")]
+
+concatPrefAppsHeaders :: [HTTP.Header] -> [HTTP.Header]
+concatPrefAppsHeaders headers = otherHeaders ++ [(HTTP.hPreferenceApplied, combinedPrefApps)]
+  where
+  (prefApps, otherHeaders) = L.partition (\(k, _) -> k == HTTP.hPreferenceApplied) headers
+  prefAppsValues = [ v | (_,v) <- prefApps]
+  combinedPrefApps = BS.intercalate ", " prefAppsValues
+
+-- | Given response headers and a preferRepresentation value, add
+--   preferRepresentation to Preference-Applied
+--
+-- >>> :{
+--  addPrefToHeaders
+--      [("Content-Type", "application/json")
+--      , ("Preference-Applied", "tx=commit")]
+--      None
+-- :}
+-- [("Content-Type","application/json"),("Preference-Applied","tx=commit, return=minimal")]
+
+addPrefToHeaders :: [HTTP.Header] -> PreferRepresentation -> [HTTP.Header]
+addPrefToHeaders headers pref = concatPrefAppsHeaders (headers ++ [toAppliedHeader pref])
+
 traceHeaderMiddleware :: AppConfig -> Wai.Middleware
 traceHeaderMiddleware AppConfig{configServerTraceHeader} app req respond =
   case configServerTraceHeader of

src/PostgREST/Response/OpenAPI.hs

@@ -12,7 +12,6 @@ import qualified Data.ByteString.Lazy  as LBS
 import qualified Data.HashMap.Strict   as HM
 import qualified Data.HashSet.InsOrd   as Set
 import qualified Data.Text             as T
-import qualified Data.Text.Encoding    as T
 
 import Control.Arrow              ((&&&))
 import Data.HashMap.Strict.InsOrd (InsOrdHashMap, fromList)
@@ -36,16 +35,16 @@ import PostgREST.SchemaCache.Routine      (Routine (..),
 import PostgREST.SchemaCache.Table        (Column (..), Table (..),
                                            TablesMap,
                                            tableColumnsList)
-import PostgREST.Version                  (docsVersion, prettyVersion)
 
 import PostgREST.MediaType
 
 import Protolude hiding (Proxy, get)
 
-encode :: AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
-encode conf sCache tables procs schemaDescription =
+encode :: (Text, Text) -> AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
+encode versions conf sCache tables procs schemaDescription =
   JSON.encode $
     postgrestSpec
+      versions
       (dbRelationships sCache)
       (concat $ HM.elems procs)
       (snd <$> HM.toList tables)
@@ -154,7 +153,7 @@ makeProcSchema pd =
   & required .~ fmap ppName (filter ppReq (pdParams pd))
 
 makeProcProperty :: RoutineParam -> (Text, Referenced Schema)
-makeProcProperty (RoutineParam n t _ _) = (n, Inline s)
+makeProcProperty (RoutineParam n t _ _ _) = (n, Inline s)
   where
     s = (mempty :: Schema)
           & type_ .~ toSwaggerType t
@@ -181,7 +180,7 @@ makePreferParam ts =
       _            -> []
 
 makeProcGetParam :: RoutineParam -> Referenced Param
-makeProcGetParam (RoutineParam n t r v) =
+makeProcGetParam (RoutineParam n t _ r v) =
   Inline $ (mempty :: Param)
     & name .~ n
     & required ?~ r
@@ -351,7 +350,7 @@ makeProcPathItem pd = ("/rpc/" ++ toS (pdName pd), pe)
       & summary .~ pSum
       & description .~ mfilter (/="") pDesc
       & tags .~ Set.fromList ["(rpc) " <> pdName pd]
-      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON]
+      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False]
       & at 200 ?~ "OK"
     getOp = procOp
       & parameters .~ makeProcGetParams (pdParams pd)
@@ -392,12 +391,12 @@ escapeHostName "*6" = "0.0.0.0"
 escapeHostName "!6" = "0.0.0.0"
 escapeHostName h    = h
 
-postgrestSpec :: RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
-postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
+postgrestSpec :: (Text, Text) -> RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
+postgrestSpec (prettyVersion, docsVersion) rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
   & basePath ?~ T.unpack b
   & schemes ?~ [s']
   & info .~ ((mempty :: Info)
-      & version .~ T.decodeUtf8 prettyVersion
+      & version .~ prettyVersion
       & title .~ fromMaybe "PostgREST API" dTitle
       & description ?~ fromMaybe "This is a dynamic API generated by PostgREST" dDesc)
   & externalDocs ?~ ((mempty :: ExternalDocs)
@@ -407,8 +406,8 @@ postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
   & definitions .~ fromList (makeTableDef rels <$> ti)
   & parameters .~ fromList (makeParamDefs ti)
   & paths .~ makePathItems pds ti
-  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
-  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
+  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False, MTTextCSV]
+  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False, MTTextCSV]
   & securityDefinitions .~ makeSecurityDefinitions securityDefName allowSecurityDef
   & security .~ [SecurityRequirement (fromList [(securityDefName, [])]) | allowSecurityDef]
     where

src/PostgREST/SchemaCache.hs

@@ -26,6 +26,8 @@ module PostgREST.SchemaCache
   , schemaDescription
   ) where
 
+import Control.Monad.Extra (whenJust)
+
 import qualified Data.Aeson                 as JSON
 import qualified Data.HashMap.Strict        as HM
 import qualified Data.HashMap.Strict.InsOrd as HMI
@@ -38,30 +40,38 @@ import qualified Hasql.Transaction          as SQL
 import Contravariant.Extras          (contrazip2)
 import Text.InterpolatedString.Perl6 (q)
 
-import PostgREST.Config.Database          (pgVersionStatement)
-import PostgREST.Config.PgVersion         (PgVersion, pgVersion100,
-                                           pgVersion110)
-import PostgREST.SchemaCache.Identifiers  (AccessSet, FieldName,
-                                           QualifiedIdentifier (..),
-                                           Schema)
-import PostgREST.SchemaCache.Relationship (Cardinality (..),
-                                           Junction (..),
-                                           Relationship (..),
-                                           RelationshipsMap)
-import PostgREST.SchemaCache.Routine      (FuncVolatility (..),
-                                           PgType (..), RetType (..),
-                                           Routine (..), RoutineMap,
-                                           RoutineParam (..))
-import PostgREST.SchemaCache.Table        (Column (..), ColumnMap,
-                                           Table (..), TablesMap)
+import PostgREST.Config                      (AppConfig (..))
+import PostgREST.Config.Database             (pgVersionStatement,
+                                              toIsolationLevel)
+import PostgREST.Config.PgVersion            (PgVersion, pgVersion100,
+                                              pgVersion110,
+                                              pgVersion120)
+import PostgREST.SchemaCache.Identifiers     (AccessSet, FieldName,
+                                              QualifiedIdentifier (..),
+                                              Schema)
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
+                                              Junction (..),
+                                              Relationship (..),
+                                              RelationshipsMap)
+import PostgREST.SchemaCache.Representations (DataRepresentation (..),
+                                              RepresentationsMap)
+import PostgREST.SchemaCache.Routine         (FuncVolatility (..),
+                                              PgType (..),
+                                              RetType (..),
+                                              Routine (..),
+                                              RoutineMap,
+                                              RoutineParam (..))
+import PostgREST.SchemaCache.Table           (Column (..), ColumnMap,
+                                              Table (..), TablesMap)
 
 import Protolude
 
 
 data SchemaCache = SchemaCache
-  { dbTables        :: TablesMap
-  , dbRelationships :: RelationshipsMap
-  , dbRoutines      :: RoutineMap
+  { dbTables          :: TablesMap
+  , dbRelationships   :: RelationshipsMap
+  , dbRoutines        :: RoutineMap
+  , dbRepresentations :: RepresentationsMap
   }
   deriving (Generic, JSON.ToJSON)
 
@@ -103,15 +113,19 @@ data KeyDep
 -- | A SQL query that can be executed independently
 type SqlQuery = ByteString
 
-querySchemaCache :: [Schema] -> [Schema] -> Bool -> SQL.Transaction SchemaCache
-querySchemaCache schemas extraSearchPath prepared = do
+querySchemaCache :: AppConfig -> SQL.Transaction SchemaCache
+querySchemaCache AppConfig{..} = do
   SQL.sql "set local schema ''" -- This voids the search path. The following queries need this for getting the fully qualified name(schema.name) of every db object
   pgVer   <- SQL.statement mempty $ pgVersionStatement prepared
   tabs    <- SQL.statement schemas $ allTables pgVer prepared
-  keyDeps <- SQL.statement (schemas, extraSearchPath) $ allViewsKeyDependencies prepared
+  keyDeps <- SQL.statement (schemas, configDbExtraSearchPath) $ allViewsKeyDependencies prepared
   m2oRels <- SQL.statement mempty $ allM2OandO2ORels pgVer prepared
   funcs   <- SQL.statement schemas $ allFunctions pgVer prepared
   cRels   <- SQL.statement mempty $ allComputedRels prepared
+  reps    <- SQL.statement schemas $ dataRepresentations prepared
+  _       <-
+    let sleepCall = SQL.Statement "select pg_sleep($1)" (param HE.int4) HD.noResult prepared in
+    whenJust configInternalSCSleep (`SQL.statement` sleepCall) -- only used for testing
 
   let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
       rels          = addInverseRels $ addM2MRels tabsWViewsPks $ addViewM2OAndO2ORels keyDeps m2oRels
@@ -120,7 +134,11 @@ querySchemaCache schemas extraSearchPath prepared = do
       dbTables = tabsWViewsPks
     , dbRelationships = getOverrideRelationshipsMap rels cRels
     , dbRoutines = funcs
+    , dbRepresentations = reps
     }
+  where
+    schemas = toList configDbSchemas
+    prepared = configDbPreparedStatements
 
 -- | overrides detected relationships with the computed relationships and gets the RelationshipsMap
 getOverrideRelationshipsMap :: [Relationship] -> [Relationship] -> RelationshipsMap
@@ -146,10 +164,11 @@ getOverrideRelationshipsMap rels cRels =
 removeInternal :: [Schema] -> SchemaCache -> SchemaCache
 removeInternal schemas dbStruct =
   SchemaCache {
-      dbTables        = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
-    , dbRelationships = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
-                        HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
-    , dbRoutines      = dbRoutines dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+      dbTables          = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
+    , dbRelationships   = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
+                          HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
+    , dbRoutines        = dbRoutines dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+    , dbRepresentations = dbRepresentations dbStruct -- no need to filter, not directly exposed through the API
     }
   where
     hasInternalJunction ComputedRelationship{} = False
@@ -240,6 +259,7 @@ decodeFuncs =
                   (RoutineParam
                   <$> compositeField HD.text
                   <*> compositeField HD.text
+                  <*> compositeField HD.text
                   <*> compositeField HD.bool
                   <*> compositeField HD.bool)
               <*> (parseRetType
@@ -250,7 +270,7 @@ decodeFuncs =
                   <*> column HD.bool)
               <*> (parseVolatility <$> column HD.char)
               <*> column HD.bool
-              <*> nullableColumn HD.text
+              <*> nullableColumn (toIsolationLevel <$> HD.text)
 
     addKey :: Routine -> (QualifiedIdentifier, Routine)
     addKey pd = (QualifiedIdentifier (pdSchema pd) (pdName pd), pd)
@@ -270,6 +290,42 @@ decodeFuncs =
                       | v == 's' = Stable
                       | otherwise = Volatile -- only 'v' can happen here
 
+decodeRepresentations :: HD.Result RepresentationsMap
+decodeRepresentations =
+  HM.fromList . map (\rep@DataRepresentation{drSourceType, drTargetType} -> ((drSourceType, drTargetType), rep)) <$> HD.rowList row
+  where
+    row = DataRepresentation
+      <$> column HD.text
+      <*> column HD.text
+      <*> column HD.text
+
+-- Selects all potential data representation transformations. To qualify the cast must be
+-- 1. to or from a domain
+-- 2. implicit
+-- For the time being it must also be to/from JSON or text, although one can imagine a future where we support special
+-- cases like CSV specific representations.
+dataRepresentations :: Bool -> SQL.Statement [Schema] RepresentationsMap
+dataRepresentations = SQL.Statement sql (arrayParam HE.text) decodeRepresentations
+  where
+    sql = [q|
+    SELECT
+      c.castsource::regtype::text,
+      c.casttarget::regtype::text,
+      c.castfunc::regproc::text
+    FROM
+      pg_catalog.pg_cast c
+    JOIN pg_catalog.pg_type src_t
+      ON c.castsource::oid = src_t.oid
+    JOIN pg_catalog.pg_type dst_t
+      ON c.casttarget::oid = dst_t.oid
+    WHERE
+      c.castcontext = 'i'
+      AND c.castmethod = 'f'
+      AND has_function_privilege(c.castfunc, 'execute')
+      AND ((src_t.typtype = 'd' AND c.casttarget IN ('json'::regtype::oid , 'text'::regtype::oid))
+       OR (dst_t.typtype = 'd' AND c.castsource IN ('json'::regtype::oid , 'text'::regtype::oid)))
+    |]
+
 allFunctions :: PgVersion -> Bool -> SQL.Statement [Schema] RoutineMap
 allFunctions pgVer = SQL.Statement sql (arrayParam HE.text) decodeFuncs
   where
@@ -312,6 +368,13 @@ funcsSqlQuery pgVer = [q|
       array_agg((
         COALESCE(name, ''), -- name
         type::regtype::text, -- type
+        CASE type
+          WHEN 'bit'::regtype THEN 'bit varying'
+          WHEN 'bit[]'::regtype THEN 'bit varying[]'
+          WHEN 'character'::regtype THEN 'character varying'
+          WHEN 'character[]'::regtype THEN 'character varying[]'
+          ELSE type::regtype::text
+        END, -- convert types that ignore the lenth and accept any value till maximum size
         idx <= (pronargs - pronargdefaults), -- is_required
         COALESCE(mode = 'v', FALSE) -- is_variadic
       ) ORDER BY idx) AS args,
@@ -506,6 +569,8 @@ tablesSqlQuery pgVer =
   -- the tbl_constraints/key_col_usage CTEs are based on the standard "information_schema.table_constraints"/"information_schema.key_column_usage" views,
   -- we cannot use those directly as they include the following privilege filter:
   -- (pg_has_role(ss.relowner, 'USAGE'::text) OR has_column_privilege(ss.roid, a.attnum, 'SELECT, INSERT, UPDATE, REFERENCES'::text));
+  -- on the "columns" CTE, left joining on pg_depend and pg_class is used to obtain the sequence name as a column default in case there are GENERATED .. AS IDENTITY,
+  -- generated columns are only available from pg >= 10 but the query is agnostic to versions. dep.deptype = 'i' is done because there are other 'a' dependencies on PKs
   [q|
   WITH
   columns AS (
@@ -514,22 +579,21 @@ tablesSqlQuery pgVer =
           c.relname::name AS table_name,
           a.attname::name AS column_name,
           d.description AS description,
-  |] <> columnDefault <>
-  [q|
+  |] <> columnDefault <> [q| AS column_default,
           not (a.attnotnull OR t.typtype = 'd' AND t.typnotnull) AS is_nullable,
+          CASE
+              WHEN t.typtype = 'd' THEN
               CASE
-                  WHEN t.typtype = 'd' THEN
-                  CASE
-                      WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
-                      ELSE format_type(a.atttypid, a.atttypmod)
-                  END
-                  ELSE
-                  CASE
-                      WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
-                      ELSE format_type(a.atttypid, a.atttypmod)
-                  END
-              END::text AS data_type,
-          t.oid AS data_type_id,
+                  WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
+                  ELSE format_type(a.atttypid, a.atttypmod)
+              END
+              ELSE
+              CASE
+                  WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
+                  ELSE format_type(a.atttypid, a.atttypmod)
+              END
+          END::text AS data_type,
+          format_type(a.atttypid, a.atttypmod)::text AS nominal_data_type,
           information_schema._pg_char_max_length(
               information_schema._pg_truetypid(a.*, t.*),
               information_schema._pg_truetypmod(a.*, t.*)
@@ -549,6 +613,12 @@ tablesSqlQuery pgVer =
               ON t.typtype = 'd' AND t.typbasetype = bt.oid
           LEFT JOIN (pg_collation co JOIN pg_namespace nco ON co.collnamespace = nco.oid)
               ON a.attcollation = co.oid AND (nco.nspname <> 'pg_catalog'::name OR co.collname <> 'default'::name)
+          LEFT JOIN pg_depend dep
+              ON dep.refobjid = a.attrelid and dep.refobjsubid = a.attnum and dep.deptype = 'i'
+          LEFT JOIN pg_class seqclass
+              ON seqclass.oid = dep.objid
+          LEFT JOIN pg_namespace seqsch
+              ON seqsch.oid = seqclass.relnamespace
       WHERE
           NOT pg_is_other_temp_schema(nc.oid)
           AND a.attnum > 0
@@ -565,7 +635,7 @@ tablesSqlQuery pgVer =
           info.description,
           info.is_nullable::boolean,
           info.data_type,
-          info.data_type_id::regtype::text,
+          info.nominal_data_type,
           info.character_maximum_length,
           info.column_default,
           coalesce(enum_info.vals, '{}')) order by info.position) as columns
@@ -697,17 +767,25 @@ tablesSqlQuery pgVer =
   "ORDER BY table_schema, table_name"
   where
     relIsPartition = if pgVer >= pgVersion100 then " AND not c.relispartition " else mempty
-    -- detect default values on columns that have GENERATED .. AS IDENTITY
-    columnDefault =
-      if pgVer >= pgVersion100
-        then [q|
-        CASE
-           WHEN nullif(a.attidentity, '') is null
-             THEN pg_get_expr(ad.adbin, ad.adrelid)::text
-             ELSE format('nextval(%s)', quote_literal(pg_get_serial_sequence(a.attrelid::regclass::text, a.attname::text)))
-        END AS column_default,|]
-        else "pg_get_expr(ad.adbin, ad.adrelid)::text AS column_default,"
-
+    columnDefault -- typbasetype and typdefaultbin handles `CREATE DOMAIN .. DEFAULT val`,  attidentity/attgenerated handles generated columns, pg_get_expr gets the default of a column
+      | pgVer >= pgVersion120 = [q|
+          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
+            WHEN a.attidentity  = 'd' THEN format('nextval(%s)', quote_literal(seqsch.nspname || '.' || seqclass.relname))
+            WHEN a.attgenerated = 's' THEN null
+            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
+          END|]
+      | pgVer >= pgVersion100 = [q|
+          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
+            WHEN a.attidentity = 'd' THEN format('nextval(%s)', quote_literal(seqsch.nspname || '.' || seqclass.relname))
+            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
+          END|]
+      | otherwise  = [q|
+          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
+            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
+          END|]
 
 -- | Gets many-to-one relationships and one-to-one(O2O) relationships, which are a refinement of the many-to-one's
 allM2OandO2ORels :: PgVersion -> Bool -> SQL.Statement () [Relationship]

src/PostgREST/SchemaCache/Identifiers.hs

@@ -24,7 +24,7 @@ data QualifiedIdentifier = QualifiedIdentifier
   { qiSchema :: Schema
   , qiName   :: TableName
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
 
 instance Hashable QualifiedIdentifier
 

src/PostgREST/SchemaCache/Relationship.hs

@@ -34,7 +34,7 @@ data Relationship = Relationship
   , relToOne        :: Bool
   , relIsSelf       :: Bool
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 -- | The relationship cardinality
 -- | https://en.wikipedia.org/wiki/Cardinality_(data_modeling)
@@ -47,7 +47,7 @@ data Cardinality
   -- ^ one-to-one, this is a refinement over M2O so operating on it is pretty much the same as M2O
   | M2M Junction
   -- ^ many-to-many
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 type FKConstraint = Text
 
@@ -59,7 +59,7 @@ data Junction = Junction
   , junColsSource  :: [(FieldName, FieldName)]
   , junColsTarget  :: [(FieldName, FieldName)]
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 -- | Key based on the source table and the foreign table schema
 type RelationshipsMap = HM.HashMap (QualifiedIdentifier, Schema)  [Relationship]

src/PostgREST/SchemaCache/Representations.hs

@@ -0,0 +1,29 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric  #-}
+
+module PostgREST.SchemaCache.Representations
+  ( DataRepresentation(..)
+  , RepresentationsMap
+  ) where
+
+import qualified Data.Aeson          as JSON
+import qualified Data.HashMap.Strict as HM
+
+
+import Protolude
+
+-- | Data representations allow user customisation of how to present and receive data through APIs, per field.
+-- This structure is used for the library of available transforms. It answers questions like:
+-- - What function, if any, should be used to present a certain field that's been selected for API output?
+-- - How do we parse incoming data for a certain field type when inserting or updating?
+-- - And similarly, how do we parse textual data in a query string to be used as a filter?
+--
+-- Support for outputting special formats like CSV and binary data would fit into the same system.
+data DataRepresentation = DataRepresentation
+  { drSourceType :: Text
+  , drTargetType :: Text
+  , drFunction   :: Text
+  } deriving (Eq, Show, Generic, JSON.ToJSON, JSON.FromJSON)
+
+-- The representation map maps from (source type, target type) to a DR.
+type RepresentationsMap = HM.HashMap (Text, Text) DataRepresentation

src/PostgREST/SchemaCache/Routine.hs

@@ -14,12 +14,16 @@ module PostgREST.SchemaCache.Routine
   , funcReturnsVoid
   , funcTableName
   , funcReturnsCompositeAlias
+  , ResultAggregate(..)
   ) where
 
-import qualified Data.Aeson          as JSON
-import qualified Data.HashMap.Strict as HM
+import           Data.Aeson                 ((.=))
+import qualified Data.Aeson                 as JSON
+import qualified Data.HashMap.Strict        as HM
+import qualified Hasql.Transaction.Sessions as SQL
 
-import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
+import PostgREST.SchemaCache.Identifiers (FieldName,
+                                          QualifiedIdentifier (..),
                                           Schema, TableName)
 
 import Protolude
@@ -27,18 +31,18 @@ import Protolude
 data PgType
   = Scalar QualifiedIdentifier
   | Composite QualifiedIdentifier Bool -- True if the composite is a domain alias(used to work around a bug in pg 11 and 12, see QueryBuilder.hs)
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 data RetType
   = Single PgType
   | SetOf PgType
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 data FuncVolatility
   = Volatile
   | Stable
   | Immutable
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 data Routine = Function
   { pdSchema      :: Schema
@@ -48,17 +52,30 @@ data Routine = Function
   , pdReturnType  :: RetType
   , pdVolatility  :: FuncVolatility
   , pdHasVariadic :: Bool
-  , pdIsoLvl      :: Maybe Text
+  , pdIsoLvl      :: Maybe SQL.IsolationLevel
   }
-  deriving (Eq, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Generic)
+-- need to define JSON manually bc SQL.IsolationLevel doesn't have a JSON instance(and we can't define one for that type without getting a compiler error)
+instance JSON.ToJSON Routine where
+  toJSON (Function sch nam desc params ret vol hasVar _) = JSON.object
+    [
+      "pdSchema"      .= sch
+    , "pdName"        .= nam
+    , "pdDescription" .= desc
+    , "pdParams"      .= JSON.toJSON params
+    , "pdReturnType"  .= JSON.toJSON ret
+    , "pdVolatility"  .= JSON.toJSON vol
+    , "pdHasVariadic" .= JSON.toJSON hasVar
+    ]
 
 data RoutineParam = RoutineParam
-  { ppName :: Text
-  , ppType :: Text
-  , ppReq  :: Bool
-  , ppVar  :: Bool
+  { ppName          :: Text
+  , ppType          :: Text
+  , ppTypeMaxLength :: Text
+  , ppReq           :: Bool
+  , ppVar           :: Bool
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 -- Order by least number of params in the case of overloaded functions
 instance Ord Routine where
@@ -71,6 +88,17 @@ instance Ord Routine where
 -- | It uses a HashMap for a faster lookup.
 type RoutineMap = HM.HashMap QualifiedIdentifier [Routine]
 
+data ResultAggregate
+   = BuiltinAggJson
+   | BuiltinAggSingleJson Bool
+   | BuiltinAggArrayJsonStrip
+   | BuiltinAggGeoJson
+   | BuiltinAggCsv
+   | BuiltinAggXml    (Maybe FieldName)
+   | BuiltinAggBinary (Maybe FieldName)
+   | NoAgg
+   deriving (Eq, Show)
+
 funcReturnsScalar :: Routine -> Bool
 funcReturnsScalar proc = case proc of
   Function{pdReturnType = Single (Scalar{})} -> True

src/PostgREST/Unix.hs

@@ -14,7 +14,6 @@ import System.Posix.Files (setFileMode)
 import System.Posix.Types (FileMode)
 
 import qualified PostgREST.AppState as AppState
-import qualified PostgREST.Workers  as Workers
 
 import Protolude
 
@@ -49,10 +48,10 @@ installSignalHandlers appState = do
 
   -- The SIGUSR1 signal updates the internal 'SchemaCache' by running
   -- 'connectionWorker' exactly as before.
-  install Signals.sigUSR1 $ Workers.connectionWorker appState
+  install Signals.sigUSR1 $ AppState.connectionWorker appState
 
   -- Re-read the config on SIGUSR2
-  install Signals.sigUSR2 $ Workers.reReadConfig False appState
+  install Signals.sigUSR2 $ AppState.reReadConfig False appState
   where
     install signal handler =
       void $ Signals.installHandler signal (Signals.Catch handler) Nothing

src/PostgREST/Workers.hs

@@ -1,343 +0,0 @@
-{-# LANGUAGE LambdaCase      #-}
-{-# LANGUAGE NamedFieldPuns  #-}
-{-# LANGUAGE RecordWildCards #-}
-
-module PostgREST.Workers
-  ( connectionWorker
-  , reReadConfig
-  , runListener
-  , runAdmin
-  ) where
-
-import qualified Data.ByteString            as BS
-import qualified Data.Text                  as T
-import qualified Hasql.Notifications        as SQL
-import qualified Hasql.Session              as SQL
-import qualified Hasql.Transaction.Sessions as SQL
-import qualified Network.HTTP.Types.Status  as HTTP
-import qualified Network.Wai                as Wai
-import qualified Network.Wai.Handler.Warp   as Warp
-
-import Control.Retry    (RetryStatus, capDelay, exponentialBackoff,
-                         retrying, rsPreviousDelay)
-import Hasql.Connection (acquire)
-
-import Network.Socket
-import Network.Socket.ByteString
-
-import PostgREST.AppState         (AppState)
-import PostgREST.Config           (AppConfig (..), readAppConfig)
-import PostgREST.Config.Database  (queryDbSettings, queryPgVersion,
-                                   queryRoleSettings)
-import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
-import PostgREST.Error            (checkIsFatal)
-import PostgREST.SchemaCache      (querySchemaCache)
-
-import qualified PostgREST.AppState as AppState
-
-import Protolude
-
-
--- | Current database connection status data ConnectionStatus
-data ConnectionStatus
-  = NotConnected
-  | Connected PgVersion
-  | FatalConnectionError Text
-  deriving (Eq)
-
--- | Schema cache status
-data SCacheStatus
-  = SCLoaded
-  | SCOnRetry
-  | SCFatalFail
-
--- | The purpose of this worker is to obtain a healthy connection to pg and an
--- up-to-date schema cache(SchemaCache).  This method is meant to be called
--- multiple times by the same thread, but does nothing if the previous
--- invocation has not terminated. In all cases this method does not halt the
--- calling thread, the work is performed in a separate thread.
---
--- Background thread that does the following :
---  1. Tries to connect to pg server and will keep trying until success.
---  2. Checks if the pg version is supported and if it's not it kills the main
---     program.
---  3. Obtains the sCache. If this fails, it goes back to 1.
-connectionWorker :: AppState -> IO ()
-connectionWorker appState = do
-  runExclusively (AppState.getWorkerSem appState) work
-  -- Prevents multiple workers to be running at the same time. Could happen on
-  -- too many SIGUSR1s.
-  where
-    runExclusively mvar action = mask_ $ do
-      success <- tryPutMVar mvar ()
-      when success $ do
-        void $ forkIO $ action `finally` takeMVar mvar
-    work = do
-      AppConfig{..} <- AppState.getConfig appState
-      AppState.logWithZTime appState "Attempting to connect to the database..."
-      connected <- establishConnection appState
-      case connected of
-        FatalConnectionError reason ->
-          -- Fatal error when connecting
-          AppState.logWithZTime appState reason >> killThread (AppState.getMainThreadId appState)
-        NotConnected ->
-          -- Unreachable because establishConnection will keep trying to connect
-          return ()
-        Connected actualPgVersion -> do
-          -- Procede with initialization
-          AppState.putPgVersion appState actualPgVersion
-          when configDbChannelEnabled $
-            AppState.signalListener appState
-          AppState.logWithZTime appState "Connection successful"
-          -- this could be fail because the connection drops, but the
-          -- loadSchemaCache will pick the error and retry again
-          when configDbConfig $ reReadConfig False appState
-          scStatus <- loadSchemaCache appState
-          case scStatus of
-            SCLoaded ->
-              -- do nothing and proceed if the load was successful
-              return ()
-            SCOnRetry ->
-              -- retry reloading the schema cache
-              work
-            SCFatalFail ->
-              -- die if our schema cache query has an error
-              killThread $ AppState.getMainThreadId appState
-
--- | Repeatedly flush the pool, and check if a connection from the
--- pool allows access to the PostgreSQL database.
---
--- Releasing the pool is key for rapid recovery. Otherwise, the pool
--- timeout would have to be reached for new healthy connections to be acquired.
--- Which might not happen if the server is busy with requests. No idle
--- connection, no pool timeout.
---
--- The connection tries are capped, but if the connection times out no error is
--- thrown, just 'False' is returned.
-establishConnection :: AppState -> IO ConnectionStatus
-establishConnection appState =
-  retrying retrySettings shouldRetry $
-    const $ AppState.flushPool appState >> getConnectionStatus
-  where
-    retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
-    delayMicroseconds = 32000000 -- 32 seconds
-    backoffMicroseconds = 1000000 -- 1 second
-
-    getConnectionStatus :: IO ConnectionStatus
-    getConnectionStatus = do
-      pgVersion <- AppState.usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
-      case pgVersion of
-        Left e -> do
-          AppState.logPgrstError appState e
-          case checkIsFatal e of
-            Just reason ->
-              return $ FatalConnectionError reason
-            Nothing ->
-              return NotConnected
-        Right version ->
-          if version < minimumPgVersion then
-            return . FatalConnectionError $
-              "Cannot run in this PostgreSQL version, PostgREST needs at least "
-              <> pgvName minimumPgVersion
-          else
-            return . Connected  $ version
-
-    shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
-    shouldRetry rs isConnSucc = do
-      let
-        delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
-        itShould = NotConnected == isConnSucc
-      when itShould . AppState.logWithZTime appState $
-        "Attempting to reconnect to the database in "
-        <> (show delay::Text)
-        <> " seconds..."
-      when itShould $ AppState.putRetryNextIn appState delay
-      return itShould
-
--- | Load the SchemaCache by using a connection from the pool.
-loadSchemaCache :: AppState -> IO SCacheStatus
-loadSchemaCache appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  result <-
-    let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
-    AppState.usePool appState . transaction SQL.ReadCommitted SQL.Read $
-      querySchemaCache (toList configDbSchemas) configDbExtraSearchPath configDbPreparedStatements
-  case result of
-    Left e -> do
-      case checkIsFatal e of
-        Just hint -> do
-          AppState.logWithZTime appState "A fatal error ocurred when loading the schema cache"
-          AppState.logPgrstError appState e
-          AppState.logWithZTime appState hint
-          return SCFatalFail
-        Nothing -> do
-          AppState.putSchemaCache appState Nothing
-          AppState.logWithZTime appState "An error ocurred when loading the schema cache"
-          AppState.logPgrstError appState e
-          return SCOnRetry
-
-    Right sCache -> do
-      AppState.putSchemaCache appState (Just sCache)
-      AppState.logWithZTime appState "Schema cache loaded"
-      return SCLoaded
-
-runListener :: AppConfig -> AppState -> IO ()
-runListener AppConfig{configDbChannelEnabled} appState =
-  when configDbChannelEnabled $ listener appState
-
--- | Starts a dedicated pg connection to LISTEN for notifications.  When a
--- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
--- cache.  It uses the connectionWorker in case the LISTEN connection dies.
-listener :: AppState -> IO ()
-listener appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  let dbChannel = toS configDbChannel
-
-  -- The listener has to wait for a signal from the connectionWorker.
-  -- This is because when the connection to the db is lost, the listener also
-  -- tries to recover the connection, but not with the same pace as the connectionWorker.
-  -- Not waiting makes stderr quickly fill with connection retries messages from the listener.
-  AppState.waitListener appState
-
-  -- forkFinally allows to detect if the thread dies
-  void . flip forkFinally (handleFinally dbChannel) $ do
-    dbOrError <- acquire $ toUtf8 configDbUri
-    case dbOrError of
-      Right db -> do
-        AppState.logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
-        AppState.putIsListenerOn appState True
-        SQL.listen db $ SQL.toPgIdentifier dbChannel
-        SQL.waitForNotifications handleNotification db
-      _ ->
-        die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
-  where
-    handleFinally dbChannel _ = do
-      -- if the thread dies, we try to recover
-      AppState.logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
-      AppState.putIsListenerOn appState False
-      -- assume the pool connection was also lost, call the connection worker
-      connectionWorker appState
-      -- retry the listener
-      listener appState
-
-    handleNotification _ msg
-      | BS.null msg            = cacheReloader
-      | msg == "reload schema" = cacheReloader
-      | msg == "reload config" = reReadConfig False appState
-      | otherwise              = pure () -- Do nothing if anything else than an empty message is sent
-
-    cacheReloader =
-      -- reloads the schema cache + restarts pool connections
-      -- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
-      connectionWorker appState
-
--- | Re-reads the config plus config options from the db
-reReadConfig :: Bool -> AppState -> IO ()
-reReadConfig startingUp appState = do
-  AppConfig{..} <- AppState.getConfig appState
-  dbSettings <-
-    if configDbConfig then do
-      qDbSettings <- AppState.usePool appState $ queryDbSettings configDbPreparedStatements
-      case qDbSettings of
-        Left e -> do
-          AppState.logWithZTime appState
-            "An error ocurred when trying to query database settings for the config parameters"
-          case checkIsFatal e of
-            Just hint -> do
-              AppState.logPgrstError appState e
-              AppState.logWithZTime appState hint
-              killThread (AppState.getMainThreadId appState)
-            Nothing -> do
-              AppState.logPgrstError appState e
-          pure mempty
-        Right x -> pure x
-    else
-      pure mempty
-  roleSettings <-
-    if configDbConfig then do
-      rSettings <- AppState.usePool appState $ queryRoleSettings configDbPreparedStatements
-      case rSettings of
-        Left e -> do
-          AppState.logWithZTime appState "An error ocurred when trying to query the role settings"
-          AppState.logPgrstError appState e
-          pure mempty
-        Right x -> pure x
-    else
-      pure mempty
-  readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings >>= \case
-    Left err   ->
-      if startingUp then
-        panic err -- die on invalid config if the program is starting up
-      else
-        AppState.logWithZTime appState $ "Failed reloading config: " <> err
-    Right newConf -> do
-      AppState.putConfig appState newConf
-      if startingUp then
-        pass
-      else
-        AppState.logWithZTime appState "Config reloaded"
-
-runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
-runAdmin conf@AppConfig{configAdminServerPort} appState settings =
-  whenJust configAdminServerPort $ \adminPort -> do
-    AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
-    void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
-  where
-    whenJust :: Applicative m => Maybe a -> (a -> m ()) -> m ()
-    whenJust mg f = maybe (pure ()) f mg
-    adminApp = admin appState conf
-
--- | PostgREST admin application
-admin :: AppState.AppState -> AppConfig -> Wai.Application
-admin appState appConfig req respond  = do
-  isMainAppReachable  <- any isRight <$> reachMainApp appConfig
-  isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
-  isConnectionUp      <-
-    if configDbChannelEnabled appConfig
-      then AppState.getIsListenerOn appState
-      else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
-
-  case Wai.pathInfo req of
-    ["ready"] ->
-      respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
-    ["live"] ->
-      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
-    _ ->
-      respond $ Wai.responseLBS HTTP.status404 [] mempty
-
--- Try to connect to the main app socket
--- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
--- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
--- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
-reachMainApp :: AppConfig -> IO [Either IOException ()]
-reachMainApp AppConfig{..} =
-  case configServerUnixSocket of
-    Just path ->  do
-      sock <- socket AF_UNIX Stream 0
-      (:[]) <$> try (do
-        connect sock $ SockAddrUnix path
-        withSocketsDo $ bracket (pure sock) close sendEmpty)
-    Nothing -> do
-      let
-        host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
-             | otherwise                                             = Just configServerHost
-        filterAddrs xs =
-          case configServerHost of
-              "*4" -> ipv4Addrs xs ++ ipv6Addrs xs
-              "!4" -> ipv4Addrs xs
-              "*6" -> ipv6Addrs xs ++ ipv4Addrs xs
-              "!6" -> ipv6Addrs xs
-              _    -> xs
-        ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
-        ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
-
-      addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
-      tryAddr `traverse` filterAddrs addrs
-  where
-    sendEmpty sock = void $ send sock mempty
-    tryAddr :: AddrInfo -> IO (Either IOException ())
-    tryAddr addr = do
-      sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
-      try $ do
-        connect sock $ addrAddress addr
-        withSocketsDo $ bracket (pure sock) close sendEmpty

stack.yaml

@@ -12,5 +12,5 @@ nix:
 extra-deps:
   - git: https://github.com/PostgREST/postgresql-libpq.git
     commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
-  - hasql-notifications-0.2.0.4
-  - hasql-pool-0.9
+  - hasql-notifications-0.2.0.5
+  - hasql-pool-0.10

stack.yaml.lock

@@ -16,19 +16,19 @@ packages:
     commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
     git: https://github.com/PostgREST/postgresql-libpq.git
 - completed:
-    hackage: hasql-notifications-0.2.0.4@sha256:9a09fa9b97feadd9492c8bd8bc6b9cffe0513510102f08374b0c45ecd479ed67,2028
+    hackage: hasql-notifications-0.2.0.5@sha256:6c67d2ee42e948162e89176a2f52a2c6bb562678b33688e96e048be9b3d74122,2028
     pantry-tree:
-      sha256: 56f9e240728e7a65711dde45fa2e2075b914e32cd370424aaa4572392378a60e
+      sha256: 8493755b0817a36b910a37d568769ca49ab715ef5dd935a0d09f0716aa39da51
       size: 452
   original:
-    hackage: hasql-notifications-0.2.0.4
+    hackage: hasql-notifications-0.2.0.5
 - completed:
-    hackage: hasql-pool-0.9@sha256:db7a37f6b3a922c37adc3c7ced47a7c10786d1f171e47a735a6e812a587ba44c,2111
+    hackage: hasql-pool-0.10@sha256:912197a328acb85505f98bb9700d61f366b87659ca45126c5c2d636687b801c3,2112
     pantry-tree:
-      sha256: 49b1181d28c6f5317e794671c2dae155754b834bdcfa30f7e5dbad28e4cf0249
+      sha256: b655c540a49764a8d16b62941137e295b936b96edc0785eb9250972f0f92dc47
       size: 346
   original:
-    hackage: hasql-pool-0.9
+    hackage: hasql-pool-0.10
 snapshots:
 - completed:
     sha256: 4905c93319aa94aa53da8f41d614d7bacdbfe6c63a8c6132d32e6e62f24a9af4

test/doc/Main.hs

@@ -11,9 +11,14 @@ main =
     [ "-XOverloadedStrings"
     , "-XNoImplicitPrelude"
     , "-XStandaloneDeriving"
+    , "-XDuplicateRecordFields"
     , "-isrc"
     , "src/PostgREST/Query/SqlFragment.hs"
     , "src/PostgREST/ApiRequest/Preferences.hs"
     , "src/PostgREST/ApiRequest/QueryParams.hs"
     , "src/PostgREST/Error.hs"
+    , "src/PostgREST/MediaType.hs"
+    , "src/PostgREST/Config.hs"
+    , "src/PostgREST/Plan.hs"
+    , "src/PostgREST/Response.hs"
     ]

test/io/configs/aliases.config

@@ -1,4 +1,5 @@
 db-schema = "provided_through_alias"
+db-pool-timeout = 5
 max-rows = 1000
 pre-request = "check_alias"
 role-claim-key = ".aliased"

test/io/configs/expected/aliases.config

@@ -7,11 +7,13 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 5
 db-pre-request = "check_alias"
 db-prepared-statements = true
 db-root-spec = "open_alias"
 db-schemas = "provided_through_alias"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true

test/io/configs/expected/boolean-numeric.config

@@ -7,11 +7,13 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
 db-pre-request = ""
 db-prepared-statements = false
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true

test/io/configs/expected/boolean-string.config

@@ -7,11 +7,13 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
 db-pre-request = ""
 db-prepared-statements = false
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true

test/io/configs/expected/defaults.config

@@ -7,11 +7,13 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
 db-pre-request = ""
 db-prepared-statements = true
 db-root-spec = ""
 db-schemas = "public"
 db-config = false
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true

test/io/configs/expected/no-defaults-with-db-other-authenticator.config

@@ -1,4 +1,4 @@
-db-anon-role = "other"
+db-anon-role = "pre_config_role"
 db-channel = "postgrest"
 db-channel-enabled = false
 db-extra-search-path = "public,extensions,other"
@@ -7,16 +7,18 @@ db-plan-enabled = true
 db-pool = 1
 db-pool-acquisition-timeout = 30
 db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
 db-pre-request = "test.other_custom_headers"
 db-prepared-statements = false
 db-root-spec = "other_root"
 db-schemas = "test,other_tenant1,other_tenant2"
 db-config = true
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "postgresql://"
 db-use-legacy-gucs = false
 jwt-aud = "https://otherexample.org"
-jwt-role-claim-key = ".\"other\".\"role\""
+jwt-role-claim-key = ".\"other\".\"pre_config_role\""
 jwt-secret = "ODERREALLYREALLYREALLYREALLYVERYSAFE"
 jwt-secret-is-base64 = true
 log-level = "info"

test/io/configs/expected/no-defaults-with-db.config

@@ -7,11 +7,13 @@ db-plan-enabled = true
 db-pool = 1
 db-pool-acquisition-timeout = 30
 db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
 db-pre-request = "test.custom_headers"
 db-prepared-statements = false
 db-root-spec = "root"
 db-schemas = "test,tenant1,tenant2"
 db-config = true
+db-pre-config = "postgrest.preconf"
 db-tx-end = "commit-allow-override"
 db-uri = "postgresql://"
 db-use-legacy-gucs = false

test/io/configs/expected/no-defaults.config

@@ -7,11 +7,13 @@ db-plan-enabled = true
 db-pool = 1
 db-pool-acquisition-timeout = 30
 db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
 db-pre-request = "please_run_fast"
 db-prepared-statements = false
 db-root-spec = "openapi_v3"
 db-schemas = "multi,tenant,setup"
 db-config = false
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "tmp_db"
 db-use-legacy-gucs = false

test/io/configs/expected/types.config

@@ -7,11 +7,13 @@ db-plan-enabled = false
 db-pool = 10
 db-pool-acquisition-timeout = 10
 db-pool-max-lifetime = 1800
+db-pool-max-idletime = 30
 db-pre-request = ""
 db-prepared-statements = true
 db-root-spec = ""
 db-schemas = "public"
 db-config = true
+db-pre-config = ""
 db-tx-end = "commit"
 db-uri = "postgresql://"
 db-use-legacy-gucs = true

test/io/configs/no-defaults-env.yaml

@@ -9,14 +9,15 @@ PGRST_DB_PLAN_ENABLED: true
 PGRST_DB_POOL: 1
 PGRST_DB_POOL_ACQUISITION_TIMEOUT: 30
 PGRST_DB_POOL_MAX_LIFETIME: 3600
+PGRST_DB_POOL_MAX_IDLETIME: 60
 PGRST_DB_PREPARED_STATEMENTS: false
 PGRST_DB_PRE_REQUEST: please_run_fast
 PGRST_DB_ROOT_SPEC: openapi_v3
 PGRST_DB_SCHEMAS: multi,   tenant,setup
 PGRST_DB_CONFIG: false
+PGRST_DB_PRE_CONFIG: "postgrest.pre_config"
 PGRST_DB_TX_END: rollback-allow-override
 PGRST_DB_URI: tmp_db
-PGRST_DB_EMBED_DEFAULT_JOIN: inner
 PGRST_DB_USE_LEGACY_GUCS: false
 PGRST_JWT_AUD: 'https://postgrest.org'
 PGRST_JWT_ROLE_CLAIM_KEY: '.user[0]."real-role"'

test/io/configs/no-defaults.config

@@ -7,11 +7,13 @@ db-plan-enabled = true
 db-pool = 1
 db-pool-acquisition-timeout = 30
 db-pool-max-lifetime = 3600
+db-pool-max-idletime = 60
 db-pre-request = "please_run_fast"
 db-prepared-statements = false
 db-root-spec = "openapi_v3"
 db-schemas = "multi,   tenant,setup"
 db-config = false
+db-pre-config = "postgrest.pre_config"
 db-tx-end = "rollback-allow-override"
 db-uri = "tmp_db"
 db-use-legacy-gucs = false

test/io/db_config.sql

@@ -9,6 +9,7 @@ ALTER ROLE db_config_authenticator SET pgrst.jwt_secret_is_base64 = 'false';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_role_claim_key = '."a"."role"';
 ALTER ROLE db_config_authenticator SET pgrst.db_anon_role = 'anonymous';
 ALTER ROLE db_config_authenticator SET pgrst.db_tx_end = 'commit-allow-override';
+ALTER ROLE db_config_authenticator SET pgrst.db_pre_config = 'postgrest.preconf';
 ALTER ROLE db_config_authenticator SET pgrst.db_schemas = 'test, tenant1, tenant2';
 ALTER ROLE db_config_authenticator SET pgrst.db_root_spec = 'root';
 ALTER ROLE db_config_authenticator SET pgrst.db_plan_enabled = 'true';
@@ -40,7 +41,10 @@ ALTER ROLE db_config_authenticator SET pgrst.db_channel_enabled = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_channel = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_pool = 'ignored';
 ALTER ROLE db_config_authenticator SET pgrst.db_pool_timeout = 'ignored';
-ALTER ROLE db_config_authenticator SET pgrst.db_config = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_acquisition_timeout = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_lifetime = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_idletime = 'ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_config = 'true';
 
 -- other authenticator reloadable config options
 CREATE ROLE other_authenticator LOGIN NOINHERIT;
@@ -49,9 +53,6 @@ ALTER ROLE other_authenticator SET pgrst.openapi_server_proxy_uri = 'https://oth
 ALTER ROLE other_authenticator SET pgrst.raw_media_types = 'application/vnd.pgrst.other-db-config';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret = 'ODERREALLYREALLYREALLYREALLYVERYSAFE';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret_is_base64 = 'true';
-ALTER ROLE other_authenticator SET pgrst.jwt_role_claim_key = '."other"."role"';
-ALTER ROLE other_authenticator SET pgrst.db_anon_role = 'other';
-ALTER ROLE other_authenticator SET pgrst.db_tx_end = 'rollback-allow-override';
 ALTER ROLE other_authenticator SET pgrst.db_schemas = 'test, other_tenant1, other_tenant2';
 ALTER ROLE other_authenticator SET pgrst.db_root_spec = 'other_root';
 ALTER ROLE other_authenticator SET pgrst.db_plan_enabled = 'true';
@@ -62,6 +63,32 @@ ALTER ROLE other_authenticator SET pgrst.db_extra_search_path = 'public, extensi
 ALTER ROLE other_authenticator SET pgrst.openapi_mode = 'disabled';
 ALTER ROLE other_authenticator SET pgrst.openapi_security_active = 'false';
 ALTER ROLE other_authenticator SET pgrst.server_trace_header = 'traceparent';
+ALTER ROLE other_authenticator SET pgrst.db_pre_config = 'postgrest.pre_config';
+
+create schema postgrest;
+grant usage on schema postgrest to db_config_authenticator;
+grant usage on schema postgrest to other_authenticator;
+
+-- pre-config hook
+create or replace function postgrest.pre_config()
+returns void as $$
+begin
+  if current_user = 'other_authenticator' then
+    perform
+      set_config('pgrst.jwt_role_claim_key', '."other"."pre_config_role"', true)
+    , set_config('pgrst.db_anon_role', 'pre_config_role', true)
+    , set_config('pgrst.db_schemas', 'will be overriden with the above ALTER ROLE.. db_schemas', true)
+    , set_config('pgrst.db_tx_end', 'rollback-allow-override', true);
+  else
+    null;
+  end if;
+end $$ language plpgsql;
+
+create or replace function postgrest.preconf()
+returns void as $$
+begin
+  null;
+end $$ language plpgsql;
 
 -- authenticator used for tests that manipulate statement timeout
 CREATE ROLE timeout_authenticator LOGIN NOINHERIT;

test/io/fixtures.sql

@@ -1,10 +1,10 @@
-\ir big_schema.sql
+-- \ir big_schema.sql big schema test currently skipped, see test_io.py
 \ir db_config.sql
 
 set search_path to public;
 
 CREATE ROLE postgrest_test_anonymous;
-ALTER ROLE :USER SET pgrst.db_anon_role = 'postgrest_test_anonymous';
+ALTER ROLE :PGUSER SET pgrst.db_anon_role = 'postgrest_test_anonymous';
 
 CREATE ROLE postgrest_test_author;
 
@@ -14,7 +14,14 @@ alter role postgrest_test_serializable set default_transaction_isolation = 'seri
 CREATE ROLE postgrest_test_repeatable_read;
 alter role postgrest_test_repeatable_read set default_transaction_isolation = 'REPEATABLE READ';
 
-GRANT postgrest_test_anonymous, postgrest_test_author, postgrest_test_serializable, postgrest_test_repeatable_read TO :USER;
+CREATE ROLE postgrest_test_w_superuser_settings;
+alter role postgrest_test_w_superuser_settings set log_min_duration_statement = 1;
+alter role postgrest_test_w_superuser_settings set log_min_messages = 'fatal';
+
+GRANT
+  postgrest_test_anonymous, postgrest_test_author,
+  postgrest_test_serializable, postgrest_test_repeatable_read,
+  postgrest_test_w_superuser_settings TO :PGUSER;
 
 CREATE SCHEMA v1;
 GRANT USAGE ON SCHEMA v1 TO postgrest_test_anonymous;
@@ -23,7 +30,7 @@ CREATE TABLE authors_only ();
 GRANT SELECT ON authors_only TO postgrest_test_author;
 
 CREATE TABLE projects AS SELECT FROM generate_series(1,5);
-GRANT SELECT ON projects TO postgrest_test_anonymous;
+GRANT SELECT ON projects TO postgrest_test_anonymous, postgrest_test_w_superuser_settings;
 
 create function get_guc_value(name text) returns text as $$
   select nullif(current_setting(name), '')::text;
@@ -90,7 +97,7 @@ create or replace function sleep(seconds double precision) returns void as $$
 $$ language sql;
 
 create or replace function hello() returns text as $$
-  select 'hello';
+  select 'hello'::text;
 $$ language sql;
 
 create table cats(id uuid primary key, name text);
@@ -142,3 +149,28 @@ returns text as $$
   select current_setting('transaction_isolation', true);
 $$
 language sql set default_transaction_isolation = 'REPEATABLE READ';
+
+create or replace function create_function() returns void as $_$
+  drop function if exists mult_them(int, int);
+  create or replace function mult_them(a int, b int) returns int as $$
+    select a*b;
+  $$ language sql;
+  notify pgrst, 'reload schema';
+$_$ language sql security definer;
+
+create or replace function migrate_function() returns void as $_$
+  drop function if exists mult_them(int, int);
+  create or replace function mult_them(c int, d int) returns int as $$
+    select c*d;
+  $$ language sql;
+  notify pgrst, 'reload schema';
+$_$ language sql security definer;
+
+create or replace function get_pgrst_version() returns text
+  language sql
+as $$
+select application_name
+from pg_stat_activity
+where application_name ilike 'postgrest%'
+limit 1;
+$$

test/io/fixtures.yaml

@@ -4,6 +4,10 @@ cli:
     args: ['--help']
   - name: help short
     args: ['-h']
+  - name: version long
+    args: ['--version']
+  - name: version short
+    args: ['-v']
   - name: example long
     args: ['--example']
   - name: example short

test/io/postgrest.py

@@ -64,6 +64,7 @@ def run(
     host=None,
     wait_for_readiness=True,
     no_pool_connection_available=False,
+    no_startup_stdout=True,
 ):
     "Run PostgREST and yield an endpoint that is ready for connections."
 
@@ -104,7 +105,8 @@ def run(
             if wait_for_readiness:
                 wait_until_ready(adminurl + "/ready")
 
-            process.stdout.read()
+            if no_startup_stdout:
+                process.stdout.read()
 
             if no_pool_connection_available:
                 sleep_pool_connection(baseurl, 10)

test/io/test_io.py

@@ -308,13 +308,11 @@ def test_db_schema_reload(tmp_path, defaultenv):
 
         # reload config
         postgrest.process.send_signal(signal.SIGUSR2)
+        time.sleep(0.1)
 
         # reload schema cache to verify that the config reload actually happened
         postgrest.process.send_signal(signal.SIGUSR1)
-
-        # takes max 1 second to load the internal cache(big_schema.sql included now)
-        # TODO this could go back to time.sleep(0.1) if the big_schema is put in another test suite
-        time.sleep(1)
+        time.sleep(0.1)
 
         response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
         assert response.text == '"\\"v1\\", \\"public\\""'
@@ -551,7 +549,7 @@ def test_pool_size(defaultenv, metapostgrest):
 
 
 def test_pool_acquisition_timeout(defaultenv, metapostgrest):
-    "Verify that PGRST_DB_POOL_ACQUISITON_TIMEOUT times out when the pool is empty"
+    "Verify that PGRST_DB_POOL_ACQUISITION_TIMEOUT times out when the pool is empty"
 
     env = {
         **defaultenv,
@@ -946,6 +944,44 @@ def test_isolation_level(defaultenv):
         assert response.text == '"serializable"'
 
 
+def test_schema_cache_reloading(defaultenv):
+    "schema cache should reload successfully"
+
+    # If DB_POOL=1, then the second request(/rpc/migrate_function) will just wait(PGRST_DB_POOL_ACQUISITION_TIMEOUT=10) for the schema cache reload to finish.
+    # This is bc the only pool connection will be busy with the PGRST_INTERNAL_SCHEMA_CACHE_SLEEP(does a pg_sleep)
+    # So this must be tested with a DB_POOL size of at least 2. That way the second request will pick the other pool connection and proceed.
+
+    env = {
+        **defaultenv,
+        "PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "1",
+        "PGRST_DB_CHANNEL_ENABLED": "true",
+        "PGRST_DB_POOL": "2",
+    }
+
+    internal_sleep = int(env["PGRST_INTERNAL_SCHEMA_CACHE_SLEEP"])
+
+    with run(env=env, wait_for_readiness=False) as postgrest:
+        time.sleep(2 * internal_sleep + 0.1)  # wait for readiness manually
+
+        response = postgrest.session.post("/rpc/create_function")
+        assert response.status_code == 204
+
+        time.sleep(
+            internal_sleep / 2
+        )  # wait to be inside the schema cache reload process
+
+        response = postgrest.session.post("/rpc/migrate_function")
+        assert response.status_code == 204
+
+        time.sleep(
+            2 * internal_sleep
+        )  # wait enough time to ensure the schema cache state remains
+
+        response = postgrest.session.get("/rpc/mult_them?c=3&d=4")
+        assert response.text == "12"
+        assert response.status_code == 200
+
+
 # TODO: This test fails now because of https://github.com/PostgREST/postgrest/pull/2122
 # The stack size of 1K(-with-rtsopts=-K1K) is not enough and this fails with "stack overflow"
 # A stack size of 200K seems to be enough for succeess
@@ -962,3 +998,68 @@ def test_openapi_in_big_schema(defaultenv):
     with run(env=env) as postgrest:
         response = postgrest.session.get("/")
         assert response.status_code == 200
+
+
+@pytest.mark.parametrize("dburi_type", ["no_params", "no_params_qmark", "with_params"])
+def test_get_pgrst_version_with_uri_connection_string(dburi_type, dburi, defaultenv):
+    "The fallback_application_name should be added to the db-uri if it has a URI format"
+    defaultenv_without_libpq = {
+        key: value
+        for key, value in defaultenv.items()
+        if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
+    }
+
+    env = {
+        "no_params": {**defaultenv, "PGRST_DB_URI": "postgresql://"},
+        "no_params_qmark": {**defaultenv, "PGRST_DB_URI": "postgresql://?"},
+        "with_params": {**defaultenv_without_libpq, "PGRST_DB_URI": dburi.decode()},
+    }
+
+    with run(env=env[dburi_type]) as postgrest:
+        response = postgrest.session.post("/rpc/get_pgrst_version")
+        version = '"%s"' % response.headers["Server"].replace(
+            "postgrest/", "PostgREST "
+        )
+        assert response.text == version
+
+
+def test_get_pgrst_version_with_keyval_connection_string(defaultenv):
+    "The fallback_application_name should be added to the db-uri if it has a keyword/value format"
+    uri = f'dbname={defaultenv["PGDATABASE"]} host={defaultenv["PGHOST"]} user={defaultenv["PGUSER"]}'
+    defaultenv_without_libpq = {
+        key: value
+        for key, value in defaultenv.items()
+        if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
+    }
+    env = {**defaultenv_without_libpq, "PGRST_DB_URI": uri}
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.post("/rpc/get_pgrst_version")
+        version = '"%s"' % response.headers["Server"].replace(
+            "postgrest/", "PostgREST "
+        )
+        assert response.text == version
+
+
+def test_log_postgrest_version(defaultenv):
+    "Should show the PostgREST version in the logs"
+
+    with run(env=defaultenv, no_startup_stdout=False) as postgrest:
+        version = postgrest.session.head("/").headers["Server"].split("/")[1]
+
+        assert (
+            "Starting PostgREST %s..." % version
+            in postgrest.process.stdout.readline().decode()
+        )
+
+
+def test_succeed_w_role_having_superuser_settings(defaultenv):
+    "Should succeed when having superuser settings on the impersonated role"
+
+    env = {**defaultenv, "PGRST_DB_CONFIG": "true", "PGRST_JWT_SECRET": SECRET}
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        headers = jwtauthheader({"role": "postgrest_test_w_superuser_settings"}, SECRET)
+        response = postgrest.session.get("/projects", headers=headers)
+        print(response.text)
+        assert response.status_code == 200

test/load/fixtures.sql

@@ -1,5 +1,5 @@
 CREATE ROLE postgrest_test_anonymous;
-GRANT postgrest_test_anonymous TO :USER;
+GRANT postgrest_test_anonymous TO :PGUSER;
 CREATE SCHEMA test;
 
 -- PUT+PATCH target needs one record and column to modify

test/spec/Feature/CorsSpec.hs

@@ -40,7 +40,7 @@ spec =
                              \Date, Location, Server, Transfer-Encoding, Range-Unit"]
           }
 
-    it "allows INFO body through even with CORS request headers present to postflight request" $
+    it "allows INFO body through even with CORS request headers present to postflight request" $ do
       request methodOptions "/items"
           [ ("Host", "localhost:3000")
           , ("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0")
@@ -54,3 +54,17 @@ spec =
         `shouldRespondWith`
           ""
           { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
+
+      request methodOptions "/items"
+          [ ("Accept", "application/json") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
+
+      request methodOptions "/shops"
+          [ ("Accept", "application/geo+json") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }

test/spec/Feature/NoSuperuserSpec.hs

@@ -0,0 +1,20 @@
+module Feature.NoSuperuserSpec where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+
+import Protolude
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "No Superuser" $ do
+    it "proves that the authenticator role is not a superuser" $ do
+      request methodGet "/rpc/is_superuser"
+          mempty
+          ""
+        `shouldRespondWith`
+          "false"
+          { matchStatus = 200 }

test/spec/Feature/Query/ComputedRelsSpec.hs

@@ -104,6 +104,47 @@ spec = describe "computed relationships" $ do
         [json|[ {"name":"Final Fantasy I","designer":{"name":"Hironobu Sakaguchi"}} ]|]
         { matchStatus  = 200 }
 
+  it "applies data representations to response" $ do
+    -- A smoke test for data reps in the presence of computed relations.
+
+    -- The data rep here title cases the designer name before presentation. So here the lowercase version will be saved,
+    -- but the title case version returned. Pulling in a computed relation should not confuse this.
+    request methodPatch "/designers?select=name,videogames:computed_videogames(name)&id=eq.1"
+      [("Prefer", "return=representation"), ("Prefer", "tx=commit")]
+      [json| {"name": "sidney k. meier"} |]
+      `shouldRespondWith`
+      [json|[{"name":"Sidney K. Meier","videogames":[{"name":"Civilization I"}, {"name":"Civilization II"}]}]|]
+      { matchStatus = 200 }
+
+    -- Verify it was saved the way we requested (there's no text data rep for this column, so if we select with the wrong casing, it should fail.)
+    get "/designers?select=id&name=eq.Sidney%20K.%20Meier"
+      `shouldRespondWith`
+      [json|[]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
+    -- But with the right casing it works.
+    get "/designers?select=id,name&name=eq.sidney%20k.%20meier"
+      `shouldRespondWith`
+      [json|[{"id": 1, "name":"Sidney K. Meier"}]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
+
+    -- Most importantly, if you read it back even via a computed relation, the data rep should be applied.
+    get "/videogames?select=name,designer:computed_designers(*)&id=eq.1"
+      `shouldRespondWith`
+      [json|[
+        {"name":"Civilization I","designer":{"id": 1, "name":"Sidney K. Meier"}}
+      ]|] { matchHeaders = [matchContentTypeJson] }
+
+    -- reset the test fixture
+    request methodPatch "/designers?id=eq.1"
+      [("Prefer", "tx=commit")]
+      [json| {"name": "Sid Meier"} |]
+      `shouldRespondWith` 204
+    -- need to poke the second one too to prevent inherent ordering from changing
+    request methodPatch "/designers?id=eq.2"
+      [("Prefer", "tx=commit")]
+      [json| {"name": "Hironobu Sakaguchi"} |]
+      `shouldRespondWith` 204
+
   it "works with self joins" $
     get "/web_content?select=name,child_web_content(name),parent_web_content(name)&id=in.(0,1)"
     `shouldRespondWith`

test/spec/Feature/Query/DeleteSpec.hs

@@ -37,7 +37,8 @@ spec =
         request methodDelete "/items?id=eq.2" [("Prefer", "return=representation"), ("Prefer", "count=exact")] ""
           `shouldRespondWith` [json|[{"id":2}]|]
           { matchStatus  = 200
-          , matchHeaders = ["Content-Range" <:> "*/1"]
+          , matchHeaders = ["Content-Range" <:> "*/1"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "ignores ?select= when return not set or return=minimal" $ do
@@ -57,7 +58,8 @@ spec =
             ""
             { matchStatus  = 204
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       it "returns the deleted item and shapes the response" $
@@ -137,7 +139,8 @@ spec =
           `shouldRespondWith`
             ""
             { matchStatus = 204
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal" ]
             }
 
       it "suceeds deleting the row with no explicit select by default" $

test/spec/Feature/Query/InsertSpec.hs

@@ -13,7 +13,8 @@ import Text.Heredoc
 
 import PostgREST.Config.PgVersion (PgVersion, pgVersion100,
                                    pgVersion110, pgVersion112,
-                                   pgVersion130)
+                                   pgVersion120, pgVersion130,
+                                   pgVersion140)
 
 import Protolude  hiding (get)
 import SpecHelper
@@ -42,7 +43,8 @@ spec actualPgVersion = do
           , "enum": "foo"
           }] |] `shouldRespondWith` [json|[{"integer":14,"varchar":"testing!"}]|]
           { matchStatus  = 201
-          , matchHeaders = [matchContentTypeJson]
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "ignores &select when return not set or using return=minimal" $ do
@@ -68,7 +70,8 @@ spec actualPgVersion = do
           `shouldRespondWith`
             ""
             { matchStatus  = 201
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
     context "non uniform json array" $ do
@@ -98,7 +101,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/1" ]
+                           , "Content-Range" <:> "*/1"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "can rename and cast the selected columns" $
@@ -109,7 +113,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/*" ]
+                           , "Content-Range" <:> "*/*"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "should not throw and return location header when selecting without PK" $
@@ -119,7 +124,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/*" ]
+                           , "Content-Range" <:> "*/*"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
     context "requesting headers only representation" $ do
@@ -132,7 +138,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/projects?id=eq.11"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
       when (actualPgVersion >= pgVersion110) $
@@ -145,7 +152,8 @@ spec actualPgVersion = do
               { matchStatus  = 201
               , matchHeaders = [ matchHeaderAbsent hContentType
                                , "Location" <:> "/car_models?name=eq.Enzo&year=eq.2021"
-                               , "Content-Range" <:> "*/*" ]
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
               }
 
     context "requesting no representation" $
@@ -192,7 +200,8 @@ spec actualPgVersion = do
               ""
               { matchStatus  = 201
               , matchHeaders = [ matchHeaderAbsent hContentType
-                               , "Location" <:> "/auto_incrementing_pk?id=eq.2" ]
+                               , "Location" <:> "/auto_incrementing_pk?id=eq.2"
+                               , "Preference-Applied" <:> "return=headers-only"]
               }
 
       context "into a table with simple pk" $
@@ -226,7 +235,8 @@ spec actualPgVersion = do
             `shouldRespondWith`
               [json| [{ "a":"bar", "b":"baz" }] |]
               { matchStatus  = 201
-              , matchHeaders = [matchHeaderAbsent hLocation]
+              , matchHeaders = [matchHeaderAbsent hLocation
+                               , "Preference-Applied" <:> "return=representation"]
               }
 
         it "returns empty array when no items inserted, and return=rep" $ do
@@ -390,6 +400,22 @@ spec actualPgVersion = do
           `shouldRespondWith` [json|[{ id: 20 }]|]
             { matchStatus  = 201 }
 
+    -- https://github.com/PostgREST/postgrest/issues/2861
+    context "bit and char columns with length" $ do
+      it "should insert to a bit column with length" $
+        request methodPost "/bitchar_with_length?select=bit"
+            [("Prefer", "return=representation")]
+            [json|{"bit": "10101"}|]
+          `shouldRespondWith` [json|[{ "bit": "10101" }]|]
+            { matchStatus  = 201 }
+
+      it "should insert to a char column with length" $
+        request methodPost "/bitchar_with_length?select=char"
+            [("Prefer", "return=representation")]
+            [json|{"char": "abcde"}|]
+          `shouldRespondWith` [json|[{ "char": "abcde" }]|]
+            { matchStatus  = 201 }
+
     context "POST with ?columns parameter" $ do
       it "ignores json keys not included in ?columns" $ do
         request methodPost "/articles?columns=id,body" [("Prefer", "return=representation")]
@@ -468,7 +494,7 @@ spec actualPgVersion = do
                   {"id": 6, "name": "Sechs", "field-with_sep": 6, "settings":null,"arr_data":[1,2,3]}
                 ]|]
                 { matchStatus  = 201
-                , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+                , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
                 }
 
         it "inserts view default values(field-with_sep) when json keys are undefined" $
@@ -483,7 +509,7 @@ spec actualPgVersion = do
                 {"id": 8, "name": "Default", "field-with_sep": 1, "settings":null,"arr_data":null}
               ]|]
               { matchStatus  = 201
-              , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
               }
 
         it "doesn't insert json duplicate keys(since it uses jsonb)" $
@@ -492,19 +518,51 @@ spec actualPgVersion = do
             `shouldRespondWith`
               [json| [ { "data": { "a": 2 }, "id": 3 } ] |]
               { matchStatus  = 201
-              , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
               }
 
         when (actualPgVersion >= pgVersion100) $
-          it "inserts a default on a generated by default as identity column" $ do
+          it "inserts a default on a generated by default as identity column" $
             request methodPost "/channels?columns=id,data,slug&select=data,slug" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
                 [json| { "slug": "foo" } |]
               `shouldRespondWith`
                 [json| [{"data":{"foo": "bar"},"slug":"foo"}] |] -- id 1 was inserted here, we don't get it for idempotence in the tests
                 { matchStatus  = 201
-                , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+                , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
                 }
 
+        when (actualPgVersion >= pgVersion120) $
+          it "fails with a good error message on generated always columns" $
+            request methodPost "/foo?columns=a,b" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+                [json| [
+                  {"a": "val"},
+                  {"a": "val", "b": "val"}
+                ]|]
+              `shouldRespondWith`
+                (if actualPgVersion < pgVersion140
+                  then [json| {
+                    "code": "42601",
+                    "details": "Column \"b\" is a generated column.",
+                    "hint": null,
+                    "message": "cannot insert into column \"b\""
+                  }|]
+                  else [json| {
+                    "code": "428C9",
+                    "details": "Column \"b\" is a generated column.",
+                    "hint": null,
+                    "message": "cannot insert a non-DEFAULT value into column \"b\""
+                  }|])
+                { matchStatus  = 400 }
+
+        it "inserts a default on a DOMAIN with default" $
+          request methodPost "/evil_friends?columns=id,name" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+              [json| { "name": "Lu" } |]
+            `shouldRespondWith`
+              [json| [{"id": 666, "name": "Lu"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+              }
+
     it "inserts json that has duplicate keys" $ do
       request methodPost "/tbl_w_json" [("Prefer", "return=representation")]
           [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
@@ -670,7 +728,8 @@ spec actualPgVersion = do
         `shouldRespondWith`
           ""
           { matchStatus = 201
-          , matchHeaders = [matchHeaderAbsent hContentType]
+          , matchHeaders = [matchHeaderAbsent hContentType
+                           , "Preference-Applied" <:> "return=minimal"]
           }
 
   describe "Inserting into VIEWs" $ do
@@ -693,7 +752,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/with_multiple_pks?pk1=eq.1&pk2=eq.2"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
     context "requesting header only representation" $ do
@@ -705,7 +765,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/compound_pk_view?k1=eq.1&k2=eq.test"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
       it "should not throw and return location header when a PK is null" $
@@ -716,5 +777,120 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/test_null_pk_competitors_sponsors?id=eq.1&sponsor_id=is.null"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
+
+
+  -- Data representations for payload parsing requires Postgres 10 or above.
+  when (actualPgVersion >= pgVersion100) $ do
+    describe "Data representations" $ do
+      context "on regular table" $ do
+        it "parses values in POST body" $
+          -- we don't check that the parsing is correct here, just that it's happening. If it doesn't happen we'll get a
+          -- an "invalid input syntax for type integer:" error.
+          request methodPost "/datarep_todos" [("Prefer", "return=headers-only")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 201
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Location" <:> "/datarep_todos?id=eq.5"
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
+              }
+
+        it "parses values in POST body and formats individually selected values in return=representation" $
+          request methodPost "/datarep_todos?select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "label_color": "#001100"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "parses values in POST body and formats values in return=representation" $
+          request methodPost "/datarep_todos" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00", "icon_image": "3q2+7w", "created_at":-15, "budget": "-100000000000000.13"} |]
+            `shouldRespondWith`
+            [json| [{"id":5,"name": "party", "label_color": "#001100", "due_at":"2018-01-03T11:00:00Z", "icon_image": "3q2+7w==", "created_at":-15, "budget": "-100000000000000.13"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+      context "with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPost "/datarep_todos?columns=id,label_color&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "invalid but should be ignored"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "name":null, "label_color": "#001100", "due_at": "2018-01-01T00:00:00Z"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "fails without parsing anything if at least one specified column doesn't exist" $
+          request methodPost "/datarep_todos?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }
+
+      context "on updatable view" $ do
+        it "parses values in POST body" $
+          -- we don't check that the parsing is correct here, just that it's happening. If it doesn't happen we'll get a
+          -- an "invalid input syntax for type integer:" error.
+          request methodPost "/datarep_todos_computed" [("Prefer", "return=headers-only")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 201
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Location" <:> "/datarep_todos_computed?id=eq.5"
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
+              }
+
+        it "parses values in POST body and formats individually selected values in return=representation" $
+          request methodPost "/datarep_todos_computed?select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "label_color": "#001100"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "parses values in POST body and formats values in return=representation" $
+          request methodPost "/datarep_todos_computed" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5,"name": "party", "label_color": "#001100", "due_at":"2018-01-03T11:00:00Z", "dark_color":"#000880"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+      context "on updatable views with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPost "/datarep_todos_computed?columns=id,label_color&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "invalid but should be ignored"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "name":null, "label_color": "#001100", "due_at": "2018-01-01T00:00:00Z"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "fails without parsing anything if at least one specified column doesn't exist" $
+          request methodPost "/datarep_todos_computed?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos_computed' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }

test/spec/Feature/Query/JsonOperatorSpec.hs

@@ -103,7 +103,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
 
       it "can get array of objects" $ do
         get "/json_arr?select=data->0->>a&id=in.(5,6)" `shouldRespondWith`
-          [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
+          [json|[{"a":"A"}, {"a":"[1,2,3]"}]|]
           { matchHeaders = [matchContentTypeJson] }
         get "/json_arr?select=data->0->a->>2&id=in.(5,6)" `shouldRespondWith`
           [json| [{"a":null}, {"a":"3"}] |]
@@ -275,7 +275,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         [json| [{"data":8}, {"data":7}] |]
         { matchHeaders = [matchContentTypeJson] }
       get "/json_arr?select=data->-2->>a&id=in.(5,6)" `shouldRespondWith`
-        [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
+        [json| [{"a":"A"}, {"a":"[1,2,3]"}] |]
         { matchHeaders = [matchContentTypeJson] }
 
     it "can filter with negative indexes" $ do

test/spec/Feature/Query/NullsStrip.hs

@@ -0,0 +1,116 @@
+module Feature.Query.NullsStrip where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import Protolude  hiding (get)
+import SpecHelper
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "Stripping null values from JSON response" $ do
+    let arrayStrip = ("Accept", "application/vnd.pgrst.array+json;nulls=stripped")
+    let singularStrip = ("Accept", "application/vnd.pgrst.object+json;nulls=stripped")
+
+    context "strip nulls from response" $ do
+      it "strips nulls when Accept: application/vnd.pgrst.array+json;nulls=stripped" $ do
+        request methodGet  "/organizations?select=*"
+          [arrayStrip]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"name":"Referee Org","manager_id":1},{"id":2,"name":"Auditor Org","manager_id":2},{"id":3,"name":"Acme","referee":1,"auditor":2,"manager_id":3},{"id":4,"name":"Umbrella","referee":1,"auditor":2,"manager_id":4},{"id":5,"name":"Cyberdyne","referee":3,"auditor":4,"manager_id":5},{"id":6,"name":"Oscorp","referee":3,"auditor":4,"manager_id":6}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPost  "/organizations?select=*"
+          [arrayStrip,("Prefer","return=representation")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7,"name":"John","manager_id":6}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPatch  "/organizations?id=eq.3&select=*"
+          [arrayStrip, ("Prefer","return=representation")]
+          [json|{"name":"John","referee":null}|]
+          `shouldRespondWith`
+          [json|[{"id":3,"name":"John","auditor":2,"manager_id":3}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.object+json;nulls=stripped" $
+        request methodGet  "/organizations?limit=1"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"id":1,"name":"Referee Org","manager_id":1}|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTSingularStrip]
+          }
+
+      it "throws error when Accept: application/vnd.pgrst.object+json;nulls=stripped and result not singular" $
+        request methodGet  "/organizations?select=*"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"details":"The result contains 6 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+          { matchStatus  = 406
+          , matchHeaders = [matchContentTypeSingular]
+          }
+
+    context "strip nulls from response even if explicitly selected" $ do
+      it "strips nulls when Accept: application/vnd.pgrst.array+json;nulls=stripped" $ do
+        request methodGet  "/organizations?select=id,referee,auditor"
+          [arrayStrip]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1},{"id":2},{"id":3,"referee":1,"auditor":2},{"id":4,"referee":1,"auditor":2},{"id":5,"referee":3,"auditor":4},{"id":6,"referee":3,"auditor":4}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPost  "/organizations?select=id,referee,auditor"
+          [arrayStrip,("Prefer","return=representation")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPatch  "/organizations?id=eq.3&select=id,name,referee,auditor"
+          [arrayStrip, ("Prefer","return=representation")]
+          [json|{"name":"John","referee":null}|]
+          `shouldRespondWith`
+          [json|[{"id":3,"name":"John","auditor":2}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.object+json;nulls=stripped" $
+        request methodGet  "/organizations?select=id,referee,auditor&limit=1"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"id":1}|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTSingularStrip]
+          }
+
+    context "doesn't strip nulls" $ do
+      it "doesn't strips nulls when Accept: application/vnd.pgrst.array+json" $
+        request methodGet  "/organizations?select=id,referee,auditor"
+          [("Accept", "application/vnd.pgrst.array+json")]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"referee":null,"auditor":null},{"id":2,"referee":null,"auditor":null},{"id":3,"referee":1,"auditor":2},{"id":4,"referee":1,"auditor":2},{"id":5,"referee":3,"auditor":4},{"id":6,"referee":3,"auditor":4}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }

test/spec/Feature/Query/PlanSpec.hs

@@ -8,7 +8,6 @@ import Network.Wai.Test (SResponse (..))
 
 import           Data.Aeson.Lens
 import           Data.Aeson.QQ
-import qualified Data.ByteString      as BS
 import qualified Data.ByteString.Lazy as LBS
 import qualified Data.Text            as T
 import           Network.HTTP.Types
@@ -33,7 +32,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe`
           if actualPgVersion > pgVersion120
@@ -49,7 +48,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe`
           if actualPgVersion > pgVersion120
@@ -65,8 +64,8 @@ spec actualPgVersion = do
               resHeaders = simpleHeaders r
 
           liftIO $ do
-            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=buffers; charset=utf-8")
-            resBody `shouldSatisfy` (\t -> T.isInfixOf "Shared Hit Blocks" (decodeUtf8 $ BS.toStrict t))
+            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=buffers; charset=utf-8")
+            resBody `shouldSatisfy` (\t -> T.isInfixOf "Shared Hit Blocks" (decodeUtf8 $ LBS.toStrict t))
         else do
           -- analyze is required for buffers on pg < 13
           r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=analyze|buffers") ""
@@ -75,7 +74,7 @@ spec actualPgVersion = do
               resHeaders = simpleHeaders r
 
           liftIO $ do
-            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=analyze|buffers; charset=utf-8")
+            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze|buffers; charset=utf-8")
             blocks `shouldBe` Just [aesonQQ| 1.0 |]
 
     when (actualPgVersion >= pgVersion120) $
@@ -86,7 +85,7 @@ spec actualPgVersion = do
             resHeaders = simpleHeaders r
 
         liftIO $ do
-          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=settings; charset=utf-8")
+          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=settings; charset=utf-8")
           searchPath `shouldBe`
             Just [aesonQQ|
               {
@@ -102,7 +101,7 @@ spec actualPgVersion = do
             resHeaders = simpleHeaders r
 
         liftIO $ do
-          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=analyze|wal; charset=utf-8")
+          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze|wal; charset=utf-8")
           walRecords `shouldBe` Just [aesonQQ|0|]
 
     it "outputs columns info when using the verbose option" $ do
@@ -112,7 +111,7 @@ spec actualPgVersion = do
           resHeaders = simpleHeaders r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=verbose; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=verbose; charset=utf-8")
         cols `shouldBe` Just [aesonQQ| ["projects.id", "projects.name", "projects.client_id"] |]
 
     it "outputs the plan for application/json " $ do
@@ -151,7 +150,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 3.27
 
@@ -164,7 +163,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 12.45
 
@@ -177,7 +176,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 15.68
 
@@ -191,7 +190,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 1.29
 
@@ -216,7 +215,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 68.56
 
@@ -241,7 +240,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         resBody `shouldSatisfy` (\t -> LBS.take 9 t == "Aggregate")
 
@@ -254,7 +253,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         resBody `shouldSatisfy` (\t -> LBS.take 9 t == "Aggregate")
 
@@ -338,7 +337,7 @@ spec actualPgVersion = do
         let resBody = simpleBody r
 
         liftIO $ do
-          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getallusers" (decodeUtf8 $ BS.toStrict t))
+          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getallusers" (decodeUtf8 $ LBS.toStrict t))
 
       it "should inline a function with arguments(the function won't appear in the plan tree)" $ do
         r <- request methodGet "/rpc/getitemrange?min=10&max=15"
@@ -347,7 +346,44 @@ spec actualPgVersion = do
         let resBody = simpleBody r
 
         liftIO $ do
-          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getitemrange" (decodeUtf8 $ BS.toStrict t))
+          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getitemrange" (decodeUtf8 $ LBS.toStrict t))
+
+    context "index usage" $ do
+      it "should use an index for a json arrow operator filter" $ do
+        r <- request methodGet "/bets?data_json->>contractId=eq.1"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index Cond" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for a jsonb arrow operator filter" $ do
+        r <- request methodGet "/bets?data_jsonb->>contractId=eq.1"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for ordering on a json arrow operator" $ do
+        r <- request methodGet "/bets?order=data_json->>contractId"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for ordering on a jsonb arrow operator" $ do
+        r <- request methodGet "/bets?order=data_jsonb->>contractId"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
 
 disabledSpec :: SpecWith ((), Application)
 disabledSpec =

test/spec/Feature/Query/QuerySpec.hs

@@ -1328,3 +1328,122 @@ spec actualPgVersion = do
       get "/articles?body=imatch(any).{stop,thing}&select=id" `shouldRespondWith`
         [json|[{"id":1}, {"id":2}]|]
         { matchHeaders = [matchContentTypeJson] }
+
+  describe "Data representations for customisable value formatting and parsing" $ do
+    it "formats a single column" $
+      get "/datarep_todos?select=id,label_color&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"label_color":"#000000"},{"id":2,"label_color":"#000100"},{"id":3,"label_color":"#01E240"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats two columns with different formatters" $
+      get "/datarep_todos?select=id,label_color,due_at&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"},{"id":2,"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"},{"id":3,"label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "fails in some reasonable way when selecting fields that don't exist" $
+      get "/datarep_todos?select=id,label_color,banana" `shouldRespondWith`
+        [json| {"code":"42703","details":null,"hint":null,"message":"column datarep_todos.banana does not exist"} |]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+    it "formats columns in views including computed columns" $
+      get "/datarep_todos_computed?select=id,label_color,dark_color" `shouldRespondWith`
+        [json| [
+          {"id":1, "label_color":"#000000", "dark_color":"#000000"},
+          {"id":2, "label_color":"#000100", "dark_color":"#000080"},
+          {"id":3, "label_color":"#01E240", "dark_color":"#00F120"},
+          {"id":4, "label_color":"", "dark_color":""}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats and allows rename" $
+      get "/datarep_todos?select=id,clr:label_color&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"clr":"#000000"},{"id":2,"clr":"#000100"},{"id":3,"clr":"#01E240"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats, renames and allows manual casting on top" $
+      get "/datarep_todos?select=id,clr:label_color::text&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"clr":"\"#000000\""},{"id":2,"clr":"\"#000100\""},{"id":3,"clr":"\"#01E240\""}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats nulls" $
+      -- due_at is formatted as NULL but label_color NULLs become empty strings-- it's up to the formatting function.
+      get "/datarep_todos?select=id,label_color,due_at&id=gt.2&id=lt.5" `shouldRespondWith`
+        [json| [{"id":3,"label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z"},{"id":4,"label_color":"","due_at":null}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats star select" $
+      get "/datarep_todos?select=*&id=lt.4" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+           {"id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+           {"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats implicit star select" $
+      get "/datarep_todos?id=lt.4" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+ {"id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+ {"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats star and explicit mix" $
+      get "/datarep_todos?select=due_at,*&id=lt.4" `shouldRespondWith`
+        [json| [
+          {"due_at":"2018-01-02T00:00:00Z","id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+           {"due_at":"2018-01-03T00:00:00Z","id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+           {"due_at":"2018-01-01T14:12:34.123456Z","id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats through join" $
+      get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)" `shouldRespondWith`
+        [json| [{"id":1,"name":"school related","first_item":{"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"}},{"id":2,"name":"do these first","first_item":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats through join with star select" $
+      get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"school related","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}},
+          {"id":2,"name":"do these first","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "uses text parser on value for filter given through query parameters" $
+      get "/datarep_todos?select=id,due_at&label_color=eq.000100" `shouldRespondWith`
+        [json| [{"id":2,"due_at":"2018-01-03T00:00:00Z"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "in the absense of text parser, does not try to use the JSON parser for query parameters" $
+      get "/datarep_todos?select=id,due_at&due_at=eq.Z" `shouldRespondWith`
+        -- we prove the parser is not used because it'd replace the Z with `+00:00` and a different error message.
+        [json| {"code":"22007","details":null,"hint":null,"message":"invalid input syntax for type timestamp with time zone: \"Z\""} |]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+    -- Before PG 11, this will fail because we need arrays of domain type values. The docs should explain data reps are
+    -- not supported in this case.
+    when (actualPgVersion >= pgVersion110) $ do
+      it "uses text parser for filter with 'IN' predicates" $
+        get "/datarep_todos?select=id,due_at&label_color=in.(000100,01E240)" `shouldRespondWith`
+          [json| [
+            {"id":2, "due_at": "2018-01-03T00:00:00Z"},
+            {"id":3, "due_at": "2018-01-01T14:12:34.123456Z"}
+          ] |]
+          { matchHeaders = [matchContentTypeJson] }
+      it "uses text parser for filter with 'NOT IN' predicates" $
+        get "/datarep_todos?select=id,due_at&label_color=not.in.(000000,01E240)" `shouldRespondWith`
+          [json| [
+            {"id":2, "due_at": "2018-01-03T00:00:00Z"}
+          ] |]
+          { matchHeaders = [matchContentTypeJson] }
+    it "uses text parser on value for filter across relations" $
+      get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100" `shouldRespondWith`
+        [json| [{"id":1,"name":"school related","datarep_todos":null},{"id":2,"name":"do these first","datarep_todos":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    -- This is not supported by data reps (would be hard to make it work with high performance). So the test just
+    -- verifies we don't panic or add inappropriate SQL to the filters.
+    it "fails safely on user trying to use ilike operator on data reps column" $
+      get "/datarep_todos?select=id,name&label_color=ilike.#*100" `shouldRespondWith` (
+        if actualPgVersion >= pgVersion110 then
+        [json|
+          {"code":"42883","details":null,"hint":"No operator matches the given name and argument types. You might need to add explicit type casts.","message":"operator does not exist: public.color ~~* unknown"}
+        |]
+        else
+        [json|
+          {"code":"42883","details":null,"hint":"No operator matches the given name and argument type(s). You might need to add explicit type casts.","message":"operator does not exist: public.color ~~* unknown"}
+        |])
+        { matchStatus  = 404
+        , matchHeaders = [matchContentTypeJson]
+        }

test/spec/Feature/Query/RelatedQueriesSpec.hs

@@ -233,18 +233,6 @@ spec = describe "related queries" $ do
         , matchHeaders = [matchContentTypeJson]
         }
 
-    it "only works with is null or is not null operators" $
-      get "/projects?select=name,clients(*)&clients=eq.3" `shouldRespondWith`
-        [json|{
-          "code":"PGRST120",
-          "details":"Only is null or not is null filters are allowed on embedded resources",
-          "hint":null,
-          "message":"Bad operator on the 'clients' embedded resource"
-        }|]
-        { matchStatus  = 400
-        , matchHeaders = [matchContentTypeJson]
-        }
-
     it "doesn't interfere filtering when embedding using the column name" $
       get "/projects?select=name,client_id,client:client_id(name)&client_id=eq.2" `shouldRespondWith`
         [json|[
@@ -254,3 +242,17 @@ spec = describe "related queries" $ do
         { matchStatus  = 200
         , matchHeaders = [matchContentTypeJson]
         }
+
+    it "doesn't interfere filtering on column names used for disambiguation" $
+      get "/user_friend?select=*,user1(*)&user1=eq.a02fb934-3a4d-469b-a6b6-4fcd88b973cf" `shouldRespondWith`
+        [json|[]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "doesn't interfere filtering on column names that are the same as the relation name" $
+      get "/tournaments?select=*,status(*)&status=eq.3" `shouldRespondWith`
+        [json|[]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }

test/spec/Feature/Query/RpcSpec.hs

@@ -475,6 +475,27 @@ spec actualPgVersion =
           `shouldRespondWith`
             [json|null|]
 
+      when (actualPgVersion >= pgVersion110) $ do
+        it "returns a record type" $ do
+          post "/rpc/returns_record"
+            ""
+           `shouldRespondWith`
+            [json|{"id":1,"name":"Windows 7","client_id":1}|]
+          post "/rpc/returns_record_params"
+            [json|{"id":1, "name": "Windows%"}|]
+           `shouldRespondWith`
+            [json|{"id":1,"name":"Windows 7","client_id":1}|]
+
+        it "returns a setof record type" $ do
+          post "/rpc/returns_setof_record"
+            ""
+           `shouldRespondWith`
+            [json|[{"id":1,"name":"Windows 7","client_id":1},{"id":2,"name":"Windows 10","client_id":1}]|]
+          post "/rpc/returns_setof_record_params"
+            [json|{"id":1,"name":"Windows%"}|]
+           `shouldRespondWith`
+            [json|[{"id":1,"name":"Windows 7","client_id":1},{"id":2,"name":"Windows 10","client_id":1}]|]
+
       context "different types when overloaded" $ do
         it "returns composite type" $
           post "/rpc/ret_point_overloaded"
@@ -1447,3 +1468,25 @@ spec actualPgVersion =
               { matchStatus  = 400
               , matchHeaders = [matchContentTypeJson]
               }
+
+    -- https://github.com/PostgREST/postgrest/issues/1586#issuecomment-696345442
+    context "a proc with bit and char parameters" $ do
+      it "modifies the param type from character to character varying" $ do
+        get "/rpc/char_param_select?char_=abcdefg&char_arr={abc,abcdefg}" `shouldRespondWith`
+          [json| [{ "char_": "abcdefg", "char_arr": [ "abc", "abcdefg" ] }] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+        post "/rpc/char_param_insert" [json| { "char_": "abcdefg", "char_arr": "{abc,abcdefg}" } |]
+           `shouldRespondWith`
+             [json| {"code":"22001","details":null,"hint":null,"message":"value too long for type character(5)"} |]
+           { matchStatus = 400 }
+
+      it "modifies the param type from bit to bit varying" $ do
+        get "/rpc/bit_param_select?bit_=101010&bit_arr={101,101010}" `shouldRespondWith`
+          [json| [{ "bit_": "101010", "bit_arr": [ "101", "101010" ] }] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+        post "/rpc/bit_param_insert" [json| { "bit_": "101010", "bit_arr": "{101,101010}" } |]
+           `shouldRespondWith`
+             [json| {"code":"22026","details":null,"hint":null,"message":"bit string length 6 does not match type bit(5)"} |]
+           { matchStatus = 400 }

test/spec/Feature/Query/SingularSpec.hs

@@ -61,7 +61,8 @@ spec =
           `shouldRespondWith`
             ""
             { matchStatus  = 204
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       it "raises an error for multiple rows" $ do
@@ -69,7 +70,7 @@ spec =
             [("Prefer", "tx=commit"), singular]
             [json| { address: "zzz" } |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 4 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 4 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
             , matchHeaders = [ matchContentTypeSingular
                              , "Preference-Applied" <:> "tx=commit" ]
@@ -85,7 +86,7 @@ spec =
             [("Prefer", "tx=commit"), ("Prefer", "return=representation"), singular]
             [json| { address: "zzz" } |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 4 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 4 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
             , matchHeaders = [ matchContentTypeSingular
                              , "Preference-Applied" <:> "tx=commit" ]
@@ -100,7 +101,7 @@ spec =
         request methodPatch "/items?id=gt.0&id=lt.0"
                 [singular] [json|{"id":1}|]
           `shouldRespondWith`
-                  [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                   { matchStatus  = 406
                   , matchHeaders = [matchContentTypeSingular]
                   }
@@ -109,7 +110,7 @@ spec =
         request methodPatch "/items?id=gt.0&id=lt.0"
                 [("Prefer", "return=representation"), singular] [json|{"id":1}|]
           `shouldRespondWith`
-                  [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                   { matchStatus  = 406
                   , matchHeaders = [matchContentTypeSingular]
                   }
@@ -133,7 +134,8 @@ spec =
             ""
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       it "raises an error when attempting to create multiple entities" $ do
@@ -141,7 +143,7 @@ spec =
             [("Prefer", "tx=commit"), singular]
             [json| [ { id: 200, address: "xxx" }, { id: 201, address: "yyy" } ] |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 2 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 2 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
             , matchHeaders = [ matchContentTypeSingular
                              , "Preference-Applied" <:> "tx=commit" ]
@@ -157,7 +159,7 @@ spec =
             [("Prefer", "tx=commit"), ("Prefer", "return=representation"), singular]
             [json| [ { id: 202, address: "xxx" }, { id: 203, address: "yyy" } ] |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 2 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 2 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
             , matchHeaders = [ matchContentTypeSingular
                              , "Preference-Applied" <:> "tx=commit" ]
@@ -173,7 +175,7 @@ spec =
             [("Prefer", "tx=commit"), ("Prefer", "return=minimal"), singular]
             [json| [ { id: 204, address: "xxx" }, { id: 205, address: "yyy" } ] |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 2 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 2 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
             , matchHeaders = [ matchContentTypeSingular
                              , "Preference-Applied" <:> "tx=commit" ]
@@ -189,7 +191,7 @@ spec =
                 [singular]
                 [json| [ ] |]
           `shouldRespondWith`
-                  [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                   { matchStatus  = 406
                   , matchHeaders = [matchContentTypeSingular]
                   }
@@ -199,7 +201,7 @@ spec =
                 [("Prefer", "return=representation"), singular]
                 [json| [ ] |]
           `shouldRespondWith`
-                  [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                   { matchStatus  = 406
                   , matchHeaders = [matchContentTypeSingular]
                   }
@@ -222,7 +224,7 @@ spec =
             [("Prefer", "tx=commit"), singular]
             ""
           `shouldRespondWith`
-            [json|{"details":"Results contain 5 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 5 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
             , matchHeaders = [ matchContentTypeSingular
                              , "Preference-Applied" <:> "tx=commit" ]
@@ -240,7 +242,7 @@ spec =
         request methodDelete "/items?id=gt.5&id=lt.11"
             [("Prefer", "tx=commit"), ("Prefer", "return=representation"), singular] ""
           `shouldRespondWith`
-            [json|{"details":"Results contain 5 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 5 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
             , matchHeaders = [ matchContentTypeSingular
                              , "Preference-Applied" <:> "tx=commit" ]
@@ -257,7 +259,7 @@ spec =
         request methodDelete "/items?id=lt.0"
                 [singular] ""
           `shouldRespondWith`
-                [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                 { matchStatus  = 406
                 , matchHeaders = [matchContentTypeSingular]
                 }
@@ -266,7 +268,7 @@ spec =
         request methodDelete "/items?id=lt.0"
                 [("Prefer", "return=representation"), singular] ""
           `shouldRespondWith`
-                [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                 { matchStatus  = 406
                 , matchHeaders = [matchContentTypeSingular]
                 }
@@ -276,7 +278,7 @@ spec =
         request methodPost "/rpc/getproject"
                 [singular] [json|{ "id": 9999999}|]
           `shouldRespondWith`
-                [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                 { matchStatus  = 406
                 , matchHeaders = [matchContentTypeSingular]
                 }
@@ -299,7 +301,7 @@ spec =
         request methodPost "/rpc/getallprojects"
                 [singular] "{}"
           `shouldRespondWith`
-                [json|{"details":"Results contain 5 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 5 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                 { matchStatus  = 406
                 , matchHeaders = [matchContentTypeSingular]
                 }
@@ -314,7 +316,7 @@ spec =
             [("Prefer", "tx=commit"), singular]
             [json| {"id_l": 1, "id_h": 2, "name": "changed"} |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 2 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 2 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
             , matchHeaders = [ matchContentTypeSingular
                              , "Preference-Applied" <:> "tx=commit" ]

test/spec/Feature/Query/UpdateSpec.hs

@@ -9,6 +9,9 @@ import Network.HTTP.Types
 import Test.Hspec.Wai
 import Test.Hspec.Wai.JSON
 
+import PostgREST.Config.PgVersion (PgVersion, pgVersion100)
+
+
 import Protolude  hiding (get)
 import SpecHelper
 
@@ -18,8 +21,8 @@ tblDataBefore = [aesonQQ|[
                 , { "id": 3, "name": "item-3" }
                 ]|]
 
-spec :: SpecWith ((), Application)
-spec = do
+spec :: PgVersion -> SpecWith ((), Application)
+spec actualPgVersion = do
   describe "Patching record" $ do
     context "to unknown uri" $
       it "indicates no table found by returning 404" $
@@ -72,7 +75,7 @@ spec = do
           `shouldRespondWith` "[]"
           {
             matchStatus  = 200,
-            matchHeaders = []
+            matchHeaders = ["Preference-Applied" <:> "return=representation"]
           }
 
       it "returns status code 200 when no rows updated" $
@@ -85,7 +88,8 @@ spec = do
           [("Prefer", "return=representation")] [json| { "id":2 } |]
           `shouldRespondWith` [json|[{"id":2}]|]
           { matchStatus  = 200,
-            matchHeaders = ["Content-Range" <:> "0-0/*"]
+            matchHeaders = ["Content-Range" <:> "0-0/*"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "can update multiple items" $ do
@@ -134,7 +138,9 @@ spec = do
             [json| { id: 100 } |]
             `shouldRespondWith` [json| [{ id: 100 }] |]
             { matchStatus  = 200,
-              matchHeaders = [matchContentTypeJson, "Content-Range" <:> "0-0/*"]
+              matchHeaders = [matchContentTypeJson
+                             ,"Content-Range" <:> "0-0/*"
+                             , "Preference-Applied" <:> "return=representation"]
             }
 
         it "returns empty array when no rows updated and return=rep" $
@@ -144,7 +150,7 @@ spec = do
             [json| { id: 100 } |]
             `shouldRespondWith` "[]"
             { matchStatus  = 200,
-              matchHeaders = []
+              matchHeaders = ["Preference-Applied" <:> "return=representation"]
             }
 
       context "with representation requested" $ do
@@ -156,7 +162,9 @@ spec = do
             [("Prefer", "return=representation")]
             [json| { id: 99 } |]
             `shouldRespondWith` [json| [{id:99}] |]
-            { matchHeaders = [matchContentTypeJson] }
+            { matchHeaders = [matchContentTypeJson
+                             , "Preference-Applied" <:> "return=representation"]
+            }
           -- put value back for other tests
           void $ request methodPatch "/items?id=eq.99" [] [json| { "id":1 } |]
 
@@ -166,7 +174,9 @@ spec = do
             [("Prefer", "return=representation")]
             [json| { id: 1 } |]
             `shouldRespondWith` [json| [{ id: 1, always_true: true }] |]
-            { matchHeaders = [matchContentTypeJson] }
+            { matchHeaders = [matchContentTypeJson
+                             , "Preference-Applied" <:> "return=representation"]
+            }
 
         it "can select overloaded computed columns" $ do
           request methodPatch
@@ -174,22 +184,28 @@ spec = do
             [("Prefer", "return=representation")]
             [json| { id: 1 } |]
             `shouldRespondWith` [json| [{ id: 1, computed_overload: true }] |]
-            { matchHeaders = [matchContentTypeJson] }
+            { matchHeaders = [matchContentTypeJson
+                             , "Preference-Applied" <:> "return=representation"]
+            }
           request methodPatch
             "/items2?id=eq.1&select=id,computed_overload"
             [("Prefer", "return=representation")]
             [json| { id: 1 } |]
             `shouldRespondWith` [json| [{ id: 1, computed_overload: true }] |]
-            { matchHeaders = [matchContentTypeJson] }
+            { matchHeaders = [matchContentTypeJson
+                             , "Preference-Applied" <:> "return=representation"]
+            }
 
       it "ignores ?select= when return not set or return=minimal" $ do
         request methodPatch "/items?id=eq.1&select=id"
-            [] [json| { id:1 } |]
+           [("Prefer", "return=minimal")]
+           [json| { id:1 } |]
           `shouldRespondWith`
             ""
             { matchStatus  = 204
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "0-0/*" ]
+                             , "Content-Range" <:> "0-0/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
         request methodPatch "/items?id=eq.1&select=id"
             [("Prefer", "return=minimal")]
@@ -198,7 +214,8 @@ spec = do
             ""
             { matchStatus  = 204
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "0-0/*" ]
+                             , "Content-Range" <:> "0-0/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       context "when patching with an empty body" $ do
@@ -269,7 +286,8 @@ spec = do
             `shouldRespondWith` "[]"
             {
               matchStatus  = 200,
-              matchHeaders = ["Content-Range" <:> "*/*"]
+              matchHeaders = ["Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=representation"]
             }
 
         it "makes no updates and returns 200 with return=rep and with ?select=" $
@@ -277,7 +295,8 @@ spec = do
             `shouldRespondWith` "[]"
             {
               matchStatus  = 200,
-              matchHeaders = ["Content-Range" <:> "*/*"]
+              matchHeaders = ["Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=representation"]
             }
 
         it "makes no updates and returns 200 with return=rep and with ?select= for overloaded computed columns" $
@@ -285,7 +304,8 @@ spec = do
             `shouldRespondWith` "[]"
             {
               matchStatus  = 200,
-              matchHeaders = ["Content-Range" <:> "*/*"]
+              matchHeaders = ["Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=representation"]
             }
 
     context "with unicode values" $
@@ -340,7 +360,7 @@ spec = do
               {"id":3,"name":"Tres","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":1}
             ]|]
             { matchStatus  = 200
-            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
             }
 
         it "updates with limit/offset using table default values(field-with_sep) when json keys are undefined" $ do
@@ -352,7 +372,7 @@ spec = do
               {"id":3,"name":"Tres"}
             ]|]
             { matchStatus  = 200
-            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
             }
 
         it "updates table default values(field-with_sep) when json keys are undefined" $ do
@@ -364,7 +384,7 @@ spec = do
               {"id":3,"name":"Tres","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":1}
             ]|]
             { matchStatus  = 200
-            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
             }
 
         it "updates view default values(field-with_sep) when json keys are undefined" $
@@ -378,7 +398,27 @@ spec = do
               {"id":3,"name":"Default","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":null,"field-with_sep":3}
             ]|]
             { matchStatus  = 200
-            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+            }
+
+    -- https://github.com/PostgREST/postgrest/issues/2861
+    context "bit and char columns with length" $ do
+      it "should update a bit column with length" $
+        request methodPatch "/bitchar_with_length?select=bit,char&char=eq.aaaaa"
+            [("Prefer", "return=representation")]
+            [json|{"bit": "11100"}|]
+          `shouldRespondWith` [json|[{ "bit": "11100", "char": "aaaaa" }]|]
+            { matchStatus  = 200
+            , matchHeaders = ["Preference-Applied" <:> "return=representation"]
+            }
+
+      it "should update a char column with length" $
+        request methodPatch "/bitchar_with_length?select=bit,char&bit=eq.00000"
+            [("Prefer", "return=representation")]
+            [json|{"char": "zzzyy"}|]
+          `shouldRespondWith` [json|[{ "bit": "00000", "char": "zzzyy" }]|]
+            { matchStatus  = 200
+            , matchHeaders = ["Preference-Applied" <:> "return=representation"]
             }
 
   context "tables with self reference foreign keys" $ do
@@ -391,8 +431,8 @@ spec = do
           [json|
             [ { "id": 0, "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" }, { "name": "bar" } ]} ]
           |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $
@@ -403,8 +443,8 @@ spec = do
           [json|
             [ { "id": 0, "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" } ]} ]
           |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
     context "embeds parent, children and grandchildren after update" $ do
@@ -425,8 +465,8 @@ spec = do
               ]
             }
           ] |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $
@@ -445,8 +485,8 @@ spec = do
               ]
             }
           ] |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
     context "embeds children after update without explicitly including the id in the ?select" $ do
@@ -458,8 +498,8 @@ spec = do
           [json|
             [ { "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" }, { "name": "bar" } ]} ]
           |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $
@@ -470,8 +510,8 @@ spec = do
           [json|
             [ { "name": "tardis-patched", "web_content": [ { "name": "bar" } ]} ]
           |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
   context "tables with foreign keys referencing other tables" $ do
@@ -492,8 +532,8 @@ spec = do
               ]
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $
@@ -510,8 +550,8 @@ spec = do
               ]
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
 
@@ -527,8 +567,8 @@ spec = do
               "students_info":{"address":"Street 1"}
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
         request methodPatch "/students_info?id=eq.1&select=address,students(name)"
                 [("Prefer", "return=representation")]
@@ -540,8 +580,8 @@ spec = do
               "students":{"name": "John Doe"}
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $ do
@@ -555,8 +595,8 @@ spec = do
               "students_info": null
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
   context "table with limited privileges" $ do
@@ -567,7 +607,8 @@ spec = do
         `shouldRespondWith`
           ""
           { matchStatus = 204
-          , matchHeaders = [matchHeaderAbsent hContentType]
+          , matchHeaders = [matchHeaderAbsent hContentType
+                           , "Preference-Applied" <:> "return=minimal"]
           }
 
     it "can update without return=minimal and no explicit select" $
@@ -742,3 +783,198 @@ spec = do
        , { "id": 2, "name": "updated-item" }
        , { "id": 3, "name": "updated-item" }
        ]|]
+
+  -- Data representations for payload parsing requires Postgres 10 or above.
+  when (actualPgVersion >= pgVersion100) $ do
+    describe "Data representations" $ do
+      context "for a single row" $ do
+        it "parses values in payload" $
+          request methodPatch "/datarep_todos?id=eq.2" [("Prefer", "return=headers-only")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 204
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Content-Range" <:> "0-0/*"]
+              }
+
+        it "parses values in payload and formats individually selected values in return=representation" $
+          request methodPatch "/datarep_todos?id=eq.2&select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+            `shouldRespondWith`
+            [json| [{"id":2, "label_color": "#221100"}] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+
+        it "parses values in payload and formats values in return=representation" $
+          request methodPatch "/datarep_todos?id=eq.2" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:20Z", "icon_image": "3q2+7w"} |]
+            `shouldRespondWith`
+            [json|  [{"id":2,"name":"Essay","label_color":"#221100","due_at":"2019-01-03T11:00:20Z","icon_image":"3q2+7w==","created_at":1513213350,"budget":"100000000000000.13"}] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+
+        it "parses values in payload and formats star mixed selected values in return=representation" $
+          request methodPatch "/datarep_todos?id=eq.2&select=due_at,*" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z", "created_at": 0} |]
+            `shouldRespondWith`
+            -- end up with due_at twice here but that's unrelated to data reps
+            [json| [{"due_at":"2019-01-03T11:00:00Z","id":2,"name":"Essay","label_color":"#221100","due_at":"2019-01-03T11:00:00Z","icon_image":null,"created_at":0,"budget":"100000000000000.13"}] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+      context "for multiple rows" $ do
+        it "parses values in payload and formats individually selected values in return=representation" $
+          request methodPatch "/datarep_todos?id=lt.4&select=id,name,label_color" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+            `shouldRespondWith`
+            [json| [
+              {"id":1, "name": "Report", "label_color": "#221100"},
+              {"id":2, "name": "Essay", "label_color": "#221100"},
+              {"id":3, "name": "Algebra", "label_color": "#221100"}
+            ] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-2/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+
+        it "parses values in payload and formats values in return=representation" $
+          request methodPatch "/datarep_todos?id=lt.4" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z", "icon_image": "3q2+7w="} |]
+            `shouldRespondWith`
+            [json| [
+              {"id":1,"name":"Report","label_color":"#221100","due_at":"2019-01-03T11:00:00Z","icon_image":"3q2+7w==","created_at":1513213350,"budget":"12.50"},
+              {"id":2,"name":"Essay","label_color":"#221100","due_at":"2019-01-03T11:00:00Z","icon_image":"3q2+7w==","created_at":1513213350,"budget":"100000000000000.13"},
+              {"id":3,"name":"Algebra","label_color":"#221100","due_at":"2019-01-03T11:00:00Z","icon_image":"3q2+7w==","created_at":1513213350,"budget":"0.00"}
+            ] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-2/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+      context "with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPatch "/datarep_todos?id=eq.2&columns=due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| [
+               {"id":2,"name":"Essay","label_color":"#000100","due_at":"2019-01-03T11:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"}
+            ] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+
+        it "fails if at least one specified column doesn't exist" $
+          request methodPatch "/datarep_todos?id=eq.2&columns=label_color,helicopters" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }
+
+        it "ignores json keys and gives 200 if no record updated" $
+          request methodPatch "/datarep_todos?id=eq.2001&columns=label_color" [("Prefer", "return=representation")]
+           [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+           `shouldRespondWith` 200
+      context "on a view" $ do
+        context "for a single row" $ do
+          it "parses values in payload" $
+            request methodPatch "/datarep_todos_computed?id=eq.2" [("Prefer", "return=headers-only")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+              `shouldRespondWith`
+              ""
+                { matchStatus  = 204
+                , matchHeaders = [ matchHeaderAbsent hContentType
+                                 , "Content-Range" <:> "0-0/*" ]
+                }
+
+          it "parses values in payload and formats individually selected values in return=representation" $
+            request methodPatch "/datarep_todos_computed?id=eq.2&select=id,label_color" [("Prefer", "return=representation")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+              `shouldRespondWith`
+              [json| [{"id":2, "label_color": "#221100"}] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+
+          it "parses values in payload and formats values in return=representation" $
+            request methodPatch "/datarep_todos_computed?id=eq.2" [("Prefer", "return=representation")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:20Z"} |]
+              `shouldRespondWith`
+              [json| [{"id":2, "name": "Essay", "label_color": "#221100", "dark_color":"#110880", "due_at":"2019-01-03T11:00:20Z"}] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+        context "for multiple rows" $ do
+          it "parses values in payload and formats individually selected values in return=representation" $
+            request methodPatch "/datarep_todos_computed?id=lt.4&select=id,name,label_color,dark_color" [("Prefer", "return=representation")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+              `shouldRespondWith`
+              [json| [
+                {"id":1, "name": "Report", "label_color": "#221100", "dark_color":"#110880"},
+                {"id":2, "name": "Essay", "label_color": "#221100", "dark_color":"#110880"},
+                {"id":3, "name": "Algebra", "label_color": "#221100", "dark_color":"#110880"}
+              ] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-2/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+
+          it "parses values in payload and formats values in return=representation" $
+            request methodPatch "/datarep_todos_computed?id=lt.4" [("Prefer", "return=representation")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+              `shouldRespondWith`
+              [json| [
+                {"id":1, "name": "Report", "label_color": "#221100", "dark_color":"#110880", "due_at":"2019-01-03T11:00:00Z"},
+                {"id":2, "name": "Essay", "label_color": "#221100", "dark_color":"#110880", "due_at":"2019-01-03T11:00:00Z"},
+                {"id":3, "name": "Algebra", "label_color": "#221100", "dark_color":"#110880", "due_at":"2019-01-03T11:00:00Z"}
+              ] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-2/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+        context "with ?columns parameter" $ do
+          it "ignores json keys not included in ?columns; parses only the ones specified" $
+            request methodPatch "/datarep_todos_computed?id=eq.2&columns=due_at" [("Prefer", "return=representation")]
+              [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+              `shouldRespondWith`
+              [json| [
+                {"id":2, "name": "Essay", "label_color": "#000100", "dark_color": "#000080", "due_at":"2019-01-03T11:00:00Z"}
+              ] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+
+          it "fails if at least one specified column doesn't exist" $
+            request methodPatch "/datarep_todos_computed?id=eq.2&columns=label_color,helicopters" [("Prefer", "return=representation")]
+              [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+              `shouldRespondWith`
+              [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos_computed' does not exist","details":null,"hint":null} |]
+                { matchStatus  = 400
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+                }
+
+          it "ignores json keys and gives 200 if no record updated" $
+            request methodPatch "/datarep_todos_computed?id=eq.2001&columns=label_color" [("Prefer", "return=representation")]
+             [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+             `shouldRespondWith` 200

test/spec/Feature/Query/UpsertSpec.hs

@@ -29,7 +29,7 @@ spec actualPgVersion =
               { "name": "C", "rank": 1 }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
             }
 
         it "INSERTs and UPDATEs row on composite pk conflict" $
@@ -42,7 +42,7 @@ spec actualPgVersion =
               { "first_name": "Peter S.", "last_name": "Yang", "salary": "$42,000.00", "company": null, "occupation": null }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
             }
 
         when (actualPgVersion >= pgVersion110) $
@@ -56,13 +56,14 @@ spec actualPgVersion =
                 { "name": "Roma", "year": 2021, "car_brand_name": "Ferrari" }
               ]|]
               { matchStatus = 201
-              , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+              , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
               }
 
         it "succeeds when the payload has no elements" $
           request methodPost "/articles" [("Prefer", "return=representation"), ("Prefer", "resolution=merge-duplicates")]
             [json|[]|] `shouldRespondWith`
-            [json|[]|] { matchStatus = 201 , matchHeaders = [matchContentTypeJson] }
+            [json|[]|] { matchStatus = 201
+                       , matchHeaders = [matchContentTypeJson] }
 
         it "INSERTs and UPDATEs rows on single unique key conflict" $
           request methodPost "/single_unique?on_conflict=unique_key" [("Prefer", "return=representation"), ("Prefer", "resolution=merge-duplicates")]
@@ -74,7 +75,7 @@ spec actualPgVersion =
               { "unique_key": 2, "value": "C" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
             }
 
         it "INSERTs and UPDATEs rows on compound unique keys conflict" $
@@ -87,7 +88,7 @@ spec actualPgVersion =
               { "key1": 1, "key2": 2, "value": "C" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
             }
 
       context "when Prefer: resolution=ignore-duplicates is specified" $ do
@@ -100,7 +101,7 @@ spec actualPgVersion =
               { "name": "PHP", "rank": 9 }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation", matchContentTypeJson]
             }
 
         it "INSERTs and ignores rows on composite pk conflict" $
@@ -112,7 +113,7 @@ spec actualPgVersion =
               { "first_name": "Sara M.", "last_name": "Torpey", "salary": "$60,000.00", "company": "Burstein-Applebee", "occupation": "Soil scientist" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation", matchContentTypeJson]
             }
 
         when (actualPgVersion >= pgVersion110) $
@@ -125,7 +126,7 @@ spec actualPgVersion =
                 { "name": "Huracán", "year": 2021, "car_brand_name": "Lamborghini" }
               ]|]
               { matchStatus = 201
-              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates", matchContentTypeJson]
+              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation", matchContentTypeJson]
               }
 
         it "INSERTs and ignores rows on single unique key conflict" $
@@ -142,7 +143,7 @@ spec actualPgVersion =
                 { "unique_key": 3, "value": "D" }
               ]|]
               { matchStatus = 201
-              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates"]
+              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation"]
               }
 
         it "INSERTs and UPDATEs rows on compound unique keys conflict" $
@@ -159,7 +160,7 @@ spec actualPgVersion =
                 { "key1": 1, "key2": 3, "value": "D" }
               ]|]
               { matchStatus = 201
-              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates"]
+              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation"]
               }
 
       it "succeeds if the table has only PK cols and no other cols" $ do
@@ -168,7 +169,7 @@ spec actualPgVersion =
           `shouldRespondWith`
           [json|[ { "id": 3} ]|]
           { matchStatus = 201 ,
-            matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates",
+            matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation",
             matchContentTypeJson] }
 
         request methodPost "/only_pk" [("Prefer", "return=representation"), ("Prefer", "resolution=merge-duplicates")]
@@ -176,22 +177,30 @@ spec actualPgVersion =
           `shouldRespondWith`
           [json|[ { "id": 1 }, { "id": 2 }, { "id": 4} ]|]
           { matchStatus = 201 ,
-            matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates",
+            matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation",
             matchContentTypeJson] }
 
       it "succeeds and ignores the Prefer: resolution header(no Preference-Applied present) if the table has no PK" $
         request methodPost "/no_pk" [("Prefer", "return=representation"), ("Prefer", "resolution=merge-duplicates")]
           [json|[ { "a": "1", "b": "0" } ]|]
           `shouldRespondWith`
-          [json|[ { "a": "1", "b": "0" } ]|] { matchStatus = 201 , matchHeaders = [matchContentTypeJson] }
+          [json|[ { "a": "1", "b": "0" } ]|]
+          { matchStatus = 201
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "return=representation"] }
 
       it "succeeds if not a single resource is created" $ do
         request methodPost "/tiobe_pls" [("Prefer", "return=representation"), ("Prefer", "resolution=ignore-duplicates")]
           [json|[ { "name": "Java", "rank": 1 } ]|] `shouldRespondWith`
-          [json|[]|] { matchStatus = 201 , matchHeaders = [matchContentTypeJson] }
+          [json|[]|]
+          { matchStatus = 201
+          , matchHeaders = [matchContentTypeJson] }
+
         request methodPost "/tiobe_pls" [("Prefer", "return=representation"), ("Prefer", "resolution=ignore-duplicates")]
           [json|[ { "name": "Java", "rank": 1 }, { "name": "C", "rank": 2 } ]|] `shouldRespondWith`
-          [json|[]|] { matchStatus = 201 , matchHeaders = [matchContentTypeJson] }
+          [json|[]|]
+          { matchStatus = 201
+          , matchHeaders = [matchContentTypeJson] }
 
     context "with PUT" $ do
       context "Restrictions" $ do
@@ -408,7 +417,7 @@ spec actualPgVersion =
               { "idUnitTest": 2, "nameUnitTest": "name of unittest 2" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates"]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation"]
             }
 
       it "works with POST and ignore-duplicates headers" $ do
@@ -423,7 +432,7 @@ spec actualPgVersion =
               { "idUnitTest": 2, "nameUnitTest": "name of unittest 2" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates"]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation"]
             }
 
       it "works with PUT" $ do
@@ -436,3 +445,16 @@ spec actualPgVersion =
             }
         get "/UnitTest?idUnitTest=eq.1" `shouldRespondWith`
           [json| [ { "idUnitTest": 1, "nameUnitTest": "unit test 1" } ]|]
+
+      it "works with request method PUT and return=minimal" $ do
+        request methodPut "/UnitTest?idUnitTest=eq.1"
+            [("Prefer", "return=minimal")]
+            [json| [ { "idUnitTest": 1, "nameUnitTest": "unit test 1" } ]|]
+          `shouldRespondWith`
+            ""
+            { matchStatus = 204
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal"]
+            }
+        get "/UnitTest?idUnitTest=eq.1" `shouldRespondWith`
+          [json| [ { "idUnitTest": 1, "nameUnitTest": "unit test 1" } ]|]

test/spec/Main.hs

@@ -3,8 +3,7 @@ module Main where
 import qualified Hasql.Pool                 as P
 import qualified Hasql.Transaction.Sessions as HT
 
-import Data.Function      (id)
-import Data.List.NonEmpty (toList)
+import Data.Function (id)
 
 import Test.Hspec
 
@@ -27,6 +26,7 @@ import qualified Feature.ConcurrentSpec
 import qualified Feature.CorsSpec
 import qualified Feature.ExtraSearchPathSpec
 import qualified Feature.LegacyGucsSpec
+import qualified Feature.NoSuperuserSpec
 import qualified Feature.ObservabilitySpec
 import qualified Feature.OpenApi.DisabledOpenApiSpec
 import qualified Feature.OpenApi.IgnorePrivOpenApiSpec
@@ -45,6 +45,7 @@ import qualified Feature.Query.HtmlRawOutputSpec
 import qualified Feature.Query.InsertSpec
 import qualified Feature.Query.JsonOperatorSpec
 import qualified Feature.Query.MultipleSchemaSpec
+import qualified Feature.Query.NullsStrip
 import qualified Feature.Query.PgSafeUpdateSpec
 import qualified Feature.Query.PlanSpec
 import qualified Feature.Query.PostGISSpec
@@ -65,14 +66,12 @@ import qualified Feature.RpcPreRequestGucsSpec
 
 main :: IO ()
 main = do
-  pool <- P.acquire 3 10 60 $ toUtf8 $ configDbUri testCfg
+  pool <- P.acquire 3 10 60 60 $ toUtf8 $ configDbUri testCfg
 
   actualPgVersion <- either (panic . show) id <$> P.use pool (queryPgVersion False)
 
-  baseSchemaCache <-
-    loadSchemaCache pool
-      (configDbSchemas testCfg)
-      (configDbExtraSearchPath testCfg)
+  -- cached schema cache so most tests run fast
+  baseSchemaCache <- loadSchemaCache pool testCfg
 
   let
     -- For tests that run with the same refSchemaCache
@@ -84,10 +83,7 @@ main = do
 
     -- For tests that run with a different SchemaCache(depends on configSchemas)
     appDbs config = do
-      customSchemaCache <-
-        loadSchemaCache pool
-          (configDbSchemas config)
-          (configDbExtraSearchPath config)
+      customSchemaCache <- loadSchemaCache pool config
       appState <- AppState.initWithPool pool config
       AppState.putPgVersion appState actualPgVersion
       AppState.putSchemaCache appState (Just customSchemaCache)
@@ -144,11 +140,13 @@ main = do
         , ("Feature.Query.RawOutputTypesSpec"            , Feature.Query.RawOutputTypesSpec.spec)
         , ("Feature.Query.RpcSpec"                       , Feature.Query.RpcSpec.spec actualPgVersion)
         , ("Feature.Query.SingularSpec"                  , Feature.Query.SingularSpec.spec)
-        , ("Feature.Query.UpdateSpec"                    , Feature.Query.UpdateSpec.spec)
+        , ("Feature.Query.NullsStrip"                    , Feature.Query.NullsStrip.spec)
+        , ("Feature.Query.UpdateSpec"                    , Feature.Query.UpdateSpec.spec actualPgVersion)
         , ("Feature.Query.UpsertSpec"                    , Feature.Query.UpsertSpec.spec actualPgVersion)
         , ("Feature.Query.ComputedRelsSpec"              , Feature.Query.ComputedRelsSpec.spec)
         , ("Feature.Query.RelatedQueriesSpec"            , Feature.Query.RelatedQueriesSpec.spec)
         , ("Feature.Query.SpreadQueriesSpec"             , Feature.Query.SpreadQueriesSpec.spec)
+        , ("Feature.NoSuperuserSpec"                     , Feature.NoSuperuserSpec.spec)
         ]
 
   hspec $ do
@@ -263,5 +261,5 @@ main = do
       describe "Feature.RollbackForcedSpec" Feature.RollbackSpec.forced
 
   where
-    loadSchemaCache pool schemas extraSearchPath =
-      either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache (toList schemas) extraSearchPath True)
+    loadSchemaCache pool conf =
+      either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache conf)

test/spec/SpecHelper.hs

@@ -40,6 +40,12 @@ matchContentTypeJson = "Content-Type" <:> "application/json; charset=utf-8"
 matchContentTypeSingular :: MatchHeader
 matchContentTypeSingular = "Content-Type" <:> "application/vnd.pgrst.object+json; charset=utf-8"
 
+matchCTArrayStrip :: MatchHeader
+matchCTArrayStrip = "Content-Type" <:> "application/vnd.pgrst.array+json;nulls=stripped; charset=utf-8"
+
+matchCTSingularStrip :: MatchHeader
+matchCTSingularStrip = "Content-Type" <:> "application/vnd.pgrst.object+json;nulls=stripped; charset=utf-8"
+
 matchHeaderAbsent :: HeaderName -> MatchHeader
 matchHeaderAbsent name = MatchHeader $ \headers _body ->
   case lookup name headers of
@@ -84,11 +90,13 @@ baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
   , configDbPoolSize            = 10
   , configDbPoolAcquisitionTimeout = 10
   , configDbPoolMaxLifetime     = 1800
+  , configDbPoolMaxIdletime     = 600
   , configDbPreRequest          = Just $ QualifiedIdentifier "test" "switch_role"
   , configDbPreparedStatements  = True
   , configDbRootSpec            = Nothing
   , configDbSchemas             = fromList ["test"]
   , configDbConfig              = False
+  , configDbPreConfig           = Nothing
   , configDbUri                 = "postgresql://"
   , configDbUseLegacyGucs       = True
   , configFilePath              = Nothing
@@ -111,6 +119,8 @@ baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
   , configDbTxRollbackAll       = True
   , configAdminServerPort       = Nothing
   , configRoleSettings          = mempty
+  , configRoleIsoLvl            = mempty
+  , configInternalSCSleep       = Nothing
   }
 
 testCfg :: AppConfig
@@ -211,7 +221,7 @@ testObservabilityCfg = baseCfg { configServerTraceHeader = Just $ mk "X-Request-
 
 analyzeTable :: Text -> IO ()
 analyzeTable tableName =
-  void $ readProcess "psql" ["--set", "ON_ERROR_STOP=1", "-a", "-c", toS $ "ANALYZE test.\"" <> tableName <> "\""] []
+  void $ readProcess "psql" ["-U", "postgres", "--set", "ON_ERROR_STOP=1", "-a", "-c", toS $ "ANALYZE test.\"" <> tableName <> "\""] []
 
 rangeHdrs :: ByteRange -> [Header]
 rangeHdrs r = [rangeUnit, (hRange, renderByteRange r)]

test/spec/fixtures/data.sql

@@ -839,3 +839,17 @@ INSERT INTO posters(id,name) VALUES (1,'Mark'), (2,'Elon'), (3,'Bill'), (4,'Jeff
 
 TRUNCATE TABLE subscriptions CASCADE;
 INSERT INTO subscriptions(subscriber,subscribed) VALUES (3,1), (4,1), (1,2);
+
+TRUNCATE TABLE datarep_todos CASCADE;
+INSERT INTO datarep_todos VALUES (1, 'Report', 0, '2018-01-02', '\x89504e470d0a1a0a0000000d4948445200000001000000010100000000376ef924000000001049444154789c62600100000000ffff03000000060005057bfabd400000000049454e44ae426082', '2017-12-14 01:02:30', 12.50); -- smallest possible PNG
+INSERT INTO datarep_todos VALUES (2, 'Essay', 256, '2018-01-03', NULL, '2017-12-14 01:02:30', 100000000000000.13); -- a number which can't be represented by a 64-bit float
+INSERT INTO datarep_todos VALUES (3, 'Algebra', 123456, '2018-01-01 14:12:34.123456');
+INSERT INTO datarep_todos VALUES (4, 'Opus Magnum', NULL, NULL);
+
+TRUNCATE TABLE datarep_next_two_todos CASCADE;
+INSERT INTO datarep_next_two_todos VALUES (1, 2, 3, 'school related');
+INSERT INTO datarep_next_two_todos VALUES (2, 1, 3, 'do these first');
+
+TRUNCATE TABLE bitchar_with_length CASCADE;
+INSERT INTO bitchar_with_length(bit, char) VALUES ('00000', 'aaaaa');
+INSERT INTO bitchar_with_length(bit, char) VALUES ('11111', 'bbbbb');

test/spec/fixtures/roles.sql

@@ -1,7 +1,6 @@
-\set AUTHENTICATOR current_user
 DROP ROLE IF EXISTS postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author;
 CREATE ROLE postgrest_test_anonymous;
 CREATE ROLE postgrest_test_default_role;
 CREATE ROLE postgrest_test_author;
 
-GRANT postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author TO :USER;
+GRANT postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author TO :PGUSER;

test/spec/fixtures/schema.sql

@@ -2774,9 +2774,20 @@ BEGIN
   LOAD 'safeupdate';
 END; $$ LANGUAGE plpgsql SECURITY DEFINER;
 
+-- This tests data representations over computed joins: even a lower case title should come back title cased.
+DROP DOMAIN IF EXISTS public.titlecasetext CASCADE;
+CREATE DOMAIN public.titlecasetext AS text;
+
+CREATE OR REPLACE FUNCTION json(public.titlecasetext) RETURNS json AS $$
+  SELECT to_json(INITCAP($1::text));
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.titlecasetext AS json) WITH FUNCTION json(public.titlecasetext) AS IMPLICIT;
+-- End of data representations specific stuff except for where the domain is used in the table.
+
 CREATE TABLE designers (
   id int primary key
-, name text
+, name public.titlecasetext
 );
 
 CREATE TABLE videogames (
@@ -3091,6 +3102,136 @@ create table test.subscriptions(
   primary key(subscriber, subscribed)
 );
 
+-- Data representations feature
+DROP DOMAIN IF EXISTS public.color CASCADE;
+CREATE DOMAIN public.color AS INTEGER CHECK (VALUE >= 0 AND VALUE <= 16777215);
+
+CREATE OR REPLACE FUNCTION color(json) RETURNS public.color AS $$
+  SELECT color($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION color(text) RETURNS public.color AS $$
+  SELECT (('x' || lpad((CASE WHEN SUBSTRING($1::text, 1, 1) = '#' THEN SUBSTRING($1::text, 2) ELSE $1::text END), 8, '0'))::bit(32)::int)::public.color;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.color) RETURNS json AS $$
+  SELECT
+    CASE WHEN $1 IS NULL THEN to_json(''::text)
+    ELSE to_json('#' || lpad(upper(to_hex($1)), 6, '0'))
+  END;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.color AS json) WITH FUNCTION json(public.color) AS IMPLICIT;
+CREATE CAST (json AS public.color) WITH FUNCTION color(json) AS IMPLICIT;
+CREATE CAST (text AS public.color) WITH FUNCTION color(text) AS IMPLICIT;
+
+DROP DOMAIN IF EXISTS public.isodate CASCADE;
+CREATE DOMAIN public.isodate AS timestamp with time zone;
+
+CREATE OR REPLACE FUNCTION isodate(json) RETURNS public.isodate AS $$
+  SELECT isodate($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION isodate(text) RETURNS public.isodate AS $$
+  SELECT (replace($1, 'Z', '+00:00')::timestamp with time zone)::public.isodate;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.isodate) RETURNS json AS $$
+  SELECT to_json(replace(to_json($1)#>>'{}', '+00:00', 'Z'));
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.isodate AS json) WITH FUNCTION json(public.isodate) AS IMPLICIT;
+CREATE CAST (json AS public.isodate) WITH FUNCTION isodate(json) AS IMPLICIT;
+-- We intentionally don't have this in order to test query string parsing doesn't try to fall back on JSON parsing.
+-- CREATE CAST (text AS public.isodate) WITH FUNCTION isodate(text) AS IMPLICIT;
+
+-- bytea_b64 is a base64-encoded binary string
+DROP DOMAIN IF EXISTS public.bytea_b64 CASCADE;
+CREATE DOMAIN public.bytea_b64 AS bytea;
+
+CREATE OR REPLACE FUNCTION bytea_b64(json) RETURNS public.bytea_b64 AS $$
+  SELECT bytea_b64($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION bytea_b64(text) RETURNS public.bytea_b64 AS $$
+  -- allow unpadded base64
+  SELECT decode($1 || repeat('=', 4 - (length($1) % 4)), 'base64')::public.bytea_b64;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.bytea_b64) RETURNS json AS $$
+  SELECT to_json(translate(encode($1, 'base64'), E'\n', ''));
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.bytea_b64 AS json) WITH FUNCTION json(public.bytea_b64) AS IMPLICIT;
+CREATE CAST (json AS public.bytea_b64) WITH FUNCTION bytea_b64(json) AS IMPLICIT;
+CREATE CAST (text AS public.bytea_b64) WITH FUNCTION bytea_b64(text) AS IMPLICIT;
+
+-- unixtz is a timestamptz represented as an integer number of seconds since the Unix epoch
+DROP DOMAIN IF EXISTS public.unixtz CASCADE;
+CREATE DOMAIN public.unixtz AS timestamp with time zone;
+
+CREATE OR REPLACE FUNCTION unixtz(json) RETURNS public.unixtz AS $$
+  SELECT unixtz($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION unixtz(text) RETURNS public.unixtz AS $$
+  SELECT (to_timestamp($1::numeric)::public.unixtz);
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.unixtz) RETURNS json AS $$
+  SELECT to_json(extract(epoch from $1)::bigint);
+$$ LANGUAGE SQL IMMUTABLE;
+
+
+CREATE CAST (public.unixtz AS json) WITH FUNCTION json(public.unixtz) AS IMPLICIT;
+CREATE CAST (json AS public.unixtz) WITH FUNCTION unixtz(json) AS IMPLICIT;
+CREATE CAST (text AS public.unixtz) WITH FUNCTION unixtz(text) AS IMPLICIT;
+
+DROP DOMAIN IF EXISTS public.monetary CASCADE;
+CREATE DOMAIN public.monetary AS numeric(17,2);
+
+CREATE OR REPLACE FUNCTION monetary(json) RETURNS public.monetary AS $$
+  SELECT monetary($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION monetary(text) RETURNS public.monetary AS $$
+  SELECT ($1::numeric)::public.monetary;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.monetary) RETURNS json AS $$
+  SELECT to_json($1::text);
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.monetary AS json) WITH FUNCTION json(public.monetary) AS IMPLICIT;
+CREATE CAST (json AS public.monetary) WITH FUNCTION monetary(json) AS IMPLICIT;
+CREATE CAST (text AS public.monetary) WITH FUNCTION monetary(text) AS IMPLICIT;
+
+CREATE TABLE datarep_todos (
+  id bigint primary key,
+  name text,
+  label_color public.color default 0,
+  due_at public.isodate default '2018-01-01'::date,
+  icon_image public.bytea_b64,
+  created_at public.unixtz default '2017-12-14 01:02:30'::timestamptz,
+  budget public.monetary default 0
+);
+
+CREATE TABLE datarep_next_two_todos (
+  id bigint primary key,
+  first_item_id bigint references datarep_todos(id),
+  second_item_id bigint references datarep_todos(id),
+  name text
+);
+
+CREATE VIEW datarep_todos_computed as (
+  SELECT id,
+    name,
+    label_color,
+    due_at,
+    (label_color / 2)::public.color as dark_color
+  FROM datarep_todos
+);
+
 -- view's name is alphabetically before projects
 create view test.alpha_projects as
   select c.id, p.name as pro_name, c.name as cli_name
@@ -3122,3 +3263,108 @@ BEGIN
   END IF;
 END
 $do$;
+
+CREATE FUNCTION test.is_superuser() RETURNS boolean
+LANGUAGE sql
+AS $$
+  select current_setting('is_superuser')::boolean;
+$$;
+
+DO $do$
+BEGIN
+  IF current_setting('server_version_num')::INT >= 120000 THEN
+    CREATE TABLE test.foo (
+      a text,
+      b text GENERATED ALWAYS AS (
+          case WHEN a = 'telegram' THEN 'im'
+               WHEN a = 'proton' THEN 'email'
+               WHEN a = 'infinity' THEN 'idea'
+               ELSE 'bad idea'
+          end) stored
+    );
+  END IF;
+END
+$do$;
+
+create domain devil_int as int
+  default 666;
+
+create table evil_friends(
+  id   devil_int
+, name text
+);
+
+create table bets (
+  id int
+, data_json  json
+, data_jsonb jsonb
+);
+
+create index bets_data_json  on bets ((data_json  ->>'contractId'));
+create index bets_data_jsonb on bets ((data_jsonb ->>'contractId'));
+
+--- https://github.com/PostgREST/postgrest/issues/2862
+create table profiles (
+  id uuid primary key,
+  username text null
+);
+
+create table user_friend (
+  id bigint primary key,
+  user1 uuid references profiles (id),
+  user2 uuid references profiles (id)
+);
+
+create table status(
+  id bigint primary key
+);
+
+create table tournaments(
+  id bigint primary key,
+  status bigint references status(id)
+);
+
+-- https://github.com/PostgREST/postgrest/issues/2861
+CREATE TABLE bitchar_with_length (
+  bit bit(5),
+  char char(5),
+  bit_arr bit(5)[],
+  char_arr char(5)[]
+);
+
+-- https://github.com/PostgREST/postgrest/issues/1586
+create or replace function char_param_select(char_ char(4), char_arr char(4)[])
+returns table(char_ char, char_arr char[]) as $$
+  select $1, $2;
+$$ language sql;
+
+create or replace function bit_param_select(bit_ char(4), bit_arr char(4)[])
+returns table(bit_ char, bit_arr char[]) as $$
+  select $1, $2;
+$$ language sql;
+
+create or replace function char_param_insert(char_ char(4), char_arr char(4)[])
+returns void as $$
+  insert into bitchar_with_length(char, char_arr) values($1, $2);
+$$ language sql;
+
+create or replace function bit_param_insert(bit_ bit(4), bit_arr bit(4)[])
+returns void as $$
+insert into bitchar_with_length(bit, bit_arr) values($1, $2);
+$$ language sql;
+
+create function returns_record() returns record as $$
+select * from projects limit 1;
+$$ language sql;
+
+create function returns_record_params(id int, name text) returns record as $$
+select * from projects p where p.id = $1 and p.name like $2;
+$$ language sql;
+
+create function returns_setof_record() returns setof record as $$
+select * from projects limit 2;
+$$ language sql;
+
+create function returns_setof_record_params(id int, name text) returns setof record as $$
+select * from projects p where p.id >= $1 and p.name like $2;
+$$ language sql;