Bump to v11.2.1

When doing Null Filtering, the to-one embed resources were not included if they had a NULL value in any of the selected fields.

.github/actions/setup-nix/action.yaml

@@ -11,7 +11,7 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: cachix/install-nix-action@v18
+    - uses: cachix/install-nix-action@v23
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12

.github/dependabot.yml

@@ -1,6 +1,11 @@
 version: 2
 updates:
-- package-ecosystem: github-actions
-  directory: /
-  schedule:
-    interval: weekly
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+
+  - package-ecosystem: github-actions
+    directory: /.github/actions/setup-nix
+    schedule:
+      interval: weekly

.github/scripts/arm/build.sh

@@ -4,13 +4,15 @@
 
 [ -z "$1" ] && { echo "Missing 1st argument: PostgREST github commit SHA"; exit 1; }
 [ -z "$2" ] && { echo "Missing 2nd argument: Build environment directory name"; exit 1; }
+[ -z "$3" ] && { echo "Missing 3rd argument: GHC version"; exit 1; }
 
 PGRST_GITHUB_COMMIT="$1"
 SCRIPT_DIR="$2"
 
 DOCKER_BUILD_DIR="$SCRIPT_DIR/docker-env"
-CABAL_VERSION="3.6.0.0"
-GHC_VERSION="9.2.4"
+# latest is a shortcut documented on https://www.haskell.org/ghcup/guide/#tags-and-shortcuts
+CABAL_VERSION="latest"
+GHC_VERSION="$3"
 
 install_packages() {
   sudo apt-get update -y

.github/scripts/arm/docker-env/Dockerfile

@@ -13,4 +13,6 @@ EXPOSE 3000
 
 USER 1000
 
-CMD postgrest
+# Use the array form to avoid running the command using bash, which does not handle `SIGTERM` properly. 
+# See https://docs.docker.com/compose/faq/#why-do-my-services-take-10-seconds-to-recreate-or-stop 
+CMD ["postgrest"]

.github/workflows/cachix.yaml

@@ -28,7 +28,7 @@ jobs:
     name: Seed ${{ matrix.os }}
     runs-on: ${{ matrix.runs-on }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:

.github/workflows/ci.yaml

@@ -17,7 +17,7 @@ jobs:
     name: Lint & check code style
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -37,7 +37,7 @@ jobs:
         # https://github.com/actions/runner/issues/241#issuecomment-842566950
         shell: script -qec "bash --noprofile --norc -eo pipefail {0}"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -63,7 +63,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pgVersion: [9.6, 10, 11, 12, 13, 14, 15]
+        pgVersion: [9.6, 10, 11, 12, 13, 14, 15, 16]
     name: Test PG ${{ matrix.pgVersion }} (Nix)
     runs-on: ubuntu-latest
     defaults:
@@ -72,7 +72,7 @@ jobs:
         # https://github.com/actions/runner/issues/241#issuecomment-842566950
         shell: script -qec "bash --noprofile --norc -eo pipefail {0}"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -91,7 +91,7 @@ jobs:
     name: Test memory (Nix)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -104,7 +104,7 @@ jobs:
     name: Build Linux static (Nix)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -135,7 +135,7 @@ jobs:
     name: Build MacOS (Nix)
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
 
@@ -175,7 +175,7 @@ jobs:
     name: Build ${{ matrix.name }} (Stack)
     runs-on: ${{ matrix.runs-on }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Stack working files cache
         uses: actions/cache@v3
         with:
@@ -199,7 +199,7 @@ jobs:
     name: Get FreeBSD build from CirrusCI
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Get FreeBSD executable from CirrusCI
         env:
           # GITHUB_SHA does weird things for pull request, so we roll our own:
@@ -216,12 +216,12 @@ jobs:
   Build-Cabal:
     strategy:
       matrix:
-        ghc: ['9.2.4']
+        ghc: ['9.0.2', '9.2.4']
       fail-fast: false
     name: Build Linux (Cabal, GHC ${{ matrix.ghc }})
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Workaround runner image issue
         # https://github.com/actions/runner-images/issues/7061
         run: sudo chown -R "$USER" /usr/local/.ghcup
@@ -247,15 +247,20 @@ jobs:
         run: cabal build --enable-tests --enable-benchmarks all
 
   Build-Cabal-Arm:
-    name: Build aarch64 (Cabal)
+    strategy:
+      matrix:
+        ghc: ['9.2.4']
+      fail-fast: false
+    name: Build aarch64 (Cabal, GHC ${{ matrix.ghc }})
     if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/rel-') }}
     runs-on: ubuntu-latest
     outputs:
       remotepath: ${{ steps.Remote-Dir.outputs.remotepath }}
     env:
       GITHUB_COMMIT: ${{ github.sha }}
+      GHC_VERSION: ${{ matrix.ghc }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - id: Remote-Dir
         name: Unique directory name for the remote build
         run: echo "remotepath=postgrest-build-$(uuidgen)" >> "$GITHUB_OUTPUT"
@@ -280,8 +285,8 @@ jobs:
           fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }}
           command_timeout: 120m
           script_stop: true
-          envs: GITHUB_COMMIT,REMOTE_DIR
-          script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR"
+          envs: GITHUB_COMMIT,REMOTE_DIR,GHC_VERSION
+          script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR" "GHC_VERSION"
       - name: Download binaries from remote server
         uses: nicklasfrahm/scp-action@main
         with:
@@ -319,7 +324,7 @@ jobs:
       version: ${{ steps.Identify-Version.outputs.version }}
       isprerelease: ${{ steps.Identify-Version.outputs.isprerelease }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - id: Identify-Version
         name: Identify the version to be released
         run: |
@@ -372,7 +377,7 @@ jobs:
     env:
       VERSION: ${{ needs.Prepare-Release.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download all artifacts
         uses: actions/download-artifact@v3
         with:
@@ -438,7 +443,7 @@ jobs:
       VERSION: ${{ needs.Prepare-Release.outputs.version }}
       ISPRERELEASE: ${{ needs.Prepare-Release.outputs.isprerelease }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -491,7 +496,7 @@ jobs:
       VERSION: ${{ needs.Prepare-Release.outputs.version }}
       ISPRERELEASE: ${{ needs.Prepare-Release.outputs.isprerelease }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Publish images for ARM builds on Docker Hub
         uses: appleboy/ssh-action@master
         env:
@@ -518,7 +523,7 @@ jobs:
     env:
       REMOTE_DIR: ${{ needs.Build-Cabal-Arm.outputs.remotepath }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Remove uploaded files from server
         uses: appleboy/ssh-action@master
         with:

.github/workflows/loadtest.yaml

@@ -15,7 +15,7 @@ jobs:
     name: Loadtest (Nix)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Setup Nix Environment

.github/workflows/report.yaml

@@ -22,7 +22,7 @@ jobs:
           name: loadtest.md
           path: artifacts
       - name: Upload to GitHub Checks
-        uses: LouisBrunner/checks-action@v1.6.0
+        uses: LouisBrunner/checks-action@v1.6.2
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sha: ${{ github.event.workflow_run.head_sha }}

CHANGELOG.md

@@ -3,7 +3,66 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
-## Unreleased
+## [11.2.1] - 2023-10-03
+
+### Fixed
+
+ - #2899, Fix `application/vnd.pgrst.array` not accepted as a valid mediatype - @taimoorzaeem
+ - #2524, Fix schema cache and configuration reloading with `NOTIFY` not working on Windows - @diogob, @laurenceisla
+ - #2915, Fix duplicate headers in response - @taimoorzaeem
+ - #2824, Fix range request with first position same as length return status 206 - @taimoorzaeem
+ - #2939, Fix wrong `Preference-Applied` with `Prefer: tx=commit` when transaction is rollbacked - @steve-chavez
+ - #2939, Fix `count=exact` not being included in `Preference-Applied` - @steve-chavez
+ - #2800, Fix not including to-one embed resources that had a `NULL` value in any of the selected fields when doing null filtering on them - @laurenceisla
+ - #2846, Fix error when requesting `Prefer: count=<type>` and doing null filtering on embedded resources - @laurenceisla
+ - #2959, Fix setting `default_transaction_isolation` unnecessarily - @steve-chavez
+ - #2929, Fix arrow filtering on RPC returning dynamic TABLE with composite type - @steve-chavez
+ - #2963, Fix RPCs not embedding correctly when using overloaded functions for computed relationships - @laurenceisla
+ - #2970, Fix regression that rejects URI connection strings with certain unescaped characters in the password - @laurenceisla, @steve-chavez
+
+## [11.2.0] - 2023-08-10
+
+### Added
+
+ - #2523, Data representations - @aljungberg
+   + Allows for flexible API output formatting and input parsing on a per-column type basis using regular SQL functions configured in the database
+   + Enables greater flexibility in the form and shape of your APIs, both for output and input, making PostgREST a more versatile general-purpose API server
+   + Examples include base64 encode/decode your binary data (like a `bytea` column containing an image), choose whether to present a timestamp column as seconds since the Unix epoch or as an ISO 8601 string, or represent fixed precision decimals as strings, not doubles, to preserve precision
+   + ...and accept the same in `POST/PUT/PATCH` by configuring the reverse transformation(s)
+   + Other use-cases include custom representation of enums, arrays, nested objects, CSS hex colour strings, gzip compressed fields, metric to imperial conversions, and much more
+   + Works when using the `select` parameter to select only a subset of columns, embedding through complex joins, renaming fields, with views and computed columns
+   + Works when filtering on a formatted column without extra indexes by parsing to the canonical representation
+   + Works for data `RETURNING` operations, such as requesting the full body in a POST/PUT/PATCH with `Prefer: return=representation`
+   + Works for batch updates and inserts
+   + Completely optional, define the functions in the database and they will be used automatically everywhere
+   + Data representations preserve the ability to write to the original column and require no extra storage or complex triggers (compared to using `GENERATED ALWAYS` columns)
+   + Note: data representations require Postgres 10 (Postgres 11 if using `IN` predicates); data representations are not implemented for RPC
+ - #2647, Allow to verify the PostgREST version in SQL: `select distinct application_name from pg_stat_activity`. - @laurenceisla
+ - #2856, Add the `--version` CLI option that prints the version information - @laurenceisla
+ - #1655, Improve `details` field of the singular error response - @taimoorzaeem
+ - #740, Add `Preference-Applied` in response for `Prefer: return=representation/headers-only/minimal` - @taimoorzaeem
+ - #1601, Add optional `nulls=stripped` parameter for mediatypes `application/vnd.pgrst.array+json` and `application/vnd.pgrst.object+json` - @taimoorzaeem
+
+### Fixed
+
+ - #2821, Fix OPTIONS not accepting all available media types - @steve-chavez
+ - #2834, Fix compilation on Ubuntu by being compatible with GHC 9.0.2 - @steve-chavez
+ - #2840, Fix `Prefer: missing=default` with DOMAIN default values - @steve-chavez
+ - #2849, Fix HEAD unnecessarily executing aggregates - @steve-chavez
+ - #2594, Fix unused index on jsonb/jsonb arrow filter and order (``/bets?data->>contractId=eq.1`` and ``/bets?order=data->>contractId``) - @steve-chavez
+ - #2861, Fix character and bit columns with fixed length not inserting/updating properly - @laurenceisla
+   + Fixes the error "value too long for type character(1)" when the char length of the column was bigger than one.
+ - #2862, Fix null filtering on embedded resource when using a column name equal to the relation name - @steve-chavez
+ - #1586, Fix function parameters of type character and bit not ignoring length - @laurenceisla
+   + Fixes the error "value too long for type character(1)" when the char length of the parameter was bigger than one.
+ - #2881, Fix error when a function returns `RECORD` or `SET OF RECORD` - @laurenceisla
+ - #2896, Fix applying superuser settings for impersonated role - @steve-chavez
+
+### Deprecated
+
+ - #2863, Deprecate resource embedding target disambiguation - @steve-chavez
+   + The `/table?select=*,other!fk(*)` must be used to disambiguate
+   + The server aids in choosing the `!fk` by sending a `hint` on the error whenever an ambiguous request happens.
 
 ## [11.1.0] - 2023-06-07
 

default.nix

@@ -50,6 +50,19 @@ let
 
   postgresqlVersions =
     [
+      {
+        name = "postgresql-16";
+        postgresql = pkgs.postgresql_16.withPackages (p: [
+          p.postgis
+          (p.pg_safeupdate.overrideAttrs (old: {
+            installPhase = ''
+              mkdir -p $out/bin
+              cp safeupdate.dylib safeupdate.so || true
+              install -D safeupdate.so -t $out/lib
+            '';
+          }))
+        ]);
+      }
       { name = "postgresql-15"; postgresql = pkgs.postgresql_15.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
       { name = "postgresql-14"; postgresql = pkgs.postgresql_14.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
       { name = "postgresql-13"; postgresql = pkgs.postgresql_13.withPackages (p: [ p.postgis p.pg_safeupdate ]); }

nix/README.md

@@ -5,24 +5,14 @@ for developing, testing and building PostgREST.
 
 ## Getting started with Nix
 
-You'll need to [get Nix](https://nixos.org/download.html). The installer will
-create your Nix store in the `/nix/` directory, where all build artifacts and
-their dependencies will be stored. It will also link the Nix executables like
-`nix-env`, `nix-build` and `nix-shell` into your PATH. Nix will manage all
-other PostgREST dependencies from here on out. To clean up older build
-artifacts from the `/nix/store`, you can run `nix-collect-garbage`.
-
-If you are on a system that does not support nix, for example Windows, you can
-run the nix development environment in a docker container. Inside the `nix/`
-directory run `docker-compose run --rm nix` to start the docker container. This
-will set up the binary cache and launch `nix-shell` automatically.
+You'll need to [get Nix](https://nixos.org/download.html). Follow the recommended installation for your operating system from the official download website.
 
 ## Building PostgREST
 
 To build PostgREST from your local checkout of the repository, run:
 
 ```bash
-nix-build --attr postgrestPackage
+$ nix-build --attr postgrestPackage
 
 ```
 
@@ -39,10 +29,10 @@ We recommend that you use the PostgREST binary cache on
 
 ```bash
 # Install cachix:
-nix-env -iA cachix -f https://cachix.org/api/v1/install
+$ nix-env -iA cachix -f https://cachix.org/api/v1/install
 
 # Set cachix up to use the PostgREST binary cache:
-cachix use postgrest
+$ cachix use postgrest
 
 ```
 
@@ -56,7 +46,7 @@ following command will put you into a new shell that has GHC and Cabal on the
 PATH:
 
 ```bash
-nix-shell
+$ nix-shell
 
 ```
 
@@ -146,10 +136,10 @@ Note: Once inside nix-shell, the utilities work from any directory inside
 the PostgREST repo. Paths are resolved relative to the repo root:
 
 ```bash
-$ cd src
+[nix-shell]$ cd src
 # Even though the current directory is ./src, the config path must still start
 # from the repo root:
-$ postgrest-run test/io/configs/simple.conf
+[nix-shell]$ postgrest-run test/io/configs/simple.conf
 ```
 
 ## Testing
@@ -177,21 +167,21 @@ run with `postgrest-test-io`. The test runner under the hood is
 
 ```bash
 # Filter the tests to run by name, including all that contain 'config':
-postgrest-test-io -k config
+[nix-shell]$ postgrest-test-io -k config
 
 # Run tests in parallel using xdist, specifying the number of processes:
-postgrest-test-io -n auto
-postgrest-test-io -n 8
+[nix-shell]$ postgrest-test-io -n auto
+[nix-shell]$ postgrest-test-io -n 8
 ```
 
 The memory tests check that we don't surpass a memory threshold for big request bodies.
 
 ```bash
 # Build the dependencies needed for the memory test
-nix-shell --arg memory true
+$ nix-shell --arg memory true
 
 # Run the memory test
-postgrest-test-memory
+[nix-shell]$ postgrest-test-memory
 ```
 
 The loadtests ensure that performance doesn't drop on a change. Underlyingly they use
@@ -199,25 +189,25 @@ The loadtests ensure that performance doesn't drop on a change. Underlyingly the
 
 ```bash
 # Run the loadtests on the latest commit(HEAD)
-postgrest-loadtest
+[nix-shell]$ postgrest-loadtest
 
 # You can loadtest comparing to a different branch
-postgrest-loadtest-against master
+[nix-shell]$ postgrest-loadtest-against master
 
 # You can simulate latency client/postgrest and postgrest/database
-PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest
+[nix-shell]$ PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest
 
 # You can build postgrest directly with cabal for faster iteration
-PGRST_BUILD_CABAL=1 postgrest-loadtest
+[nix-shell]$ PGRST_BUILD_CABAL=1 postgrest-loadtest
 
 # Produce a markdown report to be used on CI
-postgrest-loadtest-report
+[nix-shell]$ postgrest-loadtest-report
 ```
 
 doctests for some of our modules are also available:
 
 ```bash
-postgrest-test-doctest
+[nix-shell]$ postgrest-test-doctest
 ```
 
 ## Code coverage
@@ -226,11 +216,11 @@ Code coverage is available under the `postgrest-coverage` command. This will pro
 
 ```bash
 # Will run all the tests and produce a coverage dir
-postgrest-coverage
+[nix-shell]$ postgrest-coverage
 
 # Visualize the output
-cd coverage
-python -mSimpleHTTPServer 8080
+[nix-shell]$ cd coverage
+[nix-shell]$ python -mSimpleHTTPServer 8080
 ```
 
 ## Linting and styling code

nix/overlays/haskell-packages.nix

@@ -43,8 +43,8 @@ let
         (prev.callHackageDirect
           {
             pkg = "hasql-notifications";
-            ver = "0.2.0.5";
-            sha256 = "sha256-KV36zs/RTgJh/oBBPZaNiQshusmakWnjYTcdoNa4JLA=";
+            ver = "0.2.0.6";
+            sha256 = "sha256-7PyFlB2B70njudOjaX6tk1m77ol9vnF5fI0LF86kVAI=";
           }
           { });
 

nix/overlays/postgresql-future.nix

@@ -4,16 +4,16 @@ self: super:
 {
   ## Example for including a postgresql version from a specific nixpks commit:
   ##
-  #  postgresql_14 =
-  #    let
-  #      rev = "76b1e16c6659ccef7187ca69b287525fea133244";
-  #      tarballHash = "1vsahpcx80k2bgslspb0sa6j4bmhdx77sw6la455drqcrqhdqj6a";
-  #
-  #      pinnedPkgs =
-  #        builtins.fetchTarball {
-  #          url = "https://github.com/nixos/nixpkgs/archive/${rev}.tar.gz";
-  #          sha256 = tarballHash;
-  #        };
-  #    in
-  #    (import pinnedPkgs { }).pkgs.postgresql_14;
+  postgresql_16 =
+    let
+      rev = "5148520bfab61f99fd25fb9ff7bfbb50dad3c9db";
+      tarballHash = "1dfjmz65h8z4lk845724vypzmf3dbgsdndjpj8ydlhx6c7rpcq3p";
+
+      pinnedPkgs =
+        builtins.fetchTarball {
+          url = "https://github.com/nixos/nixpkgs/archive/${rev}.tar.gz";
+          sha256 = tarballHash;
+        };
+    in
+    (import pinnedPkgs { }).pkgs.postgresql_16;
 }

postgrest.cabal

@@ -1,5 +1,5 @@
 name:               postgrest
-version:            11.1.0
+version:            11.2.1
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                     for tables, views, and functions, supporting all HTTP methods that security
@@ -49,6 +49,7 @@ library
                       PostgREST.SchemaCache.Identifiers
                       PostgREST.SchemaCache.Routine
                       PostgREST.SchemaCache.Relationship
+                      PostgREST.SchemaCache.Representations
                       PostgREST.SchemaCache.Table
                       PostgREST.Error
                       PostgREST.Logger
@@ -91,7 +92,7 @@ library
                     , gitrev                    >= 1.2 && < 1.4
                     , hasql                     >= 1.6.1.1 && < 1.7
                     , hasql-dynamic-statements  >= 0.3.1 && < 0.4
-                    , hasql-notifications       >= 0.1 && < 0.3
+                    , hasql-notifications       >= 0.2.0.6 && < 0.3
                     , hasql-pool                >= 0.10 && < 0.11
                     , hasql-transaction         >= 1.0.1 && < 1.1
                     , heredoc                   >= 0.2 && < 0.3
@@ -217,6 +218,7 @@ test-suite spec
                       Feature.Query.RelatedQueriesSpec
                       Feature.Query.RpcSpec
                       Feature.Query.SingularSpec
+                      Feature.Query.NullsStrip
                       Feature.Query.SpreadQueriesSpec
                       Feature.Query.UnicodeSpec
                       Feature.Query.UpdateSpec

src/PostgREST/ApiRequest.hs

@@ -51,8 +51,7 @@ import PostgREST.ApiRequest.Types        (ApiRequestError (..),
                                           RangeError (..))
 import PostgREST.Config                  (AppConfig (..),
                                           OpenAPIMode (..))
-import PostgREST.MediaType               (MTPlanAttrs (..),
-                                          MTPlanFormat (..),
+import PostgREST.MediaType               (MTPlanFormat (..),
                                           MediaType (..))
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           convertToLimitZeroRange,
@@ -154,7 +153,7 @@ userApiRequest conf req reqBody = do
   , iRange = ranges
   , iTopLevelRange = topLevelRange
   , iPayload = payload
-  , iPreferences = Preferences.fromHeaders hdrs
+  , iPreferences = Preferences.fromHeaders (configDbTxAllowOverride conf) hdrs
   , iQueryParams = qPrms
   , iColumns = columns
   , iHeaders = iHdrs
@@ -207,7 +206,7 @@ getAction PathInfo{pathIsProc, pathIsDefSpec} method =
 
 getMediaTypes :: AppConfig -> RequestHeaders -> Action -> PathInfo -> Either ApiRequestError (MediaType, MediaType)
 getMediaTypes conf hdrs action path = do
-   acceptMediaType <- findAcceptMediaType conf action path accepts
+   acceptMediaType <- negotiateContent conf action path accepts
    pure (acceptMediaType, contentMediaType)
   where
     accepts = maybe [MTAny] (map MediaType.decodeMediaType . parseHttpAccept) $ lookupHeader "accept"
@@ -299,19 +298,6 @@ getPayload reqBody contentMediaType QueryParams{qsColumns} action PathInfo{pathI
       ActionInvoke InvPost        -> qsColumns
       _                           -> Nothing
 
-{-|
-  Find the best match from a list of media types accepted by the
-  client in order of decreasing preference and a list of types
-  producible by the server.  If there is no match but the client
-  accepts */* then return the top server pick.
--}
-mutuallyAgreeable :: [MediaType] -> [MediaType] -> Maybe MediaType
-mutuallyAgreeable sProduces cAccepts =
-  let exact = listToMaybe $ L.intersect cAccepts sProduces in
-  if isNothing exact && MTAny `elem` cAccepts
-     then listToMaybe sProduces
-     else exact
-
 type CsvData = V.Vector (M.Map Text LBS.ByteString)
 
 {-|
@@ -361,28 +347,33 @@ payloadAttributes raw json =
   where
     emptyPJArray = ProcessedJSON (JSON.encode emptyArray) S.empty
 
-findAcceptMediaType :: AppConfig -> Action -> PathInfo -> [MediaType] -> Either ApiRequestError MediaType
-findAcceptMediaType conf action path accepts =
-  case mutuallyAgreeable (requestMediaTypes conf action path) accepts of
-    Just ct ->
-      Right ct
-    Nothing ->
-      Left . MediaTypeError $ map MediaType.toMime accepts
 
-requestMediaTypes :: AppConfig -> Action -> PathInfo -> [MediaType]
-requestMediaTypes conf action path =
+-- | Do content negotiation. i.e. choose a media type based on the intersection of accepted/produced media types.
+negotiateContent :: AppConfig -> Action -> PathInfo -> [MediaType] -> Either ApiRequestError MediaType
+negotiateContent conf action path accepts =
+  case firstAcceptedPick of
+    Just MTAny -> Right MTApplicationJSON -- by default(for */*) we respond with json
+    Just mt    -> Right mt
+    Nothing    -> Left . MediaTypeError $ map MediaType.toMime accepts
+  where
+    -- if there are multiple accepted media types, pick the first
+    firstAcceptedPick = listToMaybe $ L.intersect accepts $ producedMediaTypes conf action path
+
+producedMediaTypes :: AppConfig -> Action -> PathInfo -> [MediaType]
+producedMediaTypes conf action path =
   case action of
     ActionRead _    -> defaultMediaTypes ++ rawMediaTypes
     ActionInvoke _  -> invokeMediaTypes
-    ActionInspect _ -> [MTOpenAPI, MTApplicationJSON]
-    ActionInfo      -> [MTTextCSV]
-    _               -> defaultMediaTypes
+    ActionInfo      -> defaultMediaTypes
+    ActionMutate _  -> defaultMediaTypes
+    ActionInspect _ -> inspectMediaTypes
   where
+    inspectMediaTypes = [MTOpenAPI, MTApplicationJSON, MTArrayJSONStrip, MTAny]
     invokeMediaTypes =
       defaultMediaTypes
         ++ rawMediaTypes
         ++ [MTOpenAPI | pathIsRootSpec path]
     defaultMediaTypes =
-      [MTApplicationJSON, MTSingularJSON, MTGeoJSON, MTTextCSV] ++
-      [MTPlan $ MTPlanAttrs Nothing PlanJSON mempty | configDbPlanEnabled conf]
+      [MTApplicationJSON, MTArrayJSONStrip, MTSingularJSON True, MTSingularJSON False, MTGeoJSON, MTTextCSV] ++
+      [MTPlan MTApplicationJSON PlanText mempty | configDbPlanEnabled conf] ++ [MTAny]
     rawMediaTypes = configRawMediaTypes conf `union` [MTOctetStream, MTTextPlain, MTTextXML]

src/PostgREST/ApiRequest/Preferences.hs

@@ -6,6 +6,7 @@
 --
 -- [1] https://datatracker.ietf.org/doc/html/rfc7240
 --
+{-# LANGUAGE NamedFieldPuns #-}
 module PostgREST.ApiRequest.Preferences
   ( Preferences(..)
   , PreferCount(..)
@@ -15,8 +16,8 @@ module PostgREST.ApiRequest.Preferences
   , PreferResolution(..)
   , PreferTransaction(..)
   , fromHeaders
-  , ToAppliedHeader(..)
   , shouldCount
+  , prefAppliedHeader
   ) where
 
 import qualified Data.ByteString.Char8     as BS
@@ -41,7 +42,7 @@ import Protolude
 data Preferences
   = Preferences
     { preferResolution     :: Maybe PreferResolution
-    , preferRepresentation :: PreferRepresentation
+    , preferRepresentation :: Maybe PreferRepresentation
     , preferParameters     :: Maybe PreferParameters
     , preferCount          :: Maybe PreferCount
     , preferTransaction    :: Maybe PreferTransaction
@@ -53,10 +54,10 @@ data Preferences
 --
 -- One header with comma-separated values can be used to set multiple preferences:
 --
--- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates, count=exact")]
+-- >>> pPrint $ fromHeaders True [("Prefer", "resolution=ignore-duplicates, count=exact")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
---     , preferRepresentation = None
+--     , preferRepresentation = Nothing
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
@@ -65,10 +66,10 @@ data Preferences
 --
 -- Multiple headers can also be used:
 --
--- >>> pPrint $ fromHeaders [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null")]
+-- >>> pPrint $ fromHeaders True [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
---     , preferRepresentation = None
+--     , preferRepresentation = Nothing
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Nothing
@@ -77,13 +78,13 @@ data Preferences
 --
 -- If a preference is set more than once, only the first is used:
 --
--- >>> preferTransaction $ fromHeaders [("Prefer", "tx=commit, tx=rollback")]
+-- >>> preferTransaction $ fromHeaders True [("Prefer", "tx=commit, tx=rollback")]
 -- Just Commit
 --
 -- This is also the case across multiple headers:
 --
 -- >>> :{
---   preferResolution . fromHeaders $
+--   preferResolution . fromHeaders True $
 --     [ ("Prefer", "resolution=ignore-duplicates")
 --     , ("Prefer", "resolution=merge-duplicates")
 --     ]
@@ -92,30 +93,30 @@ data Preferences
 --
 -- Preferences not recognized by the application are ignored:
 --
--- >>> preferResolution $ fromHeaders [("Prefer", "resolution=foo")]
+-- >>> preferResolution $ fromHeaders True [("Prefer", "resolution=foo")]
 -- Nothing
 --
 -- Preferences can be separated by arbitrary amounts of space, lower-case header is also recognized:
 --
--- >>> pPrint $ fromHeaders [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default")]
+-- >>> pPrint $ fromHeaders True [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default")]
 -- Preferences
 --     { preferResolution = Nothing
---     , preferRepresentation = Full
+--     , preferRepresentation = Just Full
 --     , preferParameters = Nothing
 --     , preferCount = Just ExactCount
 --     , preferTransaction = Just Commit
 --     , preferMissing = Just ApplyDefaults
 --     }
 --
-fromHeaders :: [HTTP.Header] -> Preferences
-fromHeaders headers =
+fromHeaders :: Bool -> [HTTP.Header] -> Preferences
+fromHeaders allowTxEndOverride headers =
   Preferences
-    { preferResolution = parsePrefs [MergeDuplicates, IgnoreDuplicates]
-    , preferRepresentation = fromMaybe None $ parsePrefs [Full, None, HeadersOnly]
-    , preferParameters = parsePrefs [SingleObject]
-    , preferCount = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
-    , preferTransaction = parsePrefs [Commit, Rollback]
-    , preferMissing = parsePrefs [ApplyDefaults, ApplyNulls]
+    { preferResolution     = parsePrefs [MergeDuplicates, IgnoreDuplicates]
+    , preferRepresentation = parsePrefs [Full, None, HeadersOnly]
+    , preferParameters     = parsePrefs [SingleObject]
+    , preferCount          = parsePrefs [ExactCount, PlannedCount, EstimatedCount]
+    , preferTransaction    = if allowTxEndOverride then parsePrefs [Commit, Rollback] else Nothing
+    , preferMissing        = parsePrefs [ApplyDefaults, ApplyNulls]
     }
   where
     prefHeaders = filter ((==) HTTP.hPrefer . fst) headers
@@ -128,6 +129,22 @@ fromHeaders headers =
     prefMap :: ToHeaderValue a => [a] -> Map.Map ByteString a
     prefMap = Map.fromList . fmap (\pref -> (toHeaderValue pref, pref))
 
+prefAppliedHeader :: Preferences -> Maybe HTTP.Header
+prefAppliedHeader Preferences {preferResolution, preferRepresentation, preferParameters, preferCount, preferTransaction, preferMissing } =
+  if null prefsVals
+    then Nothing
+    else Just (HTTP.hPreferenceApplied, combined)
+  where
+    combined = BS.intercalate ", " prefsVals
+    prefsVals = catMaybes [
+        toHeaderValue <$> preferResolution
+      , toHeaderValue <$> preferMissing
+      , toHeaderValue <$> preferRepresentation
+      , toHeaderValue <$> preferParameters
+      , toHeaderValue <$> preferCount
+      , toHeaderValue <$> preferTransaction
+      ]
+
 -- |
 -- Convert a preference into the value that we look for in the 'Prefer' headers.
 --
@@ -137,16 +154,6 @@ fromHeaders headers =
 class ToHeaderValue a where
   toHeaderValue :: a -> ByteString
 
--- |
--- Header to indicate that a preference has been applied.
---
--- >>> toAppliedHeader MergeDuplicates
--- ("Preference-Applied","resolution=merge-duplicates")
---
-class ToHeaderValue a => ToAppliedHeader a where
-  toAppliedHeader :: a -> HTTP.Header
-  toAppliedHeader x = (HTTP.hPreferenceApplied, toHeaderValue x)
-
 -- | How to handle duplicate values.
 data PreferResolution
   = MergeDuplicates
@@ -156,8 +163,6 @@ instance ToHeaderValue PreferResolution where
   toHeaderValue MergeDuplicates  = "resolution=merge-duplicates"
   toHeaderValue IgnoreDuplicates = "resolution=ignore-duplicates"
 
-instance ToAppliedHeader PreferResolution
-
 -- |
 -- How to return the mutated data.
 --
@@ -207,8 +212,6 @@ instance ToHeaderValue PreferTransaction where
   toHeaderValue Commit   = "tx=commit"
   toHeaderValue Rollback = "tx=rollback"
 
-instance ToAppliedHeader PreferTransaction
-
 -- |
 -- How to handle the insertion/update when the keys specified in ?columns are not present
 -- in the json body.
@@ -220,5 +223,3 @@ data PreferMissing
 instance ToHeaderValue PreferMissing where
   toHeaderValue ApplyDefaults = "missing=default"
   toHeaderValue ApplyNulls    = "missing=null"
-
-instance ToAppliedHeader PreferMissing

src/PostgREST/ApiRequest/QueryParams.hs

@@ -60,29 +60,6 @@ import PostgREST.ApiRequest.Types (EmbedParam (..), EmbedPath, Field,
 
 import Protolude hiding (try)
 
-
--- $setup
--- Setup for doctests
--- >>> import Text.Pretty.Simple (pPrint)
--- >>> deriving instance Show QPError
--- >>> deriving instance Show TrileanVal
--- >>> deriving instance Show FtsOperator
--- >>> deriving instance Show QuantOperator
--- >>> deriving instance Show SimpleOperator
--- >>> deriving instance Show OpQuantifier
--- >>> deriving instance Show Operation
--- >>> deriving instance Show OpExpr
--- >>> deriving instance Show JsonOperand
--- >>> deriving instance Show JsonOperation
--- >>> deriving instance Show Filter
--- >>> deriving instance Show JoinType
--- >>> deriving instance Show SelectItem
--- >>> deriving instance Show OrderDirection
--- >>> deriving instance Show OrderNulls
--- >>> deriving instance Show OrderTerm
--- >>> deriving instance Show LogicOperator
--- >>> deriving instance Show LogicTree
-
 data QueryParams =
   QueryParams
     { qsCanonical      :: ByteString

src/PostgREST/ApiRequest/Types.hs

@@ -62,7 +62,7 @@ data SelectItem
     , selHint     :: Maybe Hint
     , selJoinType :: Maybe JoinType
     }
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data ApiRequestError
   = AmbiguousRelBetween Text Text [Relationship]
@@ -86,12 +86,15 @@ data ApiRequestError
   | UnacceptableSchema [Text]
   | UnsupportedMethod ByteString
   | ColumnNotFound Text Text
+  deriving Show
 
 data QPError = QPError Text Text
+  deriving Show
 data RangeError
   = NegativeLimit
   | LowerGTUpper
   | OutOfBounds Text Text
+  deriving Show
 
 type NodeName = Text
 type Depth = Integer
@@ -108,17 +111,17 @@ data OrderTerm
     , otDirection :: Maybe OrderDirection
     , otNullOrder :: Maybe OrderNulls
     }
-  deriving Eq
+  deriving (Eq, Show)
 
 data OrderDirection
   = OrderAsc
   | OrderDesc
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data OrderNulls
   = OrderNullsFirst
   | OrderNullsLast
-  deriving (Eq)
+  deriving (Eq, Show)
 
 type Field = (FieldName, JsonPath)
 type Cast = Text
@@ -135,7 +138,7 @@ data EmbedParam
 data JoinType
   = JTInner
   | JTLeft
-  deriving Eq
+  deriving (Eq, Show)
 
 -- | Path of the embedded levels, e.g "clients.projects.name=eq.." gives Path
 -- ["clients", "projects"]
@@ -149,7 +152,7 @@ type JsonPath = [JsonOperation]
 data JsonOperation
   = JArrow { jOp :: JsonOperand }
   | J2Arrow { jOp :: JsonOperand }
-  deriving (Eq, Ord)
+  deriving (Eq, Show, Ord)
 
 -- | Represents the key(`->'key'`) or index(`->'1`::int`), the index is Text
 -- because we reuse our escaping functons and let pg do the casting with
@@ -157,7 +160,7 @@ data JsonOperation
 data JsonOperand
   = JKey { jVal :: Text }
   | JIdx { jVal :: Text }
-  deriving (Eq, Ord)
+  deriving (Eq, Show, Ord)
 
 -- | Boolean logic expression tree e.g. "and(name.eq.N,or(id.eq.1,id.eq.2))" is:
 --
@@ -169,28 +172,27 @@ data JsonOperand
 data LogicTree
   = Expr Bool LogicOperator [LogicTree]
   | Stmnt Filter
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data LogicOperator
   = And
   | Or
-  deriving Eq
+  deriving (Eq, Show)
 
 data Filter
   = Filter
   { field  :: Field
   , opExpr :: OpExpr
   }
-  | FilterNullEmbed Bool FieldName
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data OpExpr
   = OpExpr Bool Operation
   | NoOpExpr Text
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data OpQuantifier = QuantAny | QuantAll
-  deriving Eq
+  deriving (Eq, Show)
 
 data Operation
   = Op SimpleOperator SingleVal
@@ -199,7 +201,7 @@ data Operation
   | Is TrileanVal
   | IsDistinctFrom SingleVal
   | Fts FtsOperator (Maybe Language) SingleVal
-  deriving (Eq)
+  deriving (Eq, Show)
 
 type Language = Text
 
@@ -215,7 +217,7 @@ data TrileanVal
   | TriFalse
   | TriNull
   | TriUnknown
-  deriving Eq
+  deriving (Eq, Show)
 
 -- Operators that are quantifiable, i.e. they can be used with the any/all modifiers
 data QuantOperator
@@ -228,7 +230,7 @@ data QuantOperator
   | OpILike
   | OpMatch
   | OpIMatch
-  deriving Eq
+  deriving (Eq, Show)
 
 data SimpleOperator
   = OpNotEqual
@@ -240,7 +242,7 @@ data SimpleOperator
   | OpNotExtendsRight
   | OpNotExtendsLeft
   | OpAdjacent
-  deriving Eq
+  deriving (Eq, Show)
 
 --
 -- | Operators for full text search operators
@@ -249,4 +251,4 @@ data FtsOperator
   | FilterFtsPlain
   | FilterFtsPhrase
   | FilterFtsWebsearch
-  deriving Eq
+  deriving (Eq, Show)

src/PostgREST/App.hs

@@ -27,6 +27,7 @@ import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
 import System.Posix.Types       (FileMode)
 
 import qualified Data.HashMap.Strict        as HM
+import qualified Data.Text.Encoding         as T
 import qualified Hasql.Transaction.Sessions as SQL
 import qualified Network.Wai                as Wai
 import qualified Network.Wai.Handler.Warp   as Warp
@@ -53,7 +54,7 @@ import PostgREST.Error               (Error)
 import PostgREST.Query               (DbHandler)
 import PostgREST.SchemaCache         (SchemaCache (..))
 import PostgREST.SchemaCache.Routine (Routine (..))
-import PostgREST.Version             (prettyVersion)
+import PostgREST.Version             (docsVersion, prettyVersion)
 
 import Protolude hiding (Handler)
 
@@ -149,8 +150,7 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@
     liftEither . mapLeft Error.ApiRequestError $
       ApiRequest.userApiRequest conf req body
 
-  Response.optionalRollback conf apiRequest $
-    handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache
+  handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache
 
 runDbHandler :: AppState.AppState -> SQL.IsolationLevel -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
 runDbHandler appState isoLvl mode authenticated prepared handler = do
@@ -199,7 +199,7 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A
 
     (ActionInspect headersOnly, TargetDefaultSpec tSchema) -> do
       oaiResult <- runQuery roleIsoLvl Plan.inspectPlanTxMode $ Query.openApiQuery sCache pgVer conf tSchema
-      return $ Response.openApiResponse headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
+      return $ Response.openApiResponse (T.decodeUtf8 prettyVersion, docsVersion) headersOnly oaiResult conf sCache iSchema iNegotiatedByProfile
 
     (ActionInfo, TargetIdent identifier) ->
       return $ Response.infoIdentResponse identifier sCache

src/PostgREST/AppState.hs

@@ -34,6 +34,7 @@ import qualified Hasql.Pool                 as SQL
 import qualified Hasql.Session              as SQL
 import qualified Hasql.Transaction.Sessions as SQL
 import qualified PostgREST.Error            as Error
+import           PostgREST.Version          (prettyVersion)
 
 import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
                            updateAction)
@@ -47,6 +48,7 @@ import Data.Time          (ZonedTime, defaultTimeLocale, formatTime,
 import Data.Time.Clock    (UTCTime, getCurrentTime)
 
 import PostgREST.Config                  (AppConfig (..),
+                                          addFallbackAppName,
                                           readAppConfig)
 import PostgREST.Config.Database         (queryDbSettings,
                                           queryPgVersion,
@@ -136,7 +138,7 @@ initPool AppConfig{..} =
     (fromIntegral configDbPoolAcquisitionTimeout)
     (fromIntegral configDbPoolMaxLifetime)
     (fromIntegral configDbPoolMaxIdletime)
-    (toUtf8 configDbUri)
+    (toUtf8 $ addFallbackAppName prettyVersion configDbUri)
 
 -- | Run an action with a database connection.
 usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a)
@@ -271,6 +273,7 @@ internalConnectionWorker appState = work
   where
     work = do
       AppConfig{..} <- getConfig appState
+      logWithZTime appState $ "Starting PostgREST " <> T.decodeUtf8 prettyVersion <> "..."
       logWithZTime appState "Attempting to connect to the database..."
       connected <- establishConnection appState
       case connected of
@@ -418,7 +421,7 @@ listener appState = do
 
   -- forkFinally allows to detect if the thread dies
   void . flip forkFinally (handleFinally dbChannel) $ do
-    dbOrError <- acquire $ toUtf8 configDbUri
+    dbOrError <- acquire $ toUtf8 (addFallbackAppName prettyVersion configDbUri)
     case dbOrError of
       Right db -> do
         logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"

src/PostgREST/CLI.hs

@@ -80,7 +80,7 @@ readCLIShowHelp =
   where
     prefs = O.prefs $ O.showHelpOnError <> O.showHelpOnEmpty
     opts = O.info parser $ O.fullDesc <> progDesc
-    parser = O.helper <*> exampleParser <*> cliParser
+    parser = O.helper <*> versionFlag <*> exampleParser <*> cliParser
 
     progDesc =
       O.progDesc $
@@ -88,6 +88,12 @@ readCLIShowHelp =
         <> BS.unpack prettyVersion
         <> " / create a REST API to an existing Postgres database"
 
+    versionFlag =
+      O.infoOption ("PostgREST " <> BS.unpack prettyVersion) $
+        O.long "version"
+        <> O.short 'v'
+        <> O.help "Show the version information"
+
     exampleParser =
       O.infoOption exampleConfigFile $
         O.long "example"

src/PostgREST/Config.hs

@@ -24,6 +24,7 @@ module PostgREST.Config
   , readPGRSTEnvironment
   , toURI
   , parseSecret
+  , addFallbackAppName
   ) where
 
 import qualified Crypto.JOSE.Types      as JOSE
@@ -47,6 +48,9 @@ import Data.List               (lookup)
 import Data.List.NonEmpty      (fromList, toList)
 import Data.Maybe              (fromJust)
 import Data.Scientific         (floatingOrInteger)
+import Network.URI             (escapeURIString,
+                                isUnescapedInURIComponent, parseURI,
+                                uriQuery)
 import Numeric                 (readOct, showOct)
 import System.Environment      (getEnvironment)
 import System.Posix.Types      (FileMode)
@@ -460,3 +464,41 @@ type Environment = M.Map [Char] Text
 readPGRSTEnvironment :: IO Environment
 readPGRSTEnvironment =
   M.map T.pack . M.fromList . filter (isPrefixOf "PGRST_" . fst) <$> getEnvironment
+
+-- | Adds a `fallback_application_name` value to the connection string. This allows querying the PostgREST version on pg_stat_activity.
+--
+-- >>> let ver = "11.1.0 (5a04ec7)"::ByteString
+-- >>> let strangeVer = "11'1&0@#$%,.:\"[]{}?+^()=asdfqwer"::ByteString
+--
+-- >>> addFallbackAppName ver "postgres://user:pass@host:5432/postgres"
+-- "postgres://user:pass@host:5432/postgres?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgres://user:pass@host:5432/postgres?"
+-- "postgres://user:pass@host:5432/postgres?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgres:///postgres?host=server&port=5432"
+-- "postgres:///postgres?host=server&port=5432&fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName ver "postgresql://"
+-- "postgresql://?fallback_application_name=PostgREST%2011.1.0%20%285a04ec7%29"
+--
+-- >>> addFallbackAppName strangeVer "postgres:///postgres?host=server&port=5432"
+-- "postgres:///postgres?host=server&port=5432&fallback_application_name=PostgREST%2011%271%260%40%23%24%25%2C.%3A%22%5B%5D%7B%7D%3F%2B%5E%28%29%3Dasdfqwer"
+--
+-- >>> addFallbackAppName ver "postgres://user:invalid_chars[]#@host:5432/postgres"
+-- "postgres://user:invalid_chars[]#@host:5432/postgres"
+--
+-- >>> addFallbackAppName ver "invalid_uri1=val1 invalid_uri2=val2"
+-- "invalid_uri1=val1 invalid_uri2=val2"
+addFallbackAppName :: ByteString -> Text -> Text
+addFallbackAppName version dbUri = dbUri <>
+  case uriQuery <$> parseURI (toS dbUri) of
+    -- Does not add the application name to key=val connection strings or invalid URIs
+    Nothing  -> mempty
+    Just ""  -> "?" <> uriFmt
+    Just "?" -> uriFmt
+    _        -> "&" <> uriFmt
+  where
+    uriFmt = pKeyWord <> toS (escapeURIString isUnescapedInURIComponent $ toS pgrstVer)
+    pKeyWord = "fallback_application_name="
+    pgrstVer = "PostgREST " <> T.decodeUtf8 version

src/PostgREST/Config/Database.hs

@@ -154,8 +154,9 @@ queryRoleSettings prepared =
       select
         kv.rolname,
         i.value as iso_lvl,
-        array_agg(row(kv.key, kv.value)) filter (where key <> 'default_transation_isolation') as role_settings
+        coalesce(array_agg(row(kv.key, kv.value)) filter (where key <> 'default_transaction_isolation'), '{}') as role_settings
       from kv_settings kv
+      join pg_settings ps on ps.name = kv.key and ps.context = 'user'
       left join iso_setting i on i.rolname = kv.rolname
       group by kv.rolname, i.value;
     |]

src/PostgREST/Error.hs

@@ -478,7 +478,7 @@ instance PgrstError Error where
   headers (JwtTokenInvalid m)    = [MediaType.toContentType MTApplicationJSON, invalidTokenHeader m]
   headers JwtTokenRequired       = [MediaType.toContentType MTApplicationJSON, requiredTokenHeader]
   headers (PgErr err)            = headers err
-  headers SingularityError{}     = [MediaType.toContentType MTSingularJSON]
+  headers SingularityError{}     = [MediaType.toContentType (MTSingularJSON False)]
   headers _                      = [MediaType.toContentType MTApplicationJSON]
 
 instance JSON.ToJSON Error where
@@ -530,7 +530,7 @@ instance JSON.ToJSON Error where
   toJSON (SingularityError n) = JSON.object [
     "code"    .= ApiRequestErrorCode16,
     "message" .= ("JSON object requested, multiple (or no) rows returned" :: Text),
-    "details" .= T.unwords ["Results contain", show n, "rows,", T.decodeUtf8 (MediaType.toMime MTSingularJSON), "requires 1 row"],
+    "details" .= T.unwords ["The result contains", show n, "rows"],
     "hint"    .= JSON.Null]
 
   toJSON (PgErr err) = JSON.toJSON err

src/PostgREST/MediaType.hs

@@ -4,16 +4,13 @@ module PostgREST.MediaType
   ( MediaType(..)
   , MTPlanOption (..)
   , MTPlanFormat (..)
-  , MTPlanAttrs(..)
   , toContentType
   , toMime
   , decodeMediaType
-  , getMediaType
   ) where
 
 import qualified Data.ByteString          as BS
 import qualified Data.ByteString.Internal as BS (c2w)
-import           Data.Maybe               (fromJust)
 
 import Network.HTTP.Types.Header (Header, hContentType)
 
@@ -22,7 +19,8 @@ import Protolude
 -- | Enumeration of currently supported media types
 data MediaType
   = MTApplicationJSON
-  | MTSingularJSON
+  | MTArrayJSONStrip
+  | MTSingularJSON Bool
   | MTGeoJSON
   | MTTextCSV
   | MTTextPlain
@@ -32,18 +30,32 @@ data MediaType
   | MTOctetStream
   | MTAny
   | MTOther ByteString
-  | MTPlan MTPlanAttrs
-  deriving Eq
-
-data MTPlanAttrs = MTPlanAttrs (Maybe MediaType) MTPlanFormat [MTPlanOption]
-instance Eq MTPlanAttrs where
-  MTPlanAttrs {} == MTPlanAttrs {} = True -- we don't care about the attributes when comparing two MTPlan media types
+  -- TODO MTPlan should only have its options as [Text]. Its ResultAggregate should have the typed attributes.
+  | MTPlan MediaType MTPlanFormat [MTPlanOption]
+  deriving Show
+instance Eq MediaType where
+  MTApplicationJSON    == MTApplicationJSON = True
+  MTArrayJSONStrip     == MTArrayJSONStrip  = True
+  MTSingularJSON x     == MTSingularJSON y  = x == y
+  MTGeoJSON            == MTGeoJSON         = True
+  MTTextCSV            == MTTextCSV         = True
+  MTTextPlain          == MTTextPlain       = True
+  MTTextXML            == MTTextXML         = True
+  MTOpenAPI            == MTOpenAPI         = True
+  MTUrlEncoded         == MTUrlEncoded      = True
+  MTOctetStream        == MTOctetStream     = True
+  MTAny                == MTAny             = True
+  MTOther x            == MTOther y         = x == y
+  MTPlan{}             == MTPlan{}          = True
+  _                    == _                 = False
 
 data MTPlanOption
   = PlanAnalyze | PlanVerbose | PlanSettings | PlanBuffers | PlanWAL
+  deriving (Eq, Show)
 
 data MTPlanFormat
   = PlanJSON | PlanText
+  deriving (Eq, Show)
 
 -- | Convert MediaType to a Content-Type HTTP Header
 toContentType :: MediaType -> Header
@@ -56,20 +68,22 @@ toContentType ct = (hContentType, toMime ct <> charset)
 
 -- | Convert from MediaType to a ByteString representing the mime type
 toMime :: MediaType -> ByteString
-toMime MTApplicationJSON = "application/json"
-toMime MTGeoJSON         = "application/geo+json"
-toMime MTTextCSV         = "text/csv"
-toMime MTTextPlain       = "text/plain"
-toMime MTTextXML         = "text/xml"
-toMime MTOpenAPI         = "application/openapi+json"
-toMime MTSingularJSON    = "application/vnd.pgrst.object+json"
-toMime MTUrlEncoded      = "application/x-www-form-urlencoded"
-toMime MTOctetStream     = "application/octet-stream"
-toMime MTAny             = "*/*"
-toMime (MTOther ct)      = ct
-toMime (MTPlan (MTPlanAttrs mt fmt opts)) =
+toMime MTApplicationJSON      = "application/json"
+toMime MTArrayJSONStrip       = "application/vnd.pgrst.array+json;nulls=stripped"
+toMime MTGeoJSON              = "application/geo+json"
+toMime MTTextCSV              = "text/csv"
+toMime MTTextPlain            = "text/plain"
+toMime MTTextXML              = "text/xml"
+toMime MTOpenAPI              = "application/openapi+json"
+toMime (MTSingularJSON True)  = "application/vnd.pgrst.object+json;nulls=stripped"
+toMime (MTSingularJSON False) = "application/vnd.pgrst.object+json"
+toMime MTUrlEncoded           = "application/x-www-form-urlencoded"
+toMime MTOctetStream          = "application/octet-stream"
+toMime MTAny                  = "*/*"
+toMime (MTOther ct)           = ct
+toMime (MTPlan mt fmt opts)   =
   "application/vnd.pgrst.plan+" <> toMimePlanFormat fmt <>
-  (if isNothing mt then mempty else "; for=\"" <> toMime (fromJust mt) <> "\"") <>
+  ("; for=\"" <> toMime mt <> "\"") <>
   (if null opts then mempty else "; options=" <> BS.intercalate "|" (toMimePlanOption <$> opts))
 
 toMimePlanOption :: MTPlanOption -> ByteString
@@ -83,42 +97,73 @@ toMimePlanFormat :: MTPlanFormat -> ByteString
 toMimePlanFormat PlanJSON = "json"
 toMimePlanFormat PlanText = "text"
 
--- | Convert from ByteString to MediaType. Warning: discards MIME parameters
+-- | Convert from ByteString to MediaType.
+--
+-- >>> decodeMediaType "application/json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan;"
+-- MTPlan MTApplicationJSON PlanText []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan;for=\"application/json\""
+-- MTPlan MTApplicationJSON PlanText []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.plan+json;for=\"text/csv\""
+-- MTPlan MTTextCSV PlanJSON []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json;nulls=stripped"
+-- MTArrayJSONStrip
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json;nulls=stripped"
+-- MTSingularJSON True
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json"
+-- MTSingularJSON False
+
 decodeMediaType :: BS.ByteString -> MediaType
 decodeMediaType mt =
   case BS.split (BS.c2w ';') mt of
-    "application/json":_                   -> MTApplicationJSON
-    "application/geo+json":_               -> MTGeoJSON
-    "text/csv":_                           -> MTTextCSV
-    "text/plain":_                         -> MTTextPlain
-    "text/xml":_                           -> MTTextXML
-    "application/openapi+json":_           -> MTOpenAPI
-    "application/vnd.pgrst.object+json":_  -> MTSingularJSON
-    "application/vnd.pgrst.object":_       -> MTSingularJSON
-    "application/x-www-form-urlencoded":_  -> MTUrlEncoded
-    "application/octet-stream":_           -> MTOctetStream
-    "application/vnd.pgrst.plan":rest      -> getPlan PlanText rest
-    "application/vnd.pgrst.plan+text":rest -> getPlan PlanText rest
-    "application/vnd.pgrst.plan+json":rest -> getPlan PlanJSON rest
-    "*/*":_                                -> MTAny
-    other:_                                -> MTOther other
-    _                                      -> MTAny
+    "application/json":_                     -> MTApplicationJSON
+    "application/geo+json":_                 -> MTGeoJSON
+    "text/csv":_                             -> MTTextCSV
+    "text/plain":_                           -> MTTextPlain
+    "text/xml":_                             -> MTTextXML
+    "application/openapi+json":_             -> MTOpenAPI
+    "application/x-www-form-urlencoded":_    -> MTUrlEncoded
+    "application/octet-stream":_             -> MTOctetStream
+    "application/vnd.pgrst.plan":rest        -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+text":rest   -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+json":rest   -> getPlan PlanJSON rest
+    "application/vnd.pgrst.object+json":rest -> checkSingularNullStrip rest
+    "application/vnd.pgrst.object":rest      -> checkSingularNullStrip rest
+    "application/vnd.pgrst.array+json":rest  -> checkArrayNullStrip rest
+    "application/vnd.pgrst.array":rest       -> checkArrayNullStrip rest
+    "*/*":_                                  -> MTAny
+    other:_                                  -> MTOther other
+    _                                        -> MTAny
   where
-    getPlan fmt rest =
-     let
-       opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)
-       inOpts str   = str `elem` opts
-       mtFor        = decodeMediaType . dropAround (== BS.c2w '"') <$> (BS.stripPrefix "for=" =<< find (BS.isPrefixOf "for=") rest)
-       dropAround p = BS.dropWhile p . BS.dropWhileEnd p in
-     MTPlan $ MTPlanAttrs mtFor fmt $
-      [PlanAnalyze  | inOpts "analyze" ] ++
-      [PlanVerbose  | inOpts "verbose" ] ++
-      [PlanSettings | inOpts "settings"] ++
-      [PlanBuffers  | inOpts "buffers" ] ++
-      [PlanWAL      | inOpts "wal"     ]
+    checkArrayNullStrip ["nulls=stripped"] = MTArrayJSONStrip
+    checkArrayNullStrip  _                 = MTApplicationJSON
 
-getMediaType :: MediaType -> MediaType
-getMediaType mt = case mt of
-  MTPlan (MTPlanAttrs (Just mType) _ _) -> mType
-  MTPlan (MTPlanAttrs Nothing _ _)      -> MTApplicationJSON
-  other                                 -> other
+    checkSingularNullStrip ["nulls=stripped"] = MTSingularJSON True
+    checkSingularNullStrip  _                 = MTSingularJSON False
+
+    getPlan fmt rest =
+      let
+        opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)
+        inOpts str   = str `elem` opts
+        dropAround p = BS.dropWhile p . BS.dropWhileEnd p
+        mtFor        = fromMaybe MTApplicationJSON $ do
+          foundFor    <- find (BS.isPrefixOf "for=") rest
+          strippedFor <- BS.stripPrefix "for=" foundFor
+          pure . decodeMediaType $ dropAround (== BS.c2w '"') strippedFor
+      in
+      MTPlan mtFor fmt $
+        [PlanAnalyze  | inOpts "analyze" ] ++
+        [PlanVerbose  | inOpts "verbose" ] ++
+        [PlanSettings | inOpts "settings"] ++
+        [PlanBuffers  | inOpts "buffers" ] ++
+        [PlanWAL      | inOpts "wal"     ]

src/PostgREST/Plan.hs

@@ -13,7 +13,6 @@ resource.
 {-# LANGUAGE DuplicateRecordFields #-}
 {-# LANGUAGE LambdaCase            #-}
 {-# LANGUAGE NamedFieldPuns        #-}
-{-# LANGUAGE OverloadedRecordDot   #-}
 {-# LANGUAGE RecordWildCards       #-}
 
 module PostgREST.Plan
@@ -26,9 +25,9 @@ module PostgREST.Plan
   , inspectPlanTxMode
   ) where
 
-
 import qualified Data.ByteString.Lazy          as LBS
 import qualified Data.HashMap.Strict           as HM
+import qualified Data.HashMap.Strict.InsOrd    as HMI
 import qualified Data.List                     as L
 import qualified Data.Set                      as S
 import qualified PostgREST.SchemaCache.Routine as Routine
@@ -37,35 +36,40 @@ import Data.Either.Combinators (mapLeft, mapRight)
 import Data.List               (delete)
 import Data.Tree               (Tree (..))
 
-import PostgREST.ApiRequest               (Action (..),
-                                           ApiRequest (..),
-                                           InvokeMethod (..),
-                                           Mutation (..),
-                                           Payload (..))
-import PostgREST.Config                   (AppConfig (..))
-import PostgREST.Error                    (Error (..))
-import PostgREST.MediaType                (MTPlanAttrs (..),
-                                           MediaType (..))
-import PostgREST.Query.SqlFragment        (sourceCTEName)
-import PostgREST.RangeQuery               (NonnegRange, allRange,
-                                           convertToLimitZeroRange,
-                                           restrictRange)
-import PostgREST.SchemaCache              (SchemaCache (..))
-import PostgREST.SchemaCache.Identifiers  (FieldName,
-                                           QualifiedIdentifier (..),
-                                           Schema)
-import PostgREST.SchemaCache.Relationship (Cardinality (..),
-                                           Junction (..),
-                                           Relationship (..),
-                                           RelationshipsMap,
-                                           relIsToOne)
-import PostgREST.SchemaCache.Routine      (Routine (..), RoutineMap,
-                                           RoutineParam (..),
-                                           funcReturnsCompositeAlias,
-                                           funcReturnsScalar,
-                                           funcReturnsSetOfScalar)
-import PostgREST.SchemaCache.Table        (Table (tableName),
-                                           tablePKCols)
+import PostgREST.ApiRequest                  (Action (..),
+                                              ApiRequest (..),
+                                              InvokeMethod (..),
+                                              Mutation (..),
+                                              Payload (..))
+import PostgREST.Config                      (AppConfig (..))
+import PostgREST.Error                       (Error (..))
+import PostgREST.MediaType                   (MediaType (..))
+import PostgREST.Query.SqlFragment           (sourceCTEName)
+import PostgREST.RangeQuery                  (NonnegRange, allRange,
+                                              convertToLimitZeroRange,
+                                              restrictRange)
+import PostgREST.SchemaCache                 (SchemaCache (..))
+import PostgREST.SchemaCache.Identifiers     (FieldName,
+                                              QualifiedIdentifier (..),
+                                              Schema)
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
+                                              Junction (..),
+                                              Relationship (..),
+                                              RelationshipsMap,
+                                              relIsToOne)
+import PostgREST.SchemaCache.Representations (DataRepresentation (..),
+                                              RepresentationsMap)
+import PostgREST.SchemaCache.Routine         (ResultAggregate (..),
+                                              Routine (..),
+                                              RoutineMap,
+                                              RoutineParam (..),
+                                              funcReturnsCompositeAlias,
+                                              funcReturnsScalar,
+                                              funcReturnsSetOfScalar)
+import PostgREST.SchemaCache.Table           (Column (..), Table (..),
+                                              TablesMap,
+                                              tableColumnsList,
+                                              tablePKCols)
 
 import PostgREST.ApiRequest.Preferences
 import PostgREST.ApiRequest.Types
@@ -79,16 +83,21 @@ import qualified PostgREST.ApiRequest.QueryParams as QueryParams
 
 import Protolude hiding (from)
 
+-- $setup
+-- Setup for doctests
+-- >>> import Data.Ranged.Ranges (fullRange)
+
 data WrappedReadPlan = WrappedReadPlan {
   wrReadPlan :: ReadPlanTree
 , wrTxMode   :: SQL.Mode
-, wrBinField :: Maybe FieldName
+, wrResAgg   :: ResultAggregate
 }
 
 data MutateReadPlan = MutateReadPlan {
   mrReadPlan   :: ReadPlanTree
 , mrMutatePlan :: MutatePlan
 , mrTxMode     :: SQL.Mode
+, mrResAgg     :: ResultAggregate
 }
 
 data CallReadPlan = CallReadPlan {
@@ -96,20 +105,21 @@ data CallReadPlan = CallReadPlan {
 , crCallPlan :: CallPlan
 , crTxMode   :: SQL.Mode
 , crProc     :: Routine
-, crBinField :: Maybe FieldName
+, crResAgg   :: ResultAggregate
 }
 
 wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error WrappedReadPlan
 wrappedReadPlan  identifier conf sCache apiRequest = do
   rPlan <- readPlan identifier conf sCache apiRequest
   binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) Nothing rPlan
-  return $ WrappedReadPlan rPlan SQL.Read binField
+  return $ WrappedReadPlan rPlan SQL.Read $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
 
 mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error MutateReadPlan
 mutateReadPlan  mutation apiRequest identifier conf sCache = do
   rPlan <- readPlan identifier conf sCache apiRequest
+  binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) Nothing rPlan
   mPlan <- mutatePlan mutation identifier apiRequest sCache rPlan
-  return $ MutateReadPlan rPlan mPlan SQL.Write
+  return $ MutateReadPlan rPlan mPlan SQL.Write $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
 
 callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CallReadPlan
 callReadPlan identifier conf sCache apiRequest invMethod = do
@@ -134,7 +144,7 @@ callReadPlan identifier conf sCache apiRequest invMethod = do
           (InvPost, Routine.Volatile)  -> SQL.Write
       cPlan = callPlan proc apiRequest paramKeys args rPlan
   binField <- mapLeft ApiRequestError $ binaryField conf (iAcceptMediaType apiRequest) (Just proc) rPlan
-  return $ CallReadPlan rPlan cPlan txMode proc binField
+  return $ CallReadPlan rPlan cPlan txMode proc $ mediaToAggregate (iAcceptMediaType apiRequest) binField apiRequest
   where
     Preferences{..} = iPreferences apiRequest
     qsParams' = QueryParams.qsParams (iQueryParams apiRequest)
@@ -199,26 +209,93 @@ findProc qi argumentsKeys paramsAsSingleObject allProcs contentMediaType isInvPo
 inspectPlanTxMode :: SQL.Mode
 inspectPlanTxMode = SQL.Read
 
+-- | During planning we need to resolve Field -> CoercibleField (finding the context specific target type and map function).
+-- | ResolverContext facilitates this without the need to pass around a laundry list of parameters.
+data ResolverContext = ResolverContext
+  { tables          :: TablesMap
+  , representations :: RepresentationsMap
+  , qi              :: QualifiedIdentifier  -- ^ The table we're currently attending; changes as we recurse into joins etc.
+  , outputType      :: Text                 -- ^ The output type for the response payload; e.g. "csv", "json", "binary".
+  }
+
+resolveColumnField :: Column -> CoercibleField
+resolveColumnField col = CoercibleField (colName col) mempty False (colNominalType col) Nothing (colDefault col)
+
+resolveTableFieldName :: Table -> FieldName -> CoercibleField
+resolveTableFieldName table fieldName =
+  fromMaybe (unknownField fieldName []) $ HMI.lookup fieldName (tableColumns table) >>=
+    Just . resolveColumnField
+
+-- | Resolve a type within the context based on the given field name and JSON path. Although there are situations where failure to resolve a field is considered an error (see `resolveOrError`), there are also situations where we allow it (RPC calls). If it should be an error and `resolveOrError` doesn't fit, ensure to check the `cfIRType` isn't empty.
+resolveTypeOrUnknown :: ResolverContext -> Field -> CoercibleField
+resolveTypeOrUnknown ResolverContext{..} (fn, jp) =
+  case res of
+    -- types that are already json/jsonb don't need to be converted with `to_jsonb` for using arrow operators `data->attr`
+    -- this prevents indexes not applying https://github.com/PostgREST/postgrest/issues/2594
+    cf@CoercibleField{cfIRType="json"}  -> cf{cfJsonPath=jp, cfToJson=False}
+    cf@CoercibleField{cfIRType="jsonb"} -> cf{cfJsonPath=jp, cfToJson=False}
+    -- other types will get converted `to_jsonb(col)->attr`, even unknown types
+    cf                                  -> cf{cfJsonPath=jp, cfToJson=True}
+  where
+    res = fromMaybe (unknownField fn jp) $ HM.lookup qi tables >>=
+          Just . flip resolveTableFieldName fn
+
+-- | Install any pre-defined data representation from source to target to coerce this reference.
+--
+-- Note that we change the IR type here. This might seem unintuitive. The short of it is that for a CoercibleField without a transformer, input type == output type. A transformer maps from a -> b, so by definition the input type will be a and the output type b after. And cfIRType is the *input* type.
+--
+-- It might feel odd that once a transformer is added we 'forget' the target type (because now a /= b). You might also note there's no obvious way to stack transforms (even if there was a stack, you erased what type you're working with so it's awkward). Alas as satisfying as it would be to engineer a layered mapping system with full type information, we just don't need it.
+withTransformer :: ResolverContext -> Text -> Text -> CoercibleField -> CoercibleField
+withTransformer ResolverContext{representations} sourceType targetType field =
+  fromMaybe field $ HM.lookup (sourceType, targetType) representations >>=
+    (\fieldRepresentation -> Just field{cfIRType=sourceType, cfTransform=Just (drFunction fieldRepresentation)})
+
+-- | Map the intermediate representation type to the output type, if available.
+withOutputFormat :: ResolverContext -> CoercibleField -> CoercibleField
+withOutputFormat ctx@ResolverContext{outputType} field@CoercibleField{cfIRType} = withTransformer ctx cfIRType outputType field
+
+-- | Map text into the intermediate representation type, if available.
+withTextParse :: ResolverContext -> CoercibleField -> CoercibleField
+withTextParse ctx field@CoercibleField{cfIRType} = withTransformer ctx "text" cfIRType field
+
+-- | Map json into the intermediate representation type, if available.
+withJsonParse :: ResolverContext -> CoercibleField -> CoercibleField
+withJsonParse ctx field@CoercibleField{cfIRType} = withTransformer ctx "json" cfIRType field
+
+-- | Map the intermediate representation type to the output type defined by the resolver context (normally json), if available.
+resolveOutputField :: ResolverContext -> Field -> CoercibleField
+resolveOutputField ctx field = withOutputFormat ctx $ resolveTypeOrUnknown ctx field
+
+-- | Map the query string format of a value (text) into the intermediate representation type, if available.
+resolveQueryInputField :: ResolverContext -> Field -> CoercibleField
+resolveQueryInputField ctx field = withTextParse ctx $ resolveTypeOrUnknown ctx field
+
 -- | Builds the ReadPlan tree on a number of stages.
 -- | Adds filters, order, limits on its respective nodes.
 -- | Adds joins conditions obtained from resource embedding.
 readPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error ReadPlanTree
-readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows} SchemaCache{dbRelationships} apiRequest  =
-  mapLeft ApiRequestError $
-  treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
-  addNullEmbedFilters =<<
-  validateSpreadEmbeds =<<
-  addRelatedOrders =<<
-  addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
-  addLogicTrees apiRequest =<<
-  addRanges apiRequest =<<
-  addOrders apiRequest =<<
-  addFilters apiRequest (initReadRequest qi $ QueryParams.qsSelect $ iQueryParams apiRequest)
+readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows} SchemaCache{dbTables, dbRelationships, dbRepresentations} apiRequest  =
+  let
+    -- JSON output format hardcoded for now. In the future we might want to support other output mappings such as CSV.
+    ctx = ResolverContext dbTables dbRepresentations qi "json"
+  in
+    mapLeft ApiRequestError $
+    treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
+    addNullEmbedFilters =<<
+    validateSpreadEmbeds =<<
+    addRelatedOrders =<<
+    addDataRepresentationAliases =<<
+    expandStarsForDataRepresentations ctx =<<
+    addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
+    addLogicTrees ctx apiRequest =<<
+    addRanges apiRequest =<<
+    addOrders ctx apiRequest =<<
+    addFilters ctx apiRequest (initReadRequest ctx $ QueryParams.qsSelect $ iQueryParams apiRequest)
 
 -- Build the initial read plan tree
-initReadRequest :: QualifiedIdentifier -> [Tree SelectItem] -> ReadPlanTree
-initReadRequest qi@QualifiedIdentifier{..} =
-  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi, relName=qiName, depth=rootDepth} []
+initReadRequest :: ResolverContext -> [Tree SelectItem] -> ReadPlanTree
+initReadRequest ctx@ResolverContext{qi=QualifiedIdentifier{..}} =
+  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi ctx, relName=qiName, depth=rootDepth} []
   where
     rootDepth = 0
     defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing False rootDepth
@@ -237,7 +314,49 @@ initReadRequest qi@QualifiedIdentifier{..} =
             (Node defReadPlan{from=QualifiedIdentifier qiSchema selRelation, relName=selRelation, relHint=selHint, relJoinType=selJoinType, depth=nxtDepth, relIsSpread=True} [])
             fldForest:rForest
         SelectField{..} ->
-          Node q{select=(selField, selCast, selAlias):select q} rForest
+          Node q{select=(resolveOutputField ctx{qi=from q} selField, selCast, selAlias):select q} rForest
+
+-- | Preserve the original field name if data representation is used to coerce the value.
+addDataRepresentationAliases :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addDataRepresentationAliases rPlanTree = Right $ fmap (\rPlan@ReadPlan{select=sel} -> rPlan{select=map aliasSelectItem sel}) rPlanTree
+  where
+    aliasSelectItem :: (CoercibleField, Maybe Cast, Maybe Alias) -> (CoercibleField, Maybe Cast, Maybe Alias)
+    -- If there already is an alias, don't overwrite it.
+    aliasSelectItem (fld@(CoercibleField{cfName=fieldName, cfTransform=(Just _)}), Nothing, Nothing) = (fld, Nothing, Just fieldName)
+    aliasSelectItem fld = fld
+
+knownColumnsInContext :: ResolverContext -> [Column]
+knownColumnsInContext ResolverContext{..} =
+  fromMaybe [] $ HM.lookup qi tables >>=
+  Just . tableColumnsList
+
+-- | Expand "select *" into explicit field names of the table, if necessary to apply data representations.
+expandStarsForDataRepresentations :: ResolverContext -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+expandStarsForDataRepresentations ctx@ResolverContext{qi} rPlanTree = Right $ fmap expandStars rPlanTree
+  where
+    expandStars :: ReadPlan -> ReadPlan
+    -- When the schema is "" and the table is the source CTE, we assume the true source table is given in the from
+    -- alias and belongs to the request schema. See the bit in `addRels` with `newFrom = ...`.
+    expandStars rPlan@ReadPlan{from=(QualifiedIdentifier "" "pgrst_source"), fromAlias=(Just tblAlias)} =
+      expandStarsForTable ctx{qi=qi{qiName=tblAlias}} rPlan
+    expandStars rPlan@ReadPlan{from=fromTable} =
+      expandStarsForTable ctx{qi=fromTable} rPlan
+
+expandStarsForTable :: ResolverContext -> ReadPlan -> ReadPlan
+expandStarsForTable ctx@ResolverContext{representations, outputType} rplan@ReadPlan{select=selectItems} =
+  -- If we have a '*' select AND the target table has at least one data representation, expand.
+  if ("*" `elem` map (\(field, _, _) -> cfName field) selectItems) && any hasOutputRep knownColumns
+    then rplan{select=concatMap (expandStarSelectItem knownColumns) selectItems}
+    else rplan
+  where
+    knownColumns = knownColumnsInContext ctx
+
+    hasOutputRep :: Column -> Bool
+    hasOutputRep col = HM.member (colNominalType col, outputType) representations
+
+    expandStarSelectItem :: [Column] -> (CoercibleField, Maybe Cast, Maybe Alias) -> [(CoercibleField, Maybe Cast, Maybe Alias)]
+    expandStarSelectItem columns (CoercibleField{cfName="*", cfJsonPath=[]}, b, c) = map (\col -> (withOutputFormat ctx $ resolveColumnField col, b, c)) columns
+    expandStarSelectItem _ selectItem = [selectItem]
 
 -- | Enforces the `max-rows` config on the result
 treeRestrictRange :: Maybe Integer -> Action -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -261,7 +380,7 @@ addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,re
             Relationship{relCardinality=M2M _} -> -- m2m does internal implicit joins that don't need aliasing
               rPlan{from=relForeignTable r, relToParent=Just r, relAggAlias=aggAlias, relJoinConds=getJoinConditions Nothing parentAlias r}
             ComputedRelationship{} ->
-              rPlan{from=relForeignTable r, relToParent=Just r{relTable=maybe (relTable r) (QualifiedIdentifier mempty) parentAlias}, relAggAlias=aggAlias, fromAlias=newAlias}
+              rPlan{from=relForeignTable r, relToParent=Just r{relTableAlias=maybe (relTable r) (QualifiedIdentifier mempty) parentAlias}, relAggAlias=aggAlias, fromAlias=newAlias}
             _ ->
               rPlan{from=relForeignTable r, relToParent=Just r, relAggAlias=aggAlias, fromAlias=newAlias, relJoinConds=getJoinConditions newAlias parentAlias r}
           ) <$> rel
@@ -273,7 +392,7 @@ addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,re
       Node <$> newReadPlan <*> (updateForest . hush $ Node <$> newReadPlan <*> pure forest)
     Nothing -> -- root case
       let
-        newFrom  = QualifiedIdentifier mempty $ decodeUtf8 sourceCTEName
+        newFrom  = QualifiedIdentifier mempty sourceCTEName
         newAlias = Just (qiName $ from rPlan)
         newReadPlan = case action of
           -- the CTE for mutations/rpc is used as WITH sourceCTEName .. SELECT .. FROM sourceCTEName as alias,
@@ -363,7 +482,9 @@ findRel schema allRels origin target hint =
             target == qiName relForeignTable && isO2M relCardinality
             && matchFKRefSingleCol hnt relCardinality -- auditor
         else case hint of
+          -- DEPRECATED(remove after 2 major releases since v11.1.0): remove target
           -- target = table / view / constraint / column-from-origin (constraint/column-from-origin can only come from tables https://github.com/PostgREST/postgrest/issues/2277)
+          -- DEPRECATED(remove after 2 major releases since v11.1.0): remove hint as table/view/columns and only leave it as constraint
           -- hint   = table / view / constraint / column-from-origin / column-from-target (hint can take table / view values to aid in finding the junction in an m2m relationship)
           Nothing ->
               -- /projects?select=clients(*)
@@ -392,8 +513,8 @@ findRel schema allRels origin target hint =
             )
       ) $ fromMaybe mempty $ HM.lookup (QualifiedIdentifier schema origin, schema) allRels
 
-addFilters :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addFilters ApiRequest{..} rReq =
+addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addFilters ctx ApiRequest{..} rReq =
   foldr addFilterToNode (Right rReq) flts
   where
     QueryParams.QueryParams{..} = iQueryParams
@@ -405,10 +526,10 @@ addFilters ApiRequest{..} rReq =
 
     addFilterToNode :: (EmbedPath, Filter) -> Either ApiRequestError ReadPlanTree ->  Either ApiRequestError ReadPlanTree
     addFilterToNode =
-      updateNode (\flt (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest flt lf}  f)
+      updateNode (\flt (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest (resolveFilter ctx{qi=fromTable} flt) lf}  f)
 
-addOrders :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addOrders ApiRequest{..} rReq =
+addOrders :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addOrders ctx ApiRequest{..} rReq =
   case iAction of
     ActionMutate _ -> Right rReq
     _              -> foldr addOrderToNode (Right rReq) qsOrder
@@ -416,48 +537,115 @@ addOrders ApiRequest{..} rReq =
     QueryParams.QueryParams{..} = iQueryParams
 
     addOrderToNode :: (EmbedPath, [OrderTerm]) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
-    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=o} f)
+    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=resolveOrder ctx <$> o} f)
+
+resolveOrder :: ResolverContext -> OrderTerm -> CoercibleOrderTerm
+resolveOrder _ (OrderRelationTerm a b c d) = CoercibleOrderRelationTerm a b c d
+resolveOrder ctx (OrderTerm fld dir nulls) = CoercibleOrderTerm (resolveTypeOrUnknown ctx fld) dir nulls
 
 -- Validates that the related resource on the order is an embedded resource,
 -- e.g. if `clients` is inside the `select` in /projects?order=clients(id)&select=*,clients(*),
 -- and if it's a to-one relationship, it adds the right alias to the OrderRelationTerm so the generated query can succeed.
--- TODO might be clearer if there's an additional intermediate type
 addRelatedOrders :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
-  newOrder <- getRelOrder `traverse` order
+  newOrder <- newRelOrder `traverse` order
   Node rp{order=newOrder} <$> addRelatedOrders `traverse` forest
   where
-    getRelOrder ot@OrderTerm{}                   = Right ot
-    getRelOrder ot@OrderRelationTerm{otRelation} =
-      let foundRP = rootLabel <$> find (\(Node ReadPlan{relName, relAlias} _) -> otRelation == fromMaybe relName relAlias) forest in
+    newRelOrder cot@CoercibleOrderTerm{}                   = Right cot
+    newRelOrder cot@CoercibleOrderRelationTerm{coRelation} =
+      let foundRP = rootLabel <$> find (\(Node ReadPlan{relName, relAlias} _) -> coRelation == fromMaybe relName relAlias) forest in
       case foundRP of
         Just ReadPlan{relName,relAlias,relAggAlias,relToParent} ->
           let isToOne = relIsToOne <$> relToParent
               name    = fromMaybe relName relAlias in
           if isToOne == Just True
-            then Right $ ot{otRelation=relAggAlias}
+            then Right $ cot{coRelation=relAggAlias}
             else Left $ RelatedOrderNotToOne (qiName from) name
         Nothing ->
-          Left $ NotEmbedded otRelation
+          Left $ NotEmbedded coRelation
 
--- Searches for null filters on embeds, e.g. `clients` on /projects?select=*,clients()&clients=not.is.null.
--- If these are found, it changes the filter to use the internal aggregate name(`projects_clients_1`) so the filter can succeed.
--- It fails if operators other than is.null or not.is.null are used.
+-- | Searches for null filters on embeds, e.g. `projects=not.is.null` on `GET /clients?select=*,projects(*)&projects=not.is.null`
+--
+-- (It doesn't err but uses an Either ApiRequestError type so it can combine with the other functions that modify the read plan tree)
+--
+-- Setup:
+--
+-- >>> let nullOp    = OpExpr True (Is TriNull)
+-- >>> let nonNullOp = OpExpr False (Is TriNull)
+-- >>> let notEqOp   = OpExpr True (Op OpNotEqual "val")
+-- >>> :{
+-- -- this represents the `projects(*)` part on `/clients?select=*,projects(*)`
+-- let
+-- subForestPlan =
+--   [
+--     Node {
+--       rootLabel = ReadPlan {
+--         select = [], -- there will be fields at this stage but we just omit them for brevity
+--         from = QualifiedIdentifier {qiSchema = "test", qiName = "projects"},
+--         fromAlias = Just "projects_1", where_ = [], order = [], range_ = fullRange,
+--         relName = "projects",
+--         relToParent = Nothing,
+--         relJoinConds = [],
+--         relAlias = Nothing, relAggAlias = "clients_projects_1", relHint = Nothing, relJoinType = Nothing, relIsSpread = False, depth = 1
+--       },
+--       subForest = []
+--     }
+--   ]
+-- :}
+--
+-- >>> :{
+-- -- this represents the full URL `/clients?select=*,projects(*)&projects=not.is.null`, if subForst takes the above subForestPlan and nullOp
+-- let
+-- readPlanTree op subForst =
+--   Node {
+--     rootLabel = ReadPlan {
+--       select = [], -- there will be fields at this stage but we just omit them for brevity
+--       from = QualifiedIdentifier { qiSchema = "test", qiName = "clients"},
+--       fromAlias = Nothing,
+--       where_ = [
+--         CoercibleStmnt (
+--           CoercibleFilter {
+--            field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson=False, cfIRType = "", cfTransform = Nothing, cfDefault = Nothing},
+--            opExpr = op
+--           }
+--         )
+--       ],
+--       order = [], range_ = fullRange, relName = "clients", relToParent = Nothing, relJoinConds = [], relAlias = Nothing, relAggAlias = "", relHint = Nothing,
+--       relJoinType = Nothing, relIsSpread = False, depth = 0
+--     },
+--     subForest = subForst
+--   }
+-- :}
+--
+-- Don't do anything to the filter if there's no embedding (a subtree) on projects. Assume it's a normal filter.
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp [])
+-- Right [CoercibleStmnt (CoercibleFilter {field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson = False, cfIRType = "", cfTransform = Nothing, cfDefault = Nothing}, opExpr = OpExpr True (Is TriNull)})]
+--
+-- If there's an embedding on projects, then change the filter to use the internal aggregate name (`clients_projects_1`) so the filter can succeed later.
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp subForestPlan)
+-- Right [CoercibleStmnt (CoercibleFilterNullEmbed True "clients_projects_1")]
+--
+-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nonNullOp subForestPlan)
+-- Right [CoercibleStmnt (CoercibleFilterNullEmbed False "clients_projects_1")]
 addNullEmbedFilters :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addNullEmbedFilters (Node rp@ReadPlan{where_=oldLogic} forest) = do
-  let readPlans = rootLabel <$> forest
-  newLogic <- getFilters readPlans `traverse` oldLogic
+addNullEmbedFilters (Node rp@ReadPlan{where_=curLogic} forest) = do
+  let forestReadPlans = rootLabel <$> forest
+  newLogic <- newNullFilters forestReadPlans `traverse` curLogic
   Node rp{ReadPlan.where_= newLogic} <$> (addNullEmbedFilters `traverse` forest)
   where
-    getFilters :: [ReadPlan] -> LogicTree -> Either ApiRequestError LogicTree
-    getFilters rPlans (Expr b lOp trees) = Expr b lOp <$> (getFilters rPlans `traverse` trees)
-    getFilters rPlans flt@(Stmnt (Filter (fld, []) opExpr)) =
-      let foundRP = find (\ReadPlan{relName, relAlias} -> fld == fromMaybe relName relAlias) rPlans in
-      case (foundRP, opExpr) of
-        (Just ReadPlan{relAggAlias}, OpExpr b (Is TriNull)) -> Right $ Stmnt $ FilterNullEmbed b relAggAlias
-        (Just ReadPlan{relName}, _)                         -> Left $ UnacceptableFilter relName
-        _                                                   -> Right flt
-    getFilters _ flt@(Stmnt _)        = Right flt
+    newNullFilters :: [ReadPlan] -> CoercibleLogicTree -> Either ApiRequestError CoercibleLogicTree
+    newNullFilters rPlans = \case
+      (CoercibleExpr b lOp trees) ->
+        CoercibleExpr b lOp <$> (newNullFilters rPlans `traverse` trees)
+      flt@(CoercibleStmnt (CoercibleFilter (CoercibleField fld [] _ _ _ _) opExpr)) ->
+        let foundRP = find (\ReadPlan{relName, relAlias} -> fld == fromMaybe relName relAlias) rPlans in
+        case (foundRP, opExpr) of
+          (Just ReadPlan{relAggAlias}, OpExpr b (Is TriNull)) -> Right $ CoercibleStmnt $ CoercibleFilterNullEmbed b relAggAlias
+          _                                                   -> Right flt
+      flt@(CoercibleStmnt _) ->
+        Right flt
 
 addRanges :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRanges ApiRequest{..} rReq =
@@ -471,14 +659,21 @@ addRanges ApiRequest{..} rReq =
     addRangeToNode :: (EmbedPath, NonnegRange) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
     addRangeToNode = updateNode (\r (Node q f) -> Node q{range_=r} f)
 
-addLogicTrees :: ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
-addLogicTrees ApiRequest{..} rReq =
+addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addLogicTrees ctx ApiRequest{..} rReq =
   foldr addLogicTreeToNode (Right rReq) qsLogic
   where
     QueryParams.QueryParams{..} = iQueryParams
 
     addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
-    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{where_=lf} f) -> Node q{ReadPlan.where_=t:lf} f)
+    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)
+
+resolveLogicTree :: ResolverContext -> LogicTree -> CoercibleLogicTree
+resolveLogicTree ctx (Stmnt flt) = CoercibleStmnt $ resolveFilter ctx flt
+resolveLogicTree ctx (Expr b op lts) = CoercibleExpr b op (map (resolveLogicTree ctx) lts)
+
+resolveFilter :: ResolverContext -> Filter -> CoercibleFilter
+resolveFilter ctx (Filter fld opExpr) = CoercibleFilter{field=resolveQueryInputField ctx fld, opExpr=opExpr}
 
 -- Validates that spread embeds are only done on to-one relationships
 validateSpreadEmbeds :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
@@ -504,10 +699,10 @@ updateNode f (targetNodeName:remainingPath, a) (Right (Node rootNode forest)) =
     findNode = find (\(Node ReadPlan{relName, relAlias} _) -> relName == targetNodeName || relAlias == Just targetNodeName) forest
 
 mutatePlan :: Mutation -> QualifiedIdentifier -> ApiRequest -> SchemaCache -> ReadPlanTree -> Either Error MutatePlan
-mutatePlan mutation qi ApiRequest{iPreferences=preferences, ..} sCache readReq = mapLeft ApiRequestError $
+mutatePlan mutation qi ApiRequest{iPreferences=Preferences{..}, ..} SchemaCache{dbTables, dbRepresentations} readReq = mapLeft ApiRequestError $
   case mutation of
     MutationCreate ->
-      mapRight (\typedColumns -> Insert qi typedColumns body ((,) <$> preferences.preferResolution <*> Just confCols) [] returnings pkCols applyDefaults) typedColumnsOrError
+      mapRight (\typedColumns -> Insert qi typedColumns body ((,) <$> preferResolution <*> Just confCols) [] returnings pkCols applyDefaults) typedColumnsOrError
     MutationUpdate ->
       mapRight (\typedColumns -> Update qi typedColumns body combinedLogic iTopLevelRange rootOrder returnings applyDefaults) typedColumnsOrError
     MutationSingleUpsert ->
@@ -522,27 +717,28 @@ mutatePlan mutation qi ApiRequest{iPreferences=preferences, ..} sCache readReq =
           Left InvalidFilters
     MutationDelete -> Right $ Delete qi combinedLogic iTopLevelRange rootOrder returnings
   where
+    ctx = ResolverContext dbTables dbRepresentations qi "json"
     confCols = fromMaybe pkCols qsOnConflict
     QueryParams.QueryParams{..} = iQueryParams
     returnings =
-      if preferences.preferRepresentation == None
+      if preferRepresentation == Just None || isNothing preferRepresentation
         then []
         else inferColsEmbedNeeds readReq pkCols
-    pkCols = maybe mempty tablePKCols $ HM.lookup qi $ dbTables sCache
-    logic = map snd qsLogic
-    rootOrder = maybe [] snd $ find (\(x, _) -> null x) qsOrder
-    combinedLogic = foldr addFilterToLogicForest logic qsFiltersRoot
+    tbl = HM.lookup qi dbTables
+    pkCols = maybe mempty tablePKCols tbl
+    logic = map (resolveLogicTree ctx . snd) qsLogic
+    rootOrder = resolveOrder ctx <$> maybe [] snd (find (\(x, _) -> null x) qsOrder)
+    combinedLogic = foldr (addFilterToLogicForest . resolveFilter ctx) logic qsFiltersRoot
     body = payRaw <$> iPayload -- the body is assumed to be json at this stage(ApiRequest validates)
-    tbl = HM.lookup qi $ dbTables sCache
-    typedColumnsOrError = resolveOrError tbl `traverse` S.toList iColumns
-    applyDefaults = preferences.preferMissing == Just ApplyDefaults
+    applyDefaults = preferMissing == Just ApplyDefaults
+    typedColumnsOrError = resolveOrError ctx tbl `traverse` S.toList iColumns
 
-resolveOrError :: Maybe Table -> FieldName -> Either ApiRequestError TypedField
-resolveOrError Nothing _ = Left NotFound
-resolveOrError (Just table) field =
-  case resolveTableField table field of
-    Nothing         -> Left $ ColumnNotFound (tableName table) field
-    Just typedField -> Right typedField
+resolveOrError :: ResolverContext -> Maybe Table -> FieldName -> Either ApiRequestError CoercibleField
+resolveOrError _ Nothing _ = Left NotFound
+resolveOrError ctx (Just table) field =
+  case resolveTableFieldName table field of
+    CoercibleField{cfIRType=""} -> Left $ ColumnNotFound (tableName table) field
+    cf                          -> Right $ withJsonParse ctx cf
 
 callPlan :: Routine -> ApiRequest -> S.Set FieldName -> LBS.ByteString -> ReadPlanTree -> CallPlan
 callPlan proc ApiRequest{iPreferences=Preferences{..}} paramKeys args readReq = FunctionCall {
@@ -571,7 +767,7 @@ inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols
   | "*" `elem` fldNames = ["*"]
   | otherwise           = returnings
   where
-    fldNames = (\((fld, _), _, _) -> fld) <$> select
+    fldNames = cfName . (\(f, _, _) -> f) <$> select
     -- Without fkCols, when a mutatePlan to
     -- /projects?select=name,clients(name) occurs, the RETURNING SQL part would
     -- be `RETURNING name`(see QueryBuilder).  This would make the embedding
@@ -610,8 +806,8 @@ inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols
 
 -- Traditional filters(e.g. id=eq.1) are added as root nodes of the LogicTree
 -- they are later concatenated with AND in the QueryBuilder
-addFilterToLogicForest :: Filter -> [LogicTree] -> [LogicTree]
-addFilterToLogicForest flt lf = Stmnt flt : lf
+addFilterToLogicForest :: CoercibleFilter -> [CoercibleLogicTree] -> [CoercibleLogicTree]
+addFilterToLogicForest flt lf = CoercibleStmnt flt : lf
 
 -- | If raw(binary) output is requested, check that MediaType is one of the
 -- admitted rawMediaTypes and that`?select=...` contains only one field other
@@ -634,12 +830,43 @@ binaryField AppConfig{configRawMediaTypes} acceptMediaType proc rpTree
   where
     isRawMediaType = acceptMediaType `elem` configRawMediaTypes `L.union` [MTOctetStream, MTTextPlain, MTTextXML] || isRawPlan acceptMediaType
     isRawPlan mt = case mt of
-      MTPlan (MTPlanAttrs (Just MTOctetStream) _ _) -> True
-      MTPlan (MTPlanAttrs (Just MTTextPlain) _ _)   -> True
-      MTPlan (MTPlanAttrs (Just MTTextXML) _ _)     -> True
-      _                                             -> False
+      MTPlan MTOctetStream _ _ -> True
+      MTPlan MTTextPlain _ _   -> True
+      MTPlan MTTextXML _ _     -> True
+      _                        -> False
 
     fstFieldName :: ReadPlanTree -> Maybe FieldName
-    fstFieldName (Node ReadPlan{select=(("*", []), _, _):_} [])  = Nothing
-    fstFieldName (Node ReadPlan{select=[((fld, []), _, _)]} []) = Just fld
+    fstFieldName (Node ReadPlan{select=(CoercibleField{cfName="*", cfJsonPath=[]}, _, _):_} [])  = Nothing
+    fstFieldName (Node ReadPlan{select=[(CoercibleField{cfName=fld, cfJsonPath=[]}, _, _)]} [])  = Just fld
     fstFieldName _                                               = Nothing
+
+
+mediaToAggregate :: MediaType -> Maybe FieldName -> ApiRequest -> ResultAggregate
+mediaToAggregate mt binField apiReq@ApiRequest{iAction=act, iPreferences=Preferences{preferRepresentation=rep}} =
+  if noAgg then NoAgg
+  else case mt of
+    MTApplicationJSON     -> BuiltinAggJson
+    MTSingularJSON strip  -> BuiltinAggSingleJson strip
+    MTArrayJSONStrip      -> BuiltinAggArrayJsonStrip
+    MTGeoJSON             -> BuiltinAggGeoJson
+    MTTextCSV             -> BuiltinAggCsv
+    MTAny                 -> BuiltinAggJson
+    MTOpenAPI             -> BuiltinAggJson
+    MTUrlEncoded          -> NoAgg -- TODO: unreachable since a previous step (producedMediaTypes) whitelists the media types that can become aggregates.
+
+    -- binary types
+    MTTextPlain           -> BuiltinAggBinary binField
+    MTTextXML             -> BuiltinAggXml binField
+    MTOctetStream         -> BuiltinAggBinary binField
+    MTOther _             -> BuiltinAggBinary binField
+
+    -- Doing `Accept: application/vnd.pgrst.plan; for="application/vnd.pgrst.plan"` doesn't make sense, so we just empty the body.
+    -- TODO: fail instead to be more strict
+    MTPlan (MTPlan{}) _ _ -> NoAgg
+    MTPlan media      _ _ -> mediaToAggregate media binField apiReq
+  where
+    noAgg = case act of
+      ActionMutate _         -> rep == Just HeadersOnly || rep == Just None || isNothing rep
+      ActionRead _isHead     -> _isHead -- no need for an aggregate on HEAD https://github.com/PostgREST/postgrest/issues/2849
+      ActionInvoke invMethod -> invMethod == InvHead
+      _                      -> False

src/PostgREST/Plan/MutatePlan.hs

@@ -6,8 +6,9 @@ where
 import qualified Data.ByteString.Lazy as LBS
 
 import PostgREST.ApiRequest.Preferences  (PreferResolution)
-import PostgREST.ApiRequest.Types        (LogicTree, OrderTerm)
-import PostgREST.Plan.Types              (TypedField)
+import PostgREST.Plan.Types              (CoercibleField,
+                                          CoercibleLogicTree,
+                                          CoercibleOrderTerm)
 import PostgREST.RangeQuery              (NonnegRange)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier)
@@ -18,28 +19,28 @@ import Protolude
 data MutatePlan
   = Insert
       { in_        :: QualifiedIdentifier
-      , insCols    :: [TypedField]
+      , insCols    :: [CoercibleField]
       , insBody    :: Maybe LBS.ByteString
       , onConflict :: Maybe (PreferResolution, [FieldName])
-      , where_     :: [LogicTree]
+      , where_     :: [CoercibleLogicTree]
       , returning  :: [FieldName]
       , insPkCols  :: [FieldName]
       , applyDefs  :: Bool
       }
   | Update
       { in_       :: QualifiedIdentifier
-      , updCols   :: [TypedField]
+      , updCols   :: [CoercibleField]
       , updBody   :: Maybe LBS.ByteString
-      , where_    :: [LogicTree]
+      , where_    :: [CoercibleLogicTree]
       , mutRange  :: NonnegRange
-      , mutOrder  :: [OrderTerm]
+      , mutOrder  :: [CoercibleOrderTerm]
       , returning :: [FieldName]
       , applyDefs :: Bool
       }
   | Delete
       { in_       :: QualifiedIdentifier
-      , where_    :: [LogicTree]
+      , where_    :: [CoercibleLogicTree]
       , mutRange  :: NonnegRange
-      , mutOrder  :: [OrderTerm]
+      , mutOrder  :: [CoercibleOrderTerm]
       , returning :: [FieldName]
       }

src/PostgREST/Plan/ReadPlan.hs

@@ -6,9 +6,11 @@ module PostgREST.Plan.ReadPlan
 
 import Data.Tree (Tree (..))
 
-import PostgREST.ApiRequest.Types         (Alias, Cast, Depth, Field,
-                                           Hint, JoinType, LogicTree,
-                                           NodeName, OrderTerm)
+import PostgREST.ApiRequest.Types         (Alias, Cast, Depth, Hint,
+                                           JoinType, NodeName)
+import PostgREST.Plan.Types               (CoercibleField (..),
+                                           CoercibleLogicTree,
+                                           CoercibleOrderTerm)
 import PostgREST.RangeQuery               (NonnegRange)
 import PostgREST.SchemaCache.Identifiers  (FieldName,
                                            QualifiedIdentifier)
@@ -23,14 +25,14 @@ data JoinCondition =
   JoinCondition
     (QualifiedIdentifier, FieldName)
     (QualifiedIdentifier, FieldName)
-  deriving (Eq)
+  deriving (Eq, Show)
 
 data ReadPlan = ReadPlan
-  { select       :: [(Field, Maybe Cast, Maybe Alias)]
+  { select       :: [(CoercibleField, Maybe Cast, Maybe Alias)]
   , from         :: QualifiedIdentifier
   , fromAlias    :: Maybe Alias
-  , where_       :: [LogicTree]
-  , order        :: [OrderTerm]
+  , where_       :: [CoercibleLogicTree]
+  , order        :: [CoercibleOrderTerm]
   , range_       :: NonnegRange
   , relName      :: NodeName
   , relToParent  :: Maybe Relationship
@@ -43,4 +45,4 @@ data ReadPlan = ReadPlan
   , depth        :: Depth
   -- ^ used for aliasing
   }
-  deriving (Eq)
+  deriving (Eq, Show)

src/PostgREST/Plan/Types.hs

@@ -1,24 +1,67 @@
 module PostgREST.Plan.Types
-  ( TypedField(..)
-  , resolveTableField
+  ( CoercibleField(..)
+  , unknownField
+  , CoercibleLogicTree(..)
+  , CoercibleFilter(..)
+  , TransformerProc
+  , CoercibleOrderTerm(..)
   ) where
 
-import qualified Data.HashMap.Strict.InsOrd as HMI
+import PostgREST.ApiRequest.Types (Field, JsonPath, LogicOperator,
+                                   OpExpr, OrderDirection, OrderNulls)
 
 import PostgREST.SchemaCache.Identifiers (FieldName)
-import PostgREST.SchemaCache.Table       (Column (..), Table (..))
 
 import Protolude
 
--- | A TypedField is a field with sufficient information to be read from JSON with `json_to_recordset`.
-data TypedField = TypedField
-   { tfName    :: FieldName
-   , tfIRType  :: Text -- ^ The initial type of the field, before any casting.
-   , tfDefault :: Maybe Text
-   } deriving (Eq)
+type TransformerProc = Text
 
-resolveTableField :: Table -> FieldName -> Maybe TypedField
-resolveTableField table fieldName =
-  case HMI.lookup fieldName (tableColumns table) of
-    Just column -> Just $ TypedField (colName column) (colNominalType column) (colDefault column)
-    Nothing     -> Nothing
+-- | A CoercibleField pairs the name of a query element with any type coercion information we need for some specific use case.
+-- |
+-- | As suggested by the name, it's often a reference to a field in a table but really it can be any nameable element (function parameter, calculation with an alias, etc) with a knowable type.
+-- |
+-- | In the simplest case, it allows us to parse JSON payloads with `json_to_recordset`, for which we need to know both the name and the type of each thing we'd like to extract. At a higher level, CoercibleField generalises to reflect that any value we work with in a query may need type specific handling.
+-- |
+-- | CoercibleField is the foundation for the Data Representations feature. This feature allow user-definable mappings between database types so that the same data can be presented or interpreted in various ways as needed. Sometimes the way Postgres coerces data implicitly isn't right for the job. Different mappings might be appropriate for different situations: parsing a filter from a query string requires one function (text -> field type) while parsing a payload from JSON takes another (json -> field type). And the reverse, outputting a field as JSON, requires yet a third (field type -> json). CoercibleField is that "job specific" reference to an element paired with the type we desire for that particular purpose and the function we'll use to get there, if any.
+-- |
+-- | In the planning phase, we "resolve" generic named elements into these specialised CoercibleFields. Again this is context specific: two different CoercibleFields both representing the exact same table column in the database, even in the same query, might have two different target types and mapping functions. For example, one might represent a column in a filter, and another the very same column in an output role to be sent in the response body.
+-- |
+-- | The type value is allowed to be the empty string. The analog here is soft type checking in programming languages: sometimes we don't need a variable to have a specified type and things will work anyhow. So the empty type variant is valid when we don't know and *don't need to know* about the specific type in some context. Note that this variation should not be used if it guarantees failure: in that case you should instead raise an error at the planning stage and bail out. For example, we can't parse JSON with `json_to_recordset` without knowing the types of each recipient field, and so error out. Using the empty string for the type would be incorrect and futile. On the other hand we use the empty type for RPC calls since type resolution isn't implemented for RPC, but it's fine because the query still works with Postgres' implicit coercion. In the future, hopefully we will support data representations across the board and then the empty type may be permanently retired.
+data CoercibleField = CoercibleField
+  { cfName      :: FieldName
+  , cfJsonPath  :: JsonPath
+  , cfToJson    :: Bool
+  , cfIRType    :: Text                  -- ^ The native Postgres type of the field, the intermediate (IR) type before mapping.
+  , cfTransform :: Maybe TransformerProc -- ^ The optional mapping from irType -> targetType.
+  , cfDefault   :: Maybe Text
+  } deriving (Eq, Show)
+
+unknownField :: FieldName -> JsonPath -> CoercibleField
+unknownField name path = CoercibleField name path False "" Nothing Nothing
+
+-- | Like an API request LogicTree, but with coercible field information.
+data CoercibleLogicTree
+  = CoercibleExpr Bool LogicOperator [CoercibleLogicTree]
+  | CoercibleStmnt CoercibleFilter
+  deriving (Eq, Show)
+
+data CoercibleFilter = CoercibleFilter
+  { field  :: CoercibleField
+  , opExpr :: OpExpr
+  }
+  | CoercibleFilterNullEmbed Bool FieldName
+  deriving (Eq, Show)
+
+data CoercibleOrderTerm
+  = CoercibleOrderTerm
+    { coField     :: CoercibleField
+    , coDirection :: Maybe OrderDirection
+    , coNullOrder :: Maybe OrderNulls
+    }
+  | CoercibleOrderRelationTerm
+    { coRelation  :: FieldName
+    , coRelTerm   :: Field
+    , coDirection :: Maybe OrderDirection
+    , coNullOrder :: Maybe OrderNulls
+    }
+  deriving (Eq, Show)

src/PostgREST/Query.hs

@@ -24,16 +24,13 @@ import qualified Data.Text.Encoding                as T
 import qualified Hasql.Decoders                    as HD
 import qualified Hasql.DynamicStatements.Snippet   as SQL (Snippet)
 import qualified Hasql.DynamicStatements.Statement as SQL
-import qualified Hasql.Encoders                    as HE
-import qualified Hasql.Statement                   as SQL
 import qualified Hasql.Transaction                 as SQL
 
-import qualified PostgREST.Error               as Error
-import qualified PostgREST.Query.QueryBuilder  as QueryBuilder
-import qualified PostgREST.Query.Statements    as Statements
-import qualified PostgREST.RangeQuery          as RangeQuery
-import qualified PostgREST.SchemaCache         as SchemaCache
-import qualified PostgREST.SchemaCache.Routine as Routine
+import qualified PostgREST.Error              as Error
+import qualified PostgREST.Query.QueryBuilder as QueryBuilder
+import qualified PostgREST.Query.Statements   as Statements
+import qualified PostgREST.RangeQuery         as RangeQuery
+import qualified PostgREST.SchemaCache        as SchemaCache
 
 import Data.Scientific (FPFormat (..), formatScientific, isInteger)
 
@@ -52,8 +49,8 @@ import PostgREST.Plan                    (CallReadPlan (..),
                                           MutateReadPlan (..),
                                           WrappedReadPlan (..))
 import PostgREST.Plan.MutatePlan         (MutatePlan (..))
-import PostgREST.Query.SqlFragment       (fromQi, intercalateSnippet,
-                                          pgFmtIdentList,
+import PostgREST.Query.SqlFragment       (escapeIdentList, fromQi,
+                                          intercalateSnippet,
                                           setConfigLocal,
                                           setConfigLocalJson)
 import PostgREST.Query.Statements        (ResultSet (..))
@@ -68,7 +65,7 @@ import Protolude hiding (Handler)
 type DbHandler = ExceptT Error SQL.Transaction
 
 readQuery :: WrappedReadPlan -> AppConfig -> ApiRequest -> DbHandler ResultSet
-readQuery WrappedReadPlan{wrReadPlan, wrBinField} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} = do
+readQuery WrappedReadPlan{wrReadPlan, wrResAgg} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} = do
   let countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan
   resultSet <-
      lift . SQL.statement mempty $
@@ -82,7 +79,7 @@ readQuery WrappedReadPlan{wrReadPlan, wrBinField} conf@AppConfig{..} apiReq@ApiR
         )
         (shouldCount preferCount)
         iAcceptMediaType
-        wrBinField
+        wrResAgg
         configDbPreparedStatements
   failNotSingular iAcceptMediaType resultSet
   optionalRollback conf apiReq
@@ -153,19 +150,17 @@ deleteQuery mrPlan apiReq@ApiRequest{..} conf = do
   pure resultSet
 
 invokeQuery :: Routine -> CallReadPlan -> ApiRequest -> AppConfig -> PgVersion -> DbHandler ResultSet
-invokeQuery proc CallReadPlan{crReadPlan, crCallPlan, crBinField} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} conf@AppConfig{..} pgVer = do
+invokeQuery rout CallReadPlan{crReadPlan, crCallPlan, crResAgg} apiReq@ApiRequest{iPreferences=Preferences{..}, ..} conf@AppConfig{..} pgVer = do
   resultSet <-
     lift . SQL.statement mempty $
       Statements.prepareCall
-        (Routine.funcReturnsScalar proc)
-        (Routine.funcReturnsSingleComposite proc)
-        (Routine.funcReturnsSetOfScalar proc)
+        rout
         (QueryBuilder.callPlanToQuery crCallPlan pgVer)
         (QueryBuilder.readPlanToQuery crReadPlan)
         (QueryBuilder.readPlanToCountQuery crReadPlan)
         (shouldCount preferCount)
         iAcceptMediaType
-        crBinField
+        crResAgg
         configDbPreparedStatements
 
   optionalRollback conf apiReq
@@ -190,7 +185,7 @@ openApiQuery sCache pgVer AppConfig{..} tSchema =
       pure Nothing
 
 writeQuery :: MutateReadPlan -> ApiRequest -> AppConfig  -> DbHandler ResultSet
-writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferences=Preferences{..}} conf =
+writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan, mrResAgg} apiReq@ApiRequest{iPreferences=Preferences{..}} conf =
   let
     (isInsert, pkCols) = case mrMutatePlan of {Insert{insPkCols} -> (True, insPkCols); _ -> (False, mempty);}
   in
@@ -200,6 +195,7 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferenc
       (QueryBuilder.mutatePlanToQuery mrMutatePlan)
       isInsert
       (iAcceptMediaType apiReq)
+      mrResAgg
       preferRepresentation
       pkCols
       (configDbPreparedStatements conf)
@@ -210,7 +206,7 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan} apiReq@ApiRequest{iPreferenc
 failNotSingular :: MediaType -> ResultSet -> DbHandler ()
 failNotSingular _ RSPlan{} = pure ()
 failNotSingular mediaType RSStandard{rsQueryTotal=queryTotal} =
-  when (mediaType == MTSingularJSON && queryTotal /= 1) $ do
+  when (elem mediaType [MTSingularJSON True,MTSingularJSON False]  && queryTotal /= 1) $ do
     lift SQL.condemn
     throwError $ Error.singularityError queryTotal
 
@@ -230,9 +226,9 @@ optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} = do
     SQL.condemn
   where
     shouldCommit =
-      configDbTxAllowOverride && preferTransaction == Just Commit
+      preferTransaction == Just Commit
     shouldRollback =
-      configDbTxAllowOverride && preferTransaction == Just Rollback
+      preferTransaction == Just Rollback
 
 -- | Runs local (transaction scoped) GUCs for every request.
 setPgLocals :: AppConfig  -> KM.KeyMap JSON.Value -> BS.ByteString -> [(ByteString, ByteString)] ->
@@ -257,7 +253,7 @@ setPgLocals AppConfig{..} claims role roleSettings req actualPgVersion = lift $
     roleSettingsSql = setConfigLocal mempty <$> roleSettings
     appSettingsSql = setConfigLocal mempty <$> (join bimap toUtf8 <$> configAppSettings)
     searchPathSql =
-      let schemas = pgFmtIdentList (iSchema req : configDbExtraSearchPath) in
+      let schemas = escapeIdentList (iSchema req : configDbExtraSearchPath) in
       setConfigLocal mempty ("search_path", schemas)
     usesLegacyGucs = configDbUseLegacyGucs && actualPgVersion < pgVersion140
 
@@ -272,8 +268,7 @@ setPgLocals AppConfig{..} claims role roleSettings req actualPgVersion = lift $
 runPreReq :: AppConfig -> DbHandler ()
 runPreReq conf = lift $ traverse_ (SQL.statement mempty . stmt) (configDbPreRequest conf)
   where
-    stmt req = SQL.Statement
+    stmt req = SQL.dynamicallyParameterized
       ("select " <> fromQi req <> "()")
-      HE.noParams
       HD.noResult
       (configDbPreparedStatements conf)

src/PostgREST/Query/QueryBuilder.hs

@@ -55,7 +55,7 @@ readPlanToQuery (Node ReadPlan{select,from=mainQi,fromAlias,where_=logicForest,o
   where
     fromFrag = fromF relToParent mainQi fromAlias
     qi = getQualifiedIdentifier relToParent mainQi fromAlias
-    defSelect = [(("*", []), Nothing, Nothing)] -- gets all the columns in case of an empty select, ignoring/obtaining these columns is done at the aggregation stage
+    defSelect = [(unknownField "*" [], Nothing, Nothing)] -- gets all the columns in case of an empty select, ignoring/obtaining these columns is done at the aggregation stage
     (selects, joins) = foldr getSelectsJoins ([],[]) forest
 
 getSelectsJoins :: ReadPlanTree -> ([SQL.Snippet], [SQL.Snippet]) -> ([SQL.Snippet], [SQL.Snippet])
@@ -66,45 +66,43 @@ getSelectsJoins rr@(Node ReadPlan{select, relName, relToParent=Just rel, relAggA
     aliasOrName = pgFmtIdent $ fromMaybe relName relAlias
     aggAlias = pgFmtIdent relAggAlias
     correlatedSubquery sub al cond =
-      (if relJoinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> SQL.sql al <> " ON " <> cond
+      (if relJoinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> al <> " ON " <> cond
     (sel, joi) = if relIsToOne rel
       then
         ( if relIsSpread
-            then SQL.sql aggAlias <> ".*"
-            else SQL.sql ("row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName)
+            then aggAlias <> ".*"
+            else "row_to_json(" <> aggAlias <> ".*) AS " <> aliasOrName
         , correlatedSubquery subquery aggAlias "TRUE")
       else
-        ( SQL.sql $ "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> aliasOrName
+        ( "COALESCE( " <> aggAlias <> "." <> aggAlias <> ", '[]') AS " <> aliasOrName
         , correlatedSubquery (
-            "SELECT json_agg(" <> SQL.sql aggAlias <> ") AS " <> SQL.sql aggAlias <>
-            "FROM (" <> subquery <> " ) AS " <> SQL.sql aggAlias
-          ) aggAlias $ if relJoinType == Just JTInner then SQL.sql aggAlias <> " IS NOT NULL" else "TRUE")
+            "SELECT json_agg(" <> aggAlias <> ") AS " <> aggAlias <>
+            "FROM (" <> subquery <> " ) AS " <> aggAlias
+          ) aggAlias $ if relJoinType == Just JTInner then aggAlias <> " IS NOT NULL" else "TRUE")
   in
   (if null select && null forest then selects else sel:selects, joi:joins)
 
 mutatePlanToQuery :: MutatePlan -> SQL.Snippet
 mutatePlanToQuery (Insert mainQi iCols body onConflct putConditions returnings _ applyDefaults) =
-  "INSERT INTO " <> SQL.sql (fromQi mainQi) <> SQL.sql (if null iCols then " " else "(" <> cols <> ") ") <>
+  "INSERT INTO " <> fromQi mainQi <> (if null iCols then " " else "(" <> cols <> ") ") <>
   fromJsonBodyF body iCols True False applyDefaults <>
   -- Only used for PUT
   (if null putConditions then mempty else "WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree (QualifiedIdentifier mempty "pgrst_body") <$> putConditions)) <>
-  SQL.sql (BS.unwords [
-    maybe mempty (\(oncDo, oncCols) ->
-      if null oncCols then
-        mempty
-      else
-        " ON CONFLICT(" <> BS.intercalate ", " (pgFmtIdent <$> oncCols) <> ") " <> case oncDo of
-        IgnoreDuplicates ->
-          "DO NOTHING"
-        MergeDuplicates  ->
-          if null iCols
-             then "DO NOTHING"
-             else "DO UPDATE SET " <> BS.intercalate ", " ((pgFmtIdent . tfName) <> const " = EXCLUDED." <> (pgFmtIdent . tfName) <$> iCols)
-      ) onConflct,
-    returningF mainQi returnings
-    ])
+  maybe mempty (\(oncDo, oncCols) ->
+    if null oncCols then
+      mempty
+    else
+      " ON CONFLICT(" <> intercalateSnippet ", " (pgFmtIdent <$> oncCols) <> ") " <> case oncDo of
+      IgnoreDuplicates ->
+        "DO NOTHING"
+      MergeDuplicates  ->
+        if null iCols
+           then "DO NOTHING"
+           else "DO UPDATE SET " <> intercalateSnippet ", " ((pgFmtIdent . cfName) <> const " = EXCLUDED." <> (pgFmtIdent . cfName) <$> iCols)
+    ) onConflct <> " " <>
+  returningF mainQi returnings
   where
-    cols = BS.intercalate ", " $ pgFmtIdent . tfName <$> iCols
+    cols = intercalateSnippet ", " $ pgFmtIdent . cfName <$> iCols
 
 -- An update without a limit is always filtered with a WHERE
 mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings applyDefaults)
@@ -112,82 +110,83 @@ mutatePlanToQuery (Update mainQi uCols body logicForest range ordts returnings a
     -- if there are no columns we cannot do UPDATE table SET {empty}, it'd be invalid syntax
     -- selecting an empty resultset from mainQi gives us the column names to prevent errors when using &select=
     -- the select has to be based on "returnings" to make computed overloaded functions not throw
-    SQL.sql $ "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"
+    "SELECT " <> emptyBodyReturnedColumns <> " FROM " <> fromQi mainQi <> " WHERE false"
 
   | range == allRange =
-    "UPDATE " <> mainTbl <> " SET " <> SQL.sql nonRangeCols <> " " <>
+    "UPDATE " <> mainTbl <> " SET " <> nonRangeCols <> " " <>
     fromJsonBodyF body uCols False False applyDefaults <>
     whereLogic <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings
 
   | otherwise =
     "WITH " <>
     "pgrst_update_body AS (" <> fromJsonBodyF body uCols True True applyDefaults <> "), " <>
     "pgrst_affected_rows AS (" <>
-      "SELECT " <> SQL.sql rangeIdF <> " FROM " <> mainTbl <>
+      "SELECT " <> rangeIdF <> " FROM " <> mainTbl <>
       whereLogic <> " " <>
       orderF mainQi ordts <> " " <>
       limitOffsetF range <>
     ") " <>
-    "UPDATE " <> mainTbl <> " SET " <> SQL.sql rangeCols <>
+    "UPDATE " <> mainTbl <> " SET " <> rangeCols <>
     "FROM pgrst_affected_rows " <>
-    "WHERE " <> SQL.sql whereRangeIdF <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    "WHERE " <> whereRangeIdF <> " " <>
+    returningF mainQi returnings
 
   where
     whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
-    mainTbl = SQL.sql (fromQi mainQi)
-    emptyBodyReturnedColumns = if null returnings then "NULL" else BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
-    nonRangeCols = BS.intercalate ", " (pgFmtIdent . tfName <> const " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_body") . tfName <$> uCols)
-    rangeCols = BS.intercalate ", " ((\col -> pgFmtIdent (tfName col) <> " = (SELECT " <> pgFmtIdent (tfName col) <> " FROM pgrst_update_body) ") <$> uCols)
-    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
+    mainTbl = fromQi mainQi
+    emptyBodyReturnedColumns = if null returnings then "NULL" else intercalateSnippet ", " (pgFmtColumn (QualifiedIdentifier mempty $ qiName mainQi) <$> returnings)
+    nonRangeCols = intercalateSnippet ", " (pgFmtIdent . cfName <> const " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_body") . cfName <$> uCols)
+    rangeCols = intercalateSnippet ", " ((\col -> pgFmtIdent (cfName col) <> " = (SELECT " <> pgFmtIdent (cfName col) <> " FROM pgrst_update_body) ") <$> uCols)
+    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (cfName . coField <$> ordts)
 
 mutatePlanToQuery (Delete mainQi logicForest range ordts returnings)
   | range == allRange =
-    "DELETE FROM " <> SQL.sql (fromQi mainQi) <> " " <>
+    "DELETE FROM " <> fromQi mainQi <> " " <>
     whereLogic <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    returningF mainQi returnings
 
   | otherwise =
     "WITH " <>
     "pgrst_affected_rows AS (" <>
-      "SELECT " <> SQL.sql rangeIdF <> " FROM " <> SQL.sql (fromQi mainQi) <>
+      "SELECT " <> rangeIdF <> " FROM " <> fromQi mainQi <>
        whereLogic <> " " <>
       orderF mainQi ordts <> " " <>
       limitOffsetF range <>
     ") " <>
-    "DELETE FROM " <> SQL.sql (fromQi mainQi) <> " " <>
+    "DELETE FROM " <> fromQi mainQi <> " " <>
     "USING pgrst_affected_rows " <>
-    "WHERE " <> SQL.sql whereRangeIdF <> " " <>
-    SQL.sql (returningF mainQi returnings)
+    "WHERE " <> whereRangeIdF <> " " <>
+    returningF mainQi returnings
 
   where
     whereLogic = if null logicForest then mempty else " WHERE " <> intercalateSnippet " AND " (pgFmtLogicTree mainQi <$> logicForest)
-    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (fst . otTerm <$> ordts)
+    (whereRangeIdF, rangeIdF) = mutRangeF mainQi (cfName . coField <$> ordts)
 
 callPlanToQuery :: CallPlan -> PgVersion -> SQL.Snippet
 callPlanToQuery (FunctionCall qi params args returnsScalar returnsSetOfScalar returnsCompositeAlias returnings) pgVer =
-  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call AS pgrst_scalar " else returnedColumns) <> " " <>
+  "SELECT " <> (if returnsScalar || returnsSetOfScalar then "pgrst_call.pgrst_scalar" else returnedColumns) <> " " <>
   fromCall
   where
     fromCall = case params of
       OnePosParam prm -> "FROM " <> callIt (singleParameter args $ encodeUtf8 $ ppType prm)
       KeyParams []    -> "FROM " <> callIt mempty
-      KeyParams prms  -> fromJsonBodyF args ((\p -> TypedField (ppName p) (ppType p) Nothing) <$> prms) False True False <> ", " <>
+      KeyParams prms  -> fromJsonBodyF args ((\p -> CoercibleField (ppName p) mempty False (ppTypeMaxLength p) Nothing Nothing) <$> prms) False True False <> ", " <>
                          "LATERAL " <> callIt (fmtParams prms)
 
     callIt :: SQL.Snippet -> SQL.Snippet
-    callIt argument | pgVer < pgVersion130 && pgVer >= pgVersion110 && returnsCompositeAlias = "(SELECT (" <> SQL.sql (fromQi qi) <> "(" <> argument <> ")).*) pgrst_call"
-                    | otherwise                                                              = SQL.sql (fromQi qi) <> "(" <> argument <> ") pgrst_call"
+    callIt argument | pgVer < pgVersion130 && pgVer >= pgVersion110 && returnsCompositeAlias = "(SELECT (" <> fromQi qi <> "(" <> argument <> ")).*) pgrst_call"
+                    | returnsScalar || returnsSetOfScalar                                    = "(SELECT " <> fromQi qi <> "(" <> argument <> ") pgrst_scalar) pgrst_call"
+                    | otherwise                                                              = fromQi qi <> "(" <> argument <> ") pgrst_call"
 
     fmtParams :: [RoutineParam] -> SQL.Snippet
-    fmtParams prms = SQL.sql $ BS.intercalate ", "
+    fmtParams prms = intercalateSnippet ", "
       ((\a -> (if ppVar a then "VARIADIC " else mempty) <> pgFmtIdent (ppName a) <> " := pgrst_body." <> pgFmtIdent (ppName a)) <$> prms)
 
     returnedColumns :: SQL.Snippet
     returnedColumns
       | null returnings = "*"
-      | otherwise       = SQL.sql $ BS.intercalate ", " (pgFmtColumn (QualifiedIdentifier mempty "pgrst_call") <$> returnings)
+      | otherwise       = intercalateSnippet ", " (pgFmtColumn (QualifiedIdentifier mempty "pgrst_call") <$> returnings)
 
 -- | SQL query meant for COUNTing the root node of the Tree.
 -- It only takes WHERE into account and doesn't include LIMIT/OFFSET because it would reduce the COUNT.
@@ -204,7 +203,7 @@ readPlanToCountQuery (Node ReadPlan{from=mainQi, fromAlias=tblAlias, where_=logi
     then mempty
     else " WHERE " ) <>
   intercalateSnippet " AND " (
-    map (pgFmtLogicTree qi) logicForest ++
+    map (pgFmtLogicTreeCount qi) logicForest ++
     map pgFmtJoinCondition relJoinConds ++
     subQueries
   )
@@ -217,6 +216,18 @@ readPlanToCountQuery (Node ReadPlan{from=mainQi, fromAlias=tblAlias, where_=logi
       if joinType == Just JTInner
         then ("EXISTS (" <> readPlanToCountQuery readReq <> " )"):rest
         else rest
+    findNullEmbedRel fld = find (\(Node ReadPlan{relAggAlias} _) -> fld == relAggAlias) forest
+
+    -- https://github.com/PostgREST/postgrest/pull/2930#discussion_r1325293698
+    pgFmtLogicTreeCount :: QualifiedIdentifier -> CoercibleLogicTree -> SQL.Snippet
+    pgFmtLogicTreeCount qiCount (CoercibleExpr hasNot op frst) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTreeCount qiCount <$> frst) <> ")"
+      where
+        notOp =  if hasNot then "NOT" else mempty
+        opSql And = " AND "
+        opSql Or  = " OR "
+    pgFmtLogicTreeCount _ (CoercibleStmnt (CoercibleFilterNullEmbed hasNot fld)) =
+      maybe mempty (\x -> (if not hasNot then "NOT " else mempty) <> "EXISTS (" <> readPlanToCountQuery x <> ")") (findNullEmbedRel fld)
+    pgFmtLogicTreeCount qiCount (CoercibleStmnt flt) = pgFmtFilter qiCount flt
 
 limitedQuery :: SQL.Snippet -> Maybe Integer -> SQL.Snippet
 limitedQuery query maxRows = query <> SQL.sql (maybe mempty (\x -> " LIMIT " <> BS.pack (show x)) maxRows)
@@ -229,10 +240,12 @@ getQualifiedIdentifier rel mainQi tblAlias = case rel of
 
 -- FROM clause plus implicit joins
 fromF :: Maybe Relationship -> QualifiedIdentifier -> Maybe Alias -> SQL.Snippet
-fromF rel mainQi tblAlias = SQL.sql $ "FROM " <>
+fromF rel mainQi tblAlias = "FROM " <>
   (case rel of
-    Just ComputedRelationship{relFunction,relTable} -> fromQi relFunction <> "(" <> pgFmtIdent (qiName relTable) <> ")"
-    _                                               -> fromQi mainQi) <>
+    -- Due to the use of CTEs on RPC, we need to cast the parameter to the table name in case of function overloading.
+    -- See https://github.com/PostgREST/postgrest/issues/2963#issuecomment-1736557386
+    Just ComputedRelationship{relFunction,relTableAlias,relTable} -> fromQi relFunction <> "(" <> pgFmtIdent (qiName relTableAlias) <> "::" <> fromQi relTable <> ")"
+    _                                                             -> fromQi mainQi) <>
   maybe mempty (\a -> " AS " <> pgFmtIdent a) tblAlias <>
   (case rel of
     Just Relationship{relCardinality=M2M Junction{junTable=jt}} -> ", " <> fromQi jt

src/PostgREST/Query/SqlFragment.hs

@@ -4,18 +4,10 @@
 {-|
 Module      : PostgREST.Query.SqlFragment
 Description : Helper functions for PostgREST.QueryBuilder.
-
-Any function that outputs a SqlFragment should be in this module.
 -}
 module PostgREST.Query.SqlFragment
   ( noLocationF
-  , SqlFragment
-  , asBinaryF
-  , asCsvF
-  , asGeoJsonF
-  , asJsonF
-  , asJsonSingleF
-  , asXmlF
+  , aggF
   , countF
   , fromQi
   , limitOffsetF
@@ -23,8 +15,8 @@ module PostgREST.Query.SqlFragment
   , mutRangeF
   , orderF
   , pgFmtColumn
+  , pgFmtFilter
   , pgFmtIdent
-  , pgFmtIdentList
   , pgFmtJoinCondition
   , pgFmtLogicTree
   , pgFmtOrderTerm
@@ -34,12 +26,15 @@ module PostgREST.Query.SqlFragment
   , responseStatusF
   , returningF
   , singleParameter
+  , sourceCTE
   , sourceCTEName
   , unknownEncoder
   , intercalateSnippet
   , explainF
   , setConfigLocal
   , setConfigLocalJson
+  , escapeIdent
+  , escapeIdentList
   ) where
 
 import qualified Data.Aeson                      as JSON
@@ -56,44 +51,50 @@ import Control.Arrow ((***))
 import Data.Foldable                 (foldr1)
 import Text.InterpolatedString.Perl6 (qc)
 
-import PostgREST.ApiRequest.Types        (Alias, Cast, Field,
-                                          Filter (..),
+import PostgREST.ApiRequest.Types        (Alias, Cast,
                                           FtsOperator (..),
                                           JsonOperand (..),
                                           JsonOperation (..),
                                           JsonPath,
                                           LogicOperator (..),
-                                          LogicTree (..), OpExpr (..),
+                                          OpExpr (..),
                                           OpQuantifier (..),
                                           Operation (..),
                                           OrderDirection (..),
                                           OrderNulls (..),
-                                          OrderTerm (..),
                                           QuantOperator (..),
                                           SimpleOperator (..),
                                           TrileanVal (..))
 import PostgREST.MediaType               (MTPlanFormat (..),
                                           MTPlanOption (..))
 import PostgREST.Plan.ReadPlan           (JoinCondition (..))
-import PostgREST.Plan.Types              (TypedField (..))
+import PostgREST.Plan.Types              (CoercibleField (..),
+                                          CoercibleFilter (..),
+                                          CoercibleLogicTree (..),
+                                          CoercibleOrderTerm (..),
+                                          unknownField)
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           rangeLimit, rangeOffset)
 import PostgREST.SchemaCache.Identifiers (FieldName,
                                           QualifiedIdentifier (..))
+import PostgREST.SchemaCache.Routine     (ResultAggregate (..),
+                                          Routine (..),
+                                          funcReturnsScalar,
+                                          funcReturnsSetOfScalar,
+                                          funcReturnsSingleComposite)
 
 import Protolude hiding (cast)
 
-
--- | A part of a SQL query that cannot be executed independently
-type SqlFragment = ByteString
-
-noLocationF :: SqlFragment
-noLocationF = "array[]::text[]"
-
-sourceCTEName :: SqlFragment
+sourceCTEName :: Text
 sourceCTEName = "pgrst_source"
 
-simpleOperator :: SimpleOperator -> SqlFragment
+sourceCTE :: SQL.Snippet
+sourceCTE = "pgrst_source"
+
+noLocationF :: SQL.Snippet
+noLocationF = "array[]::text[]"
+
+simpleOperator :: SimpleOperator -> SQL.Snippet
 simpleOperator = \case
   OpNotEqual        -> "<>"
   OpContains        -> "@>"
@@ -105,7 +106,7 @@ simpleOperator = \case
   OpNotExtendsLeft  -> "&>"
   OpAdjacent        -> "-|-"
 
-quantOperator :: QuantOperator -> SqlFragment
+quantOperator :: QuantOperator -> SQL.Snippet
 quantOperator = \case
   OpEqual            -> "="
   OpGreaterThanEqual -> ">="
@@ -117,7 +118,7 @@ quantOperator = \case
   OpMatch            -> "~"
   OpIMatch           -> "~*"
 
-ftsOperator :: FtsOperator -> SqlFragment
+ftsOperator :: FtsOperator -> SQL.Snippet
 ftsOperator = \case
   FilterFts          -> "@@ to_tsquery"
   FilterFtsPlain     -> "@@ plainto_tsquery"
@@ -145,8 +146,11 @@ pgBuildArrayLiteral vals =
  "{" <> T.intercalate "," (escaped <$> vals) <> "}"
 
 -- TODO: refactor by following https://github.com/PostgREST/postgrest/pull/1631#issuecomment-711070833
-pgFmtIdent :: Text -> SqlFragment
-pgFmtIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
+pgFmtIdent :: Text -> SQL.Snippet
+pgFmtIdent x = SQL.sql $ escapeIdent x
+
+escapeIdent :: Text -> ByteString
+escapeIdent x = encodeUtf8 $ "\"" <> T.replace "\"" "\"\"" (trimNullChars x) <> "\""
 
 -- Only use it if the input comes from the database itself, like on `jsonb_build_object('column_from_a_table', val)..`
 pgFmtLit :: Text -> Text
@@ -164,12 +168,12 @@ trimNullChars = T.takeWhile (/= '\x0')
 -- |
 -- Format a list of identifiers and separate them by commas.
 --
--- >>> pgFmtIdentList ["schema_1", "schema_2", "SPECIAL \"@/\\#~_-"]
+-- >>> escapeIdentList ["schema_1", "schema_2", "SPECIAL \"@/\\#~_-"]
 -- "\"schema_1\", \"schema_2\", \"SPECIAL \"\"@/\\#~_-\""
-pgFmtIdentList :: [Text] -> SqlFragment
-pgFmtIdentList schemas = BS.intercalate ", " $ pgFmtIdent <$> schemas
+escapeIdentList :: [Text] -> ByteString
+escapeIdentList schemas = BS.intercalate ", " $ escapeIdent <$> schemas
 
-asCsvF :: SqlFragment
+asCsvF :: SQL.Snippet
 asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
   where
     asCsvHeaderF =
@@ -177,34 +181,49 @@ asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
       "  FROM (" <>
       "    SELECT json_object_keys(r)::text as k" <>
       "    FROM ( " <>
-      "      SELECT row_to_json(hh) as r from " <> sourceCTEName <> " as hh limit 1" <>
+      "      SELECT row_to_json(hh) as r from " <> sourceCTE <> " as hh limit 1" <>
       "    ) s" <>
       "  ) a" <>
       ")"
     asCsvBodyF = "coalesce(string_agg(substring(_postgrest_t::text, 2, length(_postgrest_t::text) - 2), '\n'), '')"
 
-asJsonSingleF :: Bool -> SqlFragment
-asJsonSingleF returnsScalar
-  | returnsScalar = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
-  | otherwise     = "coalesce(json_agg(_postgrest_t)->0, 'null')"
+addNullsToSnip :: Bool -> SQL.Snippet -> SQL.Snippet
+addNullsToSnip strip snip =
+  if strip then "json_strip_nulls(" <> snip <> ")" else  snip
 
-asJsonF :: Bool -> Bool -> Bool -> SqlFragment
-asJsonF returnsScalar returnsSetOfScalar returnsSingleComposite
-  | returnsSingleComposite              = "coalesce(json_agg(_postgrest_t)->0, 'null')"
-  | returnsScalar                       = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
-  | returnsSetOfScalar                  = "coalesce(json_agg(_postgrest_t.pgrst_scalar), '[]')"
-  | otherwise                           = "coalesce(json_agg(_postgrest_t), '[]')"
+asJsonSingleF :: Maybe Routine -> Bool -> SQL.Snippet
+asJsonSingleF rout strip
+  | returnsScalar = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0"  <> ", 'null')"
+  | otherwise     = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0"  <> ", 'null')"
+  where
+    returnsScalar = maybe False funcReturnsScalar rout
 
-asXmlF :: FieldName -> SqlFragment
-asXmlF fieldName = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
+asJsonF :: Maybe Routine -> Bool -> SQL.Snippet
+asJsonF rout strip
+  | returnsSingleComposite    = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0" <> ", 'null')"
+  | returnsScalar             = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0" <> ", 'null')"
+  | returnsSetOfScalar        = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)" <> ", '[]')"
+  | otherwise                 = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)" <> ", '[]')"
+  where
+    (returnsSingleComposite, returnsScalar, returnsSetOfScalar) = case rout of
+      Just r  -> (funcReturnsSingleComposite r, funcReturnsScalar r, funcReturnsSetOfScalar r)
+      Nothing -> (False, False, False)
 
-asGeoJsonF ::  SqlFragment
+
+asXmlF :: Maybe FieldName -> SQL.Snippet
+asXmlF (Just fieldName) = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
+-- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
+asXmlF Nothing          = "coalesce(xmlagg(_postgrest_t), '')"
+
+asGeoJsonF ::  SQL.Snippet
 asGeoJsonF = "json_build_object('type', 'FeatureCollection', 'features', coalesce(json_agg(ST_AsGeoJSON(_postgrest_t)::json), '[]'))"
 
-asBinaryF :: FieldName -> SqlFragment
-asBinaryF fieldName = "coalesce(string_agg(_postgrest_t." <> pgFmtIdent fieldName <> ", ''), '')"
+asBinaryF :: Maybe FieldName -> SQL.Snippet
+asBinaryF (Just fieldName) = "coalesce(string_agg(_postgrest_t." <> pgFmtIdent fieldName <> ", ''), '')"
+-- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
+asBinaryF Nothing          = "coalesce(string_agg(_postgrest_t, ''), '')"
 
-locationF :: [Text] -> SqlFragment
+locationF :: [Text] -> SQL.Snippet
 locationF pKeys = [qc|(
   WITH data AS (SELECT row_to_json(_) AS row FROM {sourceCTEName} AS _ LIMIT 1)
   SELECT array_agg(json_data.key || '=' || coalesce('eq.' || json_data.value, 'is.null'))
@@ -214,34 +233,45 @@ locationF pKeys = [qc|(
   where
     fmtPKeys = T.intercalate "','" pKeys
 
-fromQi :: QualifiedIdentifier -> SqlFragment
+fromQi :: QualifiedIdentifier -> SQL.Snippet
 fromQi t = (if T.null s then mempty else pgFmtIdent s <> ".") <> pgFmtIdent n
   where
     n = qiName t
     s = qiSchema t
 
-pgFmtColumn :: QualifiedIdentifier -> Text -> SqlFragment
+pgFmtColumn :: QualifiedIdentifier -> Text -> SQL.Snippet
 pgFmtColumn table "*" = fromQi table <> ".*"
 pgFmtColumn table c   = fromQi table <> "." <> pgFmtIdent c
 
-pgFmtField :: QualifiedIdentifier -> Field -> SQL.Snippet
-pgFmtField table (c, []) = SQL.sql (pgFmtColumn table c)
--- Using to_jsonb instead of to_json to avoid missing operator errors when filtering:
--- "operator does not exist: json = unknown"
-pgFmtField table (c, jp) = SQL.sql ("to_jsonb(" <> pgFmtColumn table c <> ")") <> pgFmtJsonPath jp
+pgFmtCallUnary :: Text -> SQL.Snippet -> SQL.Snippet
+pgFmtCallUnary f x = SQL.sql (encodeUtf8 f) <> "(" <> x <> ")"
 
-pgFmtSelectItem :: QualifiedIdentifier -> (Field, Maybe Cast, Maybe Alias) -> SQL.Snippet
-pgFmtSelectItem table (f@(fName, jp), Nothing, alias) = pgFmtField table f <> SQL.sql (pgFmtAs fName jp alias)
+pgFmtField :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
+pgFmtField table CoercibleField{cfName=fn, cfJsonPath=[]}                                = pgFmtColumn table fn
+pgFmtField table CoercibleField{cfName=fn, cfToJson=doToJson, cfJsonPath=jp} | doToJson  = "to_jsonb(" <> pgFmtColumn table fn <> ")" <> pgFmtJsonPath jp
+                                                                             | otherwise = pgFmtColumn table fn <> pgFmtJsonPath jp
+
+-- Select the value of a named element from a table, applying its optional coercion mapping if any.
+pgFmtTableCoerce :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
+pgFmtTableCoerce table fld@(CoercibleField{cfTransform=(Just formatterProc)}) = pgFmtCallUnary formatterProc (pgFmtField table fld)
+pgFmtTableCoerce table f = pgFmtField table f
+
+-- | Like the previous but now we just have a name so no namespace or JSON paths.
+pgFmtCoerceNamed :: CoercibleField -> SQL.Snippet
+pgFmtCoerceNamed CoercibleField{cfName=fn, cfTransform=(Just formatterProc)} = pgFmtCallUnary formatterProc (pgFmtIdent fn) <> " AS " <> pgFmtIdent fn
+pgFmtCoerceNamed CoercibleField{cfName=fn} = pgFmtIdent fn
+
+pgFmtSelectItem :: QualifiedIdentifier -> (CoercibleField, Maybe Cast, Maybe Alias) -> SQL.Snippet
+pgFmtSelectItem table (fld, Nothing, alias) = pgFmtTableCoerce table fld <> pgFmtAs (cfName fld) (cfJsonPath fld) alias
 -- Ideally we'd quote the cast with "pgFmtIdent cast". However, that would invalidate common casts such as "int", "bigint", etc.
 -- Try doing: `select 1::"bigint"` - it'll err, using "int8" will work though. There's some parser magic that pg does that's invalidated when quoting.
 -- Not quoting should be fine, we validate the input on Parsers.
-pgFmtSelectItem table (f@(fName, jp), Just cast, alias) = "CAST (" <> pgFmtField table f <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )" <> SQL.sql (pgFmtAs fName jp alias)
+pgFmtSelectItem table (fld, Just cast, alias) = "CAST (" <> pgFmtTableCoerce table fld <> " AS " <> SQL.sql (encodeUtf8 cast) <> " )" <> pgFmtAs (cfName fld) (cfJsonPath fld) alias
 
 -- TODO: At this stage there shouldn't be a Maybe since ApiRequest should ensure that an INSERT/UPDATE has a body
-fromJsonBodyF :: Maybe LBS.ByteString -> [TypedField] -> Bool -> Bool -> Bool -> SQL.Snippet
+fromJsonBodyF :: Maybe LBS.ByteString -> [CoercibleField] -> Bool -> Bool -> Bool -> SQL.Snippet
 fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
-  SQL.sql
-  (if includeSelect then "SELECT " <> parsedCols <> " " else mempty) <>
+  (if includeSelect then "SELECT " <> namedCols <> " " else mempty) <>
   "FROM (SELECT " <> jsonPlaceHolder <> " AS json_data) pgrst_payload, " <>
   -- convert a json object into a json array, this way we can use json_to_recordset for all json payloads
   -- Otherwise we'd have to use json_to_record for json objects and json_to_recordset for json arrays
@@ -250,22 +280,23 @@ fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
   (if includeDefaults
     then "LATERAL (SELECT jsonb_agg(jsonb_build_object(" <> defsJsonb <> ") || elem) AS val from jsonb_array_elements(pgrst_uniform_json.val) elem) pgrst_json_defs, "
     else mempty) <>
-  "LATERAL (SELECT * FROM " <>
+  "LATERAL (SELECT " <> parsedCols <> " FROM " <>
     (if null fields
       -- When we are inserting no columns (e.g. using default values), we can't use our ordinary `json_to_recordset`
       -- because it can't extract records with no columns (there's no valid syntax for the `AS (colName colType,...)`
       -- part). But we still need to ensure as many rows are created as there are array elements.
       then SQL.sql $ jsonArrayElementsF <> "(" <> finalBodyF <> ") _ "
-      else SQL.sql $ jsonToRecordsetF <> "(" <> finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
+      else jsonToRecordsetF <> "(" <> SQL.sql finalBodyF <> ") AS _(" <> typedCols <> ") " <> if includeLimitOne then "LIMIT 1" else mempty
     ) <>
   ") pgrst_body "
   where
-    parsedCols = BS.intercalate ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . tfName <$> fields
-    typedCols = BS.intercalate ", " $ pgFmtIdent . tfName <> const " " <> encodeUtf8 . tfIRType <$> fields
+    namedCols = intercalateSnippet ", " $ fromQi  . QualifiedIdentifier "pgrst_body" . cfName <$> fields
+    parsedCols = intercalateSnippet ", " $ pgFmtCoerceNamed <$> fields
+    typedCols = intercalateSnippet ", " $ pgFmtIdent . cfName <> const " " <> SQL.sql . encodeUtf8 . cfIRType <$> fields
     defsJsonb = SQL.sql $ BS.intercalate "," fieldsWDefaults
     fieldsWDefaults = mapMaybe (\case
-        TypedField{tfName=nam, tfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
-        TypedField{tfDefault=Nothing} -> Nothing
+        CoercibleField{cfName=nam, cfDefault=Just def} -> Just $ encodeUtf8 (pgFmtLit nam <> ", " <> def)
+        CoercibleField{cfDefault=Nothing} -> Nothing
       ) fields
     (finalBodyF, jsonTypeofF, jsonBuildArrayF, jsonArrayElementsF, jsonToRecordsetF) =
       if includeDefaults
@@ -273,16 +304,16 @@ fromJsonBodyF body fields includeSelect includeLimitOne includeDefaults =
         else ("pgrst_uniform_json.val", "json_typeof", "json_build_array", "json_array_elements", "json_to_recordset")
     jsonPlaceHolder = SQL.encoderAndParam (HE.nullable $ if includeDefaults then HE.jsonbLazyBytes else HE.jsonLazyBytes) body
 
-pgFmtOrderTerm :: QualifiedIdentifier -> OrderTerm -> SQL.Snippet
+pgFmtOrderTerm :: QualifiedIdentifier -> CoercibleOrderTerm -> SQL.Snippet
 pgFmtOrderTerm qi ot =
   fmtOTerm ot <> " " <>
   SQL.sql (BS.unwords [
-    maybe mempty direction $ otDirection ot,
-    maybe mempty nullOrder $ otNullOrder ot])
+    maybe mempty direction $ coDirection ot,
+    maybe mempty nullOrder $ coNullOrder ot])
   where
     fmtOTerm = \case
-      OrderTerm{otTerm}                        -> pgFmtField qi otTerm
-      OrderRelationTerm{otRelation, otRelTerm} -> pgFmtField (QualifiedIdentifier mempty otRelation) otRelTerm
+      CoercibleOrderTerm{coField=cof}                            -> pgFmtField qi cof
+      CoercibleOrderRelationTerm{coRelation, coRelTerm=(fn, jp)} -> pgFmtField (QualifiedIdentifier mempty coRelation) (unknownField fn jp)
 
     direction OrderAsc  = "ASC"
     direction OrderDesc = "DESC"
@@ -290,17 +321,31 @@ pgFmtOrderTerm qi ot =
     nullOrder OrderNullsFirst = "NULLS FIRST"
     nullOrder OrderNullsLast  = "NULLS LAST"
 
+-- | Interpret a literal in the way the planner indicated through the CoercibleField.
+pgFmtUnknownLiteralForField :: SQL.Snippet -> CoercibleField -> SQL.Snippet
+pgFmtUnknownLiteralForField value CoercibleField{cfTransform=(Just parserProc)} = pgFmtCallUnary parserProc value
+-- But when no transform is requested, we just use the literal as-is.
+pgFmtUnknownLiteralForField value _ = value
 
-pgFmtFilter :: QualifiedIdentifier -> Filter -> SQL.Snippet
-pgFmtFilter _ (FilterNullEmbed hasNot fld) = SQL.sql (pgFmtIdent fld) <> " IS " <> (if hasNot then "NOT" else mempty) <> " NULL"
-pgFmtFilter _ (Filter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
-pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
-   Op op val -> " " <> SQL.sql (simpleOperator op) <> " " <> unknownLiteral val
+-- | Array version of the above, used by ANY().
+pgFmtArrayLiteralForField :: [Text] -> CoercibleField -> SQL.Snippet
+-- When a transformation is requested, we need to apply the transformation to each element of the array. This could be done by just making a query with `parser(value)` for each value, but may lead to huge query lengths. Imagine `data_representations.color_from_text('...'::text)` for repeated for a hundred values. Instead we use `unnest()` to unpack a standard array literal and then apply the transformation to each element, like a map.
+-- Note the literals will be treated as text since in every case when we use ANY() the parameters are textual (coming from a query string). We want to rely on the `text->domain` parser to do the right thing.
+pgFmtArrayLiteralForField values CoercibleField{cfTransform=(Just parserProc)} = SQL.sql "(SELECT " <> pgFmtCallUnary parserProc (SQL.sql "unnest(" <> unknownLiteral (pgBuildArrayLiteral values) <> "::text[])") <> ")"
+-- When no transformation is requested, we don't need a subquery.
+pgFmtArrayLiteralForField values _ = unknownLiteral (pgBuildArrayLiteral values)
 
-   OpQuant op quant val -> " " <> SQL.sql (quantOperator op) <> " " <> case op of
+
+pgFmtFilter :: QualifiedIdentifier -> CoercibleFilter -> SQL.Snippet
+pgFmtFilter _ (CoercibleFilterNullEmbed hasNot fld) = pgFmtIdent fld <> " IS " <> (if not hasNot then "NOT " else mempty) <> "DISTINCT FROM NULL"
+pgFmtFilter _ (CoercibleFilter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
+pgFmtFilter table (CoercibleFilter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
+   Op op val  -> " " <> simpleOperator op <> " " <> pgFmtUnknownLiteralForField (unknownLiteral val) fld
+
+   OpQuant op quant val -> " " <> quantOperator op <> " " <> case op of
      OpLike  -> fmtQuant quant $ unknownLiteral (T.map star val)
      OpILike -> fmtQuant quant $ unknownLiteral (T.map star val)
-     _       -> fmtQuant quant $ unknownLiteral val
+     _       -> fmtQuant quant $ pgFmtUnknownLiteralForField (unknownLiteral val) fld
 
    -- IS cannot be prepared. `PREPARE boolplan AS SELECT * FROM projects where id IS $1` will give a syntax error.
    -- The above can be fixed by using `PREPARE boolplan AS SELECT * FROM projects where id IS NOT DISTINCT FROM $1;`
@@ -319,9 +364,9 @@ pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField
    -- + Can invalidate prepared statements: multiple parameters on an IN($1, $2, $3) will lead to using different prepared statements and not take advantage of caching.
    In vals -> " " <> case vals of
       [""] -> "= ANY('{}') "
-      _    -> "= ANY (" <> unknownLiteral (pgBuildArrayLiteral vals) <> ") "
+      _    -> "= ANY (" <> pgFmtArrayLiteralForField vals fld <> ") "
 
-   Fts op lang val -> " " <> SQL.sql (ftsOperator op) <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
+   Fts op lang val -> " " <> ftsOperator op <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
  where
    ftsLang = maybe mempty (\l -> unknownLiteral l <> ", ")
    notOp = if hasNot then "NOT" else mempty
@@ -333,16 +378,16 @@ pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField
 
 pgFmtJoinCondition :: JoinCondition -> SQL.Snippet
 pgFmtJoinCondition (JoinCondition (qi1, col1) (qi2, col2)) =
-  SQL.sql $ pgFmtColumn qi1 col1 <> " = " <> pgFmtColumn qi2 col2
+  pgFmtColumn qi1 col1 <> " = " <> pgFmtColumn qi2 col2
 
-pgFmtLogicTree :: QualifiedIdentifier -> LogicTree -> SQL.Snippet
-pgFmtLogicTree qi (Expr hasNot op forest) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTree qi <$> forest) <> ")"
+pgFmtLogicTree :: QualifiedIdentifier -> CoercibleLogicTree -> SQL.Snippet
+pgFmtLogicTree qi (CoercibleExpr hasNot op forest) = SQL.sql notOp <> " (" <> intercalateSnippet (opSql op) (pgFmtLogicTree qi <$> forest) <> ")"
   where
     notOp =  if hasNot then "NOT" else mempty
 
     opSql And = " AND "
     opSql Or  = " OR "
-pgFmtLogicTree qi (Stmnt flt) = pgFmtFilter qi flt
+pgFmtLogicTree qi (CoercibleStmnt flt) = pgFmtFilter qi flt
 
 pgFmtJsonPath :: JsonPath -> SQL.Snippet
 pgFmtJsonPath = \case
@@ -353,7 +398,7 @@ pgFmtJsonPath = \case
     pgFmtJsonOperand (JKey k) = unknownLiteral k
     pgFmtJsonOperand (JIdx i) = unknownLiteral i <> "::int"
 
-pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SqlFragment
+pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SQL.Snippet
 pgFmtAs _ [] Nothing = mempty
 pgFmtAs fName jp Nothing = case jOp <$> lastMay jp of
   Just (JKey key) -> " AS " <> pgFmtIdent key
@@ -365,7 +410,7 @@ pgFmtAs fName jp Nothing = case jOp <$> lastMay jp of
   Nothing -> mempty
 pgFmtAs _ _ (Just alias) = " AS " <> pgFmtIdent alias
 
-countF :: SQL.Snippet -> Bool -> (SQL.Snippet, SqlFragment)
+countF :: SQL.Snippet -> Bool -> (SQL.Snippet, SQL.Snippet)
 countF countQuery shouldCount =
   if shouldCount
     then (
@@ -375,11 +420,11 @@ countF countQuery shouldCount =
         mempty
       , "null::bigint")
 
-returningF :: QualifiedIdentifier -> [FieldName] -> SqlFragment
+returningF :: QualifiedIdentifier -> [FieldName] -> SQL.Snippet
 returningF qi returnings =
   if null returnings
     then "RETURNING 1" -- For mutation cases where there's no ?select, we return 1 to know how many rows were modified
-    else "RETURNING " <> BS.intercalate ", " (pgFmtColumn qi <$> returnings)
+    else "RETURNING " <> intercalateSnippet ", " (pgFmtColumn qi <$> returnings)
 
 limitOffsetF :: NonnegRange -> SQL.Snippet
 limitOffsetF range =
@@ -388,25 +433,25 @@ limitOffsetF range =
     limit = maybe "ALL" (\l -> unknownEncoder (BS.pack $ show l)) $ rangeLimit range
     offset = unknownEncoder (BS.pack . show $ rangeOffset range)
 
-responseHeadersF :: SqlFragment
+responseHeadersF :: SQL.Snippet
 responseHeadersF = currentSettingF "response.headers"
 
-responseStatusF :: SqlFragment
+responseStatusF :: SQL.Snippet
 responseStatusF = currentSettingF "response.status"
 
-currentSettingF :: SqlFragment -> SqlFragment
+currentSettingF :: SQL.Snippet -> SQL.Snippet
 currentSettingF setting =
   -- nullif is used because of https://gist.github.com/steve-chavez/8d7033ea5655096903f3b52f8ed09a15
   "nullif(current_setting('" <> setting <> "', true), '')"
 
-mutRangeF :: QualifiedIdentifier -> [FieldName] -> (SqlFragment, SqlFragment)
+mutRangeF :: QualifiedIdentifier -> [FieldName] -> (SQL.Snippet, SQL.Snippet)
 mutRangeF mainQi rangeId =
   (
-    BS.intercalate " AND " $ (\col -> pgFmtColumn mainQi col <> " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_affected_rows") col) <$> rangeId
-  , BS.intercalate ", " (pgFmtColumn mainQi <$> rangeId)
+    intercalateSnippet " AND " $ (\col -> pgFmtColumn mainQi col <> " = " <> pgFmtColumn (QualifiedIdentifier mempty "pgrst_affected_rows") col) <$> rangeId
+  , intercalateSnippet ", " (pgFmtColumn mainQi <$> rangeId)
   )
 
-orderF :: QualifiedIdentifier -> [OrderTerm] -> SQL.Snippet
+orderF :: QualifiedIdentifier -> [CoercibleOrderTerm] -> SQL.Snippet
 orderF _ []    = mempty
 orderF qi ordts = "ORDER BY " <> intercalateSnippet ", " (pgFmtOrderTerm qi <$> ordts)
 
@@ -434,8 +479,8 @@ explainF fmt opts snip =
     fmtPlanOpt PlanBuffers  = "BUFFERS"
     fmtPlanOpt PlanWAL      = "WAL"
 
-    fmtPlanFmt PlanJSON = "FORMAT JSON"
     fmtPlanFmt PlanText = "FORMAT TEXT"
+    fmtPlanFmt PlanJSON = "FORMAT JSON"
 
 -- | Do a pg set_config(setting, value, true) call. This is equivalent to a SET LOCAL.
 setConfigLocal :: ByteString -> (ByteString, ByteString) -> SQL.Snippet
@@ -451,3 +496,14 @@ setConfigLocalJson prefix keyVals = [setConfigLocal mempty (prefix, gucJsonVal k
     gucJsonVal = LBS.toStrict . JSON.encode . HM.fromList . arrayByteStringToText
     arrayByteStringToText :: [(ByteString, ByteString)] -> [(Text,Text)]
     arrayByteStringToText keyVal = (T.decodeUtf8 *** T.decodeUtf8) <$> keyVal
+
+aggF :: Maybe Routine -> ResultAggregate -> SQL.Snippet
+aggF rout = \case
+  BuiltinAggJson             -> asJsonF rout False
+  BuiltinAggArrayJsonStrip   -> asJsonF rout True
+  BuiltinAggSingleJson strip -> asJsonSingleF rout strip
+  BuiltinAggGeoJson          -> asGeoJsonF
+  BuiltinAggCsv              -> asCsvF
+  BuiltinAggXml    bField    -> asXmlF    bField
+  BuiltinAggBinary bField    -> asBinaryF bField
+  NoAgg                      -> "''::text"

src/PostgREST/Query/Statements.hs

@@ -23,15 +23,13 @@ import qualified Hasql.DynamicStatements.Statement as SQL
 import qualified Hasql.Statement                   as SQL
 
 import Control.Lens ((^?))
-import Data.Maybe   (fromJust)
 
 import PostgREST.ApiRequest.Preferences
-import PostgREST.MediaType               (MTPlanAttrs (..),
-                                          MTPlanFormat (..),
-                                          MediaType (..),
-                                          getMediaType)
+import PostgREST.MediaType              (MTPlanFormat (..),
+                                         MediaType (..))
 import PostgREST.Query.SqlFragment
-import PostgREST.SchemaCache.Identifiers (FieldName)
+import PostgREST.SchemaCache.Routine    (ResultAggregate (..),
+                                         Routine)
 
 import Protolude
 
@@ -55,109 +53,82 @@ data ResultSet
   | RSPlan BS.ByteString -- ^ the plan of the query
 
 
-prepareWrite :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType ->
-                PreferRepresentation -> [Text] -> Bool -> SQL.Statement () ResultSet
-prepareWrite selectQuery mutateQuery isInsert mt rep pKeys =
+prepareWrite :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> ResultAggregate ->
+                Maybe PreferRepresentation -> [Text] -> Bool -> SQL.Statement () ResultSet
+prepareWrite selectQuery mutateQuery isInsert mt rAgg rep pKeys =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
  where
   snippet =
-    "WITH " <> SQL.sql sourceCTEName <> " AS (" <> mutateQuery <> ") " <>
-    SQL.sql (
+    "WITH " <> sourceCTE <> " AS (" <> mutateQuery <> ") " <>
     "SELECT " <>
       "'' AS total_result_set, " <>
       "pg_catalog.count(_postgrest_t) AS page_total, " <>
       locF <> " AS header, " <>
-      bodyF <> " AS body, " <>
+      aggF Nothing rAgg <> " AS body, " <>
       responseHeadersF <> " AS response_headers, " <>
-      responseStatusF  <> " AS response_status "
-    ) <>
+      responseStatusF  <> " AS response_status " <>
     "FROM (" <> selectF <> ") _postgrest_t"
 
   locF =
-    if isInsert && rep == HeadersOnly
-      then BS.unwords [
-        "CASE WHEN pg_catalog.count(_postgrest_t) = 1",
-          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ")",
-          "ELSE " <> noLocationF,
-        "END"]
+    if isInsert && rep == Just HeadersOnly
+      then
+        "CASE WHEN pg_catalog.count(_postgrest_t) = 1 " <>
+          "THEN coalesce(" <> locationF pKeys <> ", " <> noLocationF <> ") " <>
+          "ELSE " <> noLocationF <> " " <>
+        "END"
       else noLocationF
 
-  bodyF
-    | rep /= Full                       = "''"
-    | getMediaType mt == MTTextCSV      = asCsvF
-    | getMediaType mt == MTGeoJSON      = asGeoJsonF
-    | getMediaType mt == MTSingularJSON = asJsonSingleF False
-    | otherwise                         = asJsonF False False False
-
   selectF
     -- prevent using any of the column names in ?select= when no response is returned from the CTE
-    | rep /= Full = SQL.sql ("SELECT * FROM " <> sourceCTEName)
-    | otherwise   = selectQuery
+    | rAgg == NoAgg = "SELECT * FROM " <> sourceCTE
+    | otherwise     = selectQuery
 
   decodeIt :: HD.Result ResultSet
   decodeIt = case mt of
     MTPlan{} -> planRow
     _        -> fromMaybe (RSStandard Nothing 0 mempty mempty Nothing Nothing) <$> HD.rowMaybe (standardRow False)
 
-prepareRead :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> Maybe FieldName -> Bool -> SQL.Statement () ResultSet
-prepareRead selectQuery countQuery countTotal mt binaryField =
+prepareRead :: SQL.Snippet -> SQL.Snippet -> Bool -> MediaType -> ResultAggregate -> Bool -> SQL.Statement () ResultSet
+prepareRead selectQuery countQuery countTotal mt rAgg =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
  where
   snippet =
-    "WITH " <>
-    SQL.sql sourceCTEName <> " AS ( " <> selectQuery <> " ) " <>
+    "WITH " <> sourceCTE <> " AS ( " <> selectQuery <> " ) " <>
     countCTEF <> " " <>
-    SQL.sql ("SELECT " <>
+    "SELECT " <>
       countResultF <> " AS total_result_set, " <>
       "pg_catalog.count(_postgrest_t) AS page_total, " <>
-      bodyF <> " AS body, " <>
+      aggF Nothing rAgg <> " AS body, " <>
       responseHeadersF <> " AS response_headers, " <>
       responseStatusF <> " AS response_status " <>
-    "FROM ( SELECT * FROM " <> sourceCTEName <> " ) _postgrest_t")
+    "FROM ( SELECT * FROM " <> sourceCTE <> " ) _postgrest_t"
 
   (countCTEF, countResultF) = countF countQuery countTotal
 
-  bodyF
-    | getMediaType mt == MTTextCSV                       = asCsvF
-    | getMediaType mt == MTSingularJSON                  = asJsonSingleF False
-    | getMediaType mt == MTGeoJSON                       = asGeoJsonF
-    | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
-    | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-    | otherwise                                          = asJsonF False False False
-
   decodeIt :: HD.Result ResultSet
   decodeIt = case mt of
     MTPlan{} -> planRow
     _        -> HD.singleRow $ standardRow True
 
-prepareCall :: Bool -> Bool -> Bool -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
-               MediaType -> Maybe FieldName -> Bool ->
+prepareCall :: Routine -> SQL.Snippet -> SQL.Snippet -> SQL.Snippet -> Bool ->
+               MediaType -> ResultAggregate -> Bool ->
                SQL.Statement () ResultSet
-prepareCall returnsScalar returnsSingleComposite returnsSetOfScalar callProcQuery selectQuery countQuery countTotal mt binaryField =
+prepareCall rout callProcQuery selectQuery countQuery countTotal mt rAgg =
   SQL.dynamicallyParameterized (mtSnippet mt snippet) decodeIt
   where
     snippet =
-      "WITH " <> SQL.sql sourceCTEName <> " AS (" <> callProcQuery <> ") " <>
+      "WITH " <> sourceCTE <> " AS (" <> callProcQuery <> ") " <>
       countCTEF <>
-      SQL.sql (
       "SELECT " <>
         countResultF <> " AS total_result_set, " <>
         "pg_catalog.count(_postgrest_t) AS page_total, " <>
-        bodyF <> " AS body, " <>
+        aggF (Just rout) rAgg <> " AS body, " <>
         responseHeadersF <> " AS response_headers, " <>
-        responseStatusF <> " AS response_status ") <>
+        responseStatusF <> " AS response_status " <>
       "FROM (" <> selectQuery <> ") _postgrest_t"
 
     (countCTEF, countResultF) = countF countQuery countTotal
 
-    bodyF
-     | getMediaType mt == MTSingularJSON                  = asJsonSingleF returnsScalar
-     | getMediaType mt == MTTextCSV                       = asCsvF
-     | getMediaType mt == MTGeoJSON                       = asGeoJsonF
-     | isJust binaryField && getMediaType mt == MTTextXML = asXmlF $ fromJust binaryField
-     | isJust binaryField                                 = asBinaryF $ fromJust binaryField
-     | otherwise                                          = asJsonF returnsScalar returnsSetOfScalar returnsSingleComposite
-
     decodeIt :: HD.Result ResultSet
     decodeIt = case mt of
       MTPlan{} -> planRow
@@ -187,8 +158,8 @@ standardRow noLocation =
 
 mtSnippet :: MediaType -> SQL.Snippet -> SQL.Snippet
 mtSnippet mediaType snippet = case mediaType of
-  MTPlan (MTPlanAttrs _ fmt opts) -> explainF fmt opts snippet
-  _                               -> snippet
+  MTPlan _ fmt opts -> explainF fmt opts snippet
+  _                 -> snippet
 
 -- | We use rowList because when doing EXPLAIN (FORMAT TEXT), the result comes as many rows. FORMAT JSON comes as one.
 planRow :: HD.Result ResultSet

src/PostgREST/RangeQuery.hs

@@ -104,9 +104,9 @@ rangeStatusHeader topLevelRange queryTotal tableTotal =
     rangeStatus :: Integer -> Integer -> Maybe Integer -> Status
     rangeStatus _ _ Nothing = status200
     rangeStatus lower upper (Just total)
-      | lower > total               = status416 -- 416 Range Not Satisfiable
-      | (1 + upper - lower) < total = status206 -- 206 Partial Content
-      | otherwise                   = status200 -- 200 OK
+      | lower >= total && lower /= upper = status416 -- 416 Range Not Satisfiable
+      | (1 + upper - lower) < total      = status206 -- 206 Partial Content
+      | otherwise                        = status200 -- 200 OK
 
 contentRangeH :: (Integral a, Show a) => a -> a -> Maybe a -> Header
 contentRangeH lower upper total =

src/PostgREST/Response.hs

@@ -1,3 +1,7 @@
+{- |
+   Module      : PostgREST.Response
+   Description : Generate HTTP Response
+-}
 {-# LANGUAGE NamedFieldPuns  #-}
 {-# LANGUAGE RecordWildCards #-}
 module PostgREST.Response
@@ -13,7 +17,6 @@ module PostgREST.Response
   , updateResponse
   , addRetryHint
   , isServiceUnavailable
-  , optionalRollback
   , traceHeaderMiddleware
   ) where
 
@@ -36,10 +39,9 @@ import qualified PostgREST.Response.OpenAPI as OpenAPI
 import PostgREST.ApiRequest              (ApiRequest (..),
                                           InvokeMethod (..))
 import PostgREST.ApiRequest.Preferences  (PreferRepresentation (..),
-                                          PreferTransaction (..),
                                           Preferences (..),
-                                          shouldCount,
-                                          toAppliedHeader)
+                                          prefAppliedHeader,
+                                          shouldCount)
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
 import PostgREST.Config                  (AppConfig (..))
 import PostgREST.MediaType               (MediaType (..))
@@ -60,12 +62,14 @@ import qualified PostgREST.SchemaCache.Routine as Routine
 import Protolude      hiding (Handler, toS)
 import Protolude.Conv (toS)
 
+
 readResponse :: Bool -> QualifiedIdentifier -> ApiRequest -> ResultSet -> Wai.Response
-readResponse headersOnly identifier ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+readResponse headersOnly identifier ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} resultSet = case resultSet of
   RSStandard{..} -> do
     let
       (status, contentRange) = RangeQuery.rangeStatusHeader iTopLevelRange rsQueryTotal rsTableTotal
       response = gucResponse rsGucStatus rsGucHeaders
+      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing Nothing Nothing preferCount preferTransaction Nothing
       headers =
         [ contentRange
         , ( "Content-Location"
@@ -75,6 +79,7 @@ readResponse headersOnly identifier ctxApiRequest@ApiRequest{..} resultSet = cas
           )
         ]
         ++ contentTypeHeaders ctxApiRequest
+        ++ prefHeader
       rsOrErrBody = if status == HTTP.status416
         then Error.errorPayload $ Error.ApiRequestError $ ApiRequestTypes.InvalidRange
           $ ApiRequestTypes.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
@@ -91,6 +96,9 @@ createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiReques
     let
       pkCols = case mrMutatePlan of { Insert{insPkCols} -> insPkCols; _ -> mempty;}
       response = gucResponse rsGucStatus rsGucHeaders
+      prefHeader = prefAppliedHeader $
+        Preferences (if null pkCols && isNothing (qsOnConflict iQueryParams) then Nothing else preferResolution)
+        preferRepresentation Nothing preferCount preferTransaction preferMissing
       headers =
         catMaybes
           [ if null rsLocation then
@@ -104,21 +112,19 @@ createResponse QualifiedIdentifier{..} MutateReadPlan{mrMutatePlan} ctxApiReques
                 )
           , Just . RangeQuery.contentRangeH 1 0 $
               if shouldCount preferCount then Just rsQueryTotal else Nothing
-          , if null pkCols && isNothing (qsOnConflict iQueryParams) then
-              Nothing
-            else
-              toAppliedHeader <$> preferResolution
-          , toAppliedHeader <$> preferMissing
+          , prefHeader
           ]
 
-    if preferRepresentation == Full then
-      response HTTP.status201 (headers ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
-    else
-      response HTTP.status201 headers mempty
+    case preferRepresentation of
+      Just Full -> response HTTP.status201 (headers ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
+      Just None -> response HTTP.status201 headers mempty
+      Just HeadersOnly -> response HTTP.status201 headers mempty
+      Nothing -> response HTTP.status201 headers mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
+
 updateResponse :: ApiRequest -> ResultSet -> Wai.Response
 updateResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of
   RSStandard{..} -> do
@@ -127,14 +133,13 @@ updateResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet
       contentRangeHeader =
         Just . RangeQuery.contentRangeH 0 (rsQueryTotal - 1) $
           if shouldCount preferCount then Just rsQueryTotal else Nothing
-      headers = catMaybes [contentRangeHeader, toAppliedHeader <$> preferMissing]
+      prefHeader = prefAppliedHeader $ Preferences Nothing preferRepresentation Nothing preferCount preferTransaction preferMissing
+      headers = catMaybes [contentRangeHeader, prefHeader]
 
-    if preferRepresentation == Full then
-        response HTTP.status200
-          (headers ++ contentTypeHeaders ctxApiRequest)
-          (LBS.fromStrict rsBody)
-      else
-        response HTTP.status204 headers mempty
+    case preferRepresentation of
+        Just Full -> response HTTP.status200 (headers ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
+        Just None -> response HTTP.status204 headers mempty
+        _ -> response HTTP.status204 headers mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
@@ -144,11 +149,12 @@ singleUpsertResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resu
   RSStandard {..} -> do
     let
       response = gucResponse rsGucStatus rsGucHeaders
+      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing preferRepresentation Nothing preferCount preferTransaction Nothing
 
-    if preferRepresentation == Full then
-      response HTTP.status200 (contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
-    else
-      response HTTP.status204 [] mempty
+    case preferRepresentation of
+      Just Full -> response HTTP.status200 (contentTypeHeaders ctxApiRequest ++ prefHeader) (LBS.fromStrict rsBody)
+      Just None -> response HTTP.status204 prefHeader mempty
+      _ -> response HTTP.status204 prefHeader mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
@@ -161,14 +167,13 @@ deleteResponse ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet
       contentRangeHeader =
         RangeQuery.contentRangeH 1 0 $
           if shouldCount preferCount then Just rsQueryTotal else Nothing
-      headers = [contentRangeHeader]
+      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing preferRepresentation Nothing preferCount preferTransaction Nothing
+      headers = contentRangeHeader : prefHeader
 
-    if preferRepresentation == Full then
-        response HTTP.status200
-          (headers ++ contentTypeHeaders ctxApiRequest)
-          (LBS.fromStrict rsBody)
-      else
-        response HTTP.status204 headers mempty
+    case preferRepresentation of
+        Just Full -> response HTTP.status200 (headers ++ contentTypeHeaders ctxApiRequest) (LBS.fromStrict rsBody)
+        Just None -> response HTTP.status204 headers mempty
+        _ -> response HTTP.status204 headers mempty
 
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
@@ -201,7 +206,7 @@ respondInfo allowHeader =
   Wai.responseLBS HTTP.status200 [allOrigins, (HTTP.hAllow, allowHeader)] mempty
 
 invokeResponse :: InvokeMethod -> Routine -> ApiRequest -> ResultSet -> Wai.Response
-invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resultSet of
+invokeResponse invMethod proc ctxApiRequest@ApiRequest{iPreferences=Preferences{..}, ..} resultSet = case resultSet of
   RSStandard {..} -> do
     let
       response = gucResponse rsGucStatus rsGucHeaders
@@ -211,7 +216,8 @@ invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resu
         then Error.errorPayload $ Error.ApiRequestError $ ApiRequestTypes.InvalidRange
           $ ApiRequestTypes.OutOfBounds (show $ RangeQuery.rangeOffset iTopLevelRange) (maybe "0" show rsTableTotal)
         else LBS.fromStrict rsBody
-      headers = [contentRange]
+      prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing Nothing preferParameters preferCount preferTransaction Nothing
+      headers = contentRange : prefHeader
 
     if Routine.funcReturnsVoid proc then
         response HTTP.status204 headers mempty
@@ -223,11 +229,11 @@ invokeResponse invMethod proc ctxApiRequest@ApiRequest{..} resultSet = case resu
   RSPlan plan ->
     Wai.responseLBS HTTP.status200 (contentTypeHeaders ctxApiRequest) $ LBS.fromStrict plan
 
-openApiResponse :: Bool -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
-openApiResponse headersOnly body conf sCache schema negotiatedByProfile =
+openApiResponse :: (Text, Text) -> Bool -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Wai.Response
+openApiResponse versions headersOnly body conf sCache schema negotiatedByProfile =
   Wai.responseLBS HTTP.status200
     (MediaType.toContentType MTOpenAPI : maybeToList (profileHeader schema negotiatedByProfile))
-    (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode conf sCache x y z) body)
+    (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache x y z) body)
 
 -- | Response with headers and status overridden from GUCs.
 gucResponse
@@ -270,25 +276,6 @@ addRetryHint delay response = do
 isServiceUnavailable :: Wai.Response -> Bool
 isServiceUnavailable response = Wai.responseStatus response == HTTP.status503
 
-optionalRollback :: AppConfig -> ApiRequest -> ExceptT Error.Error IO Wai.Response -> ExceptT Error.Error IO Wai.Response
-optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} resp = do
-  newRes <- catchError resp $ return . Error.errorResponseFor
-  return $ Wai.mapResponseHeaders preferenceApplied newRes
-  where
-    shouldCommit =
-      configDbTxAllowOverride && preferTransaction == Just Commit
-    shouldRollback =
-      configDbTxAllowOverride && preferTransaction == Just Rollback
-    preferenceApplied
-      | shouldCommit =
-          addHeadersIfNotIncluded
-            [toAppliedHeader Commit]
-      | shouldRollback =
-          addHeadersIfNotIncluded
-            [toAppliedHeader Rollback]
-      | otherwise =
-          identity
-
 -- | Add headers not already included to allow the user to override them instead of duplicating them
 addHeadersIfNotIncluded :: [HTTP.Header] -> [HTTP.Header] -> [HTTP.Header]
 addHeadersIfNotIncluded newHeaders initialHeaders =

src/PostgREST/Response/OpenAPI.hs

@@ -12,7 +12,6 @@ import qualified Data.ByteString.Lazy  as LBS
 import qualified Data.HashMap.Strict   as HM
 import qualified Data.HashSet.InsOrd   as Set
 import qualified Data.Text             as T
-import qualified Data.Text.Encoding    as T
 
 import Control.Arrow              ((&&&))
 import Data.HashMap.Strict.InsOrd (InsOrdHashMap, fromList)
@@ -36,16 +35,16 @@ import PostgREST.SchemaCache.Routine      (Routine (..),
 import PostgREST.SchemaCache.Table        (Column (..), Table (..),
                                            TablesMap,
                                            tableColumnsList)
-import PostgREST.Version                  (docsVersion, prettyVersion)
 
 import PostgREST.MediaType
 
 import Protolude hiding (Proxy, get)
 
-encode :: AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
-encode conf sCache tables procs schemaDescription =
+encode :: (Text, Text) -> AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
+encode versions conf sCache tables procs schemaDescription =
   JSON.encode $
     postgrestSpec
+      versions
       (dbRelationships sCache)
       (concat $ HM.elems procs)
       (snd <$> HM.toList tables)
@@ -154,7 +153,7 @@ makeProcSchema pd =
   & required .~ fmap ppName (filter ppReq (pdParams pd))
 
 makeProcProperty :: RoutineParam -> (Text, Referenced Schema)
-makeProcProperty (RoutineParam n t _ _) = (n, Inline s)
+makeProcProperty (RoutineParam n t _ _ _) = (n, Inline s)
   where
     s = (mempty :: Schema)
           & type_ .~ toSwaggerType t
@@ -181,7 +180,7 @@ makePreferParam ts =
       _            -> []
 
 makeProcGetParam :: RoutineParam -> Referenced Param
-makeProcGetParam (RoutineParam n t r v) =
+makeProcGetParam (RoutineParam n t _ r v) =
   Inline $ (mempty :: Param)
     & name .~ n
     & required ?~ r
@@ -351,7 +350,7 @@ makeProcPathItem pd = ("/rpc/" ++ toS (pdName pd), pe)
       & summary .~ pSum
       & description .~ mfilter (/="") pDesc
       & tags .~ Set.fromList ["(rpc) " <> pdName pd]
-      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON]
+      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False]
       & at 200 ?~ "OK"
     getOp = procOp
       & parameters .~ makeProcGetParams (pdParams pd)
@@ -392,12 +391,12 @@ escapeHostName "*6" = "0.0.0.0"
 escapeHostName "!6" = "0.0.0.0"
 escapeHostName h    = h
 
-postgrestSpec :: RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
-postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
+postgrestSpec :: (Text, Text) -> RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
+postgrestSpec (prettyVersion, docsVersion) rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
   & basePath ?~ T.unpack b
   & schemes ?~ [s']
   & info .~ ((mempty :: Info)
-      & version .~ T.decodeUtf8 prettyVersion
+      & version .~ prettyVersion
       & title .~ fromMaybe "PostgREST API" dTitle
       & description ?~ fromMaybe "This is a dynamic API generated by PostgREST" dDesc)
   & externalDocs ?~ ((mempty :: ExternalDocs)
@@ -407,8 +406,8 @@ postgrestSpec rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
   & definitions .~ fromList (makeTableDef rels <$> ti)
   & parameters .~ fromList (makeParamDefs ti)
   & paths .~ makePathItems pds ti
-  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
-  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
+  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False, MTTextCSV]
+  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False, MTTextCSV]
   & securityDefinitions .~ makeSecurityDefinitions securityDefName allowSecurityDef
   & security .~ [SecurityRequirement (fromList [(securityDefName, [])]) | allowSecurityDef]
     where

src/PostgREST/SchemaCache.hs

@@ -40,32 +40,38 @@ import qualified Hasql.Transaction          as SQL
 import Contravariant.Extras          (contrazip2)
 import Text.InterpolatedString.Perl6 (q)
 
-import PostgREST.Config                   (AppConfig (..))
-import PostgREST.Config.Database          (pgVersionStatement,
-                                           toIsolationLevel)
-import PostgREST.Config.PgVersion         (PgVersion, pgVersion100,
-                                           pgVersion110, pgVersion120)
-import PostgREST.SchemaCache.Identifiers  (AccessSet, FieldName,
-                                           QualifiedIdentifier (..),
-                                           Schema)
-import PostgREST.SchemaCache.Relationship (Cardinality (..),
-                                           Junction (..),
-                                           Relationship (..),
-                                           RelationshipsMap)
-import PostgREST.SchemaCache.Routine      (FuncVolatility (..),
-                                           PgType (..), RetType (..),
-                                           Routine (..), RoutineMap,
-                                           RoutineParam (..))
-import PostgREST.SchemaCache.Table        (Column (..), ColumnMap,
-                                           Table (..), TablesMap)
+import PostgREST.Config                      (AppConfig (..))
+import PostgREST.Config.Database             (pgVersionStatement,
+                                              toIsolationLevel)
+import PostgREST.Config.PgVersion            (PgVersion, pgVersion100,
+                                              pgVersion110,
+                                              pgVersion120)
+import PostgREST.SchemaCache.Identifiers     (AccessSet, FieldName,
+                                              QualifiedIdentifier (..),
+                                              Schema)
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
+                                              Junction (..),
+                                              Relationship (..),
+                                              RelationshipsMap)
+import PostgREST.SchemaCache.Representations (DataRepresentation (..),
+                                              RepresentationsMap)
+import PostgREST.SchemaCache.Routine         (FuncVolatility (..),
+                                              PgType (..),
+                                              RetType (..),
+                                              Routine (..),
+                                              RoutineMap,
+                                              RoutineParam (..))
+import PostgREST.SchemaCache.Table           (Column (..), ColumnMap,
+                                              Table (..), TablesMap)
 
 import Protolude
 
 
 data SchemaCache = SchemaCache
-  { dbTables        :: TablesMap
-  , dbRelationships :: RelationshipsMap
-  , dbRoutines      :: RoutineMap
+  { dbTables          :: TablesMap
+  , dbRelationships   :: RelationshipsMap
+  , dbRoutines        :: RoutineMap
+  , dbRepresentations :: RepresentationsMap
   }
   deriving (Generic, JSON.ToJSON)
 
@@ -116,6 +122,7 @@ querySchemaCache AppConfig{..} = do
   m2oRels <- SQL.statement mempty $ allM2OandO2ORels pgVer prepared
   funcs   <- SQL.statement schemas $ allFunctions pgVer prepared
   cRels   <- SQL.statement mempty $ allComputedRels prepared
+  reps    <- SQL.statement schemas $ dataRepresentations prepared
   _       <-
     let sleepCall = SQL.Statement "select pg_sleep($1)" (param HE.int4) HD.noResult prepared in
     whenJust configInternalSCSleep (`SQL.statement` sleepCall) -- only used for testing
@@ -127,6 +134,7 @@ querySchemaCache AppConfig{..} = do
       dbTables = tabsWViewsPks
     , dbRelationships = getOverrideRelationshipsMap rels cRels
     , dbRoutines = funcs
+    , dbRepresentations = reps
     }
   where
     schemas = toList configDbSchemas
@@ -156,10 +164,11 @@ getOverrideRelationshipsMap rels cRels =
 removeInternal :: [Schema] -> SchemaCache -> SchemaCache
 removeInternal schemas dbStruct =
   SchemaCache {
-      dbTables        = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
-    , dbRelationships = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
-                        HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
-    , dbRoutines      = dbRoutines dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+      dbTables          = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch `elem` schemas) $ dbTables dbStruct
+    , dbRelationships   = filter (\r -> qiSchema (relForeignTable r) `elem` schemas && not (hasInternalJunction r)) <$>
+                          HM.filterWithKey (\(QualifiedIdentifier sch _, _) _ -> sch `elem` schemas ) (dbRelationships dbStruct)
+    , dbRoutines        = dbRoutines dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
+    , dbRepresentations = dbRepresentations dbStruct -- no need to filter, not directly exposed through the API
     }
   where
     hasInternalJunction ComputedRelationship{} = False
@@ -250,6 +259,7 @@ decodeFuncs =
                   (RoutineParam
                   <$> compositeField HD.text
                   <*> compositeField HD.text
+                  <*> compositeField HD.text
                   <*> compositeField HD.bool
                   <*> compositeField HD.bool)
               <*> (parseRetType
@@ -280,6 +290,42 @@ decodeFuncs =
                       | v == 's' = Stable
                       | otherwise = Volatile -- only 'v' can happen here
 
+decodeRepresentations :: HD.Result RepresentationsMap
+decodeRepresentations =
+  HM.fromList . map (\rep@DataRepresentation{drSourceType, drTargetType} -> ((drSourceType, drTargetType), rep)) <$> HD.rowList row
+  where
+    row = DataRepresentation
+      <$> column HD.text
+      <*> column HD.text
+      <*> column HD.text
+
+-- Selects all potential data representation transformations. To qualify the cast must be
+-- 1. to or from a domain
+-- 2. implicit
+-- For the time being it must also be to/from JSON or text, although one can imagine a future where we support special
+-- cases like CSV specific representations.
+dataRepresentations :: Bool -> SQL.Statement [Schema] RepresentationsMap
+dataRepresentations = SQL.Statement sql (arrayParam HE.text) decodeRepresentations
+  where
+    sql = [q|
+    SELECT
+      c.castsource::regtype::text,
+      c.casttarget::regtype::text,
+      c.castfunc::regproc::text
+    FROM
+      pg_catalog.pg_cast c
+    JOIN pg_catalog.pg_type src_t
+      ON c.castsource::oid = src_t.oid
+    JOIN pg_catalog.pg_type dst_t
+      ON c.casttarget::oid = dst_t.oid
+    WHERE
+      c.castcontext = 'i'
+      AND c.castmethod = 'f'
+      AND has_function_privilege(c.castfunc, 'execute')
+      AND ((src_t.typtype = 'd' AND c.casttarget IN ('json'::regtype::oid , 'text'::regtype::oid))
+       OR (dst_t.typtype = 'd' AND c.castsource IN ('json'::regtype::oid , 'text'::regtype::oid)))
+    |]
+
 allFunctions :: PgVersion -> Bool -> SQL.Statement [Schema] RoutineMap
 allFunctions pgVer = SQL.Statement sql (arrayParam HE.text) decodeFuncs
   where
@@ -322,6 +368,13 @@ funcsSqlQuery pgVer = [q|
       array_agg((
         COALESCE(name, ''), -- name
         type::regtype::text, -- type
+        CASE type
+          WHEN 'bit'::regtype THEN 'bit varying'
+          WHEN 'bit[]'::regtype THEN 'bit varying[]'
+          WHEN 'character'::regtype THEN 'character varying'
+          WHEN 'character[]'::regtype THEN 'character varying[]'
+          ELSE type::regtype::text
+        END, -- convert types that ignore the lenth and accept any value till maximum size
         idx <= (pronargs - pronargdefaults), -- is_required
         COALESCE(mode = 'v', FALSE) -- is_variadic
       ) ORDER BY idx) AS args,
@@ -526,22 +579,21 @@ tablesSqlQuery pgVer =
           c.relname::name AS table_name,
           a.attname::name AS column_name,
           d.description AS description,
-  |] <> columnDefault <>
-  [q|
+  |] <> columnDefault <> [q| AS column_default,
           not (a.attnotnull OR t.typtype = 'd' AND t.typnotnull) AS is_nullable,
+          CASE
+              WHEN t.typtype = 'd' THEN
               CASE
-                  WHEN t.typtype = 'd' THEN
-                  CASE
-                      WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
-                      ELSE format_type(a.atttypid, a.atttypmod)
-                  END
-                  ELSE
-                  CASE
-                      WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
-                      ELSE format_type(a.atttypid, a.atttypmod)
-                  END
-              END::text AS data_type,
-          t.oid AS data_type_id,
+                  WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
+                  ELSE format_type(a.atttypid, a.atttypmod)
+              END
+              ELSE
+              CASE
+                  WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
+                  ELSE format_type(a.atttypid, a.atttypmod)
+              END
+          END::text AS data_type,
+          format_type(a.atttypid, a.atttypmod)::text AS nominal_data_type,
           information_schema._pg_char_max_length(
               information_schema._pg_truetypid(a.*, t.*),
               information_schema._pg_truetypmod(a.*, t.*)
@@ -583,7 +635,7 @@ tablesSqlQuery pgVer =
           info.description,
           info.is_nullable::boolean,
           info.data_type,
-          info.data_type_id::regtype::text,
+          info.nominal_data_type,
           info.character_maximum_length,
           info.column_default,
           coalesce(enum_info.vals, '{}')) order by info.position) as columns
@@ -715,19 +767,25 @@ tablesSqlQuery pgVer =
   "ORDER BY table_schema, table_name"
   where
     relIsPartition = if pgVer >= pgVersion100 then " AND not c.relispartition " else mempty
-    columnDefault
+    columnDefault -- typbasetype and typdefaultbin handles `CREATE DOMAIN .. DEFAULT val`,  attidentity/attgenerated handles generated columns, pg_get_expr gets the default of a column
       | pgVer >= pgVersion120 = [q|
           CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
             WHEN a.attidentity  = 'd' THEN format('nextval(%s)', quote_literal(seqsch.nspname || '.' || seqclass.relname))
             WHEN a.attgenerated = 's' THEN null
             ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
-          END AS column_default,|]
+          END|]
       | pgVer >= pgVersion100 = [q|
           CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
             WHEN a.attidentity = 'd' THEN format('nextval(%s)', quote_literal(seqsch.nspname || '.' || seqclass.relname))
             ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
-          END AS column_default,|]
-      | otherwise  = "pg_get_expr(ad.adbin, ad.adrelid)::text as column_default,"
+          END|]
+      | otherwise  = [q|
+          CASE
+            WHEN t.typbasetype  != 0  THEN pg_get_expr(t.typdefaultbin, 0)
+            ELSE pg_get_expr(ad.adbin, ad.adrelid)::text
+          END|]
 
 -- | Gets many-to-one relationships and one-to-one(O2O) relationships, which are a refinement of the many-to-one's
 allM2OandO2ORels :: PgVersion -> Bool -> SQL.Statement () [Relationship]
@@ -824,6 +882,7 @@ allComputedRels =
     (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
     (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
     (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
+    pure (QualifiedIdentifier mempty mempty) <*>
     column HD.bool <*>
     column HD.bool
 

src/PostgREST/SchemaCache/Identifiers.hs

@@ -24,7 +24,7 @@ data QualifiedIdentifier = QualifiedIdentifier
   { qiSchema :: Schema
   , qiName   :: TableName
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
 
 instance Hashable QualifiedIdentifier
 

src/PostgREST/SchemaCache/Relationship.hs

@@ -31,10 +31,11 @@ data Relationship = Relationship
   { relFunction     :: QualifiedIdentifier
   , relTable        :: QualifiedIdentifier
   , relForeignTable :: QualifiedIdentifier
+  , relTableAlias   :: QualifiedIdentifier
   , relToOne        :: Bool
   , relIsSelf       :: Bool
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 -- | The relationship cardinality
 -- | https://en.wikipedia.org/wiki/Cardinality_(data_modeling)
@@ -47,7 +48,7 @@ data Cardinality
   -- ^ one-to-one, this is a refinement over M2O so operating on it is pretty much the same as M2O
   | M2M Junction
   -- ^ many-to-many
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 type FKConstraint = Text
 
@@ -59,7 +60,7 @@ data Junction = Junction
   , junColsSource  :: [(FieldName, FieldName)]
   , junColsTarget  :: [(FieldName, FieldName)]
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 -- | Key based on the source table and the foreign table schema
 type RelationshipsMap = HM.HashMap (QualifiedIdentifier, Schema)  [Relationship]

src/PostgREST/SchemaCache/Representations.hs

@@ -0,0 +1,29 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric  #-}
+
+module PostgREST.SchemaCache.Representations
+  ( DataRepresentation(..)
+  , RepresentationsMap
+  ) where
+
+import qualified Data.Aeson          as JSON
+import qualified Data.HashMap.Strict as HM
+
+
+import Protolude
+
+-- | Data representations allow user customisation of how to present and receive data through APIs, per field.
+-- This structure is used for the library of available transforms. It answers questions like:
+-- - What function, if any, should be used to present a certain field that's been selected for API output?
+-- - How do we parse incoming data for a certain field type when inserting or updating?
+-- - And similarly, how do we parse textual data in a query string to be used as a filter?
+--
+-- Support for outputting special formats like CSV and binary data would fit into the same system.
+data DataRepresentation = DataRepresentation
+  { drSourceType :: Text
+  , drTargetType :: Text
+  , drFunction   :: Text
+  } deriving (Eq, Show, Generic, JSON.ToJSON, JSON.FromJSON)
+
+-- The representation map maps from (source type, target type) to a DR.
+type RepresentationsMap = HM.HashMap (Text, Text) DataRepresentation

src/PostgREST/SchemaCache/Routine.hs

@@ -14,6 +14,7 @@ module PostgREST.SchemaCache.Routine
   , funcReturnsVoid
   , funcTableName
   , funcReturnsCompositeAlias
+  , ResultAggregate(..)
   ) where
 
 import           Data.Aeson                 ((.=))
@@ -21,7 +22,8 @@ import qualified Data.Aeson                 as JSON
 import qualified Data.HashMap.Strict        as HM
 import qualified Hasql.Transaction.Sessions as SQL
 
-import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
+import PostgREST.SchemaCache.Identifiers (FieldName,
+                                          QualifiedIdentifier (..),
                                           Schema, TableName)
 
 import Protolude
@@ -29,18 +31,18 @@ import Protolude
 data PgType
   = Scalar QualifiedIdentifier
   | Composite QualifiedIdentifier Bool -- True if the composite is a domain alias(used to work around a bug in pg 11 and 12, see QueryBuilder.hs)
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 data RetType
   = Single PgType
   | SetOf PgType
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 data FuncVolatility
   = Volatile
   | Stable
   | Immutable
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 data Routine = Function
   { pdSchema      :: Schema
@@ -52,7 +54,7 @@ data Routine = Function
   , pdHasVariadic :: Bool
   , pdIsoLvl      :: Maybe SQL.IsolationLevel
   }
-  deriving (Eq, Generic)
+  deriving (Eq, Show, Generic)
 -- need to define JSON manually bc SQL.IsolationLevel doesn't have a JSON instance(and we can't define one for that type without getting a compiler error)
 instance JSON.ToJSON Routine where
   toJSON (Function sch nam desc params ret vol hasVar _) = JSON.object
@@ -67,12 +69,13 @@ instance JSON.ToJSON Routine where
     ]
 
 data RoutineParam = RoutineParam
-  { ppName :: Text
-  , ppType :: Text
-  , ppReq  :: Bool
-  , ppVar  :: Bool
+  { ppName          :: Text
+  , ppType          :: Text
+  , ppTypeMaxLength :: Text
+  , ppReq           :: Bool
+  , ppVar           :: Bool
   }
-  deriving (Eq, Ord, Generic, JSON.ToJSON)
+  deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
 
 -- Order by least number of params in the case of overloaded functions
 instance Ord Routine where
@@ -85,6 +88,17 @@ instance Ord Routine where
 -- | It uses a HashMap for a faster lookup.
 type RoutineMap = HM.HashMap QualifiedIdentifier [Routine]
 
+data ResultAggregate
+   = BuiltinAggJson
+   | BuiltinAggSingleJson Bool
+   | BuiltinAggArrayJsonStrip
+   | BuiltinAggGeoJson
+   | BuiltinAggCsv
+   | BuiltinAggXml    (Maybe FieldName)
+   | BuiltinAggBinary (Maybe FieldName)
+   | NoAgg
+   deriving (Eq, Show)
+
 funcReturnsScalar :: Routine -> Bool
 funcReturnsScalar proc = case proc of
   Function{pdReturnType = Single (Scalar{})} -> True

stack.yaml

@@ -12,5 +12,5 @@ nix:
 extra-deps:
   - git: https://github.com/PostgREST/postgresql-libpq.git
     commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
-  - hasql-notifications-0.2.0.5
+  - hasql-notifications-0.2.0.6
   - hasql-pool-0.10

stack.yaml.lock

@@ -16,12 +16,12 @@ packages:
     commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
     git: https://github.com/PostgREST/postgresql-libpq.git
 - completed:
-    hackage: hasql-notifications-0.2.0.5@sha256:6c67d2ee42e948162e89176a2f52a2c6bb562678b33688e96e048be9b3d74122,2028
+    hackage: hasql-notifications-0.2.0.6@sha256:16d783f5cd1660fad924fd3769380889de5804e057f09b304dcdc3a3ff11eb3c,2028
     pantry-tree:
-      sha256: 8493755b0817a36b910a37d568769ca49ab715ef5dd935a0d09f0716aa39da51
+      sha256: 2319743501bb3c0bef801014ce61308b8666cef86ae5a97a0a283c0c1ec12d4f
       size: 452
   original:
-    hackage: hasql-notifications-0.2.0.5
+    hackage: hasql-notifications-0.2.0.6
 - completed:
     hackage: hasql-pool-0.10@sha256:912197a328acb85505f98bb9700d61f366b87659ca45126c5c2d636687b801c3,2112
     pantry-tree:

test/doc/Main.hs

@@ -11,9 +11,14 @@ main =
     [ "-XOverloadedStrings"
     , "-XNoImplicitPrelude"
     , "-XStandaloneDeriving"
+    , "-XDuplicateRecordFields"
     , "-isrc"
     , "src/PostgREST/Query/SqlFragment.hs"
     , "src/PostgREST/ApiRequest/Preferences.hs"
     , "src/PostgREST/ApiRequest/QueryParams.hs"
     , "src/PostgREST/Error.hs"
+    , "src/PostgREST/MediaType.hs"
+    , "src/PostgREST/Config.hs"
+    , "src/PostgREST/Plan.hs"
+    , "src/PostgREST/Response.hs"
     ]

test/io/fixtures.sql

@@ -14,7 +14,14 @@ alter role postgrest_test_serializable set default_transaction_isolation = 'seri
 CREATE ROLE postgrest_test_repeatable_read;
 alter role postgrest_test_repeatable_read set default_transaction_isolation = 'REPEATABLE READ';
 
-GRANT postgrest_test_anonymous, postgrest_test_author, postgrest_test_serializable, postgrest_test_repeatable_read TO :PGUSER;
+CREATE ROLE postgrest_test_w_superuser_settings;
+alter role postgrest_test_w_superuser_settings set log_min_duration_statement = 1;
+alter role postgrest_test_w_superuser_settings set log_min_messages = 'fatal';
+
+GRANT
+  postgrest_test_anonymous, postgrest_test_author,
+  postgrest_test_serializable, postgrest_test_repeatable_read,
+  postgrest_test_w_superuser_settings TO :PGUSER;
 
 CREATE SCHEMA v1;
 GRANT USAGE ON SCHEMA v1 TO postgrest_test_anonymous;
@@ -23,7 +30,7 @@ CREATE TABLE authors_only ();
 GRANT SELECT ON authors_only TO postgrest_test_author;
 
 CREATE TABLE projects AS SELECT FROM generate_series(1,5);
-GRANT SELECT ON projects TO postgrest_test_anonymous;
+GRANT SELECT ON projects TO postgrest_test_anonymous, postgrest_test_w_superuser_settings;
 
 create function get_guc_value(name text) returns text as $$
   select nullif(current_setting(name), '')::text;
@@ -158,3 +165,12 @@ create or replace function migrate_function() returns void as $_$
   $$ language sql;
   notify pgrst, 'reload schema';
 $_$ language sql security definer;
+
+create or replace function get_pgrst_version() returns text
+  language sql
+as $$
+select application_name
+from pg_stat_activity
+where application_name ilike 'postgrest%'
+limit 1;
+$$

test/io/fixtures.yaml

@@ -4,6 +4,10 @@ cli:
     args: ['--help']
   - name: help short
     args: ['-h']
+  - name: version long
+    args: ['--version']
+  - name: version short
+    args: ['-v']
   - name: example long
     args: ['--example']
   - name: example short

test/io/postgrest.py

@@ -64,6 +64,7 @@ def run(
     host=None,
     wait_for_readiness=True,
     no_pool_connection_available=False,
+    no_startup_stdout=True,
 ):
     "Run PostgREST and yield an endpoint that is ready for connections."
 
@@ -104,7 +105,8 @@ def run(
             if wait_for_readiness:
                 wait_until_ready(adminurl + "/ready")
 
-            process.stdout.read()
+            if no_startup_stdout:
+                process.stdout.read()
 
             if no_pool_connection_available:
                 sleep_pool_connection(baseurl, 10)

test/io/test_io.py

@@ -998,3 +998,68 @@ def test_openapi_in_big_schema(defaultenv):
     with run(env=env) as postgrest:
         response = postgrest.session.get("/")
         assert response.status_code == 200
+
+
+@pytest.mark.parametrize("dburi_type", ["no_params", "no_params_qmark", "with_params"])
+def test_get_pgrst_version_with_uri_connection_string(dburi_type, dburi, defaultenv):
+    "The fallback_application_name should be added to the db-uri if it has a URI format"
+    defaultenv_without_libpq = {
+        key: value
+        for key, value in defaultenv.items()
+        if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
+    }
+
+    env = {
+        "no_params": {**defaultenv, "PGRST_DB_URI": "postgresql://"},
+        "no_params_qmark": {**defaultenv, "PGRST_DB_URI": "postgresql://?"},
+        "with_params": {**defaultenv_without_libpq, "PGRST_DB_URI": dburi.decode()},
+    }
+
+    with run(env=env[dburi_type]) as postgrest:
+        response = postgrest.session.post("/rpc/get_pgrst_version")
+        version = '"%s"' % response.headers["Server"].replace(
+            "postgrest/", "PostgREST "
+        )
+        assert response.text == version
+
+
+def test_get_pgrst_version_with_keyval_connection_string(defaultenv):
+    "The fallback_application_name should be added to the db-uri if it has a keyword/value format"
+    uri = f'dbname={defaultenv["PGDATABASE"]} host={defaultenv["PGHOST"]} user={defaultenv["PGUSER"]}'
+    defaultenv_without_libpq = {
+        key: value
+        for key, value in defaultenv.items()
+        if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
+    }
+    env = {**defaultenv_without_libpq, "PGRST_DB_URI": uri}
+
+    with run(env=env) as postgrest:
+        response = postgrest.session.post("/rpc/get_pgrst_version")
+        version = '"%s"' % response.headers["Server"].replace(
+            "postgrest/", "PostgREST "
+        )
+        assert response.text == version
+
+
+def test_log_postgrest_version(defaultenv):
+    "Should show the PostgREST version in the logs"
+
+    with run(env=defaultenv, no_startup_stdout=False) as postgrest:
+        version = postgrest.session.head("/").headers["Server"].split("/")[1]
+
+        assert (
+            "Starting PostgREST %s..." % version
+            in postgrest.process.stdout.readline().decode()
+        )
+
+
+def test_succeed_w_role_having_superuser_settings(defaultenv):
+    "Should succeed when having superuser settings on the impersonated role"
+
+    env = {**defaultenv, "PGRST_DB_CONFIG": "true", "PGRST_JWT_SECRET": SECRET}
+
+    with run(stdin=SECRET.encode(), env=env) as postgrest:
+        headers = jwtauthheader({"role": "postgrest_test_w_superuser_settings"}, SECRET)
+        response = postgrest.session.get("/projects", headers=headers)
+        print(response.text)
+        assert response.status_code == 200

test/memory/memory-tests.sh

@@ -102,7 +102,7 @@ postJsonArrayTest(){
 
 echo "Running memory usage tests.."
 
-jsonKeyTest "1M" "POST" "/rpc/leak?columns=blob" "16M"
+jsonKeyTest "1M" "POST" "/rpc/leak?columns=blob" "23M"
 jsonKeyTest "1M" "POST" "/leak?columns=blob" "16M"
 jsonKeyTest "1M" "PATCH" "/leak?id=eq.1&columns=blob" "16M"
 

test/spec/Feature/CorsSpec.hs

@@ -40,7 +40,7 @@ spec =
                              \Date, Location, Server, Transfer-Encoding, Range-Unit"]
           }
 
-    it "allows INFO body through even with CORS request headers present to postflight request" $
+    it "allows INFO body through even with CORS request headers present to postflight request" $ do
       request methodOptions "/items"
           [ ("Host", "localhost:3000")
           , ("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0")
@@ -54,3 +54,17 @@ spec =
         `shouldRespondWith`
           ""
           { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
+
+      request methodOptions "/items"
+          [ ("Accept", "application/json") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }
+
+      request methodOptions "/shops"
+          [ ("Accept", "application/geo+json") ]
+          ""
+        `shouldRespondWith`
+          ""
+          { matchHeaders = [ "Access-Control-Allow-Origin" <:> "*" ] }

test/spec/Feature/Query/ComputedRelsSpec.hs

@@ -104,6 +104,47 @@ spec = describe "computed relationships" $ do
         [json|[ {"name":"Final Fantasy I","designer":{"name":"Hironobu Sakaguchi"}} ]|]
         { matchStatus  = 200 }
 
+  it "applies data representations to response" $ do
+    -- A smoke test for data reps in the presence of computed relations.
+
+    -- The data rep here title cases the designer name before presentation. So here the lowercase version will be saved,
+    -- but the title case version returned. Pulling in a computed relation should not confuse this.
+    request methodPatch "/designers?select=name,videogames:computed_videogames(name)&id=eq.1"
+      [("Prefer", "return=representation"), ("Prefer", "tx=commit")]
+      [json| {"name": "sidney k. meier"} |]
+      `shouldRespondWith`
+      [json|[{"name":"Sidney K. Meier","videogames":[{"name":"Civilization I"}, {"name":"Civilization II"}]}]|]
+      { matchStatus = 200 }
+
+    -- Verify it was saved the way we requested (there's no text data rep for this column, so if we select with the wrong casing, it should fail.)
+    get "/designers?select=id&name=eq.Sidney%20K.%20Meier"
+      `shouldRespondWith`
+      [json|[]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
+    -- But with the right casing it works.
+    get "/designers?select=id,name&name=eq.sidney%20k.%20meier"
+      `shouldRespondWith`
+      [json|[{"id": 1, "name":"Sidney K. Meier"}]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
+
+    -- Most importantly, if you read it back even via a computed relation, the data rep should be applied.
+    get "/videogames?select=name,designer:computed_designers(*)&id=eq.1"
+      `shouldRespondWith`
+      [json|[
+        {"name":"Civilization I","designer":{"id": 1, "name":"Sidney K. Meier"}}
+      ]|] { matchHeaders = [matchContentTypeJson] }
+
+    -- reset the test fixture
+    request methodPatch "/designers?id=eq.1"
+      [("Prefer", "tx=commit")]
+      [json| {"name": "Sid Meier"} |]
+      `shouldRespondWith` 204
+    -- need to poke the second one too to prevent inherent ordering from changing
+    request methodPatch "/designers?id=eq.2"
+      [("Prefer", "tx=commit")]
+      [json| {"name": "Hironobu Sakaguchi"} |]
+      `shouldRespondWith` 204
+
   it "works with self joins" $
     get "/web_content?select=name,child_web_content(name),parent_web_content(name)&id=in.(0,1)"
     `shouldRespondWith`
@@ -151,3 +192,29 @@ spec = describe "computed relationships" $ do
           {"name":"Windows 10","computed_clients":{"name":"Microsoft"}}
         ]}
       ]|] { matchHeaders = [matchContentTypeJson] }
+
+  -- https://github.com/PostgREST/postgrest/issues/2963
+  context "can be defined using overloaded functions" $ do
+    it "tables" $ do
+      get "/items?select=*,computed_rel_overload(*)&limit=1"
+        `shouldRespondWith`
+        [json|
+          [{"id":1,"computed_rel_overload":[{"id":1}]}]
+        |] { matchHeaders = [matchContentTypeJson] }
+      get "/items2?select=*,computed_rel_overload(*)&limit=1"
+        `shouldRespondWith`
+        [json|
+          [{"id":1,"computed_rel_overload":[{"id":1},{"id":2}]}]
+        |] { matchHeaders = [matchContentTypeJson] }
+
+    it "rpc" $ do
+      get "/rpc/search?id=1&select=*,computed_rel_overload(*)"
+        `shouldRespondWith`
+        [json|
+          [{"id":1,"computed_rel_overload":[{"id":1}]}]
+        |] { matchHeaders = [matchContentTypeJson] }
+      get "/rpc/search2?id=1&select=*,computed_rel_overload(*)"
+        `shouldRespondWith`
+        [json|
+          [{"id":1,"computed_rel_overload":[{"id":1},{"id":2}]}]
+        |] { matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/DeleteSpec.hs

@@ -37,7 +37,8 @@ spec =
         request methodDelete "/items?id=eq.2" [("Prefer", "return=representation"), ("Prefer", "count=exact")] ""
           `shouldRespondWith` [json|[{"id":2}]|]
           { matchStatus  = 200
-          , matchHeaders = ["Content-Range" <:> "*/1"]
+          , matchHeaders = ["Content-Range" <:> "*/1"
+                           , "Preference-Applied" <:> "return=representation, count=exact"]
           }
 
       it "ignores ?select= when return not set or return=minimal" $ do
@@ -57,7 +58,8 @@ spec =
             ""
             { matchStatus  = 204
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       it "returns the deleted item and shapes the response" $
@@ -137,7 +139,8 @@ spec =
           `shouldRespondWith`
             ""
             { matchStatus = 204
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal" ]
             }
 
       it "suceeds deleting the row with no explicit select by default" $

test/spec/Feature/Query/InsertSpec.hs

@@ -43,7 +43,8 @@ spec actualPgVersion = do
           , "enum": "foo"
           }] |] `shouldRespondWith` [json|[{"integer":14,"varchar":"testing!"}]|]
           { matchStatus  = 201
-          , matchHeaders = [matchContentTypeJson]
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "ignores &select when return not set or using return=minimal" $ do
@@ -69,7 +70,8 @@ spec actualPgVersion = do
           `shouldRespondWith`
             ""
             { matchStatus  = 201
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
     context "non uniform json array" $ do
@@ -99,7 +101,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/1" ]
+                           , "Content-Range" <:> "*/1"
+                           , "Preference-Applied" <:> "return=representation, count=exact"]
           }
 
       it "can rename and cast the selected columns" $
@@ -110,7 +113,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/*" ]
+                           , "Content-Range" <:> "*/*"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "should not throw and return location header when selecting without PK" $
@@ -120,7 +124,8 @@ spec actualPgVersion = do
           { matchStatus  = 201
           , matchHeaders = [ matchContentTypeJson
                            , matchHeaderAbsent hLocation
-                           , "Content-Range" <:> "*/*" ]
+                           , "Content-Range" <:> "*/*"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
     context "requesting headers only representation" $ do
@@ -133,7 +138,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/projects?id=eq.11"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
       when (actualPgVersion >= pgVersion110) $
@@ -146,7 +152,8 @@ spec actualPgVersion = do
               { matchStatus  = 201
               , matchHeaders = [ matchHeaderAbsent hContentType
                                , "Location" <:> "/car_models?name=eq.Enzo&year=eq.2021"
-                               , "Content-Range" <:> "*/*" ]
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
               }
 
     context "requesting no representation" $
@@ -193,7 +200,8 @@ spec actualPgVersion = do
               ""
               { matchStatus  = 201
               , matchHeaders = [ matchHeaderAbsent hContentType
-                               , "Location" <:> "/auto_incrementing_pk?id=eq.2" ]
+                               , "Location" <:> "/auto_incrementing_pk?id=eq.2"
+                               , "Preference-Applied" <:> "return=headers-only"]
               }
 
       context "into a table with simple pk" $
@@ -227,7 +235,8 @@ spec actualPgVersion = do
             `shouldRespondWith`
               [json| [{ "a":"bar", "b":"baz" }] |]
               { matchStatus  = 201
-              , matchHeaders = [matchHeaderAbsent hLocation]
+              , matchHeaders = [matchHeaderAbsent hLocation
+                               , "Preference-Applied" <:> "return=representation"]
               }
 
         it "returns empty array when no items inserted, and return=rep" $ do
@@ -391,6 +400,22 @@ spec actualPgVersion = do
           `shouldRespondWith` [json|[{ id: 20 }]|]
             { matchStatus  = 201 }
 
+    -- https://github.com/PostgREST/postgrest/issues/2861
+    context "bit and char columns with length" $ do
+      it "should insert to a bit column with length" $
+        request methodPost "/bitchar_with_length?select=bit"
+            [("Prefer", "return=representation")]
+            [json|{"bit": "10101"}|]
+          `shouldRespondWith` [json|[{ "bit": "10101" }]|]
+            { matchStatus  = 201 }
+
+      it "should insert to a char column with length" $
+        request methodPost "/bitchar_with_length?select=char"
+            [("Prefer", "return=representation")]
+            [json|{"char": "abcde"}|]
+          `shouldRespondWith` [json|[{ "char": "abcde" }]|]
+            { matchStatus  = 201 }
+
     context "POST with ?columns parameter" $ do
       it "ignores json keys not included in ?columns" $ do
         request methodPost "/articles?columns=id,body" [("Prefer", "return=representation")]
@@ -469,7 +494,7 @@ spec actualPgVersion = do
                   {"id": 6, "name": "Sechs", "field-with_sep": 6, "settings":null,"arr_data":[1,2,3]}
                 ]|]
                 { matchStatus  = 201
-                , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+                , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
                 }
 
         it "inserts view default values(field-with_sep) when json keys are undefined" $
@@ -484,7 +509,7 @@ spec actualPgVersion = do
                 {"id": 8, "name": "Default", "field-with_sep": 1, "settings":null,"arr_data":null}
               ]|]
               { matchStatus  = 201
-              , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
               }
 
         it "doesn't insert json duplicate keys(since it uses jsonb)" $
@@ -493,7 +518,7 @@ spec actualPgVersion = do
             `shouldRespondWith`
               [json| [ { "data": { "a": 2 }, "id": 3 } ] |]
               { matchStatus  = 201
-              , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
               }
 
         when (actualPgVersion >= pgVersion100) $
@@ -503,7 +528,7 @@ spec actualPgVersion = do
               `shouldRespondWith`
                 [json| [{"data":{"foo": "bar"},"slug":"foo"}] |] -- id 1 was inserted here, we don't get it for idempotence in the tests
                 { matchStatus  = 201
-                , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+                , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
                 }
 
         when (actualPgVersion >= pgVersion120) $
@@ -529,6 +554,15 @@ spec actualPgVersion = do
                   }|])
                 { matchStatus  = 400 }
 
+        it "inserts a default on a DOMAIN with default" $
+          request methodPost "/evil_friends?columns=id,name" [("Prefer", "return=representation"), ("Prefer", "missing=default")]
+              [json| { "name": "Lu" } |]
+            `shouldRespondWith`
+              [json| [{"id": 666, "name": "Lu"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+              }
+
     it "inserts json that has duplicate keys" $ do
       request methodPost "/tbl_w_json" [("Prefer", "return=representation")]
           [json| { "data": { "a": 1, "a": 2 }, "id": 3 } |]
@@ -694,7 +728,8 @@ spec actualPgVersion = do
         `shouldRespondWith`
           ""
           { matchStatus = 201
-          , matchHeaders = [matchHeaderAbsent hContentType]
+          , matchHeaders = [matchHeaderAbsent hContentType
+                           , "Preference-Applied" <:> "return=minimal"]
           }
 
   describe "Inserting into VIEWs" $ do
@@ -717,7 +752,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/with_multiple_pks?pk1=eq.1&pk2=eq.2"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
     context "requesting header only representation" $ do
@@ -729,7 +765,8 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/compound_pk_view?k1=eq.1&k2=eq.test"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
 
       it "should not throw and return location header when a PK is null" $
@@ -740,5 +777,120 @@ spec actualPgVersion = do
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
                              , "Location" <:> "/test_null_pk_competitors_sponsors?id=eq.1&sponsor_id=is.null"
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=headers-only"]
             }
+
+
+  -- Data representations for payload parsing requires Postgres 10 or above.
+  when (actualPgVersion >= pgVersion100) $ do
+    describe "Data representations" $ do
+      context "on regular table" $ do
+        it "parses values in POST body" $
+          -- we don't check that the parsing is correct here, just that it's happening. If it doesn't happen we'll get a
+          -- an "invalid input syntax for type integer:" error.
+          request methodPost "/datarep_todos" [("Prefer", "return=headers-only")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 201
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Location" <:> "/datarep_todos?id=eq.5"
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
+              }
+
+        it "parses values in POST body and formats individually selected values in return=representation" $
+          request methodPost "/datarep_todos?select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "label_color": "#001100"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "parses values in POST body and formats values in return=representation" $
+          request methodPost "/datarep_todos" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00", "icon_image": "3q2+7w", "created_at":-15, "budget": "-100000000000000.13"} |]
+            `shouldRespondWith`
+            [json| [{"id":5,"name": "party", "label_color": "#001100", "due_at":"2018-01-03T11:00:00Z", "icon_image": "3q2+7w==", "created_at":-15, "budget": "-100000000000000.13"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+      context "with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPost "/datarep_todos?columns=id,label_color&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "invalid but should be ignored"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "name":null, "label_color": "#001100", "due_at": "2018-01-01T00:00:00Z"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "fails without parsing anything if at least one specified column doesn't exist" $
+          request methodPost "/datarep_todos?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }
+
+      context "on updatable view" $ do
+        it "parses values in POST body" $
+          -- we don't check that the parsing is correct here, just that it's happening. If it doesn't happen we'll get a
+          -- an "invalid input syntax for type integer:" error.
+          request methodPost "/datarep_todos_computed" [("Prefer", "return=headers-only")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 201
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Location" <:> "/datarep_todos_computed?id=eq.5"
+                               , "Content-Range" <:> "*/*"
+                               , "Preference-Applied" <:> "return=headers-only"]
+              }
+
+        it "parses values in POST body and formats individually selected values in return=representation" $
+          request methodPost "/datarep_todos_computed?select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "label_color": "#001100"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "parses values in POST body and formats values in return=representation" $
+          request methodPost "/datarep_todos_computed" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "2018-01-03T11:00:00+00"} |]
+            `shouldRespondWith`
+            [json| [{"id":5,"name": "party", "label_color": "#001100", "due_at":"2018-01-03T11:00:00Z", "dark_color":"#000880"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+      context "on updatable views with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPost "/datarep_todos_computed?columns=id,label_color&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"id":5, "name": "party", "label_color": "#001100", "due_at": "invalid but should be ignored"} |]
+            `shouldRespondWith`
+            [json| [{"id":5, "name":null, "label_color": "#001100", "due_at": "2018-01-01T00:00:00Z"}] |]
+              { matchStatus  = 201
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8",
+                                "Content-Range" <:> "*/*"]
+              }
+
+        it "fails without parsing anything if at least one specified column doesn't exist" $
+          request methodPost "/datarep_todos_computed?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos_computed' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }

test/spec/Feature/Query/JsonOperatorSpec.hs

@@ -103,7 +103,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
 
       it "can get array of objects" $ do
         get "/json_arr?select=data->0->>a&id=in.(5,6)" `shouldRespondWith`
-          [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
+          [json|[{"a":"A"}, {"a":"[1,2,3]"}]|]
           { matchHeaders = [matchContentTypeJson] }
         get "/json_arr?select=data->0->a->>2&id=in.(5,6)" `shouldRespondWith`
           [json| [{"a":null}, {"a":"3"}] |]
@@ -275,7 +275,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         [json| [{"data":8}, {"data":7}] |]
         { matchHeaders = [matchContentTypeJson] }
       get "/json_arr?select=data->-2->>a&id=in.(5,6)" `shouldRespondWith`
-        [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
+        [json| [{"a":"A"}, {"a":"[1,2,3]"}] |]
         { matchHeaders = [matchContentTypeJson] }
 
     it "can filter with negative indexes" $ do
@@ -300,3 +300,11 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
          "code": "PGRST100",
          "hint": null} |]
       { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
+
+  it "works when an RPC returns a dynamic TABLE with a composite type" $
+    get "/rpc/returns_complex?select=val->r&val->i=gt.0.5&order=val->>i.desc" `shouldRespondWith`
+      [json|[
+        {"r":0.3},
+        {"r":0.2}
+      ]|]
+      { matchStatus = 200, matchHeaders = [matchContentTypeJson] }

test/spec/Feature/Query/NullsStrip.hs

@@ -0,0 +1,126 @@
+module Feature.Query.NullsStrip where
+
+import Network.Wai (Application)
+
+import Network.HTTP.Types
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import Protolude  hiding (get)
+import SpecHelper
+
+spec :: SpecWith ((), Application)
+spec =
+  describe "Stripping null values from JSON response" $ do
+    let arrayStrip = ("Accept", "application/vnd.pgrst.array+json;nulls=stripped")
+    let singularStrip = ("Accept", "application/vnd.pgrst.object+json;nulls=stripped")
+
+    context "strip nulls from response" $ do
+      it "strips nulls when Accept: application/vnd.pgrst.array+json;nulls=stripped" $ do
+        request methodGet  "/organizations?select=*"
+          [arrayStrip]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"name":"Referee Org","manager_id":1},{"id":2,"name":"Auditor Org","manager_id":2},{"id":3,"name":"Acme","referee":1,"auditor":2,"manager_id":3},{"id":4,"name":"Umbrella","referee":1,"auditor":2,"manager_id":4},{"id":5,"name":"Cyberdyne","referee":3,"auditor":4,"manager_id":5},{"id":6,"name":"Oscorp","referee":3,"auditor":4,"manager_id":6}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPost  "/organizations?select=*"
+          [arrayStrip,("Prefer","return=representation")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7,"name":"John","manager_id":6}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPatch  "/organizations?id=eq.3&select=*"
+          [arrayStrip, ("Prefer","return=representation")]
+          [json|{"name":"John","referee":null}|]
+          `shouldRespondWith`
+          [json|[{"id":3,"name":"John","auditor":2,"manager_id":3}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.array;nulls=stripped" $
+        request methodGet  "/organizations?select=*"
+          [("Accept","application/vnd.pgrst.array;nulls=stripped")]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"name":"Referee Org","manager_id":1},{"id":2,"name":"Auditor Org","manager_id":2},{"id":3,"name":"Acme","referee":1,"auditor":2,"manager_id":3},{"id":4,"name":"Umbrella","referee":1,"auditor":2,"manager_id":4},{"id":5,"name":"Cyberdyne","referee":3,"auditor":4,"manager_id":5},{"id":6,"name":"Oscorp","referee":3,"auditor":4,"manager_id":6}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.object+json;nulls=stripped" $
+        request methodGet  "/organizations?limit=1"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"id":1,"name":"Referee Org","manager_id":1}|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTSingularStrip]
+          }
+
+      it "throws error when Accept: application/vnd.pgrst.object+json;nulls=stripped and result not singular" $
+        request methodGet  "/organizations?select=*"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"details":"The result contains 6 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+          { matchStatus  = 406
+          , matchHeaders = [matchContentTypeSingular]
+          }
+
+    context "strip nulls from response even if explicitly selected" $ do
+      it "strips nulls when Accept: application/vnd.pgrst.array+json;nulls=stripped" $ do
+        request methodGet  "/organizations?select=id,referee,auditor"
+          [arrayStrip]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1},{"id":2},{"id":3,"referee":1,"auditor":2},{"id":4,"referee":1,"auditor":2},{"id":5,"referee":3,"auditor":4},{"id":6,"referee":3,"auditor":4}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPost  "/organizations?select=id,referee,auditor"
+          [arrayStrip,("Prefer","return=representation")]
+          [json|{"id":7,"name":"John","referee":null,"auditor":null,"manager_id":6}|]
+          `shouldRespondWith`
+          [json|[{"id":7}]|]
+          { matchStatus  = 201
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+        request methodPatch  "/organizations?id=eq.3&select=id,name,referee,auditor"
+          [arrayStrip, ("Prefer","return=representation")]
+          [json|{"name":"John","referee":null}|]
+          `shouldRespondWith`
+          [json|[{"id":3,"name":"John","auditor":2}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTArrayStrip]
+          }
+
+      it "strips nulls when Accept: application/vnd.pgrst.object+json;nulls=stripped" $
+        request methodGet  "/organizations?select=id,referee,auditor&limit=1"
+          [singularStrip]
+          ""
+          `shouldRespondWith`
+          [json|{"id":1}|]
+          { matchStatus  = 200
+          , matchHeaders = [matchCTSingularStrip]
+          }
+
+    context "doesn't strip nulls" $ do
+      it "doesn't strips nulls when Accept: application/vnd.pgrst.array+json" $
+        request methodGet  "/organizations?select=id,referee,auditor"
+          [("Accept", "application/vnd.pgrst.array+json")]
+          ""
+          `shouldRespondWith`
+          [json|[{"id":1,"referee":null,"auditor":null},{"id":2,"referee":null,"auditor":null},{"id":3,"referee":1,"auditor":2},{"id":4,"referee":1,"auditor":2},{"id":5,"referee":3,"auditor":4},{"id":6,"referee":3,"auditor":4}]|]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson]
+          }

test/spec/Feature/Query/PlanSpec.hs

@@ -8,7 +8,6 @@ import Network.Wai.Test (SResponse (..))
 
 import           Data.Aeson.Lens
 import           Data.Aeson.QQ
-import qualified Data.ByteString      as BS
 import qualified Data.ByteString.Lazy as LBS
 import qualified Data.Text            as T
 import           Network.HTTP.Types
@@ -33,7 +32,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe`
           if actualPgVersion > pgVersion120
@@ -49,7 +48,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe`
           if actualPgVersion > pgVersion120
@@ -65,8 +64,8 @@ spec actualPgVersion = do
               resHeaders = simpleHeaders r
 
           liftIO $ do
-            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=buffers; charset=utf-8")
-            resBody `shouldSatisfy` (\t -> T.isInfixOf "Shared Hit Blocks" (decodeUtf8 $ BS.toStrict t))
+            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=buffers; charset=utf-8")
+            resBody `shouldSatisfy` (\t -> T.isInfixOf "Shared Hit Blocks" (decodeUtf8 $ LBS.toStrict t))
         else do
           -- analyze is required for buffers on pg < 13
           r <- request methodGet "/projects" (acceptHdrs "application/vnd.pgrst.plan+json; options=analyze|buffers") ""
@@ -75,7 +74,7 @@ spec actualPgVersion = do
               resHeaders = simpleHeaders r
 
           liftIO $ do
-            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=analyze|buffers; charset=utf-8")
+            resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze|buffers; charset=utf-8")
             blocks `shouldBe` Just [aesonQQ| 1.0 |]
 
     when (actualPgVersion >= pgVersion120) $
@@ -86,7 +85,7 @@ spec actualPgVersion = do
             resHeaders = simpleHeaders r
 
         liftIO $ do
-          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=settings; charset=utf-8")
+          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=settings; charset=utf-8")
           searchPath `shouldBe`
             Just [aesonQQ|
               {
@@ -102,7 +101,7 @@ spec actualPgVersion = do
             resHeaders = simpleHeaders r
 
         liftIO $ do
-          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=analyze|wal; charset=utf-8")
+          resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=analyze|wal; charset=utf-8")
           walRecords `shouldBe` Just [aesonQQ|0|]
 
     it "outputs columns info when using the verbose option" $ do
@@ -112,7 +111,7 @@ spec actualPgVersion = do
           resHeaders = simpleHeaders r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; options=verbose; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; options=verbose; charset=utf-8")
         cols `shouldBe` Just [aesonQQ| ["projects.id", "projects.name", "projects.client_id"] |]
 
     it "outputs the plan for application/json " $ do
@@ -151,7 +150,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 3.27
 
@@ -164,7 +163,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 12.45
 
@@ -177,7 +176,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 15.68
 
@@ -191,7 +190,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 1.29
 
@@ -216,7 +215,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+json; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         totalCost `shouldBe` 68.56
 
@@ -241,7 +240,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         resBody `shouldSatisfy` (\t -> LBS.take 9 t == "Aggregate")
 
@@ -254,7 +253,7 @@ spec actualPgVersion = do
           resStatus  = simpleStatus r
 
       liftIO $ do
-        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; charset=utf-8")
+        resHeaders `shouldSatisfy` elem ("Content-Type", "application/vnd.pgrst.plan+text; for=\"application/json\"; charset=utf-8")
         resStatus `shouldBe` Status { statusCode = 200, statusMessage="OK" }
         resBody `shouldSatisfy` (\t -> LBS.take 9 t == "Aggregate")
 
@@ -338,7 +337,7 @@ spec actualPgVersion = do
         let resBody = simpleBody r
 
         liftIO $ do
-          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getallusers" (decodeUtf8 $ BS.toStrict t))
+          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getallusers" (decodeUtf8 $ LBS.toStrict t))
 
       it "should inline a function with arguments(the function won't appear in the plan tree)" $ do
         r <- request methodGet "/rpc/getitemrange?min=10&max=15"
@@ -347,7 +346,44 @@ spec actualPgVersion = do
         let resBody = simpleBody r
 
         liftIO $ do
-          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getitemrange" (decodeUtf8 $ BS.toStrict t))
+          resBody `shouldSatisfy` (\t -> not $ T.isInfixOf "getitemrange" (decodeUtf8 $ LBS.toStrict t))
+
+    context "index usage" $ do
+      it "should use an index for a json arrow operator filter" $ do
+        r <- request methodGet "/bets?data_json->>contractId=eq.1"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index Cond" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for a jsonb arrow operator filter" $ do
+        r <- request methodGet "/bets?data_jsonb->>contractId=eq.1"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for ordering on a json arrow operator" $ do
+        r <- request methodGet "/bets?order=data_json->>contractId"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
+
+      it "should use an index for ordering on a jsonb arrow operator" $ do
+        r <- request methodGet "/bets?order=data_jsonb->>contractId"
+               [(hAccept, "application/vnd.pgrst.plan")] ""
+
+        let resBody = simpleBody r
+
+        liftIO $ do
+          resBody `shouldSatisfy` (\t -> T.isInfixOf "Index" (decodeUtf8 $ LBS.toStrict t))
 
 disabledSpec :: SpecWith ((), Application)
 disabledSpec =

test/spec/Feature/Query/QuerySpec.hs

@@ -1328,3 +1328,122 @@ spec actualPgVersion = do
       get "/articles?body=imatch(any).{stop,thing}&select=id" `shouldRespondWith`
         [json|[{"id":1}, {"id":2}]|]
         { matchHeaders = [matchContentTypeJson] }
+
+  describe "Data representations for customisable value formatting and parsing" $ do
+    it "formats a single column" $
+      get "/datarep_todos?select=id,label_color&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"label_color":"#000000"},{"id":2,"label_color":"#000100"},{"id":3,"label_color":"#01E240"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats two columns with different formatters" $
+      get "/datarep_todos?select=id,label_color,due_at&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"},{"id":2,"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"},{"id":3,"label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "fails in some reasonable way when selecting fields that don't exist" $
+      get "/datarep_todos?select=id,label_color,banana" `shouldRespondWith`
+        [json| {"code":"42703","details":null,"hint":null,"message":"column datarep_todos.banana does not exist"} |]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+    it "formats columns in views including computed columns" $
+      get "/datarep_todos_computed?select=id,label_color,dark_color" `shouldRespondWith`
+        [json| [
+          {"id":1, "label_color":"#000000", "dark_color":"#000000"},
+          {"id":2, "label_color":"#000100", "dark_color":"#000080"},
+          {"id":3, "label_color":"#01E240", "dark_color":"#00F120"},
+          {"id":4, "label_color":"", "dark_color":""}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats and allows rename" $
+      get "/datarep_todos?select=id,clr:label_color&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"clr":"#000000"},{"id":2,"clr":"#000100"},{"id":3,"clr":"#01E240"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats, renames and allows manual casting on top" $
+      get "/datarep_todos?select=id,clr:label_color::text&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"clr":"\"#000000\""},{"id":2,"clr":"\"#000100\""},{"id":3,"clr":"\"#01E240\""}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats nulls" $
+      -- due_at is formatted as NULL but label_color NULLs become empty strings-- it's up to the formatting function.
+      get "/datarep_todos?select=id,label_color,due_at&id=gt.2&id=lt.5" `shouldRespondWith`
+        [json| [{"id":3,"label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z"},{"id":4,"label_color":"","due_at":null}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats star select" $
+      get "/datarep_todos?select=*&id=lt.4" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+           {"id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+           {"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats implicit star select" $
+      get "/datarep_todos?id=lt.4" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+ {"id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+ {"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats star and explicit mix" $
+      get "/datarep_todos?select=due_at,*&id=lt.4" `shouldRespondWith`
+        [json| [
+          {"due_at":"2018-01-02T00:00:00Z","id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+           {"due_at":"2018-01-03T00:00:00Z","id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+           {"due_at":"2018-01-01T14:12:34.123456Z","id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats through join" $
+      get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)" `shouldRespondWith`
+        [json| [{"id":1,"name":"school related","first_item":{"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"}},{"id":2,"name":"do these first","first_item":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats through join with star select" $
+      get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"school related","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}},
+          {"id":2,"name":"do these first","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "uses text parser on value for filter given through query parameters" $
+      get "/datarep_todos?select=id,due_at&label_color=eq.000100" `shouldRespondWith`
+        [json| [{"id":2,"due_at":"2018-01-03T00:00:00Z"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "in the absense of text parser, does not try to use the JSON parser for query parameters" $
+      get "/datarep_todos?select=id,due_at&due_at=eq.Z" `shouldRespondWith`
+        -- we prove the parser is not used because it'd replace the Z with `+00:00` and a different error message.
+        [json| {"code":"22007","details":null,"hint":null,"message":"invalid input syntax for type timestamp with time zone: \"Z\""} |]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+    -- Before PG 11, this will fail because we need arrays of domain type values. The docs should explain data reps are
+    -- not supported in this case.
+    when (actualPgVersion >= pgVersion110) $ do
+      it "uses text parser for filter with 'IN' predicates" $
+        get "/datarep_todos?select=id,due_at&label_color=in.(000100,01E240)" `shouldRespondWith`
+          [json| [
+            {"id":2, "due_at": "2018-01-03T00:00:00Z"},
+            {"id":3, "due_at": "2018-01-01T14:12:34.123456Z"}
+          ] |]
+          { matchHeaders = [matchContentTypeJson] }
+      it "uses text parser for filter with 'NOT IN' predicates" $
+        get "/datarep_todos?select=id,due_at&label_color=not.in.(000000,01E240)" `shouldRespondWith`
+          [json| [
+            {"id":2, "due_at": "2018-01-03T00:00:00Z"}
+          ] |]
+          { matchHeaders = [matchContentTypeJson] }
+    it "uses text parser on value for filter across relations" $
+      get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100" `shouldRespondWith`
+        [json| [{"id":1,"name":"school related","datarep_todos":null},{"id":2,"name":"do these first","datarep_todos":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    -- This is not supported by data reps (would be hard to make it work with high performance). So the test just
+    -- verifies we don't panic or add inappropriate SQL to the filters.
+    it "fails safely on user trying to use ilike operator on data reps column" $
+      get "/datarep_todos?select=id,name&label_color=ilike.#*100" `shouldRespondWith` (
+        if actualPgVersion >= pgVersion110 then
+        [json|
+          {"code":"42883","details":null,"hint":"No operator matches the given name and argument types. You might need to add explicit type casts.","message":"operator does not exist: public.color ~~* unknown"}
+        |]
+        else
+        [json|
+          {"code":"42883","details":null,"hint":"No operator matches the given name and argument type(s). You might need to add explicit type casts.","message":"operator does not exist: public.color ~~* unknown"}
+        |])
+        { matchStatus  = 404
+        , matchHeaders = [matchContentTypeJson]
+        }

test/spec/Feature/Query/RangeSpec.hs

@@ -453,3 +453,17 @@ spec = do
             { matchStatus  = 416
             , matchHeaders = ["Content-Range" <:> "*/15"]
             }
+
+        it "refuses a range with first position the same as number of items" $
+          request methodGet "/rpc/getitemrange?min=1&max=2"
+                  (rangeHdrsWithCount $ ByteRangeFromTo 1 2) mempty
+            `shouldRespondWith`
+              [json| {
+                "message":"Requested range not satisfiable",
+                "code":"PGRST103",
+                "details":"An offset of 1 was requested, but there are only 1 rows.",
+                "hint":null
+              }|]
+            { matchStatus  = 416
+            , matchHeaders = ["Content-Range" <:> "*/1"]
+            }

test/spec/Feature/Query/RelatedQueriesSpec.hs

@@ -2,6 +2,7 @@ module Feature.Query.RelatedQueriesSpec where
 
 import Network.Wai (Application)
 
+import Network.HTTP.Types
 import Test.Hspec
 import Test.Hspec.Wai
 import Test.Hspec.Wai.JSON
@@ -233,18 +234,6 @@ spec = describe "related queries" $ do
         , matchHeaders = [matchContentTypeJson]
         }
 
-    it "only works with is null or is not null operators" $
-      get "/projects?select=name,clients(*)&clients=eq.3" `shouldRespondWith`
-        [json|{
-          "code":"PGRST120",
-          "details":"Only is null or not is null filters are allowed on embedded resources",
-          "hint":null,
-          "message":"Bad operator on the 'clients' embedded resource"
-        }|]
-        { matchStatus  = 400
-        , matchHeaders = [matchContentTypeJson]
-        }
-
     it "doesn't interfere filtering when embedding using the column name" $
       get "/projects?select=name,client_id,client:client_id(name)&client_id=eq.2" `shouldRespondWith`
         [json|[
@@ -254,3 +243,138 @@ spec = describe "related queries" $ do
         { matchStatus  = 200
         , matchHeaders = [matchContentTypeJson]
         }
+
+    it "doesn't interfere filtering on column names used for disambiguation" $
+      get "/user_friend?select=*,user1(*)&user1=eq.a02fb934-3a4d-469b-a6b6-4fcd88b973cf" `shouldRespondWith`
+        [json|[]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "doesn't interfere filtering on column names that are the same as the relation name" $
+      get "/tournaments?select=*,status(*)&status=eq.3" `shouldRespondWith`
+        [json|[]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    -- "?table=not.is.null" does a "table IS DISTINCT FROM NULL" instead of a "table IS NOT NULL"
+    -- https://github.com/PostgREST/postgrest/issues/2800#issuecomment-1720315818
+    it "embeds verifying that the entire target table row is not null" $ do
+      get "/table_b?select=name,table_a(name)&table_a=not.is.null" `shouldRespondWith`
+        [json|[
+          {"name":"Test 1","table_a":{"name":"Not null 1"}},
+          {"name":"Test 2","table_a":{"name":null}}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+      get "/table_b?select=name,table_a()&table_a=is.null" `shouldRespondWith`
+        [json|[
+          {"name":"Test 3"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [matchContentTypeJson]
+        }
+
+    it "works with count=exact" $ do
+      request methodGet "/projects?select=name,clients(name)&clients=not.is.null"
+        [("Prefer", "count=exact")] ""
+       `shouldRespondWith`
+        [json|[
+          {"name":"Windows 7", "clients":{"name":"Microsoft"}},
+          {"name":"Windows 10", "clients":{"name":"Microsoft"}},
+          {"name":"IOS", "clients":{"name":"Apple"}},
+          {"name":"OSX", "clients":{"name":"Apple"}}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-3/4" ]
+        }
+      request methodGet "/projects?select=name,clients()&clients=is.null"
+        [("Prefer", "count=exact")] ""
+       `shouldRespondWith`
+        [json|[{"name":"Orphan"}]|]
+        { matchStatus  = 200
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-0/1" ]
+        }
+      request methodGet "/client?select=*,clientinfo(),contact()&clientinfo.other=ilike.*main*&contact.name=ilike.*tabby*&or=(clientinfo.not.is.null,contact.not.is.null)"
+        [("Prefer", "count=exact")] ""
+       `shouldRespondWith`
+        [json|[
+          {"id":1,"name":"Walmart"},
+          {"id":2,"name":"Target"}
+        ]|]
+        { matchStatus  = 200
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-1/2" ]
+        }
+
+    it "works with count=planned" $ do
+      request methodGet "/projects?select=name,clients(name)&clients=not.is.null"
+        [("Prefer", "count=planned")] ""
+       `shouldRespondWith`
+        [json|[
+          {"name":"Windows 7", "clients":{"name":"Microsoft"}},
+          {"name":"Windows 10", "clients":{"name":"Microsoft"}},
+          {"name":"IOS", "clients":{"name":"Apple"}},
+          {"name":"OSX", "clients":{"name":"Apple"}}
+        ]|]
+        { matchStatus  = 206
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-3/1200" ]
+        }
+      request methodGet "/projects?select=name,clients()&clients=is.null"
+        [("Prefer", "count=planned")] ""
+       `shouldRespondWith`
+        [json|[{"name":"Orphan"}]|]
+        { matchStatus  = 200
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-0/1" ]
+        }
+      request methodGet "/client?select=*,clientinfo(),contact()&clientinfo.other=ilike.*main*&contact.name=ilike.*tabby*&or=(clientinfo.not.is.null,contact.not.is.null)"
+        [("Prefer", "count=planned")] ""
+       `shouldRespondWith`
+        [json|[
+          {"id":1,"name":"Walmart"},
+          {"id":2,"name":"Target"}
+        ]|]
+        { matchStatus  = 206
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-1/952" ]
+        }
+
+    it "works with count=estimated" $ do
+      request methodGet "/projects?select=name,clients(name)&clients=not.is.null"
+        [("Prefer", "count=estimated")] ""
+       `shouldRespondWith`
+        [json|[
+          {"name":"Windows 7", "clients":{"name":"Microsoft"}},
+          {"name":"Windows 10", "clients":{"name":"Microsoft"}},
+          {"name":"IOS", "clients":{"name":"Apple"}},
+          {"name":"OSX", "clients":{"name":"Apple"}}
+        ]|]
+        { matchStatus  = 206
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-3/1200" ]
+        }
+      request methodGet "/projects?select=name,clients()&clients=is.null"
+        [("Prefer", "count=estimated")] ""
+       `shouldRespondWith`
+        [json|[{"name":"Orphan"}]|]
+        { matchStatus  = 200
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-0/1" ]
+        }
+      request methodGet "/client?select=*,clientinfo(),contact()&clientinfo.other=ilike.*main*&contact.name=ilike.*tabby*&or=(clientinfo.not.is.null,contact.not.is.null)"
+        [("Prefer", "count=estimated")] ""
+       `shouldRespondWith`
+        [json|[
+          {"id":1,"name":"Walmart"},
+          {"id":2,"name":"Target"}
+        ]|]
+        { matchStatus  = 206
+        , matchHeaders = [ matchContentTypeJson
+                         , "Content-Range" <:> "0-1/952" ]
+        }

test/spec/Feature/Query/RpcSpec.hs

@@ -475,6 +475,27 @@ spec actualPgVersion =
           `shouldRespondWith`
             [json|null|]
 
+      when (actualPgVersion >= pgVersion110) $ do
+        it "returns a record type" $ do
+          post "/rpc/returns_record"
+            ""
+           `shouldRespondWith`
+            [json|{"id":1,"name":"Windows 7","client_id":1}|]
+          post "/rpc/returns_record_params"
+            [json|{"id":1, "name": "Windows%"}|]
+           `shouldRespondWith`
+            [json|{"id":1,"name":"Windows 7","client_id":1}|]
+
+        it "returns a setof record type" $ do
+          post "/rpc/returns_setof_record"
+            ""
+           `shouldRespondWith`
+            [json|[{"id":1,"name":"Windows 7","client_id":1},{"id":2,"name":"Windows 10","client_id":1}]|]
+          post "/rpc/returns_setof_record_params"
+            [json|{"id":1,"name":"Windows%"}|]
+           `shouldRespondWith`
+            [json|[{"id":1,"name":"Windows 7","client_id":1},{"id":2,"name":"Windows 10","client_id":1}]|]
+
       context "different types when overloaded" $ do
         it "returns composite type" $
           post "/rpc/ret_point_overloaded"
@@ -1447,3 +1468,25 @@ spec actualPgVersion =
               { matchStatus  = 400
               , matchHeaders = [matchContentTypeJson]
               }
+
+    -- https://github.com/PostgREST/postgrest/issues/1586#issuecomment-696345442
+    context "a proc with bit and char parameters" $ do
+      it "modifies the param type from character to character varying" $ do
+        get "/rpc/char_param_select?char_=abcdefg&char_arr={abc,abcdefg}" `shouldRespondWith`
+          [json| [{ "char_": "abcdefg", "char_arr": [ "abc", "abcdefg" ] }] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+        post "/rpc/char_param_insert" [json| { "char_": "abcdefg", "char_arr": "{abc,abcdefg}" } |]
+           `shouldRespondWith`
+             [json| {"code":"22001","details":null,"hint":null,"message":"value too long for type character(5)"} |]
+           { matchStatus = 400 }
+
+      it "modifies the param type from bit to bit varying" $ do
+        get "/rpc/bit_param_select?bit_=101010&bit_arr={101,101010}" `shouldRespondWith`
+          [json| [{ "bit_": "101010", "bit_arr": [ "101", "101010" ] }] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+        post "/rpc/bit_param_insert" [json| { "bit_": "101010", "bit_arr": "{101,101010}" } |]
+           `shouldRespondWith`
+             [json| {"code":"22026","details":null,"hint":null,"message":"bit string length 6 does not match type bit(5)"} |]
+           { matchStatus = 400 }

test/spec/Feature/Query/SingularSpec.hs

@@ -61,7 +61,8 @@ spec =
           `shouldRespondWith`
             ""
             { matchStatus  = 204
-            , matchHeaders = [matchHeaderAbsent hContentType]
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       it "raises an error for multiple rows" $ do
@@ -69,10 +70,9 @@ spec =
             [("Prefer", "tx=commit"), singular]
             [json| { address: "zzz" } |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 4 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 4 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
-            , matchHeaders = [ matchContentTypeSingular
-                             , "Preference-Applied" <:> "tx=commit" ]
+            , matchHeaders = [ matchContentTypeSingular ]
             }
 
         -- the rows should not be updated, either
@@ -85,10 +85,9 @@ spec =
             [("Prefer", "tx=commit"), ("Prefer", "return=representation"), singular]
             [json| { address: "zzz" } |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 4 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 4 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
-            , matchHeaders = [ matchContentTypeSingular
-                             , "Preference-Applied" <:> "tx=commit" ]
+            , matchHeaders = [ matchContentTypeSingular ]
             }
 
         -- the rows should not be updated, either
@@ -100,7 +99,7 @@ spec =
         request methodPatch "/items?id=gt.0&id=lt.0"
                 [singular] [json|{"id":1}|]
           `shouldRespondWith`
-                  [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                   { matchStatus  = 406
                   , matchHeaders = [matchContentTypeSingular]
                   }
@@ -109,7 +108,7 @@ spec =
         request methodPatch "/items?id=gt.0&id=lt.0"
                 [("Prefer", "return=representation"), singular] [json|{"id":1}|]
           `shouldRespondWith`
-                  [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                   { matchStatus  = 406
                   , matchHeaders = [matchContentTypeSingular]
                   }
@@ -133,7 +132,8 @@ spec =
             ""
             { matchStatus  = 201
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "*/*" ]
+                             , "Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       it "raises an error when attempting to create multiple entities" $ do
@@ -141,10 +141,9 @@ spec =
             [("Prefer", "tx=commit"), singular]
             [json| [ { id: 200, address: "xxx" }, { id: 201, address: "yyy" } ] |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 2 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 2 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
-            , matchHeaders = [ matchContentTypeSingular
-                             , "Preference-Applied" <:> "tx=commit" ]
+            , matchHeaders = [ matchContentTypeSingular ]
             }
 
         -- the rows should not exist, either
@@ -157,10 +156,9 @@ spec =
             [("Prefer", "tx=commit"), ("Prefer", "return=representation"), singular]
             [json| [ { id: 202, address: "xxx" }, { id: 203, address: "yyy" } ] |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 2 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 2 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
-            , matchHeaders = [ matchContentTypeSingular
-                             , "Preference-Applied" <:> "tx=commit" ]
+            , matchHeaders = [ matchContentTypeSingular ]
             }
 
         -- the rows should not exist, either
@@ -173,10 +171,9 @@ spec =
             [("Prefer", "tx=commit"), ("Prefer", "return=minimal"), singular]
             [json| [ { id: 204, address: "xxx" }, { id: 205, address: "yyy" } ] |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 2 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 2 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
-            , matchHeaders = [ matchContentTypeSingular
-                             , "Preference-Applied" <:> "tx=commit" ]
+            , matchHeaders = [ matchContentTypeSingular ]
             }
 
         -- the rows should not exist, either
@@ -189,7 +186,7 @@ spec =
                 [singular]
                 [json| [ ] |]
           `shouldRespondWith`
-                  [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                   { matchStatus  = 406
                   , matchHeaders = [matchContentTypeSingular]
                   }
@@ -199,7 +196,7 @@ spec =
                 [("Prefer", "return=representation"), singular]
                 [json| [ ] |]
           `shouldRespondWith`
-                  [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                   { matchStatus  = 406
                   , matchHeaders = [matchContentTypeSingular]
                   }
@@ -222,10 +219,9 @@ spec =
             [("Prefer", "tx=commit"), singular]
             ""
           `shouldRespondWith`
-            [json|{"details":"Results contain 5 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 5 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
-            , matchHeaders = [ matchContentTypeSingular
-                             , "Preference-Applied" <:> "tx=commit" ]
+            , matchHeaders = [ matchContentTypeSingular ]
             }
 
         -- the rows should still exist
@@ -240,10 +236,9 @@ spec =
         request methodDelete "/items?id=gt.5&id=lt.11"
             [("Prefer", "tx=commit"), ("Prefer", "return=representation"), singular] ""
           `shouldRespondWith`
-            [json|{"details":"Results contain 5 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 5 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
-            , matchHeaders = [ matchContentTypeSingular
-                             , "Preference-Applied" <:> "tx=commit" ]
+            , matchHeaders = [ matchContentTypeSingular ]
             }
 
         -- the rows should still exist
@@ -257,7 +252,7 @@ spec =
         request methodDelete "/items?id=lt.0"
                 [singular] ""
           `shouldRespondWith`
-                [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                 { matchStatus  = 406
                 , matchHeaders = [matchContentTypeSingular]
                 }
@@ -266,7 +261,7 @@ spec =
         request methodDelete "/items?id=lt.0"
                 [("Prefer", "return=representation"), singular] ""
           `shouldRespondWith`
-                [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                 { matchStatus  = 406
                 , matchHeaders = [matchContentTypeSingular]
                 }
@@ -276,7 +271,7 @@ spec =
         request methodPost "/rpc/getproject"
                 [singular] [json|{ "id": 9999999}|]
           `shouldRespondWith`
-                [json|{"details":"Results contain 0 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 0 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                 { matchStatus  = 406
                 , matchHeaders = [matchContentTypeSingular]
                 }
@@ -299,7 +294,7 @@ spec =
         request methodPost "/rpc/getallprojects"
                 [singular] "{}"
           `shouldRespondWith`
-                [json|{"details":"Results contain 5 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 5 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
                 { matchStatus  = 406
                 , matchHeaders = [matchContentTypeSingular]
                 }
@@ -314,10 +309,9 @@ spec =
             [("Prefer", "tx=commit"), singular]
             [json| {"id_l": 1, "id_h": 2, "name": "changed"} |]
           `shouldRespondWith`
-            [json|{"details":"Results contain 2 rows, application/vnd.pgrst.object+json requires 1 row","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
+            [json|{"details":"The result contains 2 rows","message":"JSON object requested, multiple (or no) rows returned","code":"PGRST116","hint":null}|]
             { matchStatus  = 406
-            , matchHeaders = [ matchContentTypeSingular
-                             , "Preference-Applied" <:> "tx=commit" ]
+            , matchHeaders = [ matchContentTypeSingular]
             }
 
         -- should rollback function

test/spec/Feature/Query/UpdateSpec.hs

@@ -9,6 +9,9 @@ import Network.HTTP.Types
 import Test.Hspec.Wai
 import Test.Hspec.Wai.JSON
 
+import PostgREST.Config.PgVersion (PgVersion, pgVersion100)
+
+
 import Protolude  hiding (get)
 import SpecHelper
 
@@ -18,8 +21,8 @@ tblDataBefore = [aesonQQ|[
                 , { "id": 3, "name": "item-3" }
                 ]|]
 
-spec :: SpecWith ((), Application)
-spec = do
+spec :: PgVersion -> SpecWith ((), Application)
+spec actualPgVersion = do
   describe "Patching record" $ do
     context "to unknown uri" $
       it "indicates no table found by returning 404" $
@@ -72,7 +75,7 @@ spec = do
           `shouldRespondWith` "[]"
           {
             matchStatus  = 200,
-            matchHeaders = []
+            matchHeaders = ["Preference-Applied" <:> "return=representation"]
           }
 
       it "returns status code 200 when no rows updated" $
@@ -85,7 +88,8 @@ spec = do
           [("Prefer", "return=representation")] [json| { "id":2 } |]
           `shouldRespondWith` [json|[{"id":2}]|]
           { matchStatus  = 200,
-            matchHeaders = ["Content-Range" <:> "0-0/*"]
+            matchHeaders = ["Content-Range" <:> "0-0/*"
+                           , "Preference-Applied" <:> "return=representation"]
           }
 
       it "can update multiple items" $ do
@@ -134,7 +138,9 @@ spec = do
             [json| { id: 100 } |]
             `shouldRespondWith` [json| [{ id: 100 }] |]
             { matchStatus  = 200,
-              matchHeaders = [matchContentTypeJson, "Content-Range" <:> "0-0/*"]
+              matchHeaders = [matchContentTypeJson
+                             ,"Content-Range" <:> "0-0/*"
+                             , "Preference-Applied" <:> "return=representation"]
             }
 
         it "returns empty array when no rows updated and return=rep" $
@@ -144,7 +150,7 @@ spec = do
             [json| { id: 100 } |]
             `shouldRespondWith` "[]"
             { matchStatus  = 200,
-              matchHeaders = []
+              matchHeaders = ["Preference-Applied" <:> "return=representation"]
             }
 
       context "with representation requested" $ do
@@ -156,7 +162,9 @@ spec = do
             [("Prefer", "return=representation")]
             [json| { id: 99 } |]
             `shouldRespondWith` [json| [{id:99}] |]
-            { matchHeaders = [matchContentTypeJson] }
+            { matchHeaders = [matchContentTypeJson
+                             , "Preference-Applied" <:> "return=representation"]
+            }
           -- put value back for other tests
           void $ request methodPatch "/items?id=eq.99" [] [json| { "id":1 } |]
 
@@ -166,7 +174,9 @@ spec = do
             [("Prefer", "return=representation")]
             [json| { id: 1 } |]
             `shouldRespondWith` [json| [{ id: 1, always_true: true }] |]
-            { matchHeaders = [matchContentTypeJson] }
+            { matchHeaders = [matchContentTypeJson
+                             , "Preference-Applied" <:> "return=representation"]
+            }
 
         it "can select overloaded computed columns" $ do
           request methodPatch
@@ -174,22 +184,28 @@ spec = do
             [("Prefer", "return=representation")]
             [json| { id: 1 } |]
             `shouldRespondWith` [json| [{ id: 1, computed_overload: true }] |]
-            { matchHeaders = [matchContentTypeJson] }
+            { matchHeaders = [matchContentTypeJson
+                             , "Preference-Applied" <:> "return=representation"]
+            }
           request methodPatch
             "/items2?id=eq.1&select=id,computed_overload"
             [("Prefer", "return=representation")]
             [json| { id: 1 } |]
             `shouldRespondWith` [json| [{ id: 1, computed_overload: true }] |]
-            { matchHeaders = [matchContentTypeJson] }
+            { matchHeaders = [matchContentTypeJson
+                             , "Preference-Applied" <:> "return=representation"]
+            }
 
       it "ignores ?select= when return not set or return=minimal" $ do
         request methodPatch "/items?id=eq.1&select=id"
-            [] [json| { id:1 } |]
+           [("Prefer", "return=minimal")]
+           [json| { id:1 } |]
           `shouldRespondWith`
             ""
             { matchStatus  = 204
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "0-0/*" ]
+                             , "Content-Range" <:> "0-0/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
         request methodPatch "/items?id=eq.1&select=id"
             [("Prefer", "return=minimal")]
@@ -198,7 +214,8 @@ spec = do
             ""
             { matchStatus  = 204
             , matchHeaders = [ matchHeaderAbsent hContentType
-                             , "Content-Range" <:> "0-0/*" ]
+                             , "Content-Range" <:> "0-0/*"
+                             , "Preference-Applied" <:> "return=minimal"]
             }
 
       context "when patching with an empty body" $ do
@@ -269,7 +286,8 @@ spec = do
             `shouldRespondWith` "[]"
             {
               matchStatus  = 200,
-              matchHeaders = ["Content-Range" <:> "*/*"]
+              matchHeaders = ["Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=representation"]
             }
 
         it "makes no updates and returns 200 with return=rep and with ?select=" $
@@ -277,7 +295,8 @@ spec = do
             `shouldRespondWith` "[]"
             {
               matchStatus  = 200,
-              matchHeaders = ["Content-Range" <:> "*/*"]
+              matchHeaders = ["Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=representation"]
             }
 
         it "makes no updates and returns 200 with return=rep and with ?select= for overloaded computed columns" $
@@ -285,7 +304,8 @@ spec = do
             `shouldRespondWith` "[]"
             {
               matchStatus  = 200,
-              matchHeaders = ["Content-Range" <:> "*/*"]
+              matchHeaders = ["Content-Range" <:> "*/*"
+                             , "Preference-Applied" <:> "return=representation"]
             }
 
     context "with unicode values" $
@@ -340,7 +360,7 @@ spec = do
               {"id":3,"name":"Tres","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":1}
             ]|]
             { matchStatus  = 200
-            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
             }
 
         it "updates with limit/offset using table default values(field-with_sep) when json keys are undefined" $ do
@@ -352,7 +372,7 @@ spec = do
               {"id":3,"name":"Tres"}
             ]|]
             { matchStatus  = 200
-            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
             }
 
         it "updates table default values(field-with_sep) when json keys are undefined" $ do
@@ -364,7 +384,7 @@ spec = do
               {"id":3,"name":"Tres","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":1}
             ]|]
             { matchStatus  = 200
-            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
             }
 
         it "updates view default values(field-with_sep) when json keys are undefined" $
@@ -378,7 +398,27 @@ spec = do
               {"id":3,"name":"Default","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":null,"field-with_sep":3}
             ]|]
             { matchStatus  = 200
-            , matchHeaders = ["Preference-Applied" <:> "missing=default"]
+            , matchHeaders = ["Preference-Applied" <:> "missing=default, return=representation"]
+            }
+
+    -- https://github.com/PostgREST/postgrest/issues/2861
+    context "bit and char columns with length" $ do
+      it "should update a bit column with length" $
+        request methodPatch "/bitchar_with_length?select=bit,char&char=eq.aaaaa"
+            [("Prefer", "return=representation")]
+            [json|{"bit": "11100"}|]
+          `shouldRespondWith` [json|[{ "bit": "11100", "char": "aaaaa" }]|]
+            { matchStatus  = 200
+            , matchHeaders = ["Preference-Applied" <:> "return=representation"]
+            }
+
+      it "should update a char column with length" $
+        request methodPatch "/bitchar_with_length?select=bit,char&bit=eq.00000"
+            [("Prefer", "return=representation")]
+            [json|{"char": "zzzyy"}|]
+          `shouldRespondWith` [json|[{ "bit": "00000", "char": "zzzyy" }]|]
+            { matchStatus  = 200
+            , matchHeaders = ["Preference-Applied" <:> "return=representation"]
             }
 
   context "tables with self reference foreign keys" $ do
@@ -391,8 +431,8 @@ spec = do
           [json|
             [ { "id": 0, "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" }, { "name": "bar" } ]} ]
           |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $
@@ -403,8 +443,8 @@ spec = do
           [json|
             [ { "id": 0, "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" } ]} ]
           |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
     context "embeds parent, children and grandchildren after update" $ do
@@ -425,8 +465,8 @@ spec = do
               ]
             }
           ] |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $
@@ -445,8 +485,8 @@ spec = do
               ]
             }
           ] |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
     context "embeds children after update without explicitly including the id in the ?select" $ do
@@ -458,8 +498,8 @@ spec = do
           [json|
             [ { "name": "tardis-patched", "web_content": [ { "name": "fezz" }, { "name": "foo" }, { "name": "bar" } ]} ]
           |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $
@@ -470,8 +510,8 @@ spec = do
           [json|
             [ { "name": "tardis-patched", "web_content": [ { "name": "bar" } ]} ]
           |]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
   context "tables with foreign keys referencing other tables" $ do
@@ -492,8 +532,8 @@ spec = do
               ]
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $
@@ -510,8 +550,8 @@ spec = do
               ]
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
 
@@ -527,8 +567,8 @@ spec = do
               "students_info":{"address":"Street 1"}
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
         request methodPatch "/students_info?id=eq.1&select=address,students(name)"
                 [("Prefer", "return=representation")]
@@ -540,8 +580,8 @@ spec = do
               "students":{"name": "John Doe"}
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
       it "with filters" $ do
@@ -555,8 +595,8 @@ spec = do
               "students_info": null
             }
           ]|]
-          { matchStatus  = 200,
-            matchHeaders = [matchContentTypeJson]
+          { matchStatus  = 200
+          , matchHeaders = [matchContentTypeJson, "Preference-Applied" <:> "return=representation"]
           }
 
   context "table with limited privileges" $ do
@@ -567,7 +607,8 @@ spec = do
         `shouldRespondWith`
           ""
           { matchStatus = 204
-          , matchHeaders = [matchHeaderAbsent hContentType]
+          , matchHeaders = [matchHeaderAbsent hContentType
+                           , "Preference-Applied" <:> "return=minimal"]
           }
 
     it "can update without return=minimal and no explicit select" $
@@ -742,3 +783,198 @@ spec = do
        , { "id": 2, "name": "updated-item" }
        , { "id": 3, "name": "updated-item" }
        ]|]
+
+  -- Data representations for payload parsing requires Postgres 10 or above.
+  when (actualPgVersion >= pgVersion100) $ do
+    describe "Data representations" $ do
+      context "for a single row" $ do
+        it "parses values in payload" $
+          request methodPatch "/datarep_todos?id=eq.2" [("Prefer", "return=headers-only")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+            `shouldRespondWith`
+            ""
+              { matchStatus  = 204
+              , matchHeaders = [ matchHeaderAbsent hContentType
+                               , "Content-Range" <:> "0-0/*"]
+              }
+
+        it "parses values in payload and formats individually selected values in return=representation" $
+          request methodPatch "/datarep_todos?id=eq.2&select=id,label_color" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+            `shouldRespondWith`
+            [json| [{"id":2, "label_color": "#221100"}] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+
+        it "parses values in payload and formats values in return=representation" $
+          request methodPatch "/datarep_todos?id=eq.2" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:20Z", "icon_image": "3q2+7w"} |]
+            `shouldRespondWith`
+            [json|  [{"id":2,"name":"Essay","label_color":"#221100","due_at":"2019-01-03T11:00:20Z","icon_image":"3q2+7w==","created_at":1513213350,"budget":"100000000000000.13"}] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+
+        it "parses values in payload and formats star mixed selected values in return=representation" $
+          request methodPatch "/datarep_todos?id=eq.2&select=due_at,*" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z", "created_at": 0} |]
+            `shouldRespondWith`
+            -- end up with due_at twice here but that's unrelated to data reps
+            [json| [{"due_at":"2019-01-03T11:00:00Z","id":2,"name":"Essay","label_color":"#221100","due_at":"2019-01-03T11:00:00Z","icon_image":null,"created_at":0,"budget":"100000000000000.13"}] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+      context "for multiple rows" $ do
+        it "parses values in payload and formats individually selected values in return=representation" $
+          request methodPatch "/datarep_todos?id=lt.4&select=id,name,label_color" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+            `shouldRespondWith`
+            [json| [
+              {"id":1, "name": "Report", "label_color": "#221100"},
+              {"id":2, "name": "Essay", "label_color": "#221100"},
+              {"id":3, "name": "Algebra", "label_color": "#221100"}
+            ] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-2/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+
+        it "parses values in payload and formats values in return=representation" $
+          request methodPatch "/datarep_todos?id=lt.4" [("Prefer", "return=representation")]
+            [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z", "icon_image": "3q2+7w="} |]
+            `shouldRespondWith`
+            [json| [
+              {"id":1,"name":"Report","label_color":"#221100","due_at":"2019-01-03T11:00:00Z","icon_image":"3q2+7w==","created_at":1513213350,"budget":"12.50"},
+              {"id":2,"name":"Essay","label_color":"#221100","due_at":"2019-01-03T11:00:00Z","icon_image":"3q2+7w==","created_at":1513213350,"budget":"100000000000000.13"},
+              {"id":3,"name":"Algebra","label_color":"#221100","due_at":"2019-01-03T11:00:00Z","icon_image":"3q2+7w==","created_at":1513213350,"budget":"0.00"}
+            ] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-2/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+      context "with ?columns parameter" $ do
+        it "ignores json keys not included in ?columns; parses only the ones specified" $
+          request methodPatch "/datarep_todos?id=eq.2&columns=due_at" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| [
+               {"id":2,"name":"Essay","label_color":"#000100","due_at":"2019-01-03T11:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"}
+            ] |]
+              { matchStatus  = 200
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+              }
+
+        it "fails if at least one specified column doesn't exist" $
+          request methodPatch "/datarep_todos?id=eq.2&columns=label_color,helicopters" [("Prefer", "return=representation")]
+            [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+            `shouldRespondWith`
+            [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos' does not exist","details":null,"hint":null} |]
+              { matchStatus  = 400
+              , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+              }
+
+        it "ignores json keys and gives 200 if no record updated" $
+          request methodPatch "/datarep_todos?id=eq.2001&columns=label_color" [("Prefer", "return=representation")]
+           [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+           `shouldRespondWith` 200
+      context "on a view" $ do
+        context "for a single row" $ do
+          it "parses values in payload" $
+            request methodPatch "/datarep_todos_computed?id=eq.2" [("Prefer", "return=headers-only")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+              `shouldRespondWith`
+              ""
+                { matchStatus  = 204
+                , matchHeaders = [ matchHeaderAbsent hContentType
+                                 , "Content-Range" <:> "0-0/*" ]
+                }
+
+          it "parses values in payload and formats individually selected values in return=representation" $
+            request methodPatch "/datarep_todos_computed?id=eq.2&select=id,label_color" [("Prefer", "return=representation")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+              `shouldRespondWith`
+              [json| [{"id":2, "label_color": "#221100"}] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+
+          it "parses values in payload and formats values in return=representation" $
+            request methodPatch "/datarep_todos_computed?id=eq.2" [("Prefer", "return=representation")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:20Z"} |]
+              `shouldRespondWith`
+              [json| [{"id":2, "name": "Essay", "label_color": "#221100", "dark_color":"#110880", "due_at":"2019-01-03T11:00:20Z"}] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+        context "for multiple rows" $ do
+          it "parses values in payload and formats individually selected values in return=representation" $
+            request methodPatch "/datarep_todos_computed?id=lt.4&select=id,name,label_color,dark_color" [("Prefer", "return=representation")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+              `shouldRespondWith`
+              [json| [
+                {"id":1, "name": "Report", "label_color": "#221100", "dark_color":"#110880"},
+                {"id":2, "name": "Essay", "label_color": "#221100", "dark_color":"#110880"},
+                {"id":3, "name": "Algebra", "label_color": "#221100", "dark_color":"#110880"}
+              ] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-2/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+
+          it "parses values in payload and formats values in return=representation" $
+            request methodPatch "/datarep_todos_computed?id=lt.4" [("Prefer", "return=representation")]
+              [json| {"label_color": "#221100", "due_at": "2019-01-03T11:00:00Z"} |]
+              `shouldRespondWith`
+              [json| [
+                {"id":1, "name": "Report", "label_color": "#221100", "dark_color":"#110880", "due_at":"2019-01-03T11:00:00Z"},
+                {"id":2, "name": "Essay", "label_color": "#221100", "dark_color":"#110880", "due_at":"2019-01-03T11:00:00Z"},
+                {"id":3, "name": "Algebra", "label_color": "#221100", "dark_color":"#110880", "due_at":"2019-01-03T11:00:00Z"}
+              ] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-2/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+        context "with ?columns parameter" $ do
+          it "ignores json keys not included in ?columns; parses only the ones specified" $
+            request methodPatch "/datarep_todos_computed?id=eq.2&columns=due_at" [("Prefer", "return=representation")]
+              [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+              `shouldRespondWith`
+              [json| [
+                {"id":2, "name": "Essay", "label_color": "#000100", "dark_color": "#000080", "due_at":"2019-01-03T11:00:00Z"}
+              ] |]
+                { matchStatus  = 200
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"
+                               , "Content-Range" <:> "0-0/*"
+                               , "Preference-Applied" <:> "return=representation"]
+                }
+
+          it "fails if at least one specified column doesn't exist" $
+            request methodPatch "/datarep_todos_computed?id=eq.2&columns=label_color,helicopters" [("Prefer", "return=representation")]
+              [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+              `shouldRespondWith`
+              [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos_computed' does not exist","details":null,"hint":null} |]
+                { matchStatus  = 400
+                , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+                }
+
+          it "ignores json keys and gives 200 if no record updated" $
+            request methodPatch "/datarep_todos_computed?id=eq.2001&columns=label_color" [("Prefer", "return=representation")]
+             [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |]
+             `shouldRespondWith` 200

test/spec/Feature/Query/UpsertSpec.hs

@@ -29,7 +29,7 @@ spec actualPgVersion =
               { "name": "C", "rank": 1 }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
             }
 
         it "INSERTs and UPDATEs row on composite pk conflict" $
@@ -42,7 +42,7 @@ spec actualPgVersion =
               { "first_name": "Peter S.", "last_name": "Yang", "salary": "$42,000.00", "company": null, "occupation": null }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
             }
 
         when (actualPgVersion >= pgVersion110) $
@@ -56,13 +56,14 @@ spec actualPgVersion =
                 { "name": "Roma", "year": 2021, "car_brand_name": "Ferrari" }
               ]|]
               { matchStatus = 201
-              , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+              , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
               }
 
         it "succeeds when the payload has no elements" $
           request methodPost "/articles" [("Prefer", "return=representation"), ("Prefer", "resolution=merge-duplicates")]
             [json|[]|] `shouldRespondWith`
-            [json|[]|] { matchStatus = 201 , matchHeaders = [matchContentTypeJson] }
+            [json|[]|] { matchStatus = 201
+                       , matchHeaders = [matchContentTypeJson] }
 
         it "INSERTs and UPDATEs rows on single unique key conflict" $
           request methodPost "/single_unique?on_conflict=unique_key" [("Prefer", "return=representation"), ("Prefer", "resolution=merge-duplicates")]
@@ -74,7 +75,7 @@ spec actualPgVersion =
               { "unique_key": 2, "value": "C" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
             }
 
         it "INSERTs and UPDATEs rows on compound unique keys conflict" $
@@ -87,7 +88,7 @@ spec actualPgVersion =
               { "key1": 1, "key2": 2, "value": "C" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation", matchContentTypeJson]
             }
 
       context "when Prefer: resolution=ignore-duplicates is specified" $ do
@@ -100,7 +101,7 @@ spec actualPgVersion =
               { "name": "PHP", "rank": 9 }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation", matchContentTypeJson]
             }
 
         it "INSERTs and ignores rows on composite pk conflict" $
@@ -112,7 +113,7 @@ spec actualPgVersion =
               { "first_name": "Sara M.", "last_name": "Torpey", "salary": "$60,000.00", "company": "Burstein-Applebee", "occupation": "Soil scientist" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates", matchContentTypeJson]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation", matchContentTypeJson]
             }
 
         when (actualPgVersion >= pgVersion110) $
@@ -125,7 +126,7 @@ spec actualPgVersion =
                 { "name": "Huracán", "year": 2021, "car_brand_name": "Lamborghini" }
               ]|]
               { matchStatus = 201
-              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates", matchContentTypeJson]
+              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation", matchContentTypeJson]
               }
 
         it "INSERTs and ignores rows on single unique key conflict" $
@@ -142,7 +143,7 @@ spec actualPgVersion =
                 { "unique_key": 3, "value": "D" }
               ]|]
               { matchStatus = 201
-              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates"]
+              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation"]
               }
 
         it "INSERTs and UPDATEs rows on compound unique keys conflict" $
@@ -159,7 +160,7 @@ spec actualPgVersion =
                 { "key1": 1, "key2": 3, "value": "D" }
               ]|]
               { matchStatus = 201
-              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates"]
+              , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation"]
               }
 
       it "succeeds if the table has only PK cols and no other cols" $ do
@@ -168,7 +169,7 @@ spec actualPgVersion =
           `shouldRespondWith`
           [json|[ { "id": 3} ]|]
           { matchStatus = 201 ,
-            matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates",
+            matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation",
             matchContentTypeJson] }
 
         request methodPost "/only_pk" [("Prefer", "return=representation"), ("Prefer", "resolution=merge-duplicates")]
@@ -176,22 +177,30 @@ spec actualPgVersion =
           `shouldRespondWith`
           [json|[ { "id": 1 }, { "id": 2 }, { "id": 4} ]|]
           { matchStatus = 201 ,
-            matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates",
+            matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation",
             matchContentTypeJson] }
 
       it "succeeds and ignores the Prefer: resolution header(no Preference-Applied present) if the table has no PK" $
         request methodPost "/no_pk" [("Prefer", "return=representation"), ("Prefer", "resolution=merge-duplicates")]
           [json|[ { "a": "1", "b": "0" } ]|]
           `shouldRespondWith`
-          [json|[ { "a": "1", "b": "0" } ]|] { matchStatus = 201 , matchHeaders = [matchContentTypeJson] }
+          [json|[ { "a": "1", "b": "0" } ]|]
+          { matchStatus = 201
+          , matchHeaders = [matchContentTypeJson
+                           , "Preference-Applied" <:> "return=representation"] }
 
       it "succeeds if not a single resource is created" $ do
         request methodPost "/tiobe_pls" [("Prefer", "return=representation"), ("Prefer", "resolution=ignore-duplicates")]
           [json|[ { "name": "Java", "rank": 1 } ]|] `shouldRespondWith`
-          [json|[]|] { matchStatus = 201 , matchHeaders = [matchContentTypeJson] }
+          [json|[]|]
+          { matchStatus = 201
+          , matchHeaders = [matchContentTypeJson] }
+
         request methodPost "/tiobe_pls" [("Prefer", "return=representation"), ("Prefer", "resolution=ignore-duplicates")]
           [json|[ { "name": "Java", "rank": 1 }, { "name": "C", "rank": 2 } ]|] `shouldRespondWith`
-          [json|[]|] { matchStatus = 201 , matchHeaders = [matchContentTypeJson] }
+          [json|[]|]
+          { matchStatus = 201
+          , matchHeaders = [matchContentTypeJson] }
 
     context "with PUT" $ do
       context "Restrictions" $ do
@@ -408,7 +417,7 @@ spec actualPgVersion =
               { "idUnitTest": 2, "nameUnitTest": "name of unittest 2" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates"]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=merge-duplicates, return=representation"]
             }
 
       it "works with POST and ignore-duplicates headers" $ do
@@ -423,7 +432,7 @@ spec actualPgVersion =
               { "idUnitTest": 2, "nameUnitTest": "name of unittest 2" }
             ]|]
             { matchStatus = 201
-            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates"]
+            , matchHeaders = ["Preference-Applied" <:> "resolution=ignore-duplicates, return=representation"]
             }
 
       it "works with PUT" $ do
@@ -436,3 +445,16 @@ spec actualPgVersion =
             }
         get "/UnitTest?idUnitTest=eq.1" `shouldRespondWith`
           [json| [ { "idUnitTest": 1, "nameUnitTest": "unit test 1" } ]|]
+
+      it "works with request method PUT and return=minimal" $ do
+        request methodPut "/UnitTest?idUnitTest=eq.1"
+            [("Prefer", "return=minimal")]
+            [json| [ { "idUnitTest": 1, "nameUnitTest": "unit test 1" } ]|]
+          `shouldRespondWith`
+            ""
+            { matchStatus = 204
+            , matchHeaders = [matchHeaderAbsent hContentType
+                             , "Preference-Applied" <:> "return=minimal"]
+            }
+        get "/UnitTest?idUnitTest=eq.1" `shouldRespondWith`
+          [json| [ { "idUnitTest": 1, "nameUnitTest": "unit test 1" } ]|]

test/spec/Feature/RollbackSpec.hs

@@ -37,8 +37,8 @@ preferCommit   = [("Prefer", "return=representation"), ("Prefer", "tx=commit")]
 preferRollback = [("Prefer", "return=representation"), ("Prefer", "tx=rollback")]
 
 withoutPreferenceApplied      = []
-withPreferenceCommitApplied   = [ "Preference-Applied" <:> "tx=commit" ]
-withPreferenceRollbackApplied = [ "Preference-Applied" <:> "tx=rollback" ]
+withPreferenceCommitApplied   = [ matchHeaderValuePresent "Preference-Applied" "tx=commit" ]
+withPreferenceRollbackApplied = [ matchHeaderValuePresent "Preference-Applied" "tx=rollback" ]
 
 shouldRespondToReads reqHeaders respHeaders = do
   it "responds to GET" $ do

test/spec/Main.hs

@@ -45,6 +45,7 @@ import qualified Feature.Query.HtmlRawOutputSpec
 import qualified Feature.Query.InsertSpec
 import qualified Feature.Query.JsonOperatorSpec
 import qualified Feature.Query.MultipleSchemaSpec
+import qualified Feature.Query.NullsStrip
 import qualified Feature.Query.PgSafeUpdateSpec
 import qualified Feature.Query.PlanSpec
 import qualified Feature.Query.PostGISSpec
@@ -139,7 +140,8 @@ main = do
         , ("Feature.Query.RawOutputTypesSpec"            , Feature.Query.RawOutputTypesSpec.spec)
         , ("Feature.Query.RpcSpec"                       , Feature.Query.RpcSpec.spec actualPgVersion)
         , ("Feature.Query.SingularSpec"                  , Feature.Query.SingularSpec.spec)
-        , ("Feature.Query.UpdateSpec"                    , Feature.Query.UpdateSpec.spec)
+        , ("Feature.Query.NullsStrip"                    , Feature.Query.NullsStrip.spec)
+        , ("Feature.Query.UpdateSpec"                    , Feature.Query.UpdateSpec.spec actualPgVersion)
         , ("Feature.Query.UpsertSpec"                    , Feature.Query.UpsertSpec.spec actualPgVersion)
         , ("Feature.Query.ComputedRelsSpec"              , Feature.Query.ComputedRelsSpec.spec)
         , ("Feature.Query.RelatedQueriesSpec"            , Feature.Query.RelatedQueriesSpec.spec)

test/spec/SpecHelper.hs

@@ -40,6 +40,18 @@ matchContentTypeJson = "Content-Type" <:> "application/json; charset=utf-8"
 matchContentTypeSingular :: MatchHeader
 matchContentTypeSingular = "Content-Type" <:> "application/vnd.pgrst.object+json; charset=utf-8"
 
+matchCTArrayStrip :: MatchHeader
+matchCTArrayStrip = "Content-Type" <:> "application/vnd.pgrst.array+json;nulls=stripped; charset=utf-8"
+
+matchCTSingularStrip :: MatchHeader
+matchCTSingularStrip = "Content-Type" <:> "application/vnd.pgrst.object+json;nulls=stripped; charset=utf-8"
+
+matchHeaderValuePresent :: HeaderName -> BS.ByteString -> MatchHeader
+matchHeaderValuePresent name val = MatchHeader $ \headers _ ->
+  case lookup name headers of
+    Just hdr -> if val `BS.isInfixOf` hdr then Nothing else Just $ "missing header value: " <> toS val <> "\n"
+    Nothing  -> Just $ "missing header: " <> toS (original name) <> "\n"
+
 matchHeaderAbsent :: HeaderName -> MatchHeader
 matchHeaderAbsent name = MatchHeader $ \headers _body ->
   case lookup name headers of

test/spec/fixtures/data.sql

@@ -839,3 +839,22 @@ INSERT INTO posters(id,name) VALUES (1,'Mark'), (2,'Elon'), (3,'Bill'), (4,'Jeff
 
 TRUNCATE TABLE subscriptions CASCADE;
 INSERT INTO subscriptions(subscriber,subscribed) VALUES (3,1), (4,1), (1,2);
+
+TRUNCATE TABLE datarep_todos CASCADE;
+INSERT INTO datarep_todos VALUES (1, 'Report', 0, '2018-01-02', '\x89504e470d0a1a0a0000000d4948445200000001000000010100000000376ef924000000001049444154789c62600100000000ffff03000000060005057bfabd400000000049454e44ae426082', '2017-12-14 01:02:30', 12.50); -- smallest possible PNG
+INSERT INTO datarep_todos VALUES (2, 'Essay', 256, '2018-01-03', NULL, '2017-12-14 01:02:30', 100000000000000.13); -- a number which can't be represented by a 64-bit float
+INSERT INTO datarep_todos VALUES (3, 'Algebra', 123456, '2018-01-01 14:12:34.123456');
+INSERT INTO datarep_todos VALUES (4, 'Opus Magnum', NULL, NULL);
+
+TRUNCATE TABLE datarep_next_two_todos CASCADE;
+INSERT INTO datarep_next_two_todos VALUES (1, 2, 3, 'school related');
+INSERT INTO datarep_next_two_todos VALUES (2, 1, 3, 'do these first');
+
+TRUNCATE TABLE bitchar_with_length CASCADE;
+INSERT INTO bitchar_with_length(bit, char) VALUES ('00000', 'aaaaa');
+INSERT INTO bitchar_with_length(bit, char) VALUES ('11111', 'bbbbb');
+
+TRUNCATE TABLE table_a CASCADE;
+INSERT INTO table_a(id, name) VALUES (1, 'Not null 1'), (2, null), (3, 'Not null 2');
+TRUNCATE TABLE table_b CASCADE;
+INSERT INTO table_b(table_a_id, name) VALUES (1, 'Test 1'), (2, 'Test 2'), (null, 'Test 3');

test/spec/fixtures/schema.sql

@@ -2774,9 +2774,20 @@ BEGIN
   LOAD 'safeupdate';
 END; $$ LANGUAGE plpgsql SECURITY DEFINER;
 
+-- This tests data representations over computed joins: even a lower case title should come back title cased.
+DROP DOMAIN IF EXISTS public.titlecasetext CASCADE;
+CREATE DOMAIN public.titlecasetext AS text;
+
+CREATE OR REPLACE FUNCTION json(public.titlecasetext) RETURNS json AS $$
+  SELECT to_json(INITCAP($1::text));
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.titlecasetext AS json) WITH FUNCTION json(public.titlecasetext) AS IMPLICIT;
+-- End of data representations specific stuff except for where the domain is used in the table.
+
 CREATE TABLE designers (
   id int primary key
-, name text
+, name public.titlecasetext
 );
 
 CREATE TABLE videogames (
@@ -3091,6 +3102,136 @@ create table test.subscriptions(
   primary key(subscriber, subscribed)
 );
 
+-- Data representations feature
+DROP DOMAIN IF EXISTS public.color CASCADE;
+CREATE DOMAIN public.color AS INTEGER CHECK (VALUE >= 0 AND VALUE <= 16777215);
+
+CREATE OR REPLACE FUNCTION color(json) RETURNS public.color AS $$
+  SELECT color($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION color(text) RETURNS public.color AS $$
+  SELECT (('x' || lpad((CASE WHEN SUBSTRING($1::text, 1, 1) = '#' THEN SUBSTRING($1::text, 2) ELSE $1::text END), 8, '0'))::bit(32)::int)::public.color;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.color) RETURNS json AS $$
+  SELECT
+    CASE WHEN $1 IS NULL THEN to_json(''::text)
+    ELSE to_json('#' || lpad(upper(to_hex($1)), 6, '0'))
+  END;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.color AS json) WITH FUNCTION json(public.color) AS IMPLICIT;
+CREATE CAST (json AS public.color) WITH FUNCTION color(json) AS IMPLICIT;
+CREATE CAST (text AS public.color) WITH FUNCTION color(text) AS IMPLICIT;
+
+DROP DOMAIN IF EXISTS public.isodate CASCADE;
+CREATE DOMAIN public.isodate AS timestamp with time zone;
+
+CREATE OR REPLACE FUNCTION isodate(json) RETURNS public.isodate AS $$
+  SELECT isodate($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION isodate(text) RETURNS public.isodate AS $$
+  SELECT (replace($1, 'Z', '+00:00')::timestamp with time zone)::public.isodate;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.isodate) RETURNS json AS $$
+  SELECT to_json(replace(to_json($1)#>>'{}', '+00:00', 'Z'));
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.isodate AS json) WITH FUNCTION json(public.isodate) AS IMPLICIT;
+CREATE CAST (json AS public.isodate) WITH FUNCTION isodate(json) AS IMPLICIT;
+-- We intentionally don't have this in order to test query string parsing doesn't try to fall back on JSON parsing.
+-- CREATE CAST (text AS public.isodate) WITH FUNCTION isodate(text) AS IMPLICIT;
+
+-- bytea_b64 is a base64-encoded binary string
+DROP DOMAIN IF EXISTS public.bytea_b64 CASCADE;
+CREATE DOMAIN public.bytea_b64 AS bytea;
+
+CREATE OR REPLACE FUNCTION bytea_b64(json) RETURNS public.bytea_b64 AS $$
+  SELECT bytea_b64($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION bytea_b64(text) RETURNS public.bytea_b64 AS $$
+  -- allow unpadded base64
+  SELECT decode($1 || repeat('=', 4 - (length($1) % 4)), 'base64')::public.bytea_b64;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.bytea_b64) RETURNS json AS $$
+  SELECT to_json(translate(encode($1, 'base64'), E'\n', ''));
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.bytea_b64 AS json) WITH FUNCTION json(public.bytea_b64) AS IMPLICIT;
+CREATE CAST (json AS public.bytea_b64) WITH FUNCTION bytea_b64(json) AS IMPLICIT;
+CREATE CAST (text AS public.bytea_b64) WITH FUNCTION bytea_b64(text) AS IMPLICIT;
+
+-- unixtz is a timestamptz represented as an integer number of seconds since the Unix epoch
+DROP DOMAIN IF EXISTS public.unixtz CASCADE;
+CREATE DOMAIN public.unixtz AS timestamp with time zone;
+
+CREATE OR REPLACE FUNCTION unixtz(json) RETURNS public.unixtz AS $$
+  SELECT unixtz($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION unixtz(text) RETURNS public.unixtz AS $$
+  SELECT (to_timestamp($1::numeric)::public.unixtz);
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.unixtz) RETURNS json AS $$
+  SELECT to_json(extract(epoch from $1)::bigint);
+$$ LANGUAGE SQL IMMUTABLE;
+
+
+CREATE CAST (public.unixtz AS json) WITH FUNCTION json(public.unixtz) AS IMPLICIT;
+CREATE CAST (json AS public.unixtz) WITH FUNCTION unixtz(json) AS IMPLICIT;
+CREATE CAST (text AS public.unixtz) WITH FUNCTION unixtz(text) AS IMPLICIT;
+
+DROP DOMAIN IF EXISTS public.monetary CASCADE;
+CREATE DOMAIN public.monetary AS numeric(17,2);
+
+CREATE OR REPLACE FUNCTION monetary(json) RETURNS public.monetary AS $$
+  SELECT monetary($1 #>> '{}');
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION monetary(text) RETURNS public.monetary AS $$
+  SELECT ($1::numeric)::public.monetary;
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION json(public.monetary) RETURNS json AS $$
+  SELECT to_json($1::text);
+$$ LANGUAGE SQL IMMUTABLE;
+
+CREATE CAST (public.monetary AS json) WITH FUNCTION json(public.monetary) AS IMPLICIT;
+CREATE CAST (json AS public.monetary) WITH FUNCTION monetary(json) AS IMPLICIT;
+CREATE CAST (text AS public.monetary) WITH FUNCTION monetary(text) AS IMPLICIT;
+
+CREATE TABLE datarep_todos (
+  id bigint primary key,
+  name text,
+  label_color public.color default 0,
+  due_at public.isodate default '2018-01-01'::date,
+  icon_image public.bytea_b64,
+  created_at public.unixtz default '2017-12-14 01:02:30'::timestamptz,
+  budget public.monetary default 0
+);
+
+CREATE TABLE datarep_next_two_todos (
+  id bigint primary key,
+  first_item_id bigint references datarep_todos(id),
+  second_item_id bigint references datarep_todos(id),
+  name text
+);
+
+CREATE VIEW datarep_todos_computed as (
+  SELECT id,
+    name,
+    label_color,
+    due_at,
+    (label_color / 2)::public.color as dark_color
+  FROM datarep_todos
+);
+
 -- view's name is alphabetically before projects
 create view test.alpha_projects as
   select c.id, p.name as pro_name, c.name as cli_name
@@ -3144,3 +3285,189 @@ BEGIN
   END IF;
 END
 $do$;
+
+create domain devil_int as int
+  default 666;
+
+create table evil_friends(
+  id   devil_int
+, name text
+);
+
+create table bets (
+  id int
+, data_json  json
+, data_jsonb jsonb
+);
+
+create index bets_data_json  on bets ((data_json  ->>'contractId'));
+create index bets_data_jsonb on bets ((data_jsonb ->>'contractId'));
+
+--- https://github.com/PostgREST/postgrest/issues/2862
+create table profiles (
+  id uuid primary key,
+  username text null
+);
+
+create table user_friend (
+  id bigint primary key,
+  user1 uuid references profiles (id),
+  user2 uuid references profiles (id)
+);
+
+create table status(
+  id bigint primary key
+);
+
+create table tournaments(
+  id bigint primary key,
+  status bigint references status(id)
+);
+
+-- https://github.com/PostgREST/postgrest/issues/2861
+CREATE TABLE bitchar_with_length (
+  bit bit(5),
+  char char(5),
+  bit_arr bit(5)[],
+  char_arr char(5)[]
+);
+
+-- https://github.com/PostgREST/postgrest/issues/1586
+create or replace function char_param_select(char_ char(4), char_arr char(4)[])
+returns table(char_ char, char_arr char[]) as $$
+  select $1, $2;
+$$ language sql;
+
+create or replace function bit_param_select(bit_ char(4), bit_arr char(4)[])
+returns table(bit_ char, bit_arr char[]) as $$
+  select $1, $2;
+$$ language sql;
+
+create or replace function char_param_insert(char_ char(4), char_arr char(4)[])
+returns void as $$
+  insert into bitchar_with_length(char, char_arr) values($1, $2);
+$$ language sql;
+
+create or replace function bit_param_insert(bit_ bit(4), bit_arr bit(4)[])
+returns void as $$
+insert into bitchar_with_length(bit, bit_arr) values($1, $2);
+$$ language sql;
+
+create function returns_record() returns record as $$
+select * from projects limit 1;
+$$ language sql;
+
+create function returns_record_params(id int, name text) returns record as $$
+select * from projects p where p.id = $1 and p.name like $2;
+$$ language sql;
+
+create function returns_setof_record() returns setof record as $$
+select * from projects limit 2;
+$$ language sql;
+
+create function returns_setof_record_params(id int, name text) returns setof record as $$
+select * from projects p where p.id >= $1 and p.name like $2;
+$$ language sql;
+
+create function raise_sqlstate_test1() returns void
+  language plpgsql
+  as $$
+begin
+    raise sqlstate 'PGRST' USING
+      message = '{"code":"123","message":"ABC","details":"DEF","hint":"XYZ"}',
+      detail = '{"status":332,"status_text":"My Custom Status","headers":{"X-Header":"str"}}';
+end
+$$;
+
+create function raise_sqlstate_test2() returns void
+  language plpgsql
+  as $$
+begin
+    raise sqlstate 'PGRST' USING
+      message = '{"code":"123","message":"ABC"}',
+      detail = '{"status":332,"headers":{"X-Header":"str"}}';
+end
+$$;
+
+create function raise_sqlstate_test3() returns void
+  language plpgsql
+  as $$
+begin
+    raise sqlstate 'PGRST' USING
+      message = '{"code":"123","message":"ABC"}',
+      detail = '{"status":404,"headers":{"X-Header":"str"}}';
+end
+$$;
+
+create function raise_sqlstate_test4() returns void
+  language plpgsql
+  as $$
+begin
+    raise sqlstate 'PGRST' USING
+      message = '{"code":"123","message":"ABC"}',
+      detail = '{"status":404,"status_text":"My Not Found","headers":{"X-Header":"str"}}';
+end
+$$;
+
+create function raise_sqlstate_invalid_json_message() returns void
+  language plpgsql
+  as $$
+begin
+    raise sqlstate 'PGRST' USING
+      message = 'INVALID',
+      detail = '{"status":332,"headers":{"X-Header":"str"}}';
+end
+$$;
+
+create function raise_sqlstate_invalid_json_details() returns void
+  language plpgsql
+  as $$
+begin
+    raise sqlstate 'PGRST' USING
+      message = '{"code":"123","message":"ABC","details":"DEF"}',
+      detail = 'INVALID';
+end
+$$;
+
+create function raise_sqlstate_missing_details() returns void
+  language plpgsql
+  as $$
+begin
+    raise sqlstate 'PGRST' USING
+      message = '{"code":"123","message":"ABC","details":"DEF"}';
+end
+$$;
+
+create table table_a (
+  id int primary key,
+  name text
+);
+
+create table table_b (
+  table_a_id int references table_a(id),
+  name text
+);
+
+create or replace function test.returns_complex()
+returns table(id int, val complex) as $$
+  select 1, row(0.1, 0.5)::complex as val
+  union
+  select 2, row(0.2, 0.6)::complex as val
+  union
+  select 3, row(0.3, 0.7)::complex as val;
+$$ language sql;
+
+create function computed_rel_overload(items) returns setof items2 as $$
+  select * from items2 limit 1
+$$ language sql;
+
+create function computed_rel_overload(items2) returns setof items2 as $$
+  select * from items2 limit 2
+$$ language sql;
+
+create function search2(id bigint) returns setof items2
+language plpgsql
+stable
+as $$ begin
+  return query select items2.id from items2 where items2.id=search2.id;
+end$$;