v0.3.2.0

Update operator documentation

CHANGELOG.md

@@ -3,6 +3,90 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+
+### Added
+
+### Fixed
+
+## [0.3.2.0] - 2016-06-10
+
+### Added
+- Reload database schema on SIGHUP - @begriffs
+- Support "-" in column names - @ruslantalpa
+- Support column/node renaming `alias:column` - @ruslantalpa
+- Accept posts from HTML forms - @begriffs
+- Ability to order embedded entities - @ruslantalpa
+- Ability to paginate using &limit and &offset parameters - @ruslantalpa
+- Ability to apply limits to embedded entities and enforce --max-rows on all levels - @ruslantalpa, @begriffs
+- Add allow response header in OPTIONS - @begriffs
+
+### Fixed
+- Return 401 or 403 for access denied rather than 404 - @begriffs
+- Omit Content-Type header for empty body - @begriffs
+- Prevent role from being changed twice - @begriffs
+- Use read-only transaction for read requests - @ruslantalpa
+- Include entities from the same parent table using two different foreign keys - @ruslantalpa
+- Ensure that Location header in 201 response is URL-encoded - @league
+- Fix garbage collector CPU leak - @ruslantalpa et al.
+- Return deleted items when return=representation header is sent - @ruslantalpa
+- Use table default values for empty object inserts - @begriffs
+
+## [0.3.1.1] - 2016-03-28
+
+### Fixed
+- Preserve unicode values in insert,update,rpc (regression) - @begriffs
+- Prevent duplicate call to stored procs (regression) - @begriffs
+- Allow SQL functions to generate registered JWT claims - @begriffs
+- Terminate gracefully on SIGTERM (for use in Docker) - @recmo
+- Relation detection fix for views that depend on multiple tables - @ruslantalpa
+- Avoid count on plurality=singular and allow multiple Prefer values - @ruslantalpa
+
+## [0.3.1.0] - 2016-02-28
+
+### Fixed
+- Prevent query error from infecting later connection - @begriffs, @ruslantalpa, @nikita-volkov, @jwiegley
+
+### Added
+- Applies range headers to RPC calls - @diogob
+
+## [0.3.0.4] - 2016-02-12
+
+### Fixed
+- Improved usage screen - @begriffs
+- Reject non-POSTs to rpc endpoints - @begriffs
+- Throw an error for OPTIONS on nonexistent tables - @calebmer
+- Remove deadlock on simultaneous contentious updates - @ruslantalpa, @begriffs
+
+## [0.3.0.3] - 2016-01-08
+
+### Fixed
+- Fix bug in many-many relation detection - @ruslantalpa
+- Inconsistent escaping of table names in read queries - @calebmer
+
+## [0.3.0.2] - 2015-12-16
+
+### Fixed
+- Miscalculation of time used for expiring tokens - @calebmer
+- Remove bcrypt dependency to fix Windows build - @begriffs
+- Detect relations event when authenticator does not have rights to intermediate tables - @ruslantalpa
+- Ensure db connections released on sigint - @begriffs
+- Fix #396 include records with missing parents - @ruslantalpa
+- `pgFmtIdent` always quotes #388 - @calebmer
+- Default schema, changed from `"1"` to `public` - @calebmer
+- #414 revert to separate count query - @ruslantalpa
+- Fix #399, allow inserting in tables with no select privileges using "Prefer: representation=minimal" - @ruslantalpa
+
+### Added
+- Allow order by computed columns - @diogob
+- Set max rows in response with --max-rows - @begriffs
+- Selection by column name (can detect if `_id` is not included) - @calebmer
+
+## [0.3.0.1] - 2015-11-27
+
+### Fixed
+- Filter columns on embedded parent items - @ruslantalpa
+
 ## [0.3.0.0] - 2015-11-24
 
 ### Fixed
@@ -25,6 +109,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 - Embed foreign keys with {} rather than () - @begriffs
+- Remove version number from binary filename in release - @begriffs
 
 ## [0.2.12.1] - 2015-11-12
 

CONTRIBUTING.md

@@ -27,6 +27,10 @@ your contributions.
 * Provide steps to reproduce the issue, including your OS version and
   the specific database schema that you are using.
 
+* Please include SQL logs for issues involving runtime problems. To obtain logs first
+  [enable logging all statements](http://www.microhowto.info/howto/log_all_queries_to_a_postgresql_server.html),
+  then [find your logs](http://blog.endpoint.com/2014/11/dear-postgresql-where-are-my-logs.html).
+
 ## Code
 
 ### Haskell Conventions

Dockerfile

@@ -0,0 +1,27 @@
+FROM debian:jessie
+
+ENV POSTGREST_VERSION 0.3.2.0
+ENV POSTGREST_SCHEMA public
+ENV POSTGREST_ANONYMOUS postgres
+ENV POSTGREST_JWT_SECRET thisisnotarealsecret
+ENV POSTGREST_MAX_ROWS 1000000
+ENV POSTGREST_POOL 200
+
+RUN apt-get update && \
+    apt-get install -y tar xz-utils wget libpq-dev && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN wget http://github.com/begriffs/postgrest/releases/download/v${POSTGREST_VERSION}/postgrest-${POSTGREST_VERSION}-ubuntu.tar.xz && \
+    tar --xz -xvf postgrest-${POSTGREST_VERSION}-ubuntu.tar.xz && \
+    mv postgrest /usr/local/bin/postgrest && \
+    rm postgrest-${POSTGREST_VERSION}-ubuntu.tar.xz
+
+CMD exec postgrest postgres://${PG_ENV_POSTGRES_USER}:${PG_ENV_POSTGRES_PASSWORD}@${PG_PORT_5432_TCP_ADDR}:${PG_PORT_5432_TCP_PORT}/${PG_ENV_POSTGRES_DB} \
+              --port 3000 \
+              --schema ${POSTGREST_SCHEMA} \
+              --anonymous ${POSTGREST_ANONYMOUS} \
+              --pool ${POSTGREST_POOL} \
+              --jwt-secret ${POSTGREST_JWT_SECRET} \
+              --max-rows ${POSTGREST_MAX_ROWS}
+
+EXPOSE 3000

README.md

@@ -5,6 +5,7 @@
   <img src="https://img.shields.io/badge/%E2%86%91_Deploy_to-Heroku-7056bf.svg" alt="Deploy">
 </a>
 [![Join the chat at https://gitter.im/begriffs/postgrest](https://img.shields.io/badge/gitter-join%20chat%20%E2%86%92-brightgreen.svg)](https://gitter.im/begriffs/postgrest)
+[![Docker Hub](https://img.shields.io/badge/Docker%20Hub-%E2%86%92-blue.svg)](https://hub.docker.com/r/begriffs/postgrest/)
 
 PostgREST serves a fully RESTful API from any existing PostgreSQL
 database. It provides a cleaner, more standards-compliant, faster
@@ -117,7 +118,7 @@ views. You run an instance of PostgREST per schema and route requests
 among them with a reverse proxy such as [nginx](http://nginx.org).
 Learn more [here](http://postgrest.com/admin/versioning/).
 
-### Self-documention
+### Self-documentation
 
 Rather than writing and maintaining separate docs yourself let the
 API explain its own affordances using HTTP. All PostgREST endpoints
@@ -125,7 +126,7 @@ respond to the OPTIONS verb and explain what they support as well
 as the data format of their JSON payload. RAML support is an upcoming
 feature.
 
-The project uses HTTP itself to commicate other metadata. For
+The project uses HTTP itself to communicate other metadata. For
 instance the number of rows returned by an endpoint is reported by -
 and limited with - range headers. More about
 [that](http://begriffs.com/posts/2014-03-06-beyond-http-header-links.html).

app.json

@@ -10,7 +10,7 @@
     },
     "POSTGREST_VER": {
       "description": "Version of PostgREST to deploy",
-      "value": "0.3.0.0"
+      "value": "0.3.2.0"
     },
     "DB_NAME": {
       "description": "Database name",
@@ -47,8 +47,8 @@
       "required": false,
       "value": "secret"
     },
-    "V1SCHEMA": {
-      "description": "DB schema selected whe no version (or version 1) requested",
+    "SCHEMA": {
+      "description": "DB schema to be exported",
       "required": false,
       "value": "1"
     }

circle.yml

@@ -1,16 +1,27 @@
-machine:
+dependencies:
+  cache_directories:
+    - "~/.stack"
+    - ".stack-work"
   pre:
+    - curl -L https://github.com/commercialhaskell/stack/releases/download/v1.1.2/stack-1.1.2-linux-x86_64.tar.gz | tar zx -C /tmp
+    - sudo mv /tmp/stack-1.1.2-linux-x86_64/stack /usr/bin
+    - sudo apt-get update; sudo apt-get install --only-upgrade binutils
     - createuser --superuser --no-password postgrest_test
     - createdb -O postgrest_test -U ubuntu postgrest_test
-  ghc:
-    version: 7.10.1
-dependencies:
   override:
-    - cabal update
-    - cabal sandbox init
-    - cabal install --upgrade-dependencies --constraint="template-haskell installed" --dependencies-only --enable-tests
-    - cabal configure --enable-tests -f ci
+    - stack setup
+    - rm -fr $(stack path --dist-dir) $(stack path --local-install-root)
+    - stack install hlint packdeps cabal-install
+    - stack build
+    - stack build --test --no-run-tests
+
 test:
-  post:
-    - cabal exec hlint -- -X QuasiQuotes src/**/*.hs test/**/*.hs
-    - cabal exec packdeps postgrest.cabal
+  override:
+    - stack test
+    - git ls-files | grep '\.l\?hs$' | xargs stack exec -- hlint -X QuasiQuotes "$@"
+    - stack exec -- cabal update
+    - stack exec --no-ghc-package-path -- cabal install --only-d --dry-run
+    - stack exec -- packdeps *.cabal || true
+    - stack exec -- cabal check
+    - stack haddock --no-haddock-deps
+    - stack sdist

docs/admin/security.md

@@ -1,12 +1,71 @@
 ## Security
 
-### SSL
+PostgREST is designed to keep the database at the center of API
+security. All authorization happens through database roles and
+permissions. It is PostgREST's job to *authenticate* requests --
+i.e. verify that a client is who they say they are -- and then let
+the database *authorize* client actions.
+
+We use [JSON Web Tokens](http://jwt.io/) to authenticate API requests.
+As you'll recall a JWT contains a list of cryptographically signed
+claims. PostgREST cares specifically about a claim called `role`.
+When request contains a valid JWT with a role claim PostgREST will
+switch to the database role with that name for the duration of the
+HTTP request.  If the client included no (or an invalid) JWT then
+PostgREST selects the "anonymous role" which is specified by a
+command line arguments to the server on startup.
+
+```js
+{
+  "role": "jdoe123"
+}
+
+// Encoded as JWT with a secret of "secret" this becomes
+// eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiamRvZTEyMyJ9.X_ZeWSS9qsKDCDczv8C-GE2fccrPQjOh_ALMZJa5jsU
+```
+
+Using JWT allows us to authenticate with external services. A login
+service needs merely to share a JWT encryption secret with the
+PostgREST server. The secret is also a server command line option.
+
+It is even possible to generate JWT from inside a stored procedure
+in your database. Any SQL stored procedure that returns a type whose
+name ends in `jwt_claims` will have its return value encoded into
+JWT.  See the [User Management](http://postgrest.com/examples/users/)
+example for details.
 
 ### Database Roles
 
-### JSON Web Tokens
+Suppose you start the server like this:
 
-#### Issuing via sql procedures
+```bash
+postgrest postgres://foo@localhost:5432/mydb --anonymous anon
+```
+
+This means that `foo` is the so-called *authenticator role* and
+`anon` is the anonymous role. When a new HTTP request arrives at the
+server the latter is connected to the database as user `foo`. If
+no JWT is present, or if it is invalid, or if it does not contain
+the role claim then the server changes to the anonymous role with
+the query
+
+```sql
+SET LOCAL ROLE anon;
+```
+
+Otherwise it sets the role to that specified by JWT. For security
+your authenticator role should have access to nothing except the
+ability to become other users. Supposing you have three roles, one
+for anonymous users, one for authors, and another for the authenticator,
+you would set it up like this
+
+```sql
+CREATE ROLE authenticator NOINHERIT LOGIN;
+CREATE ROLE anon;
+CREATE ROLE author;
+
+GRANT anon, author TO authenticator;
+```
 
 ### Row-Level Security
 
@@ -19,3 +78,5 @@
 #### Basic Auth
 
 #### Github Sign-in
+
+### SSL

docs/api/reading.md

@@ -20,7 +20,7 @@ GET /people
 ```
 
 There are no `deeply/nested/routes`. Each route provides `OPTIONS`,
-`GET`, `POST`, `PUT`, `PATCH`, and `DELETE` verbs depending entirely
+`GET`, `POST`, `PATCH`, and `DELETE` verbs depending entirely
 on database permissions.
 
 <div class="admonition note">
@@ -28,9 +28,9 @@ on database permissions.
 
   <p>Why not provide nested routes? Many APIs allow nesting to
   retrieve related information, such as <code>/films/1/director</code>.
-  We offer a more flexible mechanism instead to embed related
-  information, including many-to-many relationships. This is covered
-  in the section about Embedding.</p>
+  We offer a more flexible mechanism (inspired by GraphQL) to embed
+  related information. It can handle one-to-many and many-to-many
+  relationships. This is covered in the section about Embedding.</p>
 </div>
 
 ### Stored Procedures
@@ -47,7 +47,7 @@ POST /rpc/proc_name
 
 PostgREST supports calling procedures with [named
 arguments](http://www.postgresql.org/docs/9.4/static/sql-syntax-calling-funcs.html#SQL-SYNTAX-CALLING-FUNCS-NAMED).
-To do so include a JSON object in the request payload and each
+Include a JSON object in the request payload and each
 key/value of the object will become an argument.
 
 <div class="admonition note">
@@ -91,15 +91,20 @@ These operators are available:
 abbreviation | meaning
 ------------ | -------
 eq           | equals
-gt           | greater than
-lt           | less than
 gte          | greater than or equal
+gt           | greater than
 lte          | less than or equal
+lt           | less than
+neq          | not equal
 like         | LIKE operator (use * in place of %)
 ilike        | ILIKE operator (use * in place of %)
-@@           | full-text search using to_tsquery
-is           | checking for exact equality (null,true,false)
 in           | one of a list of values e.g. `?a=in.1,2,3`
+notin        | not one of a list of values e.g. `?a=notin.1,2,3`
+is           | checking for exact equality (null,true,false)
+isnot        | checking for exact inequality (null,true,false)
+@@           | full-text search using to_tsquery
+@>           | contains e.g. `?tags=@>.{example, new}`
+<@           | contained in e.g. `values=<@{1,2,3}`
 not          | negates another operator, see below
 
 To negate any operator, prefix it with `not` like `?a=not.eq.2`.
@@ -159,7 +164,7 @@ comma-separated list of columns and directions:
 GET /people?order=age.desc,height.asc
 ```
 
-If no direction is specified it defaults to descending order:
+If no direction is specified it defaults to ascending order:
 
 ```HTTP
 GET /people?order=age
@@ -169,8 +174,20 @@ If you care where nulls are sorted, add `nullsfirst` or `nullslast`:
 
 ```HTTP
 GET /people?order=age.nullsfirst
+GET /people?order=age.desc.nullslast
 ```
 
+To order the embedded items, you need to specify the tree path for the order param like so.
+```HTTP
+GET /projects?select=id,name,tasks{id,name}&order=id.asc&tasks.order=name.asc
+```
+
+
+You can also use [computed
+columns](http://www.postgresql.org/docs/current/interactive/xfunc-sql.html#XFUNC-SQL-COMPOSITE-FUNCTIONS)
+to order the results, even though the computed
+columns will not appear in the output.
+
 ### Limiting and Pagination
 
 #### Pagination by Limit-Offset
@@ -202,6 +219,15 @@ Range: 0-4
 You can also use open-ended ranges for an offset with no limit:
 `Range: 10-`.
 
+In addition to the `Range` header, you can use `&limit` and `&offset` parameters
+to achieve the same result.
+
+You can also set a limit (but not offset) for the embedded items like so
+```HTTP
+/posts?select=id,title,body,comments{id,email,body}&limit=10&comments.limit=3
+```
+The above request will return the first 10 posts and for each of the posts, 3 comments at most
+
 #### Suppressing Counts
 
 Sometimes knowing the total row count of a query is unnecessary and
@@ -212,7 +238,7 @@ count total using a ```Prefer``` header as:
 Prefer: count=none
 ```
 
-So the PostgREST response will be something like:
+With count suppressed the PostgREST response will look like:
 
 ```
 Range-Unit: items
@@ -221,13 +247,14 @@ Content-Range → 0-14/*
 
 ### Embedding Foreign Entities
 
-Suppose you have a `projects` table which references `clients` through
-a foreign key called `client_id`. When listing projects through the
-API you can have it embed the client within each project response.
-For example,
+To help you make fewer requests, PostgREST allows the embedding of
+traditional SQL relationships into a response. Suppose you have a
+`projects` table which references `clients` through a foreign key
+called `client_id`. When listing projects through the API you can
+have it embed the client within each project response. For example,
 
 ```HTTP
-GET /projects?id=eq.1&select=id, name, clients(*)
+GET /projects?id=eq.1&select=id, name, clients{*}
 ```
 
 Notice this is the same `select` keyword which is used to choose
@@ -240,9 +267,75 @@ The embedding works for 1-N, N-1, and N-N relationships. That means
 you could also ask for a client and all their projects:
 
 ```HTTP
-GET /clients?id=eq.42&select=id, name, projects(*)
+GET /clients?id=eq.42&select=id, name, projects{*}
 ```
 
+In the examples above we asked for all columns in the embedded resource
+but the the select query is recursive. You could for instance specify
+
+
+```HTTP
+GET /foo?select=x, y, bar{z, w, baz{*}}
+```
+
+You can select not only using table names, but also foreign key column names!
+This is especially needed when you have a table with two foreign keys pointing to the same table, for example billing_address_id and shipping_address_id.
+To embed the same foreign key row from our client example earlier
+you could do the following:
+
+```HTTP
+GET /projects?id=eq.1&select=id, name, client_id{*}
+```
+
+In the response there will be a `client_id` object containing all
+the data for that row.
+
+However, a `client_id` object doesn't make a lot of sense, so you
+could do one of two things. Tell PostgREST that you want the key renamed by using the `alias` feature like so `client:client_id{*}`, or just try `client{*}`
+in the select parameter! PostgREST supports smart ducktype checking
+for common foreign key names, so if your column name ends with
+`_id`, `_fk`, or any variation of the two (including camelcase)
+you can embed a row with just the name's beginning.
+
+So for a complete example:
+
+```HTTP
+GET /projects?id=eq.1&select=id, name, client{*}
+```
+
+Would embed in the `client` key the row referenced with `client_id`.
+
+The `alias` feature works for embedded entities and also for regular columns. This is useful in situations where for example you use different naming conventions in the database and frontend.
+
+The following request will produce the output below:
+```HTTP
+GET /orders?id=eq.1&select=orderId:id, customer:customer_id{customerId:id, customerName:name}
+```
+
+```json
+[
+  {
+    "orderId": 1,
+    "customer": {
+      "customerId": 1,
+      "customerName": "John Smith"
+    }
+  }
+]
+```
+
+If you want to apply filters to the embedded items, you can do that like so:
+```HTTP
+GET /clients?id=eq.42&select=id,name,projects{id,name,is_active}&projects.is_active=eq.true
+```
+The above request will return the client with id=42 and all the projects for that client that are still active
+
+
+<div class="admonition note">
+    <p class="admonition-title">Design Consideration</p>
+    <p>In order for this feature to work as expected after a schema change, PostgREST currently requires to be restarted.</p>
+</div>
+
 ### Response Format
 
 Query responses default to JSON but you can get them in CSV as well. Just make your request with the header
@@ -265,7 +358,8 @@ For consistency's sake all these endpoints return a JSON array,
 `/stories`, `/stories?genre=eq.mystery`, `/stories?id=eq.1`. They
 are all filtering a bigger array. However you might want the
 last one to return a single JSON object, not an array with one
-element. There is currently an open issue to enable this.
+element. To request a singular response send the header
+`Prefer: plurality=singular`.
 
 ### Data Schema
 
@@ -280,14 +374,14 @@ OPTIONS /my_view
 This will include the row names, their types, primary key
 information, and foreign keys for the given table or view.
 
-<div class="admonition danger">
-    <p class="admonition-title">Deprecation Warning</p>
+<div class="admonition warning">
+    <p class="admonition-title">Schema Changes</p>
 
-    <p>Although we currently use the OPTIONS verb for this, some
-    people <a
-    href="https://www.mnot.net/blog/2012/10/29/NO_OPTIONS">argue</a> that
-    this is inappropriate. We are considering a <code>describedby</code>
-    header link instead.</p>
+    <p>Note that when the schema of your database changes PostgREST will not reflect
+    the change. You have to either restart PostgREST or send its running process
+    a HUP signal:
+
+    <pre><code>killall -HUP postgrest</code></pre>
 </div>
 
 ### CORS

docs/api/writing.md

@@ -26,9 +26,13 @@ the server side.
 * ❌ Cannot be cached or prefetched
 * ❌ Not idempotent
 
-While regular insertion uses JSON to encode the value, bulk insertion
-uses CSV. Simply post to a table route with `Content-Type: text/csv`
-and include the names of the columns as the first row. For instance
+You can POST a JSON array or CSV to insert multiple rows in a single
+HTTP request. Note that using CSV requires less parsing on the server
+and is **much faster**.
+
+Example of CSV bulk insert. Simply post to a table route with
+`Content-Type: text/csv` and include the names of the columns as
+the first row. For instance
 
 ```HTTP
 POST /people
@@ -41,42 +45,118 @@ An empty field (`,,`) is coerced to an empty string and the reserved
 word `NULL` is mapped to the SQL null value. Note that there should
 be no spaces between the column names and commas.
 
-The server sends a multipart response for bulk insertions. Each part
-contains a Location header with URL of each created resource.
+Example of JSON bulk insert. Send an array:
 
 ```HTTP
-Content-Type: application/json
-Location: /festival?name=eq.Venice%20Film%20Festival
-
-
---postgrest_boundary
-Content-Type: application/json
-Location: /festival?name=eq.Cannes%20Film%20Festival
+POST /people
+[
+  { "name": "J Doe", "age": 62, "height": 70 },
+  { "name": "Janus", "age": 10, "height": 55 }
+]
 ```
 
-### Upsertion
-
-* ❌ Cannot be cached or prefetched
-* ✅ Idempotent
-
-To insert or update a single row use the `PUT` verb on a properly
-filtered table url:
-
-```HTTP
-PUT /table_name?primary_key=eq.foo
-{ "col1": "value1", "col2": "value2" }
-```
-
-The request must satisfy two things. First all columns must be
-specified (because a default value might be a changing value which
-would violate idempotence). Second the URL must match the URL you
-would use to get the value of the resource. This means that all
-primary key columns must be included in the filter (there are more
-than one when the primary key is compound).
-
 If you would like to get the full object back in the response to
 your request, include the header `Prefer: return=representation`.
-It will of match exactly the object you sent though.
+Chances are you only want certain information back, though, like
+created ids. You can pass a `select` parameter to affect the shape
+of the response (further documented in the [reading](/api/reading/)
+page). For instance
+
+```HTTP
+POST /people?select=id
+[...]
+```
+returns something like
+```json
+[ { "id": 1 }, { "id": 2 } ]
+```
+
+### Multiple Tables Insertion or Update
+
+The cleanest way to insert or update data into multiple tables using only one POST/PATCH request
+is to create a view that will join all target tables and present a single endpoint.
+In our example let's assume one users table and one companies table.
+In this case, we want a signup endpoint to create the first user within a company.
+And for this endpoint we want to insert with one request both user and company.
+
+```SQL
+CREATE TABLE companies (
+    id serial primary key,
+    name text unique
+);
+
+CREATE TABLE users (
+    id serial primary key,
+    name text not null,
+    pass text,
+    company_id integer not null references companies
+);
+```
+
+Having both tables created we create a view that joins them to be used
+as a ```/signup``` endpoint.
+
+```SQL
+CREATE VIEW signup AS
+    SELECT
+        c.name AS company_name,
+        u.name AS user_name,
+        u.pass
+    FROM
+        public.users u
+        JOIN public.companies c ON c.id = u.company_id;
+
+```
+
+After the signup view creation, we can issue ```GET``` requests to read data
+from users and companies, but any atempt to ```POST``` or ```PATCH``` data will fail.
+PostgreSQL won't allow any data change on views that have a ```JOIN``` 
+clause in their ```FROM``` without a proper ```INSTEAD OF``` trigger.
+So in the example bellow we create a trigger to allow insertion of data in the signup view.
+The trigger is a simple PL/pgSQL function that first inserts into the companies table and
+uses the newly create company_id to create its first user.
+
+
+```SQL
+CREATE FUNCTION signup()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  vcompany_id int;
+BEGIN
+  INSERT INTO companies (name) VALUES (new.company_name) RETURNING id INTO vcompany_id;
+  INSERT INTO users (name, pass, company_id) VALUES (new.user_name, new.pass, vcompany_id);
+RETURN new;
+END;
+$$;
+
+CREATE TRIGGER signup
+INSTEAD OF INSERT ON signup
+FOR EACH ROW
+EXECUTE PROCEDURE signup();
+```
+
+After the trigger creation we can issue a normal ```POST``` request to our signup endpoint:
+
+```HTTP
+POST /signup
+{ "company_name": "foo", "user_name": "bar" }
+```
+
+For an endpoint such as signup its usually not desirable to have a ```PATCH``` route for updates,
+and we will skip this example for the sake of brevity. But it would be implemented in a very
+similar way to our ```POST``` example.
+
+<div class="admonition note">
+    <p class="admonition-title">Design Consideration</p>
+
+    <p>It's advisable to create a separate trigger for <code>UPDATE</code> and <code>INSERT</code>
+    avoiding conditionals that decide which is the trigger current operation.
+    This makes it easier to change code for (or even disable) one operation without interfering with others while
+    improving readability.
+    </p>
+</div>
 
 ### Bulk Updates
 
@@ -106,11 +186,11 @@ basic field replacements, and not at all "incorrect."
 * ❌ Cannot be cached or prefetched
 * ✅ Idempotent
 
-Simply use the `DELETE` verb. All recors that match your filter
+Simply use the `DELETE` verb. All records that match your filter
 will be removed. For instance deleting inactive users:
 
 ```HTTP
-DELETE /user?active=eq.false
+DELETE /user?active=is.false
 ```
 
 ### Protecting Dangerous Actions
@@ -118,7 +198,6 @@ DELETE /user?active=eq.false
 Notice that it is very easy to delete or update many records at
 once. In fact forgetting a filter will affect an entire table!
 
-
 <div class="admonition warning">
     <p class="admonition-title">Invitation to Contribute</p>
 

docs/examples/blog.md

@@ -0,0 +1,181 @@
+## Multi-Tenant Blog
+
+In our blog app there will be anonymous users and authors. Each
+author can create and edit their own posts, and read (but not edit)
+the posts of other authors. Anonymous users cannot edit anything
+but can sign up for author accounts. Authors can also post comments
+on articles.
+
+This example builds off the previous previous [User Management](users/)
+one. We had previously created a signup and login system on top of
+JWT. We'll use this auth system for the blog. **Run the SQL in the
+previous example** first, before continuing with this example.
+
+For your convenience, the complete sql for the blog demo is
+[here](https://github.com/begriffs/postgrest/blob/master/schema-templates/blog.sql).
+You can try it out in this [vagrant
+image](https://github.com/ruslantalpa/blogdemo) as well.
+
+### Adding Blog-Specific Tables
+
+Storing the posts and comments is this simple. The comments do not
+form a tree, they are linear under a post.
+
+```sql
+create table if not exists
+posts (
+  id         bigserial primary key,
+  title      text not null,
+  body       text not null,
+  author     text not null references basic_auth.users (email)
+               on delete restrict on update cascade
+               default basic_auth.current_email(),
+  created_at timestamptz not null default current_date
+);
+
+create table if not exists
+comments (
+  id         bigserial primary key,
+  body       text not null,
+  author     text not null references basic_auth.users (email)
+               on delete restrict on update cascade
+               default basic_auth.current_email(),
+  post       bigint not null references posts (id)
+               on delete cascade on update cascade,
+  created_at timestamptz not null default current_date
+);
+```
+
+### Permissions
+
+On top of the `authenticator` and `anon` access granted in the
+previous example, blogs have an `author` role with extra permissions.
+
+```sql
+create role author;
+grant author to authenticator;
+
+grant usage on schema public, basic_auth to author;
+
+-- authors can edit comments/posts
+grant select, insert, update, delete
+  on basic_auth.tokens, basic_auth.users to author;
+grant select, insert, update, delete
+  on table users, posts, comments to author;
+grant usage, select on sequence posts_id_seq, comments_id_seq to author;
+```
+
+To ensure that authors cannot edit each others' posts and comments
+we'll use [row-level
+security](http://www.postgresql.org/docs/9.5/static/ddl-rowsecurity.html).
+Note that it requires PostgreSQL 9.5 or later.
+
+```sql
+grant select on posts, comments to anon;
+
+ALTER TABLE posts ENABLE ROW LEVEL SECURITY;
+ALTER TABLE comments ENABLE ROW LEVEL SECURITY;
+
+drop policy if exists posts_select_unsecure on posts;
+create policy posts_select_unsecure on posts for select
+  using (true);
+
+drop policy if exists comments_select_unsecure on comments;
+create policy comments_select_unsecure on comments for select
+  using (true);
+
+drop policy if exists authors_eigencreate on posts;
+create policy authors_eigencreate on posts for insert
+  with check (
+    author = basic_auth.current_email()
+  );
+
+drop policy if exists authors_eigencreate on comments;
+create policy authors_eigencreate on comments for insert
+  with check (
+      author = basic_auth.current_email()
+  );
+
+drop policy if exists authors_eigenedit on posts;
+create policy authors_eigenedit on posts for update
+  using (author = basic_auth.current_email())
+  with check (
+    author = basic_auth.current_email()
+  );
+
+drop policy if exists authors_eigenedit on comments;
+create policy authors_eigenedit on comments for update
+  using (author = basic_auth.current_email())
+  with check (
+    author = basic_auth.current_email()
+  );
+
+drop policy if exists authors_eigendelete on posts;
+create policy authors_eigendelete on posts for delete
+  using (author = basic_auth.current_email());
+
+drop policy if exists authors_eigendelete on comments;
+create policy authors_eigendelete on comments for delete
+  using (author = basic_auth.current_email());
+```
+
+Finally we need to modify the `users` view from the previous example.
+This is because all authors share a single db role. We could have
+chosen to assign a new role for every author (all inheriting from
+`author`) but we choose to tell them apart by their email addresses.
+The addition below prevents authors from seeing each others' info
+in the `users` view.
+
+
+```diff
+  create or replace view users as
+  select actual.role as role,
+         '***'::text as pass,
+         actual.email as email,
+         actual.verified as verified
+  from basic_auth.users as actual,
+       (select rolname
+          from pg_authid
+         where pg_has_role(current_user, oid, 'member')
+       ) as member_of
+  where actual.role = member_of.rolname
++   and (
++     actual.role <> 'author'
++     or email = basic_auth.current_email()
++   );
+```
+
+### Example client queries
+
+* Top ten most recent posts
+
+```HTTP
+  GET /posts?order=created_at.desc
+  Range: 0-9
+```
+
+* Single post (randomly chose id=1) with its comments
+
+```HTTP
+  GET /posts?id=eq.1&select=*,comments{*}
+```
+
+* Add a new post
+
+```HTTP
+  POST /posts
+  Authorization: Bearer [JWT TOKEN]
+
+  {
+    "title": "My first post",
+    "body": "Meh, forgot what I wanted to say."
+  }
+```
+
+### Conclusion
+
+Voilà, a blog API. Most of the code ended up being for defining
+security. Once you have set up an authentication system, the code
+to do application specific things like blog posts and comments is
+short.  All the front-end routes and verbs are created automatically
+for you.

docs/examples/external_auth.md

@@ -0,0 +1,193 @@
+## External Authentication
+
+API clients authenticate with [JSON Web Tokens](http://jwt.io).
+PostgREST does not support any other authentication mechanism
+directly, but they can be built on top. In this demo we will build
+a system that works with an external authentication server
+and integrates with a PostgREST server by sharing the same JWT secret.
+
+For a better understanding of JWT and PostgREST authentication system you should read
+the [User Management](users/) example as well.
+
+I'll use a [Rails](http://rubyonrails.org) application using [Devise](https://github.com/plataformatec/devise)
+just to make the example more concrete, but this could be replicated for
+any other external authentication system using the same principles.
+In case Rails is not your cup of tea you can continue reading and
+just skip the Ruby code samples. I'll also assume
+the use of JQuery for some client-side code samples for the sake of simplicity.
+
+I won't delve into Devise authentication details, for this would require a tutorial on its own,
+so I'm assuming that the reader's authentication system is already working.
+
+### Sharing the JWT Secret
+
+Allowing a third party to generate valid JWTs for your PostgREST API
+is just a matter of sharing a secret. So you need to give your authenticator
+software the same secret that was used in your API server under the ```--jwt-secret```
+parameter.
+
+This could be done easly using environment variables. You set a ```JWT_SECRET``` variable
+in the environment where you run your rails app and it will be accessible in the global
+variable ```ENV['JWT_SECRET']```.
+
+### User Model
+
+We will map each user in this example to two database roles.
+So our application users are either ```admin``` or ```customer```.
+If they are just visitors (not logged in) to our website they will be ```anonymous```.
+One way of mapping users is to add a field in our users table indicating their database role.
+I'll add a text field called role to my users table:
+
+```sql
+ALTER TABLE users ADD role text NOT NULL DEFAULT 'customer';
+```
+
+Besides the main user that PostgREST uses to connect to PostgreSQL
+and the anonymous user, we will need two additional roles for our example:
+
+* admin - to be used by users that access all the system rows.
+* customer - to be used when user has restricted access to database rows.
+
+Bellow we have the commands to create all roles that will be used:
+```sql
+CREATE USER authenticator NOINHERIT;
+CREATE ROLE anonymous;
+CREATE ROLE admin;
+CREATE ROLE customer;
+
+GRANT customer, admin, anonymous TO authenticator;
+```
+
+### Generating a JWT
+
+Several libraries are available to generate JWT, you will find a very handy list in [their website](http://jwt.io)
+under **Libraries**.
+To continue our Rails example I'll use the ruby library [json_web_token](https://github.com/garyf/json_web_token).
+
+In order to make the gem available in my Rails project I add the following line to my Gemfile:
+
+```
+gem 'json_web_token'
+```
+
+Then we create a Rails controller to serve JWTs for my authenticated users.
+For this I just open a file ```app/controllers/api_tokens_controller.rb``` with the content:
+
+```ruby
+class ApiTokensController < ApplicationController
+  TOKEN_TTL = 1.hour
+
+  def show
+    unless ENV['JWT_SECRET'].present?
+      return render json: {error: "you need to have JWT_SECRET configured to get an API token"}, status: 500
+    end
+
+    unless current_user.present?
+      return render json: {error: "only authenticated users can request the API token"}, status: 401
+    end
+
+    expires_in TOKEN_TTL, public: false
+    render json: {token: jwt}, status: 200
+  end
+
+  private
+  def jwt
+    JsonWebToken.sign(claims, key: ENV['JWT_SECRET'])
+  end
+
+  def claims
+    # This token will expire 1 hour after being issued
+    {
+      role: current_user.role,
+      user_id: current_user.id.to_s,
+      exp: (Time.now + TOKEN_TTL).to_i
+    }
+  end
+end
+```
+
+<div class="admonition note">
+    <p class="admonition-title">Token Time to Live</p>
+    <p>
+    In the code above we leverage the HTTP time based cache headers to expire the
+    endpoint cache at the same time as the token. In this example we have a token
+    that will be refresh one hour after its issuing time.
+    That's why both are based on the <code>TOKEN_TTL</code> constant.
+    </p>
+</div>
+
+We also need to create a route in the ```config/routes.rb``` file:
+
+```ruby
+resource :api_token, only: [:show]
+```
+
+Now, any authenticated user in our rails application can request an api_token making a GET
+request to ```/api_token```. This endpoint will return a json object with one property
+whose value is the token the API requests should use.
+
+### Orders Endpoint
+
+Here is how to create a view to generate an endpoint ```/orders``` filtered by
+the logged in user:
+
+```sql
+ALTER DATABASE mydb SET postgrest.claims.user_id TO '';
+
+CREATE OR REPLACE FUNCTION current_user_id()
+RETURNS integer
+STABLE
+LANGUAGE SQL
+AS $$
+    SELECT nullif(current_setting('postgrest.claims.user_id'), '')::integer;
+$$;
+
+CREATE SCHEMA private;
+
+CREATE TABLE private.orders (
+    id serial primary key,
+    user_id int references users,
+    created_at timestamp not null default current_timestamp,
+    updated_at timestamp not null default current_timestamp
+);
+
+CREATE VIEW orders AS
+SELECT
+    id, user_id, created_at, updated_at
+FROM
+    private.orders o
+WHERE
+    current_user = 'admin' OR o.user_id = current_user_id();
+```
+
+<div class="admonition note">
+    <p class="admonition-title">DRY priviledge checking conditions</p>
+    <p>
+    You can encapsulate conditions that will be commonly used to check for privileges while reading a database row.
+    We used a function <code>current_user_id()</code> but we could add more conditions to functions
+    as the system becomes more complex.<br/>
+    Remeber to mark your functions as <code>STABLE</code> so that PostgreSQL can inline then while planning the query.
+    </p>
+</div>
+
+### Using the JWT
+
+Now whenever you are authenticated in your Rails application you can use some Javascript
+ code to get the token and use it:
+```javascript
+$.getJSON('/api_token').done(function(data){
+    $.ajax('/orders', {'Authorization': 'Bearer ' + data.token}).done(function(data){
+        console.log('Visible Orders: ', data);
+    })
+}).fail(function(){
+    console.log('Error fetching API token');
+})
+```
+We could also store the token to avoid having to fetch it again in the same page.
+
+### Conclusion
+
+This section explained the implementation details for building an
+external authentication system working with PostgREST.
+With the previous [User Management](users/) example this should give a clearer
+idea of how to set up authentication for your API.

docs/examples/python-requests-jwt.md

@@ -0,0 +1,40 @@
+## Python Client for PostgREST API
+
+### Setup PostgreSQL
+
+This code relies on setting up the PostgreSQL auth functions and grants correctly first. Follow [these instructions](http://postgrest.com/examples/users/).
+
+After completing the PostgreSQL configuration, be sure to create a user with email, password, role, and verified flag. We'll use that user to login in the code below.
+
+### Setup PostgREST
+
+Next, setup PostgREST according to the documentation [http://postgrest.com/install/server/](here).
+
+### Setup Python Client
+
+Finally, we'll install and configure the python client. Follow the instructions in the [README](https://github.com/davidthewatson/postgrest_python_requests_client/blob/master/README.md). Be sure to set the [credentials](https://github.com/davidthewatson/postgrest_python_requests_client/blob/master/config.in#L3-L5) and [urls](https://github.com/davidthewatson/postgrest_python_requests_client/blob/master/config.in#L7-L9) in config.py.
+
+### Python Client Functions
+
+There are four primary functions to the python client:
+
+* login
+* construct_jwt_auth
+* get_result_size
+* get_range
+
+The *login* and *construct_jwt_auth* functions will be required for any REST client using a PostgREST server, since a JWT auth instance is presumed.
+
+The *get_result_size* and *get_range* functions are designed specifically for result sets where pagination is required. You can certainly use them for a single page result set that does not require pagination, but that may be overkill.
+
+### Login
+The [login function](https://github.com/davidthewatson/postgrest_python_requests_client/blob/master/client.py#L12-L17) takes email and password strings (credentials.email and credentials.password, respectively from the config.py) and return the response.
+
+### Construct JWT Auth
+The [construct_jwt_auth](https://github.com/davidthewatson/postgrest_python_requests_client/blob/master/client.py#L20-L23) function takes the auth response returned by the login function, retrieves the token in the response, and returns a JWT auth instance to the caller. The JWT auth instance can then be used for successive calls to the same PostgREST service.
+
+### Get Result Size
+The [get_result_size](https://github.com/davidthewatson/postgrest_python_requests_client/blob/master/client.py#L26-L30) function takes a JWT auth instance calls the URL at urls.data, extracts the size of the result set from the response object and returns the size.
+
+### Get Range
+The [get_range](https://github.com/davidthewatson/postgrest_python_requests_client/blob/master/client.py#L26-L30) function takes a beginning range, ending range, page size, and JWT auth instance, gets only that range of the available result set and returns JSON for that result set.

docs/examples/start.md

@@ -25,12 +25,10 @@ CREATE TABLE film
   id serial PRIMARY KEY,
   title text NOT NULL,
   year date NOT NULL,
-  director text,
+  director text REFERENCES director (name)
+    ON UPDATE CASCADE ON DELETE CASCADE,
   rating real NOT NULL DEFAULT 0,
-  language text NOT NULL,
-  CONSTRAINT film_director_fkey FOREIGN KEY (director)
-      REFERENCES director (name) MATCH SIMPLE
-      ON UPDATE CASCADE ON DELETE CASCADE
+  language text NOT NULL
 );
 
 CREATE TABLE festival
@@ -42,27 +40,19 @@ CREATE TABLE competition
 (
   id serial PRIMARY KEY,
   name text NOT NULL,
-  festival text NOT NULL,
-  year date NOT NULL,
-
-  CONSTRAINT comp_festival_fkey FOREIGN KEY (festival)
-      REFERENCES festival (name) MATCH SIMPLE
-      ON UPDATE CASCADE ON DELETE CASCADE
+  festival text NOT NULL REFERENCES festival (name)
+    ON UPDATE CASCADE ON DELETE CASCADE,
+  year date NOT NULL
 );
 
 CREATE TABLE film_nomination
 (
   id serial PRIMARY KEY,
-  competition integer NOT NULL,
-  film integer NOT NULL,
-  won boolean NOT NULL DEFAULT true,
-
-  CONSTRAINT nomination_competition_fkey FOREIGN KEY (competition)
-     REFERENCES competition (id) MATCH SIMPLE
-     ON UPDATE NO ACTION ON DELETE NO ACTION,
-  CONSTRAINT nomination_film_fkey FOREIGN KEY (film)
-     REFERENCES film (id) MATCH SIMPLE
-     ON UPDATE CASCADE ON DELETE CASCADE
+  competition integer NOT NULL REFERENCES competition (id)
+    ON UPDATE NO ACTION ON DELETE NO ACTION,
+  film integer NOT NULL REFERENCES film (id)
+    ON UPDATE CASCADE ON DELETE CASCADE,
+  won boolean NOT NULL DEFAULT true
 );
 
 COMMIT;
@@ -78,10 +68,10 @@ pbpaste | psql demo1
 # xclip -selection clipboard -o | psql demo1
 ```
 
-Start the PostgREST server and point it at the new database.
+Start the PostgREST server and point it at the new database. (See the [installation instructions](/install/server/).)
 
 ```sh
-postgrest -d demo1 -U postgres -a postgres --v1schema public
+postgrest postgres://postgres:@localhost:5432/demo1 -a postgres --schema public
 ```
 
 <div class="admonition note">
@@ -92,6 +82,8 @@ postgrest -d demo1 -U postgres -a postgres --v1schema public
     <code>postgres</code>.</p>
 </div>
 
+### Populating Data
+
 Let's use PostgREST to populate the database. Install a REST client such as [Postman](https://chrome.google.com/webstore/detail/postman/fhbjgbiflinjbdggehcddcbncdddomop?hl=en). Now let's insert some data as a bulk post in CSV format:
 
 ```HTTP
@@ -107,21 +99,11 @@ In Postman it will look like this
 
 ![Festival bulk insert in postman](/img/post-festivals.png)
 
-Notice that the post type is `raw` and that `Content-Type: text/csv` set in the Headers tab. 
+Notice that the post type is `raw` and that `Content-Type: text/csv` set in the Headers tab.
 
-Note that the server returns a multipart response with URL of each created resource.
+The server returns HTTP 201 Created. Because we inserted more than one item at once there is no `Location` header in the response. However sometimes you want to learn more about items which you just inserted. To have the server include the full results, include the header `Prefer: return=representation`.
 
-```HTTP
-Content-Type: application/json
-Location: /festival?name=eq.Venice%20Film%20Festival
-
-
---postgrest_boundary
-Content-Type: application/json
-Location: /festival?name=eq.Cannes%20Film%20Festival
-```
-
-If you send a GET request to `/festival` it should return
+At this point if you send a GET request to `/festival` it should return
 
 ```json
 [
@@ -267,80 +249,261 @@ competition,film,won
 2,35,f
 ```
 
-At this point nominations are fully specified but it's not a convenient interface for a rest client. Let's make a view they can use. Paste this into `psql demo1`.
+### Getting and Embedding Data
 
-```sql
-create or replace view nomination as
-select comp.festival,  
-       comp.name as competition,
-       comp.year,
-       film.title,
-       film.director,
-       film.rating
- from film_nomination as nom
- left join film on nom.film = film.id
- left join competition as comp on nom.competition = comp.id
- order by comp.year desc, comp.festival, competition;
-```
-
-Time to try it out. Let's get the contents of the new view, ordered by film rating
-
-```
-GET http://localhost:3000/nomination?order=rating.desc
-```
-
-If you find it more human readable, add an `Accept: text/csv` header.
-
-### Releasing a New Version
-
-Suppose we want this endpoint to cater to those moviegoers with attention deficit disorder. In today's busy world we don't have time to read an extra couple words or compare nuanced reviews. In API version two we will truncate the names and round the ratings!
-
-Each version lives in a numbered schema, so let's make a schema for version two.
-
-```sql
-CREATE SCHEMA "2";
-GRANT USAGE ON SCHEMA "2" TO PUBLIC;
-ALTER DATABASE demo1 SET search_path = "2", "public";
-```
-
-To override the `films` endpoint create a view in the "2" schema with that name:
-
-```sql
-create or replace view "2".film as
-select id, substring(f.title from 1 for 10) as title,
-       year, director, round(f.rating) as rating, language
-from "public".film as f;
-```
-
-We select the desired version as part of content negotiation. Try this get request:
-
-```HTTP
+First let's review which films are stored in the database:
+```http
 GET http://localhost:3000/film
-Accept: text/csv; version=2
+```
+It gives us back a list of JSON objects. What if we care only about the film titles? Use `select` to shape the output:
+
+```http
+GET http://localhost:3000/film?select=title
+```
+```json
+[
+  {
+    "title": "Chuang ru zhe"
+  },
+  {
+    "title": "The Look of Silence"
+  },
+  {
+    "title": "Fires on the Plain"
+  },
+  ...
+]
 ```
 
-Then try toggling the version string in the Accept header and watch the results change. Pretty good, now how about writing values? PostgreSQL's nice feature called auto-updatable views allows writes to pass through views. Sadly this view is not eligible because truncation and rounding cannot be uniquely reversed. If we attempt to post a new result it complains:
+Here is where it gets cool. PostgREST can embed objects in its response through foreign key relationships. Earlier we created a join table called `film_nomination`. It joins films and competitions. We can ask the server about the structure of this table:
+
+```
+OPTIONS http://localhost:3000/film_nomination
+```
 
 ```json
 {
-  "hint": null,
-  "details": "View columns that are not columns of their base relation are not updatable.",
-  "code": "0A000",
-  "message": "cannot insert into column \"title\" of view \"film\""
+  "pkey": [
+    "id"
+  ],
+  "columns": [
+    {
+      "references": null,
+      "default": "nextval('film_nomination_id_seq'::regclass)",
+      "precision": 32,
+      "updatable": true,
+      "schema": "public",
+      "name": "id",
+      "type": "integer",
+      "maxLen": null,
+      "enum": [],
+      "nullable": false,
+      "position": 1
+    },
+    {
+      "references": {
+        "schema": "public",
+        "column": "id",
+        "table": "competition"
+      },
+      "default": null,
+      "precision": 32,
+      "updatable": true,
+      "schema": "public",
+      "name": "competition",
+      "type": "integer",
+      "maxLen": null,
+      "enum": [],
+      "nullable": false,
+      "position": 2
+    },
+    {
+      "references": {
+        "schema": "public",
+        "column": "id",
+        "table": "film"
+      },
+      "default": null,
+      "precision": 32,
+      "updatable": true,
+      "schema": "public",
+      "name": "film",
+      "type": "integer",
+      "maxLen": null,
+      "enum": [],
+      "nullable": false,
+      "position": 3
+    },
+    {
+      "references": null,
+      "default": "true",
+      "precision": null,
+      "updatable": true,
+      "schema": "public",
+      "name": "won",
+      "type": "boolean",
+      "maxLen": null,
+      "enum": [],
+      "nullable": false,
+      "position": 4
+    }
+  ]
 }
 ```
 
-This is a case where we need explicit triggers
+From this you can see that the columns `film` and `competition` reference their eponymous tables. Let's ask the server for each film along with names of the competitions it entered. You don't have to do any custom coding. Send this query:
 
-```sql
--- TODO - FIX THIS
-
--- CREATE OR REPLACE RULE insert_v2_films AS
---   ON INSERT TO "2".film
---   DO INSTEAD
---      INSERT INTO public.film (id, title, year, director, rating, language)
---      VALUES (NEW.id,     NEW.title,
---              NEW.year,   NEW.director,
---              NEW.rating, NEW.language)
---      RETURNING public.film.*;
+```http
+GET http://localhost:3000/film?select=title,competition{name}
 ```
+
+```json
+[
+  {
+    "title": "Chuang ru zhe",
+    "competition": [
+      {
+        "name": "Golden Lion"
+      }
+    ]
+  },
+  {
+    "title": "The Look of Silence",
+    "competition": [
+      {
+        "name": "Golden Lion"
+      }
+    ]
+  },
+  ...
+]
+```
+
+The relation flows both ways. Here is how to get the name of each competition's name and the movies shown at it.
+
+```http
+GET http://localhost:3000/competition?select=name,film{title}
+```
+
+```json
+[
+  {
+    "name": "Golden Lion",
+    "film": [
+      {
+        "title": "Chuang ru zhe"
+      },
+      {
+        "title": "The Look of Silence"
+      },
+      ...
+    ]
+  },
+  {
+    "name": "Palme d'Or",
+    "film": [
+      {
+        "title": "The Wonders"
+      },
+      {
+        "title": "Foxcatcher"
+      },
+      ...
+    ]
+  }
+]
+```
+
+Why not learn about the directors too? There is a many-to-one relation directly between films and directors. We can alter our previous query to include directors in its results.
+
+
+```http
+GET http://localhost:3000/competition?select=name,film{title,director{*}}
+```
+
+```json
+[
+  {
+    "name": "Golden Lion",
+    "film": [
+      {
+        "title": "Manglehorn",
+        "director": {
+          "name": "David Gordon Green"
+        }
+      },
+      {
+        "title": "Belye nochi pochtalona Alekseya Tryapitsyna",
+        "director": {
+          "name": "Andrey Konchalovskiy"
+        }
+      },
+      ...
+    ]
+  },
+  ...
+]
+```
+
+### Singular Responses
+
+How do we ask for a single film, for instance the second one we inserted?
+
+```http
+GET http://localhost:3000/film?id=eq.2
+```
+It returns
+```json
+[
+  {
+    "id": 2,
+    "title": "The Look of Silence",
+    "year": "2014-01-01",
+    "director": "Joshua Oppenheimer",
+    "rating": 8.3,
+    "language": "Indonesian"
+  }
+]
+```
+
+Like any query, it gives us a result *set*, in this case an array with one element. However you and I know that `id` is a primary key, it will never return more than one result. We might want it returned as a JSON object, not an array. To express this preference include the header `Prefer: plurality=singular`. It will respond with
+
+
+```json
+{
+  "id": 2,
+  "title": "The Look of Silence",
+  "year": "2014-01-01",
+  "director": "Joshua Oppenheimer",
+  "rating": 8.3,
+  "language": "Indonesian"
+}
+```
+
+<div class="admonition note">
+    <p class="admonition-title">Why this approach to singular responses?</p>
+
+    <p>
+    PostgREST knows which columns comprise a primary key for a
+    table, so why not automatically choose plurality=singular when
+    these column filters are present? The fact is it could come as a
+    shock to a client that by adding one more filter condition it can
+    change the entire response format.
+    </p>
+    <p>
+    Then why not expose another kind of route such as /film/2 to indicate
+    one particular film? Because this does not accommodate compound keys.
+    The convention complects a plurality preference with table key
+    assumptions. We should separate concerns.
+    </p>
+    <p>
+    It turns out you can still have routes like /film/2.  Use a
+    proxy such as Nginx. It can rewrite routes such as /films/2
+    into /films?id=eq.2 and add the Prefer header to make the results
+    singular.
+    </p>
+</div>
+
+### Conclusion
+
+This tutorial showed how to create a database with a basic schema, run PostgREST, and interact with the API. The next tutorial will show how to enable security for a multi-tenant blogging API.

docs/examples/users.md

@@ -0,0 +1,513 @@
+## User Management
+
+API clients authenticate with [JSON Web Tokens](http://jwt.io).
+PostgREST does not support any other authentication mechanism
+directly, but they can be built on top. In this demo we will build
+a username and password system on top of JWT using only plpgsql.
+
+Future examples such as the multi-tenant blogging platform will use
+the results from this example for their auth. We will build a system
+for users to sign up, log in, manage their accounts, and for admins
+to manage other people's accounts. We will also see how to trigger
+outside events like sending password reset emails.
+
+Before jumping into the code, a little more about how the tokens
+work. Every JWT contains cryptographically signed *claims*. PostgREST
+cares specifically about a claim called `role`. When a client includes
+a `role` claim PostgREST executes their request using that database
+role.
+
+How would a client include a role claim, or claims in general?
+Without knowing the server JWT secret a client cannot create a
+claim. The only place to get a JWT is from the PostgREST server or
+from another service sharing the secret and acting on its behalf.
+We'll use a stored procedure returning type `jwt_claims` which is
+a special type causing the server to encrypt and sign the return
+value.
+
+### Storing Users and Passwords
+
+We create a database schema especially for auth information. We'll
+also need the postgres extension
+[pgcrypto](http://www.postgresql.org/docs/current/static/pgcrypto.html).
+
+```sql
+create extension if not exists pgcrypto;
+
+-- We put things inside the basic_auth schema to hide
+-- them from public view. Certain public procs/views will
+-- refer to helpers and tables inside.
+create schema if not exists basic_auth;
+```
+
+Next a table to store the mapping from usernames and passwords to
+database roles. The code below includes triggers and functions to
+encrypt the password and ensure the role exists.
+
+```sql
+create table if not exists
+basic_auth.users (
+  email    text primary key check ( email ~* '^.+@.+\..+$' ),
+  pass     text not null check (length(pass) < 512),
+  role     name not null check (length(role) < 512),
+  verified boolean not null default false
+  -- If you like add more columns, or a json column
+);
+
+create or replace function
+basic_auth.check_role_exists() returns trigger
+  language plpgsql
+  as $$
+begin
+  if not exists (select 1 from pg_roles as r where r.rolname = new.role) then
+    raise foreign_key_violation using message =
+      'unknown database role: ' || new.role;
+    return null;
+  end if;
+  return new;
+end
+$$;
+
+drop trigger if exists ensure_user_role_exists on basic_auth.users;
+create constraint trigger ensure_user_role_exists
+  after insert or update on basic_auth.users
+  for each row
+  execute procedure basic_auth.check_role_exists();
+
+create or replace function
+basic_auth.encrypt_pass() returns trigger
+  language plpgsql
+  as $$
+begin
+  if tg_op = 'INSERT' or new.pass <> old.pass then
+    new.pass = crypt(new.pass, gen_salt('bf'));
+  end if;
+  return new;
+end
+$$;
+
+drop trigger if exists encrypt_pass on basic_auth.users;
+create trigger encrypt_pass
+  before insert or update on basic_auth.users
+  for each row
+  execute procedure basic_auth.encrypt_pass();
+```
+
+With the table in place we can make a helper to check passwords.
+It returns the database role for a user if the email and password
+are correct.
+
+```sql
+create or replace function
+basic_auth.user_role(email text, pass text) returns name
+  language plpgsql
+  as $$
+begin
+  return (
+  select role from basic_auth.users
+   where users.email = user_role.email
+     and users.pass = crypt(user_role.pass, users.pass)
+  );
+end;
+$$;
+```
+
+### Password Reset
+
+When a user requests a password reset or signs up we create a token
+they will use later to prove their identity. The tokens go in this
+table.
+
+```sql
+drop type if exists token_type_enum cascade;
+create type token_type_enum as enum ('validation', 'reset');
+
+create table if not exists
+basic_auth.tokens (
+  token       uuid primary key,
+  token_type  token_type_enum not null,
+  email       text not null references basic_auth.users (email)
+                on delete cascade on update cascade,
+  created_at  timestamptz not null default current_date
+);
+```
+
+In the main schema (as opposed to the `basic_auth` schema) we expose
+a password reset request function. HTTP clients will call it. The
+function takes the email address of the user.
+
+```sql
+create or replace function
+request_password_reset(email text) returns void
+  language plpgsql
+  as $$
+declare
+  tok uuid;
+begin
+  delete from basic_auth.tokens
+   where token_type = 'reset'
+     and tokens.email = request_password_reset.email;
+
+  select gen_random_uuid() into tok;
+  insert into basic_auth.tokens (token, token_type, email)
+         values (tok, 'reset', request_password_reset.email);
+  perform pg_notify('reset',
+    json_build_object(
+      'email', request_password_reset.email,
+      'token', tok,
+      'token_type', 'reset'
+    )::text
+  );
+end;
+$$;
+```
+
+This function does not send any emails. It sends a postgres
+[NOTIFY](http://www.postgresql.org/docs/current/static/sql-notify.html)
+command. External programs such as a mailer listen for this event
+and do the work. The most robust way to process these signals is
+by pushing them onto work queues. Here are two programs to do that:
+
+1. [aweber/pgsql-listen-exchange](https://github.com/aweber/pgsql-listen-exchange) for RabbitMQ
+2. [SpiderOak/skeeter](https://github.com/SpiderOak/skeeter) for ZeroMQ
+
+For experimentation you don't need that though. Here's a sample
+Node program that listens for the events and logs them to stdout.
+
+```js
+var PS = require('pg-pubsub');
+
+if(process.argv.length !== 3) {
+  console.log("USAGE: DB_URL");
+  process.exit(2);
+}
+var url  = process.argv[2],
+    ps   = new PS(url);
+
+// password reset request events
+ps.addChannel('reset', console.log);
+// email validation required event
+ps.addChannel('validate', console.log);
+
+// modify me to send emails
+```
+
+Once the user has a reset token they can use it as an argument to
+the password reset function, calling it through the PostgREST RPC
+interface.
+
+```sql
+create or replace function
+reset_password(email text, token uuid, pass text)
+  returns void
+  language plpgsql
+  as $$
+declare
+  tok uuid;
+begin
+  if exists(select 1 from basic_auth.tokens
+             where tokens.email = reset_password.email
+               and tokens.token = reset_password.token
+               and token_type = 'reset') then
+    update basic_auth.users set pass=reset_password.pass
+     where users.email = reset_password.email;
+
+    delete from basic_auth.tokens
+     where tokens.email = reset_password.email
+       and tokens.token = reset_password.token
+       and token_type = 'reset';
+  else
+    raise invalid_password using message =
+      'invalid user or token';
+  end if;
+  delete from basic_auth.tokens
+   where token_type = 'reset'
+     and tokens.email = reset_password.email;
+
+  select gen_random_uuid() into tok;
+  insert into basic_auth.tokens (token, token_type, email)
+         values (tok, 'reset', reset_password.email);
+  perform pg_notify('reset',
+    json_build_object(
+      'email', reset_password.email,
+      'token', tok
+    )::text
+  );
+end;
+$$;
+```
+
+### Email Validation
+
+This is similar to password resets. Once again we generate a token.
+It differs in that there is a trigger to send validations when a
+new login is added to the users table.
+
+```sql
+create or replace function
+basic_auth.send_validation() returns trigger
+  language plpgsql
+  as $$
+declare
+  tok uuid;
+begin
+  select gen_random_uuid() into tok;
+  insert into basic_auth.tokens (token, token_type, email)
+         values (tok, 'validation', new.email);
+  perform pg_notify('validate',
+    json_build_object(
+      'email', new.email,
+      'token', tok,
+      'token_type', 'validation'
+    )::text
+  );
+  return new;
+end
+$$;
+
+drop trigger if exists send_validation on basic_auth.users;
+create trigger send_validation
+  after insert on basic_auth.users
+  for each row
+  execute procedure basic_auth.send_validation();
+```
+
+### Editing Own User
+
+We'll construct a redacted view for users. It hides passwords and
+shows only those users whose roles the currently logged in user has
+db permission to access.
+
+```sql
+create or replace view users as
+select actual.role as role,
+       '***'::text as pass,
+       actual.email as email,
+       actual.verified as verified
+from basic_auth.users as actual,
+     (select rolname
+        from pg_authid
+       where pg_has_role(current_user, oid, 'member')
+     ) as member_of
+where actual.role = member_of.rolname;
+  -- can also add restriction that current_setting('postgrest.claims.email')
+  -- is equal to email so that user can only see themselves
+```
+
+Using this view clients can see themselves and any other users with
+the right db roles. This view does not yet support inserts or updates
+because not all the columns refer directly to underlying columns.
+Nor do we want it to be auto-updatable because it would allow an escalation
+of privileges. Someone could update their own row and change their
+role to become more powerful.
+
+We'll handle updates with a trigger, but we'll need a helper function
+to prevent an escalation of privileges.
+
+```sql
+create or replace function
+basic_auth.clearance_for_role(u name) returns void as
+$$
+declare
+  ok boolean;
+begin
+  select exists (
+    select rolname
+      from pg_authid
+     where pg_has_role(current_user, oid, 'member')
+       and rolname = u
+  ) into ok;
+  if not ok then
+    raise invalid_password using message =
+      'current user not member of role ' || u;
+  end if;
+end
+$$ LANGUAGE plpgsql;
+```
+
+With the above function we can now make a safe trigger to allow
+user updates.
+
+```sql
+create or replace function
+update_users() returns trigger
+language plpgsql
+AS $$
+begin
+  if tg_op = 'INSERT' then
+    perform basic_auth.clearance_for_role(new.role);
+
+    insert into basic_auth.users
+      (role, pass, email, verified)
+    values
+      (new.role, new.pass, new.email,
+      coalesce(new.verified, false));
+    return new;
+  elsif tg_op = 'UPDATE' then
+    -- no need to check clearance for old.role because
+    -- an ineligible row would not have been available to update (http 404)
+    perform basic_auth.clearance_for_role(new.role);
+
+    update basic_auth.users set
+      email  = new.email,
+      role   = new.role,
+      pass   = new.pass,
+      verified = coalesce(new.verified, old.verified, false)
+      where email = old.email;
+    return new;
+  elsif tg_op = 'DELETE' then
+    -- no need to check clearance for old.role (see previous case)
+
+    delete from basic_auth.users
+     where basic_auth.email = old.email;
+    return null;
+  end if;
+end
+$$;
+
+drop trigger if exists update_users on users;
+create trigger update_users
+  instead of insert or update or delete on
+    users for each row execute procedure update_users();
+```
+
+Finally add a public function people can use to sign up. You can
+hard code a default db role in it. It alters the underlying
+`basic_auth.users` so you can set whatever role you want without
+restriction.
+
+```sql
+create or replace function
+signup(email text, pass text) returns void
+as $$
+  insert into basic_auth.users (email, pass, role) values
+    (signup.email, signup.pass, 'hardcoded-role-here');
+$$ language sql;
+```
+
+### Generating JWT
+
+As mentioned at the start, clients authenticate with JWT. PostgREST
+has a special convention to allow your sql functions to return JWT.
+Any function that returns a type whose name ends in `jwt_claims` will
+have its return value encoded. For instance, let's make a login function
+which consults our users table.
+
+First create a return type:
+
+```sql
+drop type if exists basic_auth.jwt_claims cascade;
+create type basic_auth.jwt_claims AS (role text, email text);
+```
+
+And now the function:
+
+```sql
+create or replace function
+login(email text, pass text) returns basic_auth.jwt_claims
+  language plpgsql
+  as $$
+declare
+  _role name;
+  _verified boolean;
+  _email text;
+  result basic_auth.jwt_claims;
+begin
+  -- check email and password
+  select basic_auth.user_role(email, pass) into _role;
+  if _role is null then
+    raise invalid_password using message = 'invalid user or password';
+  end if;
+  -- check verified flag whether users
+  -- have validated their emails
+  _email := email;
+  select verified from basic_auth.users as u where u.email=_email limit 1 into _verified;
+  if not _verified then
+    raise invalid_authorization_specification using message = 'user is not verified';
+  end if;
+  select _role as role, login.email as email into result;
+  return result;
+end;
+$$;
+```
+
+An API request to login would look like this.
+
+```HTTP
+POST /rpc/login
+
+{ "email": "foo@bar.com", "pass": "foobar" }
+```
+
+Response
+```json
+{
+  "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImZvb0BiYXIuY29tIiwicm9sZSI6ImF1dGhvciJ9.KHwYdK9dAMAg-MGCQXuDiFuvbmW-y8FjfYIcMrETnto"
+}
+```
+
+Try decoding the token at [jwt.io](http://jwt.io/). (It was encoded
+with a secret of `secret` which is the default.) To use this token
+in a future API request include it in an `Authorization` request
+header.
+
+```HTTP
+Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImZvb0BiYXIuY29tIiwicm9sZSI6ImF1dGhvciJ9.KHwYdK9dAMAg-MGCQXuDiFuvbmW-y8FjfYIcMrETnto
+```
+
+### Same-Role Users
+
+You may not want a separate db role for every user. You can distinguish
+one user from another in SQL by examining the JWT claims which
+PostgREST makes available in the SQL variable `postgrest.claims`.
+Here's a function to get the email of the currently authenticated
+user.
+
+```sql
+-- Prevent current_setting('postgrest.claims.email') from raising
+-- an exception if the setting is not present. Default it to ''.
+ALTER DATABASE your_db_name SET postgrest.claims.email TO '';
+
+create or replace function
+basic_auth.current_email() returns text
+  language plpgsql
+  as $$
+begin
+  return current_setting('postgrest.claims.email');
+end;
+$$;
+```
+
+Remember that the `login` function set the claims `email` and `role`.
+You can modify `login` to set other claims as well if they are
+useful for your other SQL functions to reference later.
+
+### Permissions
+
+Basic table-level permissions. We'll add an the `authenticator`
+role which can't do anything itself other than switch into other
+roles as directed by JWT.
+
+```sql
+create role anon;
+create role authenticator noinherit;
+grant anon to authenticator;
+
+grant usage on schema public, basic_auth to anon;
+
+-- anon can create new logins
+grant insert on table basic_auth.users, basic_auth.tokens to anon;
+grant select on table pg_authid, basic_auth.users to anon;
+grant execute on function
+  login(text,text),
+  request_password_reset(text),
+  reset_password(text,uuid,text),
+  signup(text, text)
+  to anon;
+```
+
+### Conclusion
+
+This section explained the implementation details for building a
+password based authentication system in pure sql. The next example
+will put it to work in a multi-tenant blogging API.

docs/index.md

@@ -1,3 +1,18 @@
+<style>
+.videoWrapper {
+  position: relative;
+  padding-bottom: 56.25%; /* 16:9 */
+  padding-top: 25px;
+  height: 0;
+}
+.videoWrapper iframe {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+}
+</style>
 ![PostgREST logo](img/logo.png)
 
 ## Introduction
@@ -35,6 +50,14 @@ PostgREST has a focused scope. It works well with other tools like Nginx. This f
 
 As with any open source project, we all gain from features and fixes in the tool. It's more beneficial than improvements locked inextricably within custom codebases.
 
+### Intro Video
+
+Some things have changed since this video was created but the basics are the same. Learn the big vision behind automating APIs.
+
+<div class="videoWrapper">
+<iframe src="https://player.vimeo.com/video/115668217" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+</div>
+
 ### Myths
 
 #### You have to make tons of stored procs and triggers

docs/install/ecosystem.md

@@ -2,9 +2,11 @@
 
 ### Client-Side Libraries
 
+* [calebmer/postgrest-client](https://github.com/calebmer/postgrest-client) - Advanced JS client for the PostgREST API
 * [mithril.postgrest](https://github.com/catarse/mithril.postgrest) - Mithril plugin to create and authenticate requests
 * [lewisjared/postgrest-request](https://github.com/lewisjared/postgrest-request) - node interface to postgrest instances
 * [JarvusInnovations/jarvus-postgrest-apikit](https://github.com/JarvusInnovations/jarvus-postgrest-apikit) - Sencha framework package for binding models/stores/proxies to PostgREST tables
+* [davidthewatson/postgrest_python_requests_client](https://github.com/davidthewatson/postgrest_python_requests_client) - python client featuring JWT auth and pagination of result sets
 
 ### Extensions
 
@@ -12,6 +14,7 @@
 
 ### Example Apps
 
+* [ruslantalpa/blogdemo](https://github.com/ruslantalpa/blogdemo) - blog api demo in a vagrant image
 * [timwis/ext-postgrest-crud](https://github.com/timwis/ext-postgrest-crud) - browser-based spreadsheet
 * [srid/chronicle](https://github.com/srid/chronicle#deploying-to-heroku) - tracking a tree of personal memories
 * [begriffs/postgrest-example](https://github.com/begriffs/postgrest-example) - how to configure a db for use as an API

docs/install/server.md

@@ -2,7 +2,10 @@
 
 ### Installing from Pre-Built Release
 
-The [release page](https://github.com/begriffs/postgrest/releases/latest) has precompiled binaries for Mac OS X and 64-bit Ubuntu. Next extract the tarball and run the binary inside with no arguments to see usage instructions:
+The [release page](https://github.com/begriffs/postgrest/releases/latest)
+has precompiled binaries for Mac OS X, Windows, and several Linux
+distros.  Extract the tarball and run the binary inside with no
+arguments to see usage instructions:
 
 ```sh
 # Untar the release (available at https://github.com/begriffs/postgrest/releases/latest)
@@ -18,45 +21,133 @@ $ ./postgrest
 <div class="admonition warning">
     <p class="admonition-title">Invitation to Contribute</p>
 
-    <p>I currently build the binaries manually for each version. We need to set up an automated build matrix for various architectures. It should support 32- and 64-bit versions of
+    <p>I currently build the binaries manually for each architecture.
+    It would be nice to set up an automated build matrix for various
+    architectures. It should support Mac, Windows and 32- and 64-bit
+    versions of
 
-    <ul><li>Scientific Linux 6</li><li>CentOS</li><li>RHEL 6</li></ul>
-
-    Also it would be good to create a package for apt.</p>
+    <ul><li>Scientific Linux 6</li><li>CentOS</li><li>RHEL 6</li></ul></p>
 </div>
 
-We'll learn the meaning of the command line flags later, but here is a minimal example of running the app. It does all operations as user `postgres`, including for unauthenticated requests.
-
-```sh
-$ ./postgrest -d dbname -U postgres -a postgres --v1schema public
-```
-
 ### Building from Source
 
-When a prebuilt binary does not exist for your system you can build the project from source. You'll also need to do this if you want to help with development. [Stack](https://github.com/commercialhaskell/stack) makes it easy. It will install any necessary Haskell dependencies on your system.
+When a prebuilt binary does not exist for your system you can build
+the project from source. You'll also need to do this if you want
+to help with development.
+[Stack](https://github.com/commercialhaskell/stack) makes it easy.
+It will install any necessary Haskell dependencies on your system.
 
-* [Install Stack](https://github.com/commercialhaskell/stack#how-to-install) for your platform
+* [Install Stack](http://docs.haskellstack.org/en/stable/README.html#how-to-install) for your platform
 ```bash
 #ubuntu example
+#See the link above for other operating systems
+
 wget -q -O- https://s3.amazonaws.com/download.fpcomplete.com/ubuntu/fpco.key | sudo apt-key add -
 echo 'deb http://download.fpcomplete.com/ubuntu/trusty stable main'|sudo tee /etc/apt/sources.list.d/fpco.list
 sudo apt-get update && sudo apt-get install stack -y
 ```
+* Install libpq-dev
+```
+sudo apt-get install -y libpq-dev
+```
 * Build & install in one step
 
 ```bash
 git clone https://github.com/begriffs/postgrest.git
 cd postgrest
-sudo stack install --install-ghc --local-bin-path /usr/local/bin
+stack build --install-ghc
+sudo stack install --allow-different-user --local-bin-path /usr/local/bin
 ```
 
 * Run the server
 
+If you want to run the test suite, stack can do that too: `stack test`.
+
+### Running the Server
+
 ```bash
-postgrest dbconnectionstring arg1 arg2
+postgrest postgres://user:pass@host:port/db -a anon_user [other flags]
 ```
 
-If you want to run the test suite, stack can do that too: `stack test`.
+The user in the connection string is the "authenticator role," i.e.
+a role which is used temporarily to switch into other roles depending
+on the authentication request JWT. For simple API's you can use the
+same role for authenticator and anonymous.
+
+The complete list of options:
+
+<dl>
+<dt>-p, --port</dt>
+<dd>The port on which the server will listen for HTTP requests.
+    Defaults to 3000.</dd>
+
+<dt>-a, --anonymous (required)</dt>
+<dd>The database role used to execute commands for those requests
+    which provide no JWT authorization.</dd>
+
+<dt>-s, --schema</dt>
+<dd>The db schema which you want to expose as an API. For historical
+    reasons it defaults to <code>1</code>, but you're more likely
+    to want to choose a value of <code>public</code>.</dd>
+
+<dt>-j, --jwt-secret</dt>
+<dd>The secret passphrase used to encrypt JWT tokens. Defaults to
+    <code>secret</code> but do not use the default in production!
+    Load-balanced PostgREST servers should share the same secret.</dd>
+
+<dt>-o, --pool</dt>
+<dd>Max connections to use in db pool. Defaults to to 10, but you
+    should find an optimal value for your db by running the SQL
+    command <code>show max_connections;</code></dd>
+
+<dt>-m, --max-rows</dt>
+<dd>Max number of rows to return in a read request. The default is
+    no limit.</dd>
+</dl>
+
+<div class="admonition note">
+    <p class="admonition-title">Hiding Password from Process List</p>
+
+    <p>Passing the database password and JWT secret as naked
+    parameters might not be a good idea because the parameters are
+    visible in a <code>ps</code> listing. One solution is to set
+    environment variables such as PASS and use <code>$PASS</code>
+    in the connection string.  Another is to use a user-specific
+    <a
+    href="http://www.postgresql.org/docs/current/static/libpq-pgpass.html">.pgpass</a>
+    file.</p>
+</div>
+
+When running `postgrest` on the same machine as PostgreSQL, it is also
+possible to connect to the database using the [Unix socket]
+(https://en.wikipedia.org/wiki/Unix_domain_socket) and the
+[Peer Authentication method]
+(http://www.postgresql.org/docs/current/static/auth-methods.html#AUTH-PEER)
+as an alternative to TCP/IP communication and authentication with a password.
+
+The Peer Authentication grants access to the database to any Unix user
+who connects as a user of the same name in the database.
+Since the empty host resolves to the Unix socket]
+(http://www.postgresql.org/docs/current/static/libpq-connect.html#AEN42494)
+and the password can be omitted in this case,
+the command line is reduced to:
+
+```sh
+sudo -u user postgrest postgres://user@/db [flags]
+```
+
+where the `sudo -u user` command runs the following command as given `user`.
+
+If you create a Unix user `postgrest` and a database user `postgrest`
+for example, the command becomes:
+
+```sh
+sudo -u postgrest postgrest postgres://postgrest@/db [flags]
+```
+
+The first `postgrest` is the Unix user name, the second `postgrest`
+is the name of the executable, the third `postgrest` is the name
+of the database user.
 
 ### Install via Homebrew (Mac OS X)
 
@@ -73,7 +164,7 @@ brew doctor
 brew install postgrest
 ```
 
-This will automatically install PostgreSQL as a dependency (see the [Installing PostgreSQL](#installing-postgresql) section for setup instructions). The process tends to take around 15 minutes to install the package and its dependencies.
+This will automatically install PostgreSQL as a dependency (see the [Installing PostgreSQL](#installing-postgresql) section for setup instructions). The process tends to take up to 15 minutes to install the package and its dependencies.
 
 After installation completes, the tool is added to your $PATH and can be used from anywhere with:
 
@@ -83,7 +174,8 @@ postgrest --help
 
 ### Installing PostgreSQL
 
-To use PostgREST you will need an underlying database. You can use something like Amazon [RDS](https://aws.amazon.com/rds/) but installing your own locally is cheaper and more convenient for development.
+To use PostgREST you will need an underlying database (PostgreSQL version 9.3 or greater is required). You can use something like Amazon [RDS](https://aws.amazon.com/rds/) but installing your own locally is cheaper and more convenient for development.
 
 * [Instructions for OS X](http://exponential.io/blog/2015/02/21/install-postgresql-on-mac-os-x-via-brew/)
 * [Instructions for Ubuntu 14.04](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-14-04)
+* [Installer for Windows](http://www.enterprisedb.com/products-services-training/pgdownload#windows)

main/Main.hs

@@ -0,0 +1,87 @@
+{-# LANGUAGE CPP #-}
+
+module Main where
+
+
+import           PostgREST.App
+import           PostgREST.Config                     (AppConfig (..),
+                                                       minimumPgVersion,
+                                                       prettyVersion,
+                                                       readOptions)
+import           PostgREST.DbStructure
+
+import           Control.Monad
+import           Data.Monoid                          ((<>))
+import           Data.String.Conversions              (cs)
+import qualified Hasql.Query                          as H
+import qualified Hasql.Session                        as H
+import qualified Hasql.Decoders                       as HD
+import qualified Hasql.Encoders                       as HE
+import qualified Hasql.Pool                           as P
+import           Network.Wai.Handler.Warp
+import           System.IO                            (BufferMode (..),
+                                                       hSetBuffering, stderr,
+                                                       stdin, stdout)
+import           Web.JWT                              (secret)
+import           Data.IORef
+#ifndef mingw32_HOST_OS
+import           Control.Monad.IO.Class               (liftIO)
+import           System.Posix.Signals
+import           Control.Concurrent                   (myThreadId)
+import           Control.Exception.Base               (throwTo, AsyncException(..))
+#endif
+
+isServerVersionSupported :: H.Session Bool
+isServerVersionSupported = do
+  ver <- H.query () pgVersion
+  return $ read (cs ver) >= minimumPgVersion
+ where
+  pgVersion =
+    H.statement "SHOW server_version_num"
+      HE.unit (HD.singleRow $ HD.value HD.text) True
+
+main :: IO ()
+main = do
+  hSetBuffering stdout LineBuffering
+  hSetBuffering stdin  LineBuffering
+  hSetBuffering stderr NoBuffering
+
+  conf <- readOptions
+  let port = configPort conf
+      pgSettings = cs (configDatabase conf)
+      appSettings = setPort port
+                  . setServerName (cs $ "postgrest/" <> prettyVersion)
+                  $ defaultSettings
+
+  unless (secret "secret" /= configJwtSecret conf) $
+    putStrLn "WARNING, running in insecure mode, JWT secret is the default value"
+  Prelude.putStrLn $ "Listening on port " ++
+    (show $ configPort conf :: String)
+
+  pool <- P.acquire (configPool conf, 10, pgSettings)
+
+  result <- P.use pool $ do
+    supported <- isServerVersionSupported
+    unless supported $ error (
+      "Cannot run in this PostgreSQL version, PostgREST needs at least "
+      <> show minimumPgVersion)
+    getDbStructure (cs $ configSchema conf)
+
+  refDbStructure <- newIORef $ either (error.show) id result
+
+#ifndef mingw32_HOST_OS
+  tid <- myThreadId
+  forM_ [sigINT, sigTERM] $ \sig ->
+    void $ installHandler sig (Catch $ do
+        P.release pool
+        throwTo tid UserInterrupt
+      ) Nothing
+
+  void $ installHandler sigHUP (
+      Catch . void . P.use pool $ do
+        s <- getDbStructure (cs $ configSchema conf)
+        liftIO $ atomicWriteIORef refDbStructure s
+   ) Nothing
+#endif
+
+  runSettings appSettings $ postgrest conf refDbStructure pool

mkdocs.yml

@@ -22,3 +22,7 @@ pages:
   - Performance: admin/performance.md
 - Examples:
   - Getting Started: examples/start.md
+  - User Management: examples/users.md
+  - Multi-Tenant Blog: examples/blog.md
+  - External Authentication: examples/external_auth.md
+  - Python Client:  examples/python-requests-jwt.md

postgrest.cabal

@@ -2,7 +2,7 @@ name:                  postgrest
 description:           Reads the schema of a PostgreSQL database and creates RESTful routes
                        for the tables and views, supporting all HTTP verbs that security
                        permits.
-version:               0.3.0.0
+version:               0.3.2.0
 synopsis:              REST API for any Postgres database
 license:               MIT
 license-file:          LICENSE
@@ -22,109 +22,93 @@ Flag CI
   Default:     False
 
 executable postgrest
-  if flag(ci)
-    ghc-options:       -Wall -W -Werror
-  else
-    ghc-options:       -Wall -W -O2
-
-  main-is:             PostgREST/Main.hs
+  main-is:             Main.hs
   default-extensions:  OverloadedStrings, ScopedTypeVariables, QuasiQuotes
+  ghc-options:
+    -threaded
+    -rtsopts
+    "-with-rtsopts=-N -I2"
   default-language:    Haskell2010
-  build-depends:       base >= 4.8 && < 5
-                     , postgrest
-                     , hasql >= 0.7.3 && < 0.8
-                     , hasql-backend >= 0.4.1 && < 0.5
-                     , hasql-postgres >= 0.10.4 && < 0.11
-                     , warp >= 3.0.2, wai >= 3.0.1
-                     , wai-extra, wai-cors
-                     , wai-middleware-static >= 0.6.0
-                     , HTTP, convertible, http-types
-                     , case-insensitive
-                     , scientific, time
-                     , aeson >= 0.8, network >= 2.6
-                     , aeson-pretty >= 0.7 && < 0.8
-                     , bytestring, text, split, string-conversions
-                     , stringsearch
-                     , containers, unordered-containers
-                     , optparse-applicative >= 0.11 && < 0.13
-                     , regex-base, regex-tdfa
-                     , Ranged-sets
-                     , transformers, MissingH
-                     , bcrypt >= 0.0.6, base64-string
-                     , network-uri >= 2.6
-                     , resource-pool
-                     , blaze-builder
-                     , vector
-                     , mtl
-                     , cassava
-                     , jwt
-                     , parsec
-                     , errors
-                     , bifunctors
-  hs-source-dirs:      src
-  other-modules:       Paths_postgrest
-                     , PostgREST.App
-                     , PostgREST.Auth
-                     , PostgREST.Config
-                     , PostgREST.Error
-                     , PostgREST.Middleware
-                     , PostgREST.Parsers
-                     , PostgREST.DbStructure
-                     , PostgREST.QueryBuilder
-                     , PostgREST.RangeQuery
-                     , PostgREST.ApiRequest
-                     , PostgREST.Types
-
-library
-  if flag(ci)
-    ghc-options:       -Wall -W -Werror
-  else
-    ghc-options:       -Wall -W -O2
-
-  default-language:    Haskell2010
-  default-extensions:  OverloadedStrings, ScopedTypeVariables, QuasiQuotes
-  build-depends:       HTTP
-                     , MissingH
-                     , Ranged-sets
-                     , aeson
-                     , base >=4.6 && <5
-                     , base64-string
-                     , bcrypt
-                     , bifunctors
-                     , blaze-builder
+  build-depends:       aeson (>= 0.8 && < 0.10) || (>= 0.11 && < 0.12)
+                     , base >= 4.8 && < 6
                      , bytestring
+                     , bytestring-tree-builder == 0.2.7
                      , case-insensitive
                      , cassava
                      , containers
-                     , convertible
+                     , contravariant
                      , errors
-                     , hasql
-                     , hasql-backend
-                     , hasql-postgres
+                     , hasql == 0.19.12
+                     , hasql-pool == 0.4.1
+                     , hasql-transaction == 0.4.5
                      , http-types
+                     , interpolatedstring-perl6
                      , jwt
+                     , microlens >= 0.4.2 && < 0.5
+                     , microlens-aeson >= 2.1.1 && < 2.2
                      , mtl
-                     , network
-                     , network-uri
-                     , optparse-applicative
+                     , optparse-applicative >= 0.11 && < 0.13
                      , parsec
-                     , regex-base
+                     , postgresql-binary == 0.9.0.1
+                     , postgrest
                      , regex-tdfa
-                     , resource-pool
+                     , safe >= 0.3 && < 0.4
                      , scientific
-                     , split
                      , string-conversions
-                     , stringsearch
                      , text
                      , time
                      , transformers
                      , unordered-containers
                      , vector
-                     , wai
+                     , wai >= 3.0.1
                      , wai-cors
                      , wai-extra
-                     , wai-middleware-static
-                     , warp
+                     , wai-middleware-static >= 0.6.0
+                     , warp >= 3.1.0
+                     , HTTP
+                     , Ranged-sets
+  if !os(windows)
+    build-depends:     unix >= 2.7 && < 3
+
+  hs-source-dirs:      main
+
+library
+  default-language:    Haskell2010
+  default-extensions:  OverloadedStrings, ScopedTypeVariables, QuasiQuotes
+  build-depends:       aeson
+                     , base
+                     , bytestring
+                     , case-insensitive
+                     , cassava
+                     , containers
+                     , contravariant
+                     , errors
+                     , hasql
+                     , hasql-transaction
+                     , hasql-pool
+                     , http-types
+                     , interpolatedstring-perl6
+                     , jwt
+                     , microlens
+                     , microlens-aeson
+                     , mtl
+                     , optparse-applicative
+                     , parsec
+                     , regex-tdfa
+                     , safe
+                     , scientific
+                     , string-conversions
+                     , text
+                     , time
+                     , unordered-containers
+                     , vector
+                     , HTTP
+                     , Ranged-sets
+                     , wai >= 3.0.1
+                     , wai-cors
+                     , wai-extra
+                     , wai-middleware-static >= 0.6.0
+                     , warp >= 3.1.0
 
   Other-Modules:       Paths_postgrest
   Exposed-Modules:     PostgREST.App
@@ -144,64 +128,63 @@ Test-Suite spec
   Type:                exitcode-stdio-1.0
   Default-Language:    Haskell2010
   default-extensions:  OverloadedStrings, ScopedTypeVariables, QuasiQuotes
-  Hs-Source-Dirs:      test, src
-  if flag(ci)
-    ghc-options:       -Wall -W -Werror
-  else
-    ghc-options:       -Wall -W -O2
+  ghc-options:         -threaded -rtsopts -with-rtsopts=-N
+  Hs-Source-Dirs:      test
   Main-Is:             Main.hs
   Other-Modules:       Feature.AuthSpec
+                     , Feature.ConcurrentSpec
                      , Feature.CorsSpec
                      , Feature.DeleteSpec
                      , Feature.InsertSpec
                      , Feature.QuerySpec
+                     , Feature.QueryLimitedSpec
                      , Feature.RangeSpec
                      , Feature.StructureSpec
-                     , Paths_postgrest
-                     , PostgREST.App
-                     , PostgREST.Auth
-                     , PostgREST.Config
-                     , PostgREST.Error
-                     , PostgREST.Middleware
-                     , PostgREST.Parsers
-                     , PostgREST.DbStructure
-                     , PostgREST.QueryBuilder
-                     , PostgREST.RangeQuery
-                     , PostgREST.ApiRequest
-                     , PostgREST.Types
-                     , Spec
+                     , Feature.UnicodeSpec
                      , SpecHelper
                      , TestTypes
-  Build-Depends:       base, hspec == 2.2.*, QuickCheck
-                     , hspec-wai, hspec-wai-json
-                     , hasql, hasql-backend
-                     , hasql-postgres
-                     , warp, wai
-                     , packdeps, hlint
-                     , HTTP, convertible
+  Build-Depends:       aeson
+                     , async
+                     , base
+                     , base64-string
+                     , bytestring
                      , case-insensitive
-                     , wai-extra, wai-cors, containers
-                     , wai-middleware-static
-                     , http-types, scientific, time
-                     , bytestring, aeson, network
-                     , text, optparse-applicative
-                     , stringsearch
-                     , unordered-containers
-                     , regex-base
-                     , string-conversions
-                     , http-media, regex-tdfa
-                     , Ranged-sets
-                     , transformers, MissingH, split
-                     , bcrypt, base64-string
-                     , network-uri
-                     , resource-pool
-                     , blaze-builder
-                     , vector
-                     , mtl
                      , cassava
-                     , process
-                     , heredoc
-                     , jwt
-                     , parsec
+                     , containers
+                     , contravariant
                      , errors
-                     , bifunctors
+                     , hasql
+                     , hasql-pool
+                     , hasql-transaction
+                     , heredoc
+                     , hspec
+                     , hspec-wai
+                     , hspec-wai-json
+                     , http-types
+                     , interpolatedstring-perl6
+                     , jwt
+                     , microlens
+                     , microlens-aeson
+                     , monad-control
+                     , mtl
+                     , optparse-applicative
+                     , parsec
+                     , postgrest
+                     , process
+                     , regex-tdfa
+                     , safe
+                     , scientific
+                     , string-conversions
+                     , text
+                     , time
+                     , transformers
+                     , transformers-base
+                     , unordered-containers
+                     , vector
+                     , wai
+                     , wai-cors
+                     , wai-extra
+                     , wai-middleware-static
+                     , warp
+                     , HTTP
+                     , Ranged-sets

schema-templates/blog.sql

@@ -5,11 +5,12 @@ begin;
 
 -- comment out the role creation statements if
 -- you want to run this script more than once
-create role anon noinherit;
+create role anon;
 create role author;
+create role authenticator noinherit;
+grant anon, author to authenticator;
 
 create extension if not exists pgcrypto;
-create extension if not exists "uuid-ossp";
 
 -- We put things inside the basic_auth schema to hide
 -- them from public view. Certain public procs/views will
@@ -95,7 +96,7 @@ basic_auth.send_validation() returns trigger
 declare
   tok uuid;
 begin
-  select uuid_generate_v4() into tok;
+  select gen_random_uuid() into tok;
   insert into basic_auth.tokens (token, token_type, email)
          values (tok, 'validation', new.email);
   perform pg_notify('validate',
@@ -173,7 +174,7 @@ begin
    where token_type = 'reset'
      and tokens.email = request_password_reset.email;
 
-  select uuid_generate_v4() into tok;
+  select gen_random_uuid() into tok;
   insert into basic_auth.tokens (token, token_type, email)
          values (tok, 'reset', request_password_reset.email);
   perform pg_notify('reset',
@@ -213,7 +214,7 @@ begin
    where token_type = 'reset'
      and tokens.email = reset_password.email;
 
-  select uuid_generate_v4() into tok;
+  select gen_random_uuid() into tok;
   insert into basic_auth.tokens (token, token_type, email)
          values (tok, 'reset', reset_password.email);
   perform pg_notify('reset',
@@ -323,7 +324,8 @@ posts (
   title      text not null,
   body       text not null,
   author     text not null references basic_auth.users (email)
-               on delete restrict on update cascade,
+               on delete restrict on update cascade
+               default basic_auth.current_email(),
   created_at timestamptz not null default current_date
 );
 
@@ -332,7 +334,8 @@ comments (
   id         bigserial primary key,
   body       text not null,
   author     text not null references basic_auth.users (email)
-               on delete restrict on update cascade,
+               on delete restrict on update cascade
+               default basic_auth.current_email(),
   post       bigint not null references posts (id)
                on delete cascade on update cascade,
   created_at timestamptz not null default current_date
@@ -350,7 +353,6 @@ grant execute on function
   signup(text, text)
   to anon;
 
-grant author to anon;
 grant select, insert, update, delete
   on basic_auth.tokens, basic_auth.users to anon, author;
 grant select, insert, update, delete

src/PostgREST/ApiRequest.hs

@@ -1,25 +1,32 @@
 module PostgREST.ApiRequest where
 
-import qualified Data.Aeson              as JSON
-import qualified Data.ByteString         as BS
-import qualified Data.ByteString.Lazy    as BL
-import qualified Data.Csv                as CSV
-import           Data.List               (find)
-import qualified Data.HashMap.Strict     as M
-import qualified Data.Set                as S
-import           Data.Maybe              (fromMaybe, isJust, isNothing,
-                                          listToMaybe, fromJust)
-import           Control.Monad           (join)
-import           Data.Monoid             ((<>))
-import           Data.String.Conversions (cs)
-import qualified Data.Text               as T
-import qualified Data.Vector             as V
-import           Network.Wai             (Request (..))
-import           Network.Wai.Parse       (parseHttpAccept)
-import           PostgREST.RangeQuery    (NonnegRange, rangeRequested)
-import           PostgREST.Types         (QualifiedIdentifier (..),
-                                          Schema, Payload(..),
-                                          UniformObjects(..))
+import qualified Data.Aeson                as JSON
+import qualified Data.ByteString           as BS
+import qualified Data.ByteString.Lazy      as BL
+import qualified Data.Csv                  as CSV
+import           Data.List                 (find, sortBy)
+import qualified Data.HashMap.Strict       as M
+import qualified Data.Set                  as S
+import           Data.Maybe                (fromMaybe, isJust, isNothing,
+                                            listToMaybe, fromJust)
+import           Control.Arrow             ((***))
+import           Control.Monad             (join)
+import           Data.Monoid               ((<>))
+import           Data.Ord                  (comparing)
+import           Data.String.Conversions   (cs)
+import qualified Data.Text                 as T
+import           Text.Read                 (readMaybe)
+import qualified Data.Vector               as V
+import           Network.HTTP.Base         (urlEncodeVars)
+import           Network.HTTP.Types.Header (hAuthorization)
+import           Network.HTTP.Types.URI    (parseSimpleQuery)
+import           Network.Wai               (Request (..))
+import           Network.Wai.Parse         (parseHttpAccept)
+import           PostgREST.RangeQuery      (NonnegRange, rangeRequested, restrictRange, rangeGeq, allRange)
+import           PostgREST.Types           (QualifiedIdentifier (..),
+                                            Schema, Payload(..),
+                                            UniformObjects(..))
+import           Data.Ranged.Ranges        (singletonRange, rangeIntersection)
 
 type RequestBody = BL.ByteString
 
@@ -27,17 +34,21 @@ type RequestBody = BL.ByteString
 data Action = ActionCreate | ActionRead
             | ActionUpdate | ActionDelete
             | ActionInfo   | ActionInvoke
-            | ActionUnknown BS.ByteString deriving Eq
+            | ActionInappropriate
+            deriving Eq
 -- | The target db object of a user action
 data Target = TargetIdent QualifiedIdentifier
+            | TargetProc  QualifiedIdentifier
             | TargetRoot
             | TargetUnknown [T.Text]
+-- | How to return the inserted data
+data PreferRepresentation = Full | HeadersOnly | None deriving Eq
 -- | Enumeration of currently supported content types for
 -- route responses and upload payloads
 data ContentType = ApplicationJSON | TextCSV deriving Eq
 instance Show ContentType where
-  show ApplicationJSON = "application/json"
-  show TextCSV         = "text/csv"
+  show ApplicationJSON = "application/json; charset=utf-8"
+  show TextCSV         = "text/csv; charset=utf-8"
 
 {-|
   Describes what the user wants to do. This data type is a
@@ -47,18 +58,18 @@ instance Show ContentType where
   if it is an action we are able to perform.
 -}
 data ApiRequest = ApiRequest {
-  -- | Set to Nothing for unknown HTTP verbs
+  -- | Similar but not identical to HTTP verb, e.g. Create/Invoke both POST
     iAction :: Action
-  -- | Set to Nothing for malformed range
-  , iRange  :: Maybe NonnegRange
-  -- | Set to Nothing for strangely nested urls
+  -- | Requested range of rows within response
+  , iRange  :: M.HashMap String NonnegRange
+  -- | The target, be it calling a proc or accessing a table
   , iTarget :: Target
   -- | The content type the client most desires (or JSON if undecided)
   , iAccepts :: Either BS.ByteString ContentType
   -- | Data sent by client and used for mutation actions
   , iPayload :: Maybe Payload
   -- | If client wants created items echoed back
-  , iPreferRepresentation :: Bool
+  , iPreferRepresentation :: PreferRepresentation
   -- | If client wants first row as raw object
   , iPreferSingular :: Bool
   -- | Whether the client wants a result count (slower)
@@ -67,27 +78,36 @@ data ApiRequest = ApiRequest {
   , iFilters :: [(String, String)]
   -- | &select parameter used to shape the response
   , iSelect :: String
-  -- | &order parameter
-  , iOrder :: Maybe String
+  -- | &order parameters for each level
+  , iOrder :: [(String,String)]
+  -- | Alphabetized (canonical) request query string for response URLs
+  , iCanonicalQS :: String
+  -- | JSON Web Token
+  , iJWT :: T.Text
   }
 
 -- | Examines HTTP request and translates it into user intent.
 userApiRequest :: Schema -> Request -> RequestBody -> ApiRequest
 userApiRequest schema req reqBody =
-  let action = case method of
-                 "GET"     -> ActionRead
-                 "POST"    -> if isTargetingProc
-                                then ActionInvoke
-                                else ActionCreate
-                 "PATCH"   -> ActionUpdate
-                 "DELETE"  -> ActionDelete
-                 "OPTIONS" -> ActionInfo
-                 other     -> ActionUnknown other
+  let action =
+        if isTargetingProc
+          then
+            if method == "POST"
+               then ActionInvoke
+               else ActionInappropriate
+          else
+            case method of
+               "GET"     -> ActionRead
+               "POST"    -> ActionCreate
+               "PATCH"   -> ActionUpdate
+               "DELETE"  -> ActionDelete
+               "OPTIONS" -> ActionInfo
+               _         -> ActionInappropriate
       target = case path of
                  []            -> TargetRoot
                  [table]       -> TargetIdent
                                   $ QualifiedIdentifier schema table
-                 ["rpc", proc] -> TargetIdent
+                 ["rpc", proc] -> TargetProc
                                   $ QualifiedIdentifier schema proc
                  other         -> TargetUnknown other
       payload = case pickContentType (lookupHeader "content-type") of
@@ -103,6 +123,13 @@ userApiRequest schema req reqBody =
               Nothing -> PayloadParseError "All lines must have same number of fields"
               Just json -> PayloadJSON json)
             (CSV.decodeByName reqBody)
+        -- This is a Left value because form-urlencoded is not a content
+        -- type which we ever use for responses, only something we handle
+        -- just this once for requests
+        Left "application/x-www-form-urlencoded" ->
+          PayloadJSON . UniformObjects . V.singleton . M.fromList
+                      . map (cs *** JSON.String . cs) . parseSimpleQuery
+                      $ cs reqBody
         Left accept ->
           PayloadParseError $
             "Content-type not acceptable: " <> accept
@@ -114,18 +141,23 @@ userApiRequest schema req reqBody =
 
   ApiRequest {
     iAction = action
-  , iRange  = if singular then Nothing else rangeRequested hdrs
   , iTarget = target
+  , iRange = M.insert "limit" (rangeIntersection headerRange urlRange) $
+      M.fromList [ (cs k, restrictRange (readMaybe =<< v) allRange) | (k,v) <- qParams, isJust v, endingIn ["limit"] k ]
   , iAccepts = pickContentType $ lookupHeader "accept"
   , iPayload = relevantPayload
-  , iPreferRepresentation = hasPrefer "return=representation"
+  , iPreferRepresentation = representation
   , iPreferSingular = singular
-  , iPreferCount = not $ hasPrefer "count=none"
-  , iFilters = [ (k, fromJust v) | (k,v) <- qParams, k `notElem` ["select", "order"], isJust v ]
-  , iSelect = if method == "DELETE"
-              then "*"
-              else fromMaybe "*" $ fromMaybe (Just "*") $ lookup "select" qParams
-  , iOrder = join $ lookup "order" qParams
+  , iPreferCount = not $ singular || hasPrefer "count=none"
+  , iFilters = [ (cs k, fromJust v) | (k,v) <- qParams, isJust v, k /= "select", k /= "offset", not (endingIn ["order", "limit"] k) ]
+  , iSelect = fromMaybe "*" $ fromMaybe (Just "*") $ lookup "select" qParams
+  , iOrder = [(cs k, fromJust v) | (k,v) <- qParams, isJust v, endingIn ["order"] k ]
+  , iCanonicalQS = urlEncodeVars
+     . sortBy (comparing fst)
+     . map (join (***) cs)
+     . parseSimpleQuery
+     $ rawQueryString req
+  , iJWT = tokenStr
   }
 
  where
@@ -135,10 +167,30 @@ userApiRequest schema req reqBody =
   hdrs            = requestHeaders req
   qParams         = [(cs k, cs <$> v)|(k,v) <- queryString req]
   lookupHeader    = flip lookup hdrs
-  hasPrefer val   = any (\(h,v) -> h == "Prefer" && v == val) hdrs
+  hasPrefer :: T.Text -> Bool
+  hasPrefer val   = any (\(h,v) -> h == "Prefer" && val `elem` split v) hdrs
+    where
+        split :: BS.ByteString -> [T.Text]
+        split = map T.strip . T.split (==';') . cs
   singular        = hasPrefer "plurality=singular"
+  representation
+    | hasPrefer "return=representation" = Full
+    | hasPrefer "return=minimal" = None
+    | otherwise = HeadersOnly
+  auth = fromMaybe "" $ lookupHeader hAuthorization
+  tokenStr = case T.split (== ' ') (cs auth) of
+    ("Bearer" : t : _) -> t
+    _                  -> ""
+  endingIn:: [T.Text] -> T.Text -> Bool
+  endingIn xx key = lastWord `elem` xx
+    where lastWord = last $ T.split (=='.') key
 
-
+  headerRange = if singular then singletonRange 0 else rangeRequested hdrs
+  urlOffsetRange = rangeGeq . fromMaybe (0::Integer) $
+    readMaybe =<< join (lookup "offset" qParams)
+  urlRange = restrictRange
+    (readMaybe =<< join (lookup "limit" qParams))
+    urlOffsetRange
 
 -- PRIVATE ---------------------------------------------------------------
 

src/PostgREST/App.hs

@@ -3,65 +3,94 @@
 {-# LANGUAGE TupleSections #-}
 --module PostgREST.App where
 module PostgREST.App (
-  app
+  postgrest
 ) where
 
 import           Control.Applicative
-import           Control.Arrow             ((***))
-import           Control.Monad             (join)
 import           Data.Bifunctor            (first)
-import qualified Data.ByteString.Lazy      as BL
-import           Data.Functor.Identity
-import           Data.List                 (find, sortBy, delete)
+import qualified Data.ByteString.Char8   as BS
+import           Data.IORef                (IORef, readIORef)
+import           Data.List                 (find, delete)
 import           Data.Maybe                (fromMaybe, fromJust, mapMaybe)
-import           Data.Ord                  (comparing)
 import           Data.Ranged.Ranges        (emptyRange)
 import           Data.String.Conversions   (cs)
 import           Data.Text                 (Text, replace, strip)
 import           Data.Tree
 
+import qualified Hasql.Pool                as P
+import qualified Hasql.Transaction         as HT
+
 import           Text.Parsec.Error
 import           Text.ParserCombinators.Parsec (parse)
 
-import           Network.HTTP.Base         (urlEncodeVars)
 import           Network.HTTP.Types.Header
 import           Network.HTTP.Types.Status
-import           Network.HTTP.Types.URI    (parseSimpleQuery)
+import           Network.HTTP.Types.URI (renderSimpleQuery)
 import           Network.Wai
+import           Network.Wai.Middleware.RequestLogger (logStdout)
 
 import           Data.Aeson
 import           Data.Aeson.Types (emptyArray)
 import           Data.Monoid
+import           Data.Time.Clock.POSIX     (getPOSIXTime)
 import qualified Data.Vector               as V
-import qualified Hasql                     as H
-import qualified Hasql.Backend             as B
-import qualified Hasql.Postgres            as P
+import qualified Hasql.Transaction         as H
+
+import qualified Data.HashMap.Strict       as M
 
-import           PostgREST.Config          (AppConfig (..))
-import           PostgREST.Parsers
-import           PostgREST.DbStructure
-import           PostgREST.RangeQuery
 import           PostgREST.ApiRequest   (ApiRequest(..), ContentType(..)
                                             , Action(..), Target(..)
+                                            , PreferRepresentation (..)
                                             , userApiRequest)
-import           PostgREST.Types
-import           PostgREST.Auth            (tokenJWT)
-import           PostgREST.Error           (errResponse)
-
-import           PostgREST.QueryBuilder ( asJson
-                                        , callProc
+import           PostgREST.Auth            (tokenJWT, jwtClaims, containsRole)
+import           PostgREST.Config          (AppConfig (..))
+import           PostgREST.DbStructure
+import           PostgREST.Error           (errResponse, pgErrResponse)
+import           PostgREST.Parsers
+import           PostgREST.RangeQuery      (NonnegRange, allRange, rangeOffset, restrictRange)
+import           PostgREST.Middleware
+import           PostgREST.QueryBuilder ( callProc
                                         , addJoinConditions
-                                        , sourceSubqueryName
+                                        , sourceCTEName
                                         , requestToQuery
+                                        , requestToCountQuery
                                         , addRelations
                                         , createReadStatement
                                         , createWriteStatement
+                                        , ResultsWithCount
                                         )
+import           PostgREST.Types
 
 import           Prelude
 
-app :: DbStructure -> AppConfig -> RequestBody -> Request -> H.Tx P.Postgres s Response
-app dbStructure conf reqBody req =
+
+postgrest :: AppConfig -> IORef DbStructure -> P.Pool -> Application
+postgrest conf refDbStructure pool =
+  let middle = (if configQuiet conf then id else logStdout) . defaultMiddle in
+
+  middle $ \ req respond -> do
+    time <- getPOSIXTime
+    body <- strictRequestBody req
+    dbStructure <- readIORef refDbStructure
+
+    let schema = cs $ configSchema conf
+        apiRequest = userApiRequest schema req body
+        eClaims = jwtClaims (configJwtSecret conf) (iJWT apiRequest) time
+        authed = containsRole eClaims
+        handleReq = runWithClaims conf eClaims (app dbStructure conf) apiRequest
+        txMode = transactionMode $ iAction apiRequest
+
+    resp <- either (pgErrResponse authed) id <$> P.use pool
+      (HT.run handleReq HT.ReadCommitted txMode)
+    respond resp
+
+transactionMode :: Action -> H.Mode
+transactionMode ActionRead = HT.Read
+transactionMode ActionInfo = HT.Read
+transactionMode _ = HT.Write
+
+app :: DbStructure -> AppConfig -> ApiRequest -> H.Transaction Response
+app dbStructure conf apiRequest =
   let
       -- TODO: blow up for Left values (there is a middleware that checks the headers)
       contentType = either (const ApplicationJSON) id (iAccepts apiRequest)
@@ -70,123 +99,124 @@ app dbStructure conf reqBody req =
   case (iAction apiRequest, iTarget apiRequest, iPayload apiRequest) of
 
     (ActionRead, TargetIdent qi, Nothing) ->
-      case selectQuery of
+      case readSqlParts of
         Left e -> return $ responseLBS status400 [jsonH] $ cs e
-        Right q -> do
-          let range = iRange apiRequest
-              singular = iPreferSingular apiRequest
-              stm = createReadStatement q range singular
-                    (iPreferCount apiRequest) (contentType == TextCSV)
-          if range == Just emptyRange
-          then return $ errResponse status416 "HTTP Range error"
-          else do
-            row <- H.maybeEx stm
-            let (tableTotal, queryTotal, _ , body) = extractQueryResult row
+        Right (q, cq) -> do
+          let singular = iPreferSingular apiRequest
+              stm = createReadStatement q cq singular
+                    shouldCount (contentType == TextCSV)
+          respondToRange $ do
+            row <- H.query () stm
+            let (tableTotal, queryTotal, _ , body) = row
             if singular
             then return $ if queryTotal <= 0
               then responseLBS status404 [] ""
-              else responseLBS status200 [contentTypeH] (fromMaybe "{}" body)
+              else responseLBS status200 [contentTypeH] (cs body)
             else do
-              let frm = fromMaybe 0 $ rangeOffset <$> range
-                  to = frm+queryTotal-1
-                  contentRange = contentRangeH frm to tableTotal
-                  status = rangeStatus frm to tableTotal
-                  canonical = urlEncodeVars -- should this be moved to the dbStructure (location)?
-                    . sortBy (comparing fst)
-                    . map (join (***) cs)
-                    . parseSimpleQuery
-                    $ rawQueryString req
+              let (status, contentRange) = rangeHeader queryTotal tableTotal
+                  canonical = iCanonicalQS apiRequest
               return $ responseLBS status
                 [contentTypeH, contentRange,
                   ("Content-Location",
                     "/" <> cs (qiName qi) <>
                       if Prelude.null canonical then "" else "?" <> cs canonical
                   )
-                ] (fromMaybe "[]" body)
+                ] (cs body)
 
-    (ActionCreate, TargetIdent (QualifiedIdentifier _ table),
-     Just payload@(PayloadJSON (UniformObjects rows))) ->
-      case queries of
+    (ActionCreate, TargetIdent qi@(QualifiedIdentifier _ table),
+     Just payload@(PayloadJSON uniform@(UniformObjects rows))) ->
+      case mutateSqlParts of
         Left e -> return $ responseLBS status400 [jsonH] $ cs e
         Right (sq,mq) -> do
           let isSingle = (==1) $ V.length rows
           let pKeys = map pkName $ filter (filterPk schema table) allPrKeys -- would it be ok to move primary key detection in the query itself?
-          let stm = createWriteStatement sq mq isSingle (iPreferRepresentation apiRequest) pKeys (contentType == TextCSV) payload
-          row <- H.maybeEx stm
-          let (_, _, location, body) = extractQueryResult row
-          return $ responseLBS status201
-            [
-              contentTypeH,
-              (hLocation, "/" <> cs table <> "?" <> cs (fromMaybe "" location))
-            ]
-            $ if iPreferRepresentation apiRequest then fromMaybe "[]" body else ""
+          let stm = createWriteStatement qi sq mq isSingle (iPreferRepresentation apiRequest) pKeys (contentType == TextCSV) payload
+          row <- H.query uniform stm
+          let (_, _, fs, body) = extractQueryResult row
+              header =
+                if null fs then []
+                else [(hLocation, "/" <> cs table <> renderLocationFields fs)]
 
-    (ActionUpdate, TargetIdent _, Just payload@(PayloadJSON _)) ->
-      case queries of
+          return $ if iPreferRepresentation apiRequest == Full
+            then responseLBS status201 (contentTypeH : header) (cs body)
+            else responseLBS status201 header ""
+
+    (ActionUpdate, TargetIdent qi, Just payload@(PayloadJSON uniform)) ->
+      case mutateSqlParts of
         Left e -> return $ responseLBS status400 [jsonH] $ cs e
         Right (sq,mq) -> do
-          let stm = createWriteStatement sq mq False (iPreferRepresentation apiRequest) [] (contentType == TextCSV) payload
-          row <- H.maybeEx stm
+          let stm = createWriteStatement qi sq mq False (iPreferRepresentation apiRequest) [] (contentType == TextCSV) payload
+          row <- H.query uniform stm
           let (_, queryTotal, _, body) = extractQueryResult row
-              r = contentRangeH 0 (queryTotal-1) (Just queryTotal)
+              r = contentRangeH 0 (toInteger $ queryTotal-1) (toInteger <$> Just queryTotal)
               s = case () of _ | queryTotal == 0 -> status404
-                               | iPreferRepresentation apiRequest -> status200
+                               | iPreferRepresentation apiRequest == Full -> status200
                                | otherwise -> status204
-          return $ responseLBS s [contentTypeH, r]
-            $ if iPreferRepresentation apiRequest then fromMaybe "[]" body else ""
+          return $ if iPreferRepresentation apiRequest == Full
+            then responseLBS s [contentTypeH, r] (cs body)
+            else responseLBS s [r] ""
 
-    (ActionDelete, TargetIdent _, Nothing) ->
-      case queries of
+    (ActionDelete, TargetIdent qi, Nothing) ->
+      case mutateSqlParts of
         Left e -> return $ responseLBS status400 [jsonH] $ cs e
         Right (sq,mq) -> do
-          let fakeload = PayloadJSON $ UniformObjects V.empty
-          let stm = createWriteStatement sq mq False False [] (contentType == TextCSV) fakeload
-          row <- H.maybeEx stm
-          let (_, queryTotal, _, _) = extractQueryResult row
+          let emptyUniform = UniformObjects V.empty
+              fakeload = PayloadJSON emptyUniform
+              stm = createWriteStatement qi sq mq False (iPreferRepresentation apiRequest) [] (contentType == TextCSV) fakeload
+          row <- H.query emptyUniform stm
+          let (_, queryTotal, _, body) = extractQueryResult row
+              r = contentRangeH 1 0 (toInteger <$> Just queryTotal)
           return $ if queryTotal == 0
             then notFound
-            else responseLBS status204 [("Content-Range", "*/"<> cs (show queryTotal))] ""
+            else if iPreferRepresentation apiRequest == Full
+              then responseLBS status200 [contentTypeH, r] (cs body)
+              else responseLBS status204 [r] ""
 
-    (ActionInfo, TargetIdent (QualifiedIdentifier tSchema tTable), Nothing) -> do
-      let cols = filter (filterCol tSchema tTable) $ dbColumns dbStructure
-          pkeys = map pkName $ filter (filterPk tSchema tTable) allPrKeys
-          body = encode (TableOptions cols pkeys)
-          filterCol :: Schema -> TableName -> Column -> Bool
-          filterCol sc tb (Column{colTable=Table{tableSchema=s, tableName=t}}) = s==sc && t==tb
-          filterCol _ _ _ =  False
-      return $ responseLBS status200 [jsonH, allOrigins] $ cs body
+    (ActionInfo, TargetIdent (QualifiedIdentifier tSchema tTable), Nothing) ->
+      let mTable = find (\t -> tableName t == tTable && tableSchema t == tSchema) (dbTables dbStructure) in
+      case mTable of
+        Nothing -> return notFound
+        Just table ->
+          let cols = filter (filterCol tSchema tTable) $ dbColumns dbStructure
+              pkeys = map pkName $ filter (filterPk tSchema tTable) allPrKeys
+              body = encode (TableOptions cols pkeys)
+              filterCol :: Schema -> TableName -> Column -> Bool
+              filterCol sc tb Column{colTable=Table{tableSchema=s, tableName=t}} = s==sc && t==tb
+              filterCol _ _ _ =  False
+              acceptH = (hAllow, if tableInsertable table then "GET,POST,PATCH,DELETE" else "GET") in
+          return $ responseLBS status200 [jsonH, allOrigins, acceptH] $ cs body
 
-    (ActionInvoke, TargetIdent qi,
+    (ActionInvoke, TargetProc qi,
      Just (PayloadJSON (UniformObjects payload))) -> do
-      exists <- doesProcExist qi
+      exists <- H.query qi doesProcExist
       if exists
         then do
           let p = V.head payload
-              call = B.Stmt "select " V.empty True <>
-                asJson (callProc qi p)
               jwtSecret = configJwtSecret conf
-
-          bodyJson :: Maybe (Identity Value) <- H.maybeEx call
-          returnJWT <- doesProcReturnJWT qi
-          return $ responseLBS status200 [jsonH]
-                 (let body = fromMaybe emptyArray $ runIdentity <$> bodyJson in
-                    if returnJWT
-                    then "{\"token\":\"" <> cs (tokenJWT jwtSecret body) <> "\"}"
-                    else cs $ encode body)
+          respondToRange $ do
+            row <- H.query () (callProc qi p topLevelRange shouldCount)
+            returnJWT <- H.query qi doesProcReturnJWT
+            let (tableTotal, queryTotal, body) = fromMaybe (Just 0, 0, emptyArray) row
+                (status, contentRange) = rangeHeader queryTotal tableTotal
+              in
+              return $ responseLBS status [jsonH, contentRange]
+                      (if returnJWT
+                      then "{\"token\":\"" <> cs (tokenJWT jwtSecret body) <> "\"}"
+                      else cs $ encode body)
         else return notFound
 
     (ActionRead, TargetRoot, Nothing) -> do
-      body <- encode <$> accessibleTables (filter ((== cs schema) . tableSchema) (dbTables dbStructure))
+      body <- encode <$> H.query schema accessibleTables
       return $ responseLBS status200 [jsonH] $ cs body
 
-    (ActionUnknown _, _, _) -> return notFound
-
-    (_, TargetUnknown _, _) -> return notFound
+    (ActionInappropriate, _, _) -> return $ responseLBS status405 [] ""
 
     (_, _, Just (PayloadParseError e)) ->
       return $ responseLBS status400 [jsonH] $
         cs (formatGeneralError "Cannot parse request payload" (cs e))
 
+    (_, TargetUnknown _, _) -> return notFound
+
     (_, _, _) -> return notFound
 
  where
@@ -195,19 +225,40 @@ app dbStructure conf reqBody req =
   allPrKeys = dbPrimaryKeys dbStructure
   allOrigins = ("Access-Control-Allow-Origin", "*") :: Header
   schema = cs $ configSchema conf
-  apiRequest = userApiRequest schema req reqBody
-  selectQuery = requestToQuery schema <$> (DbRead <$> buildReadRequest (dbRelations dbStructure) apiRequest)
-  mutateQuery = requestToQuery schema <$> (DbMutate <$> buildMutateRequest apiRequest)
-  queries = (,) <$> selectQuery <*> mutateQuery
+  shouldCount = iPreferCount apiRequest
+  topLevelRange = fromMaybe allRange $ M.lookup "limit" $ iRange apiRequest
+  readDbRequest = DbRead <$> buildReadRequest (configMaxRows conf) (dbRelations dbStructure) apiRequest
+  mutateDbRequest = DbMutate <$> buildMutateRequest apiRequest
+  selectQuery = requestToQuery schema <$> readDbRequest
+  countQuery = requestToCountQuery schema <$> readDbRequest
+  mutateQuery = requestToQuery schema <$> mutateDbRequest
+  readSqlParts = (,) <$> selectQuery <*> countQuery
+  mutateSqlParts = (,) <$> selectQuery <*> mutateQuery
+  respondToRange response = if topLevelRange == emptyRange
+                            then return $ errResponse status416 "HTTP Range error"
+                            else response
+  rangeHeader queryTotal tableTotal = let frm = rangeOffset topLevelRange
+                                          to = frm + toInteger queryTotal - 1
+                                          contentRange = contentRangeH frm to (toInteger <$> tableTotal)
+                                          status = rangeStatus frm to (toInteger <$> tableTotal)
+                                      in (status, contentRange)
 
-rangeStatus :: Int -> Int -> Maybe Int -> Status
+splitKeyValue :: BS.ByteString -> (BS.ByteString, BS.ByteString)
+splitKeyValue kv = (k, BS.tail v)
+  where (k, v) = BS.break (== '=') kv
+
+renderLocationFields :: [BS.ByteString] -> BS.ByteString
+renderLocationFields fields =
+  renderSimpleQuery True $ map splitKeyValue fields
+
+rangeStatus :: Integer -> Integer -> Maybe Integer -> Status
 rangeStatus _ _ Nothing = status200
 rangeStatus frm to (Just total)
   | frm > total            = status416
   | (1 + to - frm) < total = status206
   | otherwise               = status200
 
-contentRangeH :: Int -> Int -> Maybe Int -> Header
+contentRangeH :: Integer -> Integer -> Maybe Integer -> Header
 contentRangeH frm to total =
     ("Content-Range", cs headerValue)
     where
@@ -220,7 +271,7 @@ contentRangeH frm to total =
       fromInRange   = frm <= to
 
 jsonH :: Header
-jsonH = (hContentType, "application/json")
+jsonH = (hContentType, "application/json; charset=utf-8")
 
 formatRelationError :: Text -> Text
 formatRelationError = formatGeneralError
@@ -243,68 +294,122 @@ augumentRequestWithJoin schema allRels request =
   (first formatRelationError . addRelations schema allRels Nothing) request
   >>= addJoinConditions schema
 
-buildReadRequest :: [Relation] -> ApiRequest -> Either Text ReadRequest
-buildReadRequest allRels apiRequest  =
-  augumentRequestWithJoin schema rels =<< first formatParserError (foldr addFilter <$> (addOrder <$> readRequest <*> ord) <*> flts)
+addFiltersOrdersRanges :: ApiRequest -> Either ParseError (ReadRequest -> ReadRequest)
+addFiltersOrdersRanges apiRequest = foldr1 (liftA2 (.)) [
+    flip (foldr addFilter) <$> filters,
+    flip (foldr addOrder) <$> orders,
+    flip (foldr addRange) <$> ranges
+  ]
+  {-
+  The esence of what is going on above is that we are composing tree functions
+  of type (ReadRequest->ReadRequest) that are in (Either ParseError a) context
+  -}
+  where
+    filters :: Either ParseError [(Path, Filter)]
+    filters = mapM pRequestFilter flts
+      where
+        action = iAction apiRequest
+        flts = if action == ActionRead
+          then iFilters apiRequest
+          else filter (( '.' `elem` ) . fst) $ iFilters apiRequest -- there can be no filters on the root table whre we are doing insert/update
+    orders :: Either ParseError [(Path, [OrderTerm])]
+    orders = mapM pRequestOrder $ iOrder apiRequest
+    ranges :: Either ParseError [(Path, NonnegRange)]
+    ranges = mapM pRequestRange $ M.toList $ iRange apiRequest
+
+treeRestrictRange :: Maybe Integer -> ReadRequest -> Either Text ReadRequest
+treeRestrictRange maxRows_ request = pure $ nodeRestrictRange maxRows_ `fmap` request
+  where
+    nodeRestrictRange :: Maybe Integer -> ReadNode -> ReadNode
+    nodeRestrictRange m (q@Select {range_=r}, i) = (q{range_=restrictRange m r }, i)
+
+buildReadRequest :: Maybe Integer -> [Relation] -> ApiRequest -> Either Text ReadRequest
+buildReadRequest maxRows allRels apiRequest  =
+  treeRestrictRange maxRows =<<
+  augumentRequestWithJoin schema relations =<<
+  first formatParserError readRequest
   where
-    selStr = iSelect apiRequest
-    orderS = iOrder apiRequest
-    action = iAction apiRequest
-    target = iTarget apiRequest
     (schema, rootTableName) = fromJust $ -- Make it safe
+      let target = iTarget apiRequest in
       case target of
         (TargetIdent (QualifiedIdentifier s t) ) -> Just (s, t)
         _ -> Nothing
 
-    rootName = if action == ActionRead
-      then rootTableName
-      else sourceSubqueryName
-    filters = if action == ActionRead
-      then iFilters apiRequest
-      else filter (( '.' `elem` ) . fst) $ iFilters apiRequest -- there can be no filters on the root table whre we are doing insert/update
-    rels = case action of
+    action :: Action
+    action = iAction apiRequest
+
+    readRequest :: Either ParseError ReadRequest
+    readRequest = addFiltersOrdersRanges apiRequest <*>
+      parse (pRequestSelect rootName) ("failed to parse select parameter <<"++selStr++">>") selStr
+      where
+        selStr = iSelect apiRequest
+        rootName = if action == ActionRead
+          then rootTableName
+          else sourceCTEName
+
+    relations :: [Relation]
+    relations = case action of
       ActionCreate -> fakeSourceRelations ++ allRels
       ActionUpdate -> fakeSourceRelations ++ allRels
+      ActionDelete -> fakeSourceRelations ++ allRels
       _       -> allRels
       where fakeSourceRelations = mapMaybe (toSourceRelation rootTableName) allRels -- see comment in toSourceRelation
-    readRequest = parse (pRequestSelect rootName) ("failed to parse select parameter <<"++selStr++">>") selStr
-    addOrder (Node (q,i) f) o = Node (q{order=o}, i) f
-    flts = mapM pRequestFilter filters
-    ord = traverse (parse pOrder ("failed to parse order parameter <<"++fromMaybe "" orderS++">>")) orderS
 
 buildMutateRequest :: ApiRequest -> Either Text MutateRequest
-buildMutateRequest apiRequest =
-  mutateApiRequest
+buildMutateRequest apiRequest = case action of
+  ActionCreate -> Insert rootTableName <$> pure payload
+  ActionUpdate -> Update rootTableName <$> pure payload <*> filters
+  ActionDelete -> Delete rootTableName <$> filters
+  _        -> Left "Unsupported HTTP verb"
   where
     action = iAction apiRequest
-    target = iTarget apiRequest
     payload = fromJust $ iPayload apiRequest
     rootTableName = -- TODO: Make it safe
+      let target = iTarget apiRequest in
       case target of
         (TargetIdent (QualifiedIdentifier _ t) ) -> t
         _ -> undefined
-    mutateApiRequest = case action of
-      ActionCreate -> Insert rootTableName <$> pure payload
-      ActionUpdate -> Update rootTableName <$> pure payload <*> cond
-      ActionDelete -> Delete rootTableName <$> cond
-      _        -> Left "Unsupported HTTP verb"
-    mutateFilters = filter (not . ( '.' `elem` ) . fst) $ iFilters apiRequest -- update/delete filters can be only on the root table
-    cond = first formatParserError $ map snd <$> mapM pRequestFilter mutateFilters
+    filters = first formatParserError $ map snd <$> mapM pRequestFilter mutateFilters
+      where mutateFilters = filter (not . ( '.' `elem` ) . fst) $ iFilters apiRequest -- update/delete filters can be only on the root table
+
+addFilterToNode :: Filter -> ReadRequest -> ReadRequest
+addFilterToNode flt (Node (q@Select {flt_=flts}, i) f) = Node (q {flt_=flt:flts}, i) f
 
 addFilter :: (Path, Filter) -> ReadRequest -> ReadRequest
-addFilter ([], flt) (Node (q@(Select {flt_=flts}), i) forest) = Node (q {flt_=flt:flts}, i) forest
-addFilter (path, flt) (Node rn forest) =
+addFilter = addProperty addFilterToNode
+
+addOrderToNode :: [OrderTerm] -> ReadRequest -> ReadRequest
+addOrderToNode o (Node (q,i) f) = Node (q{order=Just o}, i) f
+
+addOrder :: (Path, [OrderTerm]) -> ReadRequest -> ReadRequest
+addOrder = addProperty addOrderToNode
+
+addRangeToNode :: NonnegRange -> ReadRequest -> ReadRequest
+addRangeToNode r (Node (q,i) f) = Node (q{range_=r}, i) f
+
+addRange :: (Path, NonnegRange) -> ReadRequest -> ReadRequest
+addRange = addProperty addRangeToNode
+
+addProperty :: (a -> ReadRequest -> ReadRequest) -> (Path, a) -> ReadRequest -> ReadRequest
+addProperty f ([], a) n = f a n
+addProperty f (path, a) (Node rn forest) =
   case targetNode of
-    Nothing -> Node rn forest -- the filter is silenty dropped in the Request does not contain the required path
-    Just tn -> Node rn (addFilter (remainingPath, flt) tn:restForest)
+    Nothing -> Node rn forest -- the property is silenty dropped in the Request does not contain the required path
+    Just tn -> Node rn (addProperty f (remainingPath, a) tn:restForest)
   where
     targetNodeName:remainingPath = path
     (targetNode,restForest) = splitForest targetNodeName forest
+    splitForest :: NodeName -> Forest ReadNode -> (Maybe ReadRequest, Forest ReadNode)
     splitForest name forst =
       case maybeNode of
         Nothing -> (Nothing,forest)
         Just node -> (Just node, delete node forest)
-      where maybeNode = find ((name==).fst.snd.rootLabel) forst
+      where
+        maybeNode :: Maybe ReadRequest
+        maybeNode = find fnd forst
+          where
+            fnd :: ReadRequest -> Bool
+            fnd (Node (_,(n,_,_)) _) = n == name
 
 -- in a relation where one of the tables mathces "TableName"
 -- replace the name to that table with pg_source
@@ -313,9 +418,9 @@ addFilter (path, flt) (Node rn forest) =
 -- as just another table that has relations with other tables
 toSourceRelation :: TableName -> Relation -> Maybe Relation
 toSourceRelation mt r@(Relation t _ ft _ _ rt _ _)
-  | mt == tableName t = Just $ r {relTable=t {tableName=sourceSubqueryName}}
-  | mt == tableName ft = Just $ r {relFTable=t {tableName=sourceSubqueryName}}
-  | Just mt == (tableName <$> rt) = Just $ r {relLTable=(\tbl -> tbl {tableName=sourceSubqueryName}) <$> rt}
+  | mt == tableName t = Just $ r {relTable=t {tableName=sourceCTEName}}
+  | mt == tableName ft = Just $ r {relFTable=t {tableName=sourceCTEName}}
+  | Just mt == (tableName <$> rt) = Just $ r {relLTable=(\tbl -> tbl {tableName=sourceCTEName}) <$> rt}
   | otherwise = Nothing
 
 data TableOptions = TableOptions {
@@ -329,6 +434,5 @@ instance ToJSON TableOptions where
     , "pkey"   .= tblOptpkey t ]
 
 
-extractQueryResult :: Maybe (Maybe Int, Int, Maybe BL.ByteString, Maybe BL.ByteString)
-                         -> (Maybe Int, Int, Maybe BL.ByteString, Maybe BL.ByteString)
-extractQueryResult = fromMaybe (Just 0, 0, Just "", Just "")
+extractQueryResult :: Maybe ResultsWithCount -> ResultsWithCount
+extractQueryResult = fromMaybe (Nothing, 0, [], "")

src/PostgREST/Auth.hs

@@ -12,73 +12,77 @@ In the test suite there is an example of simple login function that can be used
 very simple authentication system inside the PostgreSQL database.
 -}
 module PostgREST.Auth (
-    setRole
-  , claimsToSQL
+    claimsToSQL
+  , containsRole
   , jwtClaims
   , tokenJWT
   ) where
 
-import           Control.Monad           (join)
-import           Data.Aeson              (Value (..), Object)
-import           Data.Aeson.Types        (emptyObject, emptyArray)
-import           Data.Vector             as V (null, head)
-import           Data.Map                as M (fromList, toList)
+import           Lens.Micro
+import           Lens.Micro.Aeson
+import           Data.Aeson              (Value (..), parseJSON, toJSON)
+import           Data.Aeson.Types        (parseMaybe, emptyObject, emptyArray)
+import qualified Data.ByteString         as BS
+import qualified Data.Vector             as V
+import qualified Data.HashMap.Strict     as M
+import           Data.Maybe              (fromMaybe, maybeToList, fromJust)
 import           Data.Monoid             ((<>))
 import           Data.String.Conversions (cs)
 import           Data.Text               (Text)
 import           Data.Time.Clock         (NominalDiffTime)
-import           PostgREST.QueryBuilder  (pgFmtLit, pgFmtIdent, unquoted)
+import           PostgREST.QueryBuilder  (pgFmtIdent, pgFmtLit, unquoted)
 import qualified Web.JWT                 as JWT
-import qualified Data.HashMap.Lazy       as H
 
 {-|
-  Receives a map of JWT claims and returns a list
-  of PostgreSQL statements to set the claims as user defined GUCs.
-  Except if we have a claim called role,
-  this one is mapped to a SET ROLE statement.
-  In case there is any problem decoding the JWT it returns Nothing.
+  Receives a map of JWT claims and returns a list of PostgreSQL
+  statements to set the claims as user defined GUCs.  Except if we
+  have a claim called role, this one is mapped to a SET ROLE
+  statement.
 -}
-claimsToSQL :: JWT.ClaimsMap -> [Text]
-claimsToSQL = map setVar . toList
-  where
-    setVar ("role", String val) = setRole val
-    setVar (k, val) = "set local postgrest.claims." <> pgFmtIdent k <>
-                      " = " <> valueToVariable val <> ";"
-    valueToVariable = pgFmtLit . unquoted
+claimsToSQL :: M.HashMap Text Value -> [BS.ByteString]
+claimsToSQL claims = roleStmts <> varStmts
+ where
+  roleStmts = maybeToList $
+    (\r -> "set local role " <> r <> ";") . cs . valueToVariable <$> M.lookup "role" claims
+  varStmts = map setVar $ M.toList (M.delete "role" claims)
+  setVar (k, val) = "set local " <> cs (pgFmtIdent $ "postgrest.claims." <> k)
+                    <> " = " <> cs (valueToVariable val) <> ";"
+  valueToVariable = pgFmtLit . unquoted
 
 {-|
   Receives the JWT secret (from config) and a JWT and
   returns a map of JWT claims
-  In case there is any problem decoding the JWT it returns Nothing.
+  In case there is any problem decoding the JWT it returns an error Text
 -}
-jwtClaims :: JWT.Secret -> Text -> NominalDiffTime -> Maybe JWT.ClaimsMap
-jwtClaims secret input time =
-  case join $ claim JWT.exp of
-    Just expires ->
-      if JWT.secondsSinceEpoch expires > time
-        then customClaims
-        else Nothing
-    _ -> customClaims
-  where
-    decoded = JWT.decodeAndVerifySignature secret input
-    claim :: (JWT.JWTClaimsSet -> a) -> Maybe a
-    claim prop = prop . JWT.claims <$> decoded
-    customClaims = claim JWT.unregisteredClaims
-
--- | Receives the name of a role and returns a SET ROLE statement
-setRole :: Text -> Text
-setRole role = "set local role " <> cs (pgFmtLit role) <> ";"
-
+jwtClaims :: JWT.Secret -> Text -> NominalDiffTime -> Either Text (M.HashMap Text Value)
+jwtClaims _ "" _ = Right M.empty
+jwtClaims secret jwt time =
+  case isExpired <$> mClaims of
+    Just True -> Left "JWT expired"
+    Nothing -> Left "Invalid JWT"
+    Just False -> Right $ value2map $ fromJust mClaims
+ where
+  isExpired claims =
+    let mExp = claims ^? key "exp" . _Integer
+    in fromMaybe False $ (<= time) . fromInteger <$> mExp
+  mClaims = toJSON . JWT.claims <$> JWT.decodeAndVerifySignature secret jwt
+  value2map (Object o) = o
+  value2map _          = M.empty
 
 {-|
   Receives the JWT secret (from config) and a JWT and a JSON value
   and returns a signed JWT.
 -}
 tokenJWT :: JWT.Secret -> Value -> Text
-tokenJWT secret (Array a) = JWT.encodeSigned JWT.HS256 secret
-                               JWT.def { JWT.unregisteredClaims = fromHashMap o }
-                          where
-                            Object o = if V.null a then emptyObject else V.head a
-                            fromHashMap :: Object -> JWT.ClaimsMap
-                            fromHashMap = M.fromList . H.toList
-tokenJWT secret _          = tokenJWT secret emptyArray
+tokenJWT secret (Array arr) =
+  let obj = if V.null arr then emptyObject else V.head arr
+      jcs = parseMaybe parseJSON obj :: Maybe JWT.JWTClaimsSet in
+  JWT.encodeSigned JWT.HS256 secret $ fromMaybe JWT.def jcs
+tokenJWT secret _ = tokenJWT secret emptyArray
+
+{-|
+  Whether a response from jwtClaims contains a role claim
+-}
+containsRole :: Either Text (M.HashMap Text Value) -> Bool
+containsRole (Left _) = False
+containsRole (Right claims) = M.member "role" claims

src/PostgREST/Config.hs

@@ -30,29 +30,33 @@ import           Network.Wai
 import           Network.Wai.Middleware.Cors (CorsResourcePolicy (..))
 import           Options.Applicative
 import           Paths_postgrest             (version)
-import           Web.JWT                     (Secret, secret)
 import           Prelude
+import           Safe                        (readMay)
+import           Web.JWT                     (Secret, secret)
 
 -- | Data type to store all command line options
 data AppConfig = AppConfig {
     configDatabase  :: String
-  , configPort      :: Int
   , configAnonRole  :: String
   , configSchema    :: String
+  , configPort      :: Int
   , configJwtSecret :: Secret
   , configPool      :: Int
+  , configMaxRows   :: Maybe Integer
+  , configQuiet     :: Bool
   }
 
 argParser :: Parser AppConfig
 argParser = AppConfig
-  <$> argument str (help "database connection string" <> metavar "STRING")
-
+  <$> argument str (help "(REQUIRED) database connection string, e.g. postgres://user:pass@host:port/db" <> metavar "DB_URL")
+  <*> strOption    (long "anonymous"  <> short 'a' <> help "(REQUIRED) postgres role to use for non-authenticated requests" <> metavar "ROLE")
+  <*> strOption    (long "schema"     <> short 's' <> help "schema to use for API routes" <> metavar "NAME" <> value "public" <> showDefault)
   <*> option auto  (long "port"       <> short 'p' <> help "port number on which to run HTTP server" <> metavar "PORT" <> value 3000 <> showDefault)
-  <*> strOption    (long "anonymous"  <> short 'a' <> help "postgres role to use for non-authenticated requests" <> metavar "ROLE")
-  <*> strOption    (long "schema"     <> short 's' <> help "schema to use for API routes" <> metavar "NAME" <> value "1" <> showDefault)
   <*> (secret . cs <$>
       strOption    (long "jwt-secret" <> short 'j' <> help "secret used to encrypt and decrypt JWT tokens" <> metavar "SECRET" <> value "secret" <> showDefault))
   <*> option auto  (long "pool"       <> short 'o' <> help "max connections in database pool" <> metavar "COUNT" <> value 10 <> showDefault)
+  <*> (readMay <$> strOption  (long "max-rows"   <> short 'm' <> help "max rows in response" <> metavar "COUNT" <> value "infinity" <> showDefault))
+  <*> pure False
 
 defaultCorsPolicy :: CorsResourcePolicy
 defaultCorsPolicy =  CorsResourcePolicy Nothing

src/PostgREST/DbStructure.hs

@@ -10,28 +10,34 @@ module PostgREST.DbStructure (
 , doesProcReturnJWT
 ) where
 
-import           Control.Applicative
-import           Control.Monad          (join)
-import           Data.Functor.Identity
-import           Data.List              (elemIndex, find, subsequences, sort, transpose)
-import           Data.Maybe             (fromMaybe, fromJust, isJust, mapMaybe, listToMaybe)
-import           Data.Monoid
-import           Data.Text              (Text, split)
-import qualified Hasql                  as H
-import qualified Hasql.Postgres         as P
-import qualified Hasql.Backend          as B
-import           PostgREST.Types
+import qualified Hasql.Decoders                as HD
+import qualified Hasql.Encoders                as HE
+import qualified Hasql.Query                   as H
 
-import           GHC.Exts               (groupWith)
+import           Control.Applicative
+import           Control.Monad                 (join, replicateM)
+import           Data.Functor.Contravariant    (contramap)
+import           Data.List                     (elemIndex, find, sort,
+                                                subsequences, transpose)
+import           Data.Maybe                    (fromJust, fromMaybe, isJust,
+                                                listToMaybe, mapMaybe)
+import           Data.Monoid
+import           Data.Text                     (Text, split)
+import qualified Hasql.Session                 as H
+import           PostgREST.Types
+import           Text.InterpolatedString.Perl6 (q)
+
+import           Data.Int                      (Int32)
+import           GHC.Exts                      (groupWith)
 import           Prelude
 
-getDbStructure :: Schema -> H.Tx P.Postgres s DbStructure
+getDbStructure :: Schema -> H.Session DbStructure
 getDbStructure schema = do
-  tabs <- allTables
-  cols <- allColumns tabs
-  syns <- allSynonyms cols
-  rels <- allRelations tabs cols
-  keys <- allPrimaryKeys tabs
+  tabs <- H.query () allTables
+  cols <- H.query () $ allColumns tabs
+  syns <- H.query () $ allSynonyms cols
+  rels <- H.query () $ allRelations tabs cols
+  keys <- H.query () $ allPrimaryKeys tabs
 
   let rels' = (addManyToManyRelations . raiseRelations schema syns . addParentRelations . addSynonymousRelations syns) rels
       cols' = addForeignKeys rels' cols
@@ -44,52 +50,113 @@ getDbStructure schema = do
     , dbPrimaryKeys = keys'
     }
 
-doesProc :: forall c s. B.CxValue c Int =>
-            (Text -> Text -> B.Stmt c) -> QualifiedIdentifier -> H.Tx c s Bool
-doesProc stmt qi = do
-  row :: Maybe (Identity Int) <- H.maybeEx $ stmt (qiSchema qi) (qiName qi)
-  return $ isJust row
+encodeQi :: HE.Params QualifiedIdentifier
+encodeQi =
+  contramap qiSchema (HE.value HE.text) <>
+  contramap qiName   (HE.value HE.text)
 
-doesProcExist :: QualifiedIdentifier -> H.Tx P.Postgres s Bool
-doesProcExist = doesProc [H.stmt|
+decodeTables :: HD.Result [Table]
+decodeTables =
+  HD.rowsList tblRow
+ where
+  tblRow = Table <$> HD.value HD.text <*> HD.value HD.text
+                 <*> HD.value HD.bool
+
+decodeColumns :: [Table] -> HD.Result [Column]
+decodeColumns tables =
+  mapMaybe (columnFromRow tables) <$> HD.rowsList colRow
+ where
+  colRow =
+    (,,,,,,,,,,)
+      <$> HD.value HD.text <*> HD.value HD.text
+      <*> HD.value HD.text <*> HD.value HD.int4
+      <*> HD.value HD.bool <*> HD.value HD.text
+      <*> HD.value HD.bool
+      <*> HD.nullableValue HD.int4
+      <*> HD.nullableValue HD.int4
+      <*> HD.nullableValue HD.text
+      <*> HD.nullableValue HD.text
+
+decodeRelations :: [Table] -> [Column] -> HD.Result [Relation]
+decodeRelations tables cols =
+  mapMaybe (relationFromRow tables cols) <$> HD.rowsList relRow
+ where
+  relRow = (,,,,,)
+    <$> HD.value HD.text
+    <*> HD.value HD.text
+    <*> HD.value (HD.array (HD.arrayDimension replicateM (HD.arrayValue HD.text)))
+    <*> HD.value HD.text
+    <*> HD.value HD.text
+    <*> HD.value (HD.array (HD.arrayDimension replicateM (HD.arrayValue HD.text)))
+
+decodePks :: [Table] -> HD.Result [PrimaryKey]
+decodePks tables =
+  mapMaybe (pkFromRow tables) <$> HD.rowsList pkRow
+ where
+  pkRow = (,,) <$> HD.value HD.text <*> HD.value HD.text <*> HD.value HD.text
+
+decodeSynonyms :: [Column] -> HD.Result [(Column,Column)]
+decodeSynonyms cols =
+  mapMaybe (synonymFromRow cols) <$> HD.rowsList synRow
+ where
+  synRow = (,,,,,)
+    <$> HD.value HD.text <*> HD.value HD.text
+    <*> HD.value HD.text <*> HD.value HD.text
+    <*> HD.value HD.text <*> HD.value HD.text
+
+doesProcExist :: H.Query QualifiedIdentifier Bool
+doesProcExist =
+  H.statement sql encodeQi (HD.singleRow (HD.value HD.bool)) True
+ where
+  sql = [q| SELECT EXISTS (
       SELECT 1
       FROM   pg_catalog.pg_namespace n
       JOIN   pg_catalog.pg_proc p
       ON     pronamespace = n.oid
-      WHERE  nspname = ?
-      AND    proname = ?
-    |]
+      WHERE  nspname = $1
+      AND    proname = $2
+    ) |]
 
-doesProcReturnJWT :: QualifiedIdentifier -> H.Tx P.Postgres s Bool
-doesProcReturnJWT = doesProc [H.stmt|
+doesProcReturnJWT :: H.Query QualifiedIdentifier Bool
+doesProcReturnJWT =
+  H.statement sql encodeQi (HD.singleRow (HD.value HD.bool)) True
+ where
+  sql = [q| SELECT EXISTS (
       SELECT 1
       FROM   pg_catalog.pg_namespace n
       JOIN   pg_catalog.pg_proc p
       ON     pronamespace = n.oid
-      WHERE  nspname = ?
-      AND    proname = ?
+      WHERE  nspname = $1
+      AND    proname = $2
       AND    pg_catalog.pg_get_function_result(p.oid) like '%jwt_claims'
-    |]
+    ) |]
 
-accessibleTables :: [Table] -> H.Tx P.Postgres s [Table]
-accessibleTables allTabs = do
-  accessible <- H.listEx $ [H.stmt|
-      SELECT
-        n.nspname AS table_schema,
-        c.relname AS table_name
-      FROM pg_class c
-      JOIN pg_namespace n ON n.oid = c.relnamespace
-      WHERE
-        c.relkind IN ('v','r','m') AND
-        n.nspname NOT IN ('pg_catalog', 'information_schema') AND (
-          pg_has_role(c.relowner, 'USAGE'::text) OR
-          has_table_privilege(c.oid, 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER'::text) OR
-          has_any_column_privilege(c.oid, 'SELECT, INSERT, UPDATE, REFERENCES'::text)
-        )
-      ORDER BY table_schema, table_name
-    |]
-  let isAccessible table = isJust $ find (\(s,n) -> tableSchema table == s && tableName table == n) accessible
-  return $ filter isAccessible allTabs
+accessibleTables :: H.Query Schema [Table]
+accessibleTables =
+  H.statement sql (HE.value HE.text) decodeTables True
+ where
+  sql = [q|
+    select
+      n.nspname as table_schema,
+      relname as table_name,
+      c.relkind = 'r' or (c.relkind IN ('v', 'f')) and (pg_relation_is_updatable(c.oid::regclass, false) & 8) = 8
+      or (exists (
+         select 1
+         from pg_trigger
+         where pg_trigger.tgrelid = c.oid and (pg_trigger.tgtype::integer & 69) = 69)
+      ) as insertable
+    from
+      pg_class c
+      join pg_namespace n on n.oid = c.relnamespace
+    where
+      c.relkind in ('v', 'r', 'm')
+      and n.nspname = $1
+      and (
+        pg_has_role(c.relowner, 'USAGE'::text)
+        or has_table_privilege(c.oid, 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER'::text)
+        or has_any_column_privilege(c.oid, 'SELECT, INSERT, UPDATE, REFERENCES'::text)
+      )
+    order by relname |]
 
 synonymousColumns :: [(Column,Column)] -> [Column] -> [[Column]]
 synonymousColumns allSyns cols = synCols'
@@ -107,9 +174,9 @@ addForeignKeys rels = map addFk
     addFk col = col { colFK = fk col }
     fk col = join $ relToFk col <$> find (lookupFn col) rels
     lookupFn :: Column -> Relation -> Bool
-    lookupFn c (Relation{relColumns=cs, relType=rty}) = c `elem` cs && rty==Child
+    lookupFn c Relation{relColumns=cs, relType=rty} = c `elem` cs && rty==Child
     -- lookupFn _ _ = False
-    relToFk col (Relation{relColumns=cols, relFColumns=colsF}) = ForeignKey <$> colF
+    relToFk col Relation{relColumns=cols, relFColumns=colsF} = ForeignKey <$> colF
       where
         pos = elemIndex col cols
         colF = (colsF !!) <$> pos
@@ -127,12 +194,14 @@ addParentRelations [] = []
 addParentRelations (rel@(Relation t c ft fc _ _ _ _):rels) = Relation ft fc t c Parent Nothing Nothing Nothing : rel : addParentRelations rels
 
 addManyToManyRelations :: [Relation] -> [Relation]
-addManyToManyRelations rels = rels ++ mapMaybe link2Relation links
+addManyToManyRelations rels = rels ++ addMirrorRelation (mapMaybe link2Relation links)
   where
     links = join $ map (combinations 2) $ filter (not . null) $ groupWith groupFn $ filter ( (==Child). relType) rels
     groupFn :: Relation -> Text
-    groupFn (Relation{relTable=Table{tableSchema=s, tableName=t}}) = s<>"_"<>t
+    groupFn Relation{relTable=Table{tableSchema=s, tableName=t}} = s<>"_"<>t
     combinations k ns = filter ((k==).length) (subsequences ns)
+    addMirrorRelation [] = []
+    addMirrorRelation (rel@(Relation t c ft fc _ lt lc1 lc2):rels') = Relation ft fc t c Many lt lc2 lc1 : rel : addMirrorRelation rels'
     link2Relation [
       Relation{relTable=lt, relColumns=lc1, relFTable=t,  relFColumns=c},
       Relation{             relColumns=lc2, relFTable=ft, relFColumns=fc}
@@ -161,80 +230,170 @@ synonymousPrimaryKeys syns (key:keys) = key : newKeys ++ synonymousPrimaryKeys s
     keySyns = filter ((\c -> colTable c == pkTable key && colName c == pkName key) . fst) syns
     newKeys = map ((\c -> PrimaryKey{pkTable=colTable c,pkName=colName c}) . snd) keySyns
 
-allTables :: H.Tx P.Postgres s [Table]
-allTables = do
-    rows <- H.listEx $ [H.stmt|
-      SELECT
-        n.nspname AS table_schema,
-        c.relname AS table_name,
-        c.relkind = 'r' OR (c.relkind IN ('v','f'))
-        AND (pg_relation_is_updatable(c.oid::regclass, FALSE) & 8) = 8
-        OR (EXISTS
-          ( SELECT 1
-            FROM pg_trigger
-            WHERE pg_trigger.tgrelid = c.oid
-            AND (pg_trigger.tgtype::integer & 69) = 69) ) AS insertable
-      FROM pg_class c
-      JOIN pg_namespace n ON n.oid = c.relnamespace
-      WHERE c.relkind IN ('v','r','m')
-        AND n.nspname NOT IN ('pg_catalog', 'information_schema')
-      GROUP BY table_schema, table_name, insertable
-      ORDER BY table_schema, table_name
-    |]
-    return $ map tableFromRow rows
+allTables :: H.Query () [Table]
+allTables =
+  H.statement sql HE.unit decodeTables True
+ where
+  sql = [q|
+    SELECT
+      n.nspname AS table_schema,
+      c.relname AS table_name,
+      c.relkind = 'r' OR (c.relkind IN ('v','f'))
+      AND (pg_relation_is_updatable(c.oid::regclass, FALSE) & 8) = 8
+      OR (EXISTS
+        ( SELECT 1
+          FROM pg_trigger
+          WHERE pg_trigger.tgrelid = c.oid
+          AND (pg_trigger.tgtype::integer & 69) = 69) ) AS insertable
+    FROM pg_class c
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE c.relkind IN ('v','r','m')
+      AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+    GROUP BY table_schema, table_name, insertable
+    ORDER BY table_schema, table_name |]
 
-tableFromRow :: (Text, Text, Bool) -> Table
-tableFromRow (s, n, i) = Table s n i
-
-allColumns :: [Table] -> H.Tx P.Postgres s [Column]
-allColumns tabs = do
-  cols <- H.listEx $ [H.stmt|
-      SELECT DISTINCT
-          info.table_schema AS schema,
-          info.table_name AS table_name,
-          info.column_name AS name,
-          info.ordinal_position AS position,
-          info.is_nullable::boolean AS nullable,
-          info.data_type AS col_type,
-          info.is_updatable::boolean AS updatable,
-          info.character_maximum_length AS max_len,
-          info.numeric_precision AS precision,
-          info.column_default AS default_value,
-          array_to_string(enum_info.vals, ',') AS enum
-      FROM (
-          SELECT
-              table_schema,
-              table_name,
-              column_name,
-              ordinal_position,
-              is_nullable,
-              data_type,
-              is_updatable,
-              character_maximum_length,
-              numeric_precision,
-              column_default,
-              udt_name
-          FROM information_schema.columns
-          WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
-      ) AS info
-      LEFT OUTER JOIN (
-          SELECT
-              n.nspname AS s,
-              t.typname AS n,
-              array_agg(e.enumlabel ORDER BY e.enumsortorder) AS vals
-          FROM pg_type t
-          JOIN pg_enum e ON t.oid = e.enumtypid
-          JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
-          GROUP BY s,n
-      ) AS enum_info ON (info.udt_name = enum_info.n)
-      ORDER BY schema, position
-  |]
-  return $ mapMaybe (columnFromRow tabs) cols
+allColumns :: [Table] -> H.Query () [Column]
+allColumns tabs =
+  H.statement sql HE.unit (decodeColumns tabs) True
+ where
+  sql = [q|
+    SELECT DISTINCT
+        info.table_schema AS schema,
+        info.table_name AS table_name,
+        info.column_name AS name,
+        info.ordinal_position AS position,
+        info.is_nullable::boolean AS nullable,
+        info.data_type AS col_type,
+        info.is_updatable::boolean AS updatable,
+        info.character_maximum_length AS max_len,
+        info.numeric_precision AS precision,
+        info.column_default AS default_value,
+        array_to_string(enum_info.vals, ',') AS enum
+    FROM (
+        /*
+        -- CTE based on information_schema.columns to remove the owner filter
+        */
+        WITH columns AS (
+            SELECT current_database()::information_schema.sql_identifier AS table_catalog,
+                nc.nspname::information_schema.sql_identifier AS table_schema,
+                c.relname::information_schema.sql_identifier AS table_name,
+                a.attname::information_schema.sql_identifier AS column_name,
+                a.attnum::information_schema.cardinal_number AS ordinal_position,
+                pg_get_expr(ad.adbin, ad.adrelid)::information_schema.character_data AS column_default,
+                    CASE
+                        WHEN a.attnotnull OR t.typtype = 'd'::"char" AND t.typnotnull THEN 'NO'::text
+                        ELSE 'YES'::text
+                    END::information_schema.yes_or_no AS is_nullable,
+                    CASE
+                        WHEN t.typtype = 'd'::"char" THEN
+                        CASE
+                            WHEN bt.typelem <> 0::oid AND bt.typlen = (-1) THEN 'ARRAY'::text
+                            WHEN nbt.nspname = 'pg_catalog'::name THEN format_type(t.typbasetype, NULL::integer)
+                            ELSE 'USER-DEFINED'::text
+                        END
+                        ELSE
+                        CASE
+                            WHEN t.typelem <> 0::oid AND t.typlen = (-1) THEN 'ARRAY'::text
+                            WHEN nt.nspname = 'pg_catalog'::name THEN format_type(a.atttypid, NULL::integer)
+                            ELSE 'USER-DEFINED'::text
+                        END
+                    END::information_schema.character_data AS data_type,
+                information_schema._pg_char_max_length(information_schema._pg_truetypid(a.*, t.*), information_schema._pg_truetypmod(a.*, t.*))::information_schema.cardinal_number AS character_maximum_length,
+                information_schema._pg_char_octet_length(information_schema._pg_truetypid(a.*, t.*), information_schema._pg_truetypmod(a.*, t.*))::information_schema.cardinal_number AS character_octet_length,
+                information_schema._pg_numeric_precision(information_schema._pg_truetypid(a.*, t.*), information_schema._pg_truetypmod(a.*, t.*))::information_schema.cardinal_number AS numeric_precision,
+                information_schema._pg_numeric_precision_radix(information_schema._pg_truetypid(a.*, t.*), information_schema._pg_truetypmod(a.*, t.*))::information_schema.cardinal_number AS numeric_precision_radix,
+                information_schema._pg_numeric_scale(information_schema._pg_truetypid(a.*, t.*), information_schema._pg_truetypmod(a.*, t.*))::information_schema.cardinal_number AS numeric_scale,
+                information_schema._pg_datetime_precision(information_schema._pg_truetypid(a.*, t.*), information_schema._pg_truetypmod(a.*, t.*))::information_schema.cardinal_number AS datetime_precision,
+                information_schema._pg_interval_type(information_schema._pg_truetypid(a.*, t.*), information_schema._pg_truetypmod(a.*, t.*))::information_schema.character_data AS interval_type,
+                NULL::integer::information_schema.cardinal_number AS interval_precision,
+                NULL::character varying::information_schema.sql_identifier AS character_set_catalog,
+                NULL::character varying::information_schema.sql_identifier AS character_set_schema,
+                NULL::character varying::information_schema.sql_identifier AS character_set_name,
+                    CASE
+                        WHEN nco.nspname IS NOT NULL THEN current_database()
+                        ELSE NULL::name
+                    END::information_schema.sql_identifier AS collation_catalog,
+                nco.nspname::information_schema.sql_identifier AS collation_schema,
+                co.collname::information_schema.sql_identifier AS collation_name,
+                    CASE
+                        WHEN t.typtype = 'd'::"char" THEN current_database()
+                        ELSE NULL::name
+                    END::information_schema.sql_identifier AS domain_catalog,
+                    CASE
+                        WHEN t.typtype = 'd'::"char" THEN nt.nspname
+                        ELSE NULL::name
+                    END::information_schema.sql_identifier AS domain_schema,
+                    CASE
+                        WHEN t.typtype = 'd'::"char" THEN t.typname
+                        ELSE NULL::name
+                    END::information_schema.sql_identifier AS domain_name,
+                current_database()::information_schema.sql_identifier AS udt_catalog,
+                COALESCE(nbt.nspname, nt.nspname)::information_schema.sql_identifier AS udt_schema,
+                COALESCE(bt.typname, t.typname)::information_schema.sql_identifier AS udt_name,
+                NULL::character varying::information_schema.sql_identifier AS scope_catalog,
+                NULL::character varying::information_schema.sql_identifier AS scope_schema,
+                NULL::character varying::information_schema.sql_identifier AS scope_name,
+                NULL::integer::information_schema.cardinal_number AS maximum_cardinality,
+                a.attnum::information_schema.sql_identifier AS dtd_identifier,
+                'NO'::character varying::information_schema.yes_or_no AS is_self_referencing,
+                'NO'::character varying::information_schema.yes_or_no AS is_identity,
+                NULL::character varying::information_schema.character_data AS identity_generation,
+                NULL::character varying::information_schema.character_data AS identity_start,
+                NULL::character varying::information_schema.character_data AS identity_increment,
+                NULL::character varying::information_schema.character_data AS identity_maximum,
+                NULL::character varying::information_schema.character_data AS identity_minimum,
+                NULL::character varying::information_schema.yes_or_no AS identity_cycle,
+                'NEVER'::character varying::information_schema.character_data AS is_generated,
+                NULL::character varying::information_schema.character_data AS generation_expression,
+                CASE
+                    WHEN c.relkind = 'r'::"char" OR (c.relkind = ANY (ARRAY['v'::"char", 'f'::"char"])) AND pg_column_is_updatable(c.oid::regclass, a.attnum, false) THEN 'YES'::text
+                    ELSE 'NO'::text
+                END::information_schema.yes_or_no AS is_updatable
+            FROM pg_attribute a
+               LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum
+               JOIN (pg_class c
+               JOIN pg_namespace nc ON c.relnamespace = nc.oid) ON a.attrelid = c.oid
+               JOIN (pg_type t
+               JOIN pg_namespace nt ON t.typnamespace = nt.oid) ON a.atttypid = t.oid
+               LEFT JOIN (pg_type bt
+               JOIN pg_namespace nbt ON bt.typnamespace = nbt.oid) ON t.typtype = 'd'::"char" AND t.typbasetype = bt.oid
+               LEFT JOIN (pg_collation co
+               JOIN pg_namespace nco ON co.collnamespace = nco.oid) ON a.attcollation = co.oid AND (nco.nspname <> 'pg_catalog'::name OR co.collname <> 'default'::name)
+            WHERE NOT pg_is_other_temp_schema(nc.oid) AND a.attnum > 0 AND NOT a.attisdropped AND (c.relkind = ANY (ARRAY['r'::"char", 'v'::"char", 'f'::"char"]))
+              /*--AND (pg_has_role(c.relowner, 'USAGE'::text) OR has_column_privilege(c.oid, a.attnum, 'SELECT, INSERT, UPDATE, REFERENCES'::text))*/
+        )
+        SELECT
+            table_schema,
+            table_name,
+            column_name,
+            ordinal_position,
+            is_nullable,
+            data_type,
+            is_updatable,
+            character_maximum_length,
+            numeric_precision,
+            column_default,
+            udt_name
+        /*-- FROM information_schema.columns*/
+        FROM columns
+        WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
+    ) AS info
+    LEFT OUTER JOIN (
+        SELECT
+            n.nspname AS s,
+            t.typname AS n,
+            array_agg(e.enumlabel ORDER BY e.enumsortorder) AS vals
+        FROM pg_type t
+        JOIN pg_enum e ON t.oid = e.enumtypid
+        JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
+        GROUP BY s,n
+    ) AS enum_info ON (info.udt_name = enum_info.n)
+    ORDER BY schema, position |]
 
 columnFromRow :: [Table] ->
-                 (Text,       Text,      Text,
-                  Int,        Bool,      Text,
-                  Bool,       Maybe Int, Maybe Int,
+                 (Text,       Text,        Text,
+                  Int32,      Bool,        Text,
+                  Bool,       Maybe Int32, Maybe Int32,
                   Maybe Text, Maybe Text)
                  -> Maybe Column
 columnFromRow tabs (s, t, n, pos, nul, typ, u, l, p, d, e) = buildColumn <$> table
@@ -244,9 +403,11 @@ columnFromRow tabs (s, t, n, pos, nul, typ, u, l, p, d, e) = buildColumn <$> tab
     parseEnum :: Maybe Text -> [Text]
     parseEnum str = fromMaybe [] $ split (==',') <$> str
 
-allRelations :: [Table] -> [Column] -> H.Tx P.Postgres s [Relation]
-allRelations tabs cols = do
-  rels <- H.listEx $ [H.stmt|
+allRelations :: [Table] -> [Column] -> H.Query () [Relation]
+allRelations tabs cols =
+  H.statement sql HE.unit (decodeRelations tabs cols) True
+ where
+  sql = [q|
     SELECT ns1.nspname AS table_schema,
            tab.relname AS table_name,
            column_info.cols AS columns,
@@ -270,80 +431,203 @@ allRelations tabs cols = do
        LATERAL (SELECT * FROM pg_class WHERE pg_class.oid = confrelid) AS other,
        LATERAL (SELECT * FROM pg_namespace WHERE pg_namespace.oid = other.relnamespace) AS ns2
     WHERE confrelid != 0
-    ORDER BY (conrelid, column_info.nums)
-  |]
-  return $ mapMaybe (relationFromRow tabs cols) rels
+    ORDER BY (conrelid, column_info.nums) |]
 
 relationFromRow :: [Table] -> [Column] -> (Text, Text, [Text], Text, Text, [Text]) -> Maybe Relation
 relationFromRow allTabs allCols (rs, rt, rcs, frs, frt, frcs) =
-  if isJust table && isJust tableF && length cols == length rcs && length colsF == length frcs
-    then Just $ Relation (fromJust table) cols (fromJust tableF) colsF Child Nothing Nothing Nothing
-    else Nothing
+  Relation <$> table <*> cols <*> tableF <*> colsF <*> pure Child <*> pure Nothing <*> pure Nothing <*> pure Nothing
   where
     findTable s t = find (\tbl -> tableSchema tbl == s && tableName tbl == t) allTabs
-    findCols s t cs = filter (\col -> tableSchema (colTable col) == s && tableName (colTable col) == t && colName col `elem` cs) allCols
+    findCol s t c = find (\col -> tableSchema (colTable col) == s && tableName (colTable col) == t && colName col == c) allCols
     table  = findTable rs rt
     tableF = findTable frs frt
-    cols  = findCols rs rt rcs
-    colsF = findCols frs frt frcs
+    cols  = mapM (findCol rs rt) rcs
+    colsF = mapM (findCol frs frt) frcs
 
-allPrimaryKeys :: [Table] -> H.Tx P.Postgres s [PrimaryKey]
-allPrimaryKeys tabs = do
-  pks <- H.listEx $ [H.stmt|
+allPrimaryKeys :: [Table] -> H.Query () [PrimaryKey]
+allPrimaryKeys tabs =
+  H.statement sql HE.unit (decodePks tabs) True
+ where
+  sql = [q|
+    /*
+    -- CTE to replace information_schema.table_constraints to remove owner limit
+    */
+    WITH tc AS (
+        SELECT current_database()::information_schema.sql_identifier AS constraint_catalog,
+            nc.nspname::information_schema.sql_identifier AS constraint_schema,
+            c.conname::information_schema.sql_identifier AS constraint_name,
+            current_database()::information_schema.sql_identifier AS table_catalog,
+            nr.nspname::information_schema.sql_identifier AS table_schema,
+            r.relname::information_schema.sql_identifier AS table_name,
+                CASE c.contype
+                    WHEN 'c'::"char" THEN 'CHECK'::text
+                    WHEN 'f'::"char" THEN 'FOREIGN KEY'::text
+                    WHEN 'p'::"char" THEN 'PRIMARY KEY'::text
+                    WHEN 'u'::"char" THEN 'UNIQUE'::text
+                    ELSE NULL::text
+                END::information_schema.character_data AS constraint_type,
+                CASE
+                    WHEN c.condeferrable THEN 'YES'::text
+                    ELSE 'NO'::text
+                END::information_schema.yes_or_no AS is_deferrable,
+                CASE
+                    WHEN c.condeferred THEN 'YES'::text
+                    ELSE 'NO'::text
+                END::information_schema.yes_or_no AS initially_deferred
+        FROM pg_namespace nc,
+            pg_namespace nr,
+            pg_constraint c,
+            pg_class r
+        WHERE nc.oid = c.connamespace AND nr.oid = r.relnamespace AND c.conrelid = r.oid AND (c.contype <> ALL (ARRAY['t'::"char", 'x'::"char"])) AND r.relkind = 'r'::"char" AND NOT pg_is_other_temp_schema(nr.oid)
+        /*--AND (pg_has_role(r.relowner, 'USAGE'::text) OR has_table_privilege(r.oid, 'INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER'::text) OR has_any_column_privilege(r.oid, 'INSERT, UPDATE, REFERENCES'::text))*/
+        UNION ALL
+        SELECT current_database()::information_schema.sql_identifier AS constraint_catalog,
+            nr.nspname::information_schema.sql_identifier AS constraint_schema,
+            (((((nr.oid::text || '_'::text) || r.oid::text) || '_'::text) || a.attnum::text) || '_not_null'::text)::information_schema.sql_identifier AS constraint_name,
+            current_database()::information_schema.sql_identifier AS table_catalog,
+            nr.nspname::information_schema.sql_identifier AS table_schema,
+            r.relname::information_schema.sql_identifier AS table_name,
+            'CHECK'::character varying::information_schema.character_data AS constraint_type,
+            'NO'::character varying::information_schema.yes_or_no AS is_deferrable,
+            'NO'::character varying::information_schema.yes_or_no AS initially_deferred
+        FROM pg_namespace nr,
+            pg_class r,
+            pg_attribute a
+        WHERE nr.oid = r.relnamespace AND r.oid = a.attrelid AND a.attnotnull AND a.attnum > 0 AND NOT a.attisdropped AND r.relkind = 'r'::"char" AND NOT pg_is_other_temp_schema(nr.oid)
+        /*--AND (pg_has_role(r.relowner, 'USAGE'::text) OR has_table_privilege(r.oid, 'INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER'::text) OR has_any_column_privilege(r.oid, 'INSERT, UPDATE, REFERENCES'::text))*/
+    ),
+    /*
+    -- CTE to replace information_schema.key_column_usage to remove owner limit
+    */
+    kc AS (
+        SELECT current_database()::information_schema.sql_identifier AS constraint_catalog,
+            ss.nc_nspname::information_schema.sql_identifier AS constraint_schema,
+            ss.conname::information_schema.sql_identifier AS constraint_name,
+            current_database()::information_schema.sql_identifier AS table_catalog,
+            ss.nr_nspname::information_schema.sql_identifier AS table_schema,
+            ss.relname::information_schema.sql_identifier AS table_name,
+            a.attname::information_schema.sql_identifier AS column_name,
+            (ss.x).n::information_schema.cardinal_number AS ordinal_position,
+                CASE
+                    WHEN ss.contype = 'f'::"char" THEN information_schema._pg_index_position(ss.conindid, ss.confkey[(ss.x).n])
+                    ELSE NULL::integer
+                END::information_schema.cardinal_number AS position_in_unique_constraint
+        FROM pg_attribute a,
+            ( SELECT r.oid AS roid,
+                r.relname,
+                r.relowner,
+                nc.nspname AS nc_nspname,
+                nr.nspname AS nr_nspname,
+                c.oid AS coid,
+                c.conname,
+                c.contype,
+                c.conindid,
+                c.confkey,
+                c.confrelid,
+                information_schema._pg_expandarray(c.conkey) AS x
+               FROM pg_namespace nr,
+                pg_class r,
+                pg_namespace nc,
+                pg_constraint c
+              WHERE nr.oid = r.relnamespace AND r.oid = c.conrelid AND nc.oid = c.connamespace AND (c.contype = ANY (ARRAY['p'::"char", 'u'::"char", 'f'::"char"])) AND r.relkind = 'r'::"char" AND NOT pg_is_other_temp_schema(nr.oid)) ss
+        WHERE ss.roid = a.attrelid AND a.attnum = (ss.x).x AND NOT a.attisdropped
+        /*--AND (pg_has_role(ss.relowner, 'USAGE'::text) OR has_column_privilege(ss.roid, a.attnum, 'SELECT, INSERT, UPDATE, REFERENCES'::text))*/
+    )
     SELECT
         kc.table_schema,
         kc.table_name,
         kc.column_name
     FROM
-        information_schema.table_constraints tc,
-        information_schema.key_column_usage kc
+        /*
+        --information_schema.table_constraints tc,
+        --information_schema.key_column_usage kc
+        */
+        tc, kc
     WHERE
         tc.constraint_type = 'PRIMARY KEY' AND
         kc.table_name = tc.table_name AND
         kc.table_schema = tc.table_schema AND
         kc.constraint_name = tc.constraint_name AND
-        kc.table_schema NOT IN ('pg_catalog', 'information_schema')
-    |]
-  return $ mapMaybe (pkFromRow tabs) pks
+        kc.table_schema NOT IN ('pg_catalog', 'information_schema') |]
 
 pkFromRow :: [Table] -> (Schema, Text, Text) -> Maybe PrimaryKey
 pkFromRow tabs (s, t, n) = PrimaryKey <$> table <*> pure n
   where table = find (\tbl -> tableSchema tbl == s && tableName tbl == t) tabs
 
-allSynonyms :: [Column] -> H.Tx P.Postgres s [(Column,Column)]
-allSynonyms allCols = do
-  syns <- H.listEx $ [H.stmt|
-    WITH synonyms AS (
-      SELECT
-        vcu.table_schema AS src_table_schema,
-        vcu.table_name AS src_table_name,
-        vcu.column_name AS src_column_name,
-        view.table_schema AS syn_table_schema,
-        view.table_name AS syn_table_name,
-        view.view_definition AS view_definition
-      FROM
-        information_schema.views AS view,
-        information_schema.view_column_usage AS vcu
-      WHERE
-        view.table_schema = vcu.view_schema AND
-        view.table_name = vcu.view_name AND
-        view.table_schema NOT IN ('pg_catalog', 'information_schema') AND
-        (SELECT COUNT(*) FROM information_schema.view_table_usage WHERE view_schema = view.table_schema AND view_name = view.table_name) = 1
+allSynonyms :: [Column] -> H.Query () [(Column,Column)]
+allSynonyms cols =
+  H.statement sql HE.unit (decodeSynonyms cols) True
+ where
+  -- query explanation at https://gist.github.com/ruslantalpa/2eab8c930a65e8043d8f
+  sql = [q|
+    WITH view_columns AS (
+    	SELECT
+    		c.oid AS view_oid,
+    		a.attname::information_schema.sql_identifier AS column_name
+    	FROM pg_attribute a
+    	JOIN pg_class c ON a.attrelid = c.oid
+    	JOIN pg_namespace nc ON c.relnamespace = nc.oid
+    	WHERE
+    		NOT pg_is_other_temp_schema(nc.oid)
+    		AND a.attnum > 0
+    		AND NOT a.attisdropped
+    		AND (c.relkind = 'v'::"char")
+    		AND nc.nspname NOT IN ('information_schema', 'pg_catalog')
+    ),
+    view_column_usage AS (
+    	SELECT DISTINCT
+    		v.oid as view_oid,
+    		nv.nspname::information_schema.sql_identifier AS view_schema,
+    		v.relname::information_schema.sql_identifier AS view_name,
+    		nt.nspname::information_schema.sql_identifier AS table_schema,
+    		t.relname::information_schema.sql_identifier AS table_name,
+    		a.attname::information_schema.sql_identifier AS column_name,
+    		pg_get_viewdef(v.oid)::information_schema.character_data AS view_definition
+    	FROM pg_namespace nv
+    	JOIN pg_class v ON nv.oid = v.relnamespace
+    	JOIN pg_depend dv ON v.oid = dv.refobjid
+    	JOIN pg_depend dt ON dv.objid = dt.objid
+    	JOIN pg_class t ON dt.refobjid = t.oid
+    	JOIN pg_namespace nt ON t.relnamespace = nt.oid
+    	JOIN pg_attribute a ON t.oid = a.attrelid AND dt.refobjsubid = a.attnum
+
+    	WHERE
+    		nv.nspname not in ('information_schema', 'pg_catalog')
+    		AND v.relkind = 'v'::"char"
+    		AND dv.refclassid = 'pg_class'::regclass::oid
+    		AND dv.classid = 'pg_rewrite'::regclass::oid
+    		AND dv.deptype = 'i'::"char"
+    		AND dv.refobjid <> dt.refobjid
+    		AND dt.classid = 'pg_rewrite'::regclass::oid
+    		AND dt.refclassid = 'pg_class'::regclass::oid
+    		AND (t.relkind = ANY (ARRAY['r'::"char", 'v'::"char", 'f'::"char"]))
+    ),
+    candidates AS (
+    	SELECT
+    		vcu.*,
+    		(
+    			SELECT CASE WHEN match IS NOT NULL THEN coalesce(match[7], match[4]) END
+    			FROM REGEXP_MATCHES(
+    				CONCAT('SELECT ', SPLIT_PART(vcu.view_definition, 'SELECT', 2)),
+    				CONCAT('SELECT.*?((',vcu.table_name,')|(\w+))\.(', vcu.column_name, ')(\sAS\s(")?([^"]+)\6)?.*?FROM.*?',vcu.table_schema,'\.(\2|',vcu.table_name,'\s+(AS\s)?\3)'),
+    				'ns'
+    			) match
+    		) AS view_column_name
+    	FROM view_column_usage AS vcu
     )
     SELECT
-      src_table_schema, src_table_name, src_column_name,
-      syn_table_schema, syn_table_name,
-      (regexp_matches(view_definition, CONCAT('\.(', src_column_name, ')(?=,|$)'), 'gn'))[1]
-    FROM synonyms
-    UNION (
-      SELECT
-        src_table_schema, src_table_name, src_column_name,
-        syn_table_schema, syn_table_name,
-        (regexp_matches(view_definition, CONCAT('\.', src_column_name, '\sAS\s("?)(.+?)\1(,|$)'), 'gn'))[2] /* " <- for syntax highlighting */
-      FROM synonyms
-    )
+    	c.table_schema,
+    	c.table_name,
+    	c.column_name AS table_column_name,
+    	c.view_schema,
+    	c.view_name,
+    	c.view_column_name
+    FROM view_columns AS vc, candidates AS c
+    WHERE
+    	vc.view_oid = c.view_oid AND
+    	vc.column_name = c.view_column_name
+    ORDER BY c.view_schema, c.view_name, c.table_name, c.view_column_name
     |]
-  return $ mapMaybe (synonymFromRow allCols) syns
 
 synonymFromRow :: [Column] -> (Text,Text,Text,Text,Text,Text) -> Maybe (Column,Column)
 synonymFromRow allCols (s1,t1,c1,s2,t2,c2) = (,) <$> col1 <*> col2

src/PostgREST/Error.hs

@@ -2,58 +2,80 @@
 {-# LANGUAGE FlexibleInstances    #-}
 {-# LANGUAGE TypeSynonymInstances #-}
 
-module PostgREST.Error (PgError, pgErrResponse, errResponse) where
+module PostgREST.Error (pgErrResponse, errResponse) where
 
 
 import           Data.Aeson                ((.=))
 import qualified Data.Aeson                as JSON
+import           Data.Maybe                (fromMaybe)
+import           Data.Monoid               ((<>))
 import           Data.String.Conversions   (cs)
-import           Data.String.Utils         (replace)
 import           Data.Text                 (Text)
 import qualified Data.Text                 as T
-import qualified Hasql                     as H
-import qualified Hasql.Postgres            as P
+import qualified Hasql.Pool                as P
+import qualified Hasql.Session             as H
 import           Network.HTTP.Types.Header
 import qualified Network.HTTP.Types.Status as HT
 import           Network.Wai               (Response, responseLBS)
 
-type PgError = H.SessionError P.Postgres
-
 errResponse :: HT.Status -> Text -> Response
 errResponse status message = responseLBS status [(hContentType, "application/json")] (cs $ T.concat ["{\"message\":\"",message,"\"}"])
 
-pgErrResponse :: PgError -> Response
-pgErrResponse e = responseLBS (httpStatus e)
-  [(hContentType, "application/json")] (JSON.encode e)
+pgErrResponse :: Bool -> P.UsageError -> Response
+pgErrResponse authed e =
+  let status = httpStatus authed e
+      jsonType = (hContentType, "application/json")
+      wwwAuth = ("WWW-Authenticate", "Bearer")
+      hdrs = if status == HT.status401
+                then [jsonType, wwwAuth]
+                else [jsonType] in
+  responseLBS status hdrs (JSON.encode e)
 
-instance JSON.ToJSON PgError where
-  toJSON (H.TxError (P.ErroneousResult c m d h)) = JSON.object [
+instance JSON.ToJSON P.UsageError where
+  toJSON (P.ConnectionError e) = JSON.object [
+    "code" .= ("" :: T.Text),
+    "message" .= ("Connection error" :: T.Text),
+    "details" .= (cs (fromMaybe "" e) :: T.Text)]
+  toJSON (P.SessionError e) = JSON.toJSON e -- H.Error
+
+instance JSON.ToJSON H.Error where
+  toJSON (H.ResultError (H.ServerError c m d h)) = JSON.object [
     "code" .= (cs c::T.Text),
     "message" .= (cs m::T.Text),
     "details" .= (fmap cs d::Maybe T.Text),
     "hint" .= (fmap cs h::Maybe T.Text)]
-  toJSON (H.TxError (P.NoResult d)) = JSON.object [
-    "message" .= ("No response from server"::T.Text),
+  toJSON (H.ResultError (H.UnexpectedResult m)) = JSON.object [
+    "message" .= (cs m::T.Text)]
+  toJSON (H.ResultError (H.RowError i H.EndOfInput)) = JSON.object [
+    "message" .= ("Row error: end of input"::String),
+    "details" .=
+      ("Attempt to parse more columns than there are in the result"::String),
+    "details" .= ("Row number " <> show i)]
+  toJSON (H.ResultError (H.RowError i H.UnexpectedNull)) = JSON.object [
+    "message" .= ("Row error: unexpected null"::String),
+    "details" .= ("Attempt to parse a NULL as some value."::String),
+    "details" .= ("Row number " <> show i)]
+  toJSON (H.ResultError (H.RowError i (H.ValueError d))) = JSON.object [
+    "message" .= ("Row error: Wrong value parser used"::String),
+    "details" .= d,
+    "details" .= ("Row number " <> show i)]
+  toJSON (H.ResultError (H.UnexpectedAmountOfRows i)) = JSON.object [
+    "message" .= ("Unexpected amount of rows"::String),
+    "details" .= i]
+  toJSON (H.ClientError d) = JSON.object [
+    "message" .= ("Database client error"::String),
     "details" .= (fmap cs d::Maybe T.Text)]
-  toJSON (H.TxError (P.UnexpectedResult m)) = JSON.object ["message" .= m]
-  toJSON (H.TxError P.NotInTransaction) = JSON.object [
-    "message" .= ("Not in transaction"::T.Text)]
-  toJSON (H.CxError (P.CantConnect d)) = JSON.object [
-    "message" .= ("Can't connect to the database"::T.Text),
-    "details" .= (fmap cs d::Maybe T.Text)]
-  toJSON (H.CxError (P.UnsupportedVersion v)) = JSON.object [
-    "message" .= ("Postgres version "++version++" is not supported") ]
-      where version = replace "0" "." (show v)
-  toJSON (H.ResultError m) = JSON.object ["message" .= m]
 
-httpStatus :: PgError -> HT.Status
-httpStatus (H.TxError (P.ErroneousResult codeBS _ _ _)) =
-  let code = cs codeBS in
-  case code of
+httpStatus :: Bool -> P.UsageError -> HT.Status
+httpStatus _ (P.ConnectionError _) = HT.status500
+httpStatus authed (P.SessionError (H.ResultError (H.ServerError c _ _ _))) =
+  case cs c of
     '0':'8':_ -> HT.status503 -- pg connection err
     '0':'9':_ -> HT.status500 -- triggered action exception
     '0':'L':_ -> HT.status403 -- invalid grantor
     '0':'P':_ -> HT.status403 -- invalid role specification
+    "23503"   -> HT.status409 -- foreign_key_violation
+    "23505"   -> HT.status409 -- unique_violation
     '2':'5':_ -> HT.status500 -- invalid tx state
     '2':'8':_ -> HT.status403 -- invalid auth specification
     '2':'D':_ -> HT.status500 -- invalid tx termination
@@ -70,8 +92,8 @@ httpStatus (H.TxError (P.ErroneousResult codeBS _ _ _)) =
     'H':'V':_ -> HT.status500 -- foreign data wrapper error
     'P':'0':_ -> HT.status500 -- PL/pgSQL Error
     'X':'X':_ -> HT.status500 -- internal Error
-    "42P01" -> HT.status404 -- undefined table
-    "42501" -> HT.status404 -- insufficient privilege
-    _ -> HT.status400
-httpStatus (H.TxError (P.NoResult _)) = HT.status503
-httpStatus _ = HT.status500
+    "42P01"   -> HT.status404 -- undefined table
+    "42501"   -> if authed then HT.status403 else HT.status401 -- insufficient privilege
+    _         -> HT.status400
+httpStatus _ (P.SessionError (H.ResultError _)) = HT.status500
+httpStatus _ (P.SessionError (H.ClientError _)) = HT.status503

src/PostgREST/Main.hs

@@ -1,79 +0,0 @@
-module Main where
-
-
-import           PostgREST.App
-import           PostgREST.Config                     (AppConfig (..),
-                                                       minimumPgVersion,
-                                                       prettyVersion,
-                                                       readOptions)
-import           PostgREST.Error                      (pgErrResponse, PgError)
-import           PostgREST.Middleware
-import           PostgREST.DbStructure
-
-import           Control.Monad                        (unless)
-import           Control.Monad.IO.Class               (liftIO)
-import           Data.Aeson                           (encode)
-import           Data.Functor.Identity
-import           Data.Monoid                          ((<>))
-import           Data.String.Conversions              (cs)
-import           Data.Text                            (Text)
-import qualified Hasql                                as H
-import qualified Hasql.Postgres                       as P
-import           Network.Wai
-import           Network.Wai.Handler.Warp             hiding (Connection)
-import           Network.Wai.Middleware.RequestLogger (logStdout)
-import           System.IO                            (BufferMode (..),
-                                                       hSetBuffering, stderr,
-                                                       stdin, stdout)
-import           Web.JWT                              (secret)
-
-isServerVersionSupported :: H.Session P.Postgres IO Bool
-isServerVersionSupported = do
-  Identity (row :: Text) <- H.tx Nothing $ H.singleEx [H.stmt|SHOW server_version_num|]
-  return $ read (cs row) >= minimumPgVersion
-
-hasqlError :: PgError -> IO a
-hasqlError = error . cs . encode
-
-main :: IO ()
-main = do
-  hSetBuffering stdout LineBuffering
-  hSetBuffering stdin  LineBuffering
-  hSetBuffering stderr NoBuffering
-
-  conf <- readOptions
-  let port = configPort conf
-
-  unless (secret "secret" /= configJwtSecret conf) $
-    putStrLn "WARNING, running in insecure mode, JWT secret is the default value"
-  Prelude.putStrLn $ "Listening on port " ++
-    (show $ configPort conf :: String)
-
-  let pgSettings = P.StringSettings $ cs (configDatabase conf)
-      appSettings = setPort port
-                  . setServerName (cs $ "postgrest/" <> prettyVersion)
-                  $ defaultSettings
-      middle = logStdout . defaultMiddle
-
-  poolSettings <- maybe (fail "Improper session settings") return $
-    H.poolSettings (fromIntegral $ configPool conf) 30
-  pool :: H.Pool P.Postgres <- H.acquirePool pgSettings poolSettings
-
-  supportedOrError <- H.session pool isServerVersionSupported
-  either hasqlError
-    (\supported ->
-      unless supported $
-        error (
-          "Cannot run in this PostgreSQL version, PostgREST needs at least "
-          <> show minimumPgVersion)
-    ) supportedOrError
-
-  let txSettings = Just (H.ReadCommitted, Just True)
-  dbOrError <- H.session pool $ H.tx txSettings $ getDbStructure (cs $ configSchema conf)
-  dbStructure <- either hasqlError return dbOrError
-
-  runSettings appSettings $ middle $ \ req respond -> do
-    body <- strictRequestBody req
-    resOrError <- liftIO $ H.session pool $ H.tx txSettings $
-      runWithClaims conf (app dbStructure conf body) req
-    either (respond . pgErrResponse) respond resOrError

src/PostgREST/Middleware.hs

@@ -3,59 +3,40 @@
 
 module PostgREST.Middleware where
 
-import           Data.Maybe                    (fromMaybe)
-import           Data.Text
+import           Data.Aeson                    (Value (..))
+import qualified Data.HashMap.Strict           as M
 import           Data.String.Conversions       (cs)
-import           Data.Time.Clock.POSIX         (getPOSIXTime)
-import qualified Hasql                         as H
-import qualified Hasql.Postgres                as P
+import           Data.Text
+import qualified Hasql.Transaction             as H
 
-import           Network.HTTP.Types.Header     (hAccept, hAuthorization)
-import           Network.HTTP.Types.Status     (status415, status400)
-import           Network.Wai                   (Application, Request (..), Response,
-                                                requestHeaders)
+import           Network.HTTP.Types.Header     (hAccept)
+import           Network.HTTP.Types.Status     (status400, status415)
+import           Network.Wai                   (Application, Request (..),
+                                                Response, requestHeaders)
 import           Network.Wai.Middleware.Cors   (cors)
 import           Network.Wai.Middleware.Gzip   (def, gzip)
 import           Network.Wai.Middleware.Static (only, staticPolicy)
 
-import           PostgREST.ApiRequest       (pickContentType)
-import           PostgREST.Auth                (setRole, jwtClaims, claimsToSQL)
+import           PostgREST.ApiRequest          (ApiRequest(..), pickContentType)
+import           PostgREST.Auth                (claimsToSQL)
 import           PostgREST.Config              (AppConfig (..), corsPolicy)
 import           PostgREST.Error               (errResponse)
 
-import           System.IO.Unsafe              (unsafePerformIO)
+import           Prelude                       hiding (concat, null)
 
-import           Prelude hiding(concat)
-
-import qualified Data.Vector             as V
-import qualified Hasql.Backend           as B
-import qualified Data.Map.Lazy           as M
-
-runWithClaims :: forall s. AppConfig ->
-                 (Request -> H.Tx P.Postgres s Response) ->
-                 Request -> H.Tx P.Postgres s Response
-runWithClaims conf app req = do
-    _ <- H.unitEx $ stmt setAnon
-    let time = unsafePerformIO getPOSIXTime
-    case split (== ' ') (cs auth) of
-      ("Bearer" : tokenStr : _) ->
-        case jwtClaims jwtSecret tokenStr time of
-          Just claims ->
-            if M.member "role" claims
-            then do
-              mapM_ H.unitEx $ stmt <$> claimsToSQL claims
-              app req
-            else invalidJWT
-          _ -> invalidJWT
-      _ -> app req
+runWithClaims :: AppConfig -> Either Text (M.HashMap Text Value) ->
+                 (ApiRequest -> H.Transaction Response) ->
+                 ApiRequest -> H.Transaction Response
+runWithClaims conf eClaims app req =
+  case eClaims of
+    Left e -> clientErr e
+    Right claims -> do
+          -- role claim defaults to anon if not specified in jwt
+          H.sql . mconcat . claimsToSQL $ M.union claims (M.singleton "role" anon)
+          app req
   where
-    stmt c = B.Stmt c V.empty True
-    hdrs = requestHeaders req
-    jwtSecret = configJwtSecret conf
-    auth = fromMaybe "" $ lookup hAuthorization hdrs
-    anon = cs $ configAnonRole conf
-    setAnon = setRole anon
-    invalidJWT = return $ errResponse status400 "Invalid JWT"
+    anon = String . cs $ configAnonRole conf
+    clientErr = return . errResponse status400
 
 unsupportedAccept :: Application -> Application
 unsupportedAccept app req respond =

src/PostgREST/Parsers.hs

@@ -3,25 +3,30 @@ module PostgREST.Parsers
 -- )
 where
 
-import           Control.Applicative hiding ((<$>))
+import           Control.Applicative           hiding ((<$>))
 import           Data.Monoid
 import           Data.String.Conversions       (cs)
-import           Data.Text                     (Text)
+import           Data.Text                     (Text, intercalate)
 import           Data.Tree
+import           PostgREST.QueryBuilder        (operators)
 import           PostgREST.Types
 import           Text.ParserCombinators.Parsec hiding (many, (<|>))
-import           PostgREST.QueryBuilder (operators)
+import           PostgREST.RangeQuery      (NonnegRange,allRange)
 
 pRequestSelect :: Text -> Parser ReadRequest
 pRequestSelect rootNodeName = do
   fieldTree <- pFieldForest
-  return $ foldr treeEntry (Node (Select [] [rootNodeName] [] Nothing, (rootNodeName, Nothing)) []) fieldTree
+  return $ foldr treeEntry (Node (readQuery, (rootNodeName, Nothing, Nothing)) []) fieldTree
   where
+    readQuery = Select [] [rootNodeName] [] Nothing allRange
     treeEntry :: Tree SelectItem -> ReadRequest -> ReadRequest
-    treeEntry (Node fld@((fn, _),_) fldForest) (Node (q, i) rForest) =
+    treeEntry (Node fld@((fn, _),_,alias) fldForest) (Node (q, i) rForest) =
       case fldForest of
         [] -> Node (q {select=fld:select q}, i) rForest
-        _  -> Node (q, i) (foldr treeEntry (Node (Select [] [fn] [] Nothing, (fn, Nothing)) []) fldForest:rForest)
+        _  -> Node (q, i) newForest
+          where
+            newForest =
+              foldr treeEntry (Node (Select [] [fn] [] Nothing allRange, (fn, Nothing, alias)) []) fldForest:rForest
 
 pRequestFilter :: (String, String) -> Either ParseError (Path, Filter)
 pRequestFilter (k, v) = (,) <$> path <*> (Filter <$> fld <*> op <*> val)
@@ -33,6 +38,19 @@ pRequestFilter (k, v) = (,) <$> path <*> (Filter <$> fld <*> op <*> val)
     op = fst <$> opVal
     val = snd <$> opVal
 
+pRequestOrder :: (String, String) -> Either ParseError (Path, [OrderTerm])
+pRequestOrder (k, v) = (,) <$> path <*> ord
+  where
+    treePath = parse pTreePath ("failed to parser tree path (" ++ k ++ ")") k
+    path = fst <$> treePath
+    ord = parse pOrder ("failed to parse order (" ++ v ++ ")") v
+
+pRequestRange :: (String, NonnegRange) -> Either ParseError (Path, NonnegRange)
+pRequestRange (k, v) = (,) <$> path <*> pure v
+  where
+    treePath = parse pTreePath ("failed to parser tree path (" ++ k ++ ")") k
+    path = fst <$> treePath
+
 ws :: Parser Text
 ws = cs <$> many (oneOf " \t")
 
@@ -51,15 +69,23 @@ pFieldForest :: Parser [Tree SelectItem]
 pFieldForest = pFieldTree `sepBy1` lexeme (char ',')
 
 pFieldTree :: Parser (Tree SelectItem)
-pFieldTree = try (Node <$> pSelect <*> between (char '{') (char '}') pFieldForest)
+pFieldTree = try (Node <$> pSimpleSelect <*> between (char '{') (char '}') pFieldForest)
           <|>     Node <$> pSelect <*> pure []
 
 pStar :: Parser Text
 pStar = cs <$> (string "*" *> pure ("*"::String))
 
+
 pFieldName :: Parser Text
-pFieldName =  cs <$> (many1 (letter <|> digit <|> oneOf "_")
-          <?> "field name (* or [a..z0..9_])")
+pFieldName = do
+  matches <- (many1 (letter <|> digit <|> oneOf "_") `sepBy1` dash) <?> "field name (* or [a..z0..9_])"
+  return $ intercalate "-" $ map cs matches
+  where
+    isDash :: GenParser Char st ()
+    isDash = try ( char '-' >> notFollowedBy (char '>') )
+    dash :: Parser Char
+    dash = isDash *> pure '-'
+
 
 pJsonPathStep :: Parser Text
 pJsonPathStep = cs <$> try (string "->" *> pFieldName)
@@ -70,12 +96,28 @@ pJsonPath = (++) <$> many pJsonPathStep <*> ( (:[]) <$> (string "->>" *> pFieldN
 pField :: Parser Field
 pField = lexeme $ (,) <$> pFieldName <*> optionMaybe pJsonPath
 
+aliasSeparator :: Parser ()
+aliasSeparator = char ':' >> notFollowedBy (char ':')
+
+pSimpleSelect :: Parser SelectItem
+pSimpleSelect = lexeme $ try ( do
+    alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
+    fld <- pField
+    return (fld, Nothing, alias)
+  )
+
 pSelect :: Parser SelectItem
 pSelect = lexeme $
-  try ((,) <$> pField <*>((cs <$>) <$> optionMaybe (string "::" *> many letter)) )
+  try (
+    do
+      alias <- optionMaybe ( try(pFieldName <* aliasSeparator) )
+      fld <- pField
+      cast <- optionMaybe (string "::" *> many letter)
+      return (fld, cs <$> cast, alias)
+  )
   <|> do
     s <- pStar
-    return ((s, Nothing), Nothing)
+    return ((s, Nothing), Nothing, Nothing)
 
 pOperator :: Parser Operator
 pOperator = cs <$> (pOp <?> "operator (eq, gt, ...)")

src/PostgREST/QueryBuilder.hs

@@ -1,5 +1,6 @@
 {-# LANGUAGE FlexibleInstances    #-}
-{-# LANGUAGE TupleSections #-}
+{-# LANGUAGE FlexibleContexts     #-}
+{-# LANGUAGE TupleSections        #-}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 {-|
 Module      : PostgREST.QueryBuilder
@@ -14,7 +15,6 @@ Any function that outputs a SQL fragment should be in this module.
 module PostgREST.QueryBuilder (
     addRelations
   , addJoinConditions
-  , asJson
   , callProc
   , createReadStatement
   , createWriteStatement
@@ -22,31 +22,36 @@ module PostgREST.QueryBuilder (
   , pgFmtIdent
   , pgFmtLit
   , requestToQuery
-  , sourceSubqueryName
+  , requestToCountQuery
+  , sourceCTEName
   , unquoted
+  , ResultsWithCount
   ) where
 
-import qualified Hasql                   as H
-import qualified Hasql.Backend           as B
-import qualified Hasql.Postgres          as P
+import qualified Hasql.Query             as H
+import qualified Hasql.Encoders          as HE
+import qualified Hasql.Decoders          as HD
 
 import qualified Data.Aeson              as JSON
+import           Data.Int                (Int64)
 
-import           PostgREST.RangeQuery    (NonnegRange, rangeLimit, rangeOffset)
-import           Control.Error           (note, fromMaybe, mapMaybe)
-import           Data.Maybe              (isNothing)
-import           Control.Monad           (join)
+import           PostgREST.RangeQuery    (NonnegRange, rangeLimit, rangeOffset, allRange)
+import           Control.Error           (note, fromMaybe)
+import           Data.Functor.Contravariant (contramap)
 import qualified Data.HashMap.Strict     as HM
 import           Data.List               (find)
 import           Data.Monoid             ((<>))
 import           Data.Text               (Text, intercalate, unwords, replace, isInfixOf, toLower, split)
-import qualified Data.Text as T          (map, takeWhile)
+import qualified Data.Text as T          (map, takeWhile, null)
+import qualified Data.Text.Encoding as T
 import           Data.String.Conversions (cs)
-import           Control.Applicative     (empty, (<|>))
+import           Control.Applicative     ((<|>))
+import           Control.Monad           (replicateM)
 import           Data.Tree               (Tree(..))
 import qualified Data.Vector as V
 import           PostgREST.Types
 import qualified Data.Map as M
+import           Text.InterpolatedString.Perl6 (qc)
 import           Text.Regex.TDFA         ((=~))
 import qualified Data.ByteString.Char8   as BS
 import           Data.Scientific         ( FPFormat (..)
@@ -54,101 +59,173 @@ import           Data.Scientific         ( FPFormat (..)
                                          , isInteger
                                          )
 import           Prelude hiding          (unwords)
+import           PostgREST.ApiRequest    (PreferRepresentation (..))
 
-import           Data.Ranged.Ranges      (singletonRange)
+{-| The generic query result format used by API responses. The location header
+    is represented as a list of strings containing variable bindings like
+    @"k1=eq.42"@, or the empty list if there is no location header.
+-}
+type ResultsWithCount = (Maybe Int64, Int64, [BS.ByteString], BS.ByteString)
 
-type PStmt = H.Stmt P.Postgres
-instance Monoid PStmt where
-  mappend (B.Stmt query params prep) (B.Stmt query' params' prep') =
-    B.Stmt (query <> query') (params <> params') (prep && prep')
-  mempty = B.Stmt "" empty True
-type StatementT = PStmt -> PStmt
+standardRow :: HD.Row ResultsWithCount
+standardRow = (,,,) <$> HD.nullableValue HD.int8 <*> HD.value HD.int8
+                    <*> HD.value header <*> HD.value HD.bytea
+  where
+    header = HD.array $ HD.arrayDimension replicateM $ HD.arrayValue HD.bytea
 
-createReadStatement :: SqlQuery -> Maybe NonnegRange -> Bool -> Bool -> Bool -> B.Stmt P.Postgres
-createReadStatement selectQuery range isSingle countTable asCsv =
-  B.Stmt (
-    wrapQuery selectQuery [
-      if countTable then countAllF else countNoneF,
-      countF,
-      "null", -- location header can not be calucalted
-      if asCsv
-        then asCsvF
-        else if isSingle then asJsonSingleF else asJsonF
-    ] selectStarF (if isNothing range && isSingle then Just $ singletonRange 0 else range)
-  ) V.empty True
+noLocationF :: Text
+noLocationF = "array[]::text[]"
 
-createWriteStatement :: SqlQuery -> SqlQuery -> Bool -> Bool ->
-                        [Text] -> Bool -> Payload -> B.Stmt P.Postgres
-createWriteStatement _ _ _ _ _ _ (PayloadParseError _) = undefined
-createWriteStatement selectQuery mutateQuery isSingle echoRequested
-                     pKeys asCsv (PayloadJSON (UniformObjects rows)) =
-  B.Stmt (
-    wrapQuery mutateQuery [
-      countNoneF, -- when updateing it does not make sense
-      countF,
-      if isSingle then locationF pKeys else "null",
-      if echoRequested
-      then
-        if asCsv
-        then asCsvF
-        else if isSingle then asJsonSingleF else asJsonF
-      else "null"
+{-| Read and Write api requests use a similar response format which includes
+    various record counts and possible location header. This is the decoder
+    for that common type of query.
+-}
+decodeStandard :: HD.Result ResultsWithCount
+decodeStandard =
+  HD.singleRow standardRow
 
-    ] selectQuery Nothing
-  ) (V.singleton . B.encodeValue . JSON.Array . V.map JSON.Object $ rows) True
+decodeStandardMay :: HD.Result (Maybe ResultsWithCount)
+decodeStandardMay =
+  HD.maybeRow standardRow
+
+{-| JSON and CSV payloads from the client are given to us as
+    UniformObjects (objects who all have the same keys),
+    and we turn this into an old fasioned JSON array
+-}
+encodeUniformObjs :: HE.Params UniformObjects
+encodeUniformObjs =
+  contramap (JSON.Array . V.map JSON.Object . unUniformObjects) (HE.value HE.json)
+
+createReadStatement :: SqlQuery -> SqlQuery -> Bool -> Bool -> Bool ->
+                       H.Query () ResultsWithCount
+createReadStatement selectQuery countQuery isSingle countTotal asCsv =
+  unicodeStatement sql HE.unit decodeStandard True
+ where
+  sql = [qc|
+      WITH {sourceCTEName} AS ({selectQuery}) SELECT {cols}
+      FROM ( SELECT * FROM {sourceCTEName}) t |]
+  countResultF = if countTotal then "("<>countQuery<>")" else "null"
+  cols = intercalate ", " [
+      countResultF <> " AS total_result_set",
+      "pg_catalog.count(t) AS page_total",
+      noLocationF <> " AS header",
+      bodyF <> " AS body"
+    ]
+  bodyF
+    | asCsv = asCsvF
+    | isSingle = asJsonSingleF
+    | otherwise = asJsonF
+
+createWriteStatement :: QualifiedIdentifier -> SqlQuery -> SqlQuery -> Bool ->
+                        PreferRepresentation -> [Text] -> Bool -> Payload ->
+                        H.Query UniformObjects (Maybe ResultsWithCount)
+createWriteStatement _ _ _ _ _ _ _ (PayloadParseError _) = undefined
+createWriteStatement _ _ mutateQuery _ None
+                     _ _ (PayloadJSON (UniformObjects _)) =
+  unicodeStatement sql encodeUniformObjs decodeStandardMay True
+ where
+  sql = [qc|
+      WITH {sourceCTEName} AS ({mutateQuery})
+      SELECT '', 0, {noLocationF}, '' |]
+
+createWriteStatement qi _ mutateQuery isSingle HeadersOnly
+                     pKeys _ (PayloadJSON (UniformObjects _)) =
+  unicodeStatement sql encodeUniformObjs decodeStandardMay True
+ where
+  sql = [qc|
+      WITH {sourceCTEName} AS ({mutateQuery} RETURNING {fromQi qi}.*)
+      SELECT {cols}
+      FROM (SELECT 1 FROM {sourceCTEName}) t |]
+  cols = intercalate ", " [
+      "'' AS total_result_set",
+      "pg_catalog.count(t) AS page_total",
+      if isSingle then locationF pKeys else noLocationF,
+      "''"
+    ]
+
+createWriteStatement qi selectQuery mutateQuery isSingle Full
+                     pKeys asCsv (PayloadJSON (UniformObjects _)) =
+  unicodeStatement sql encodeUniformObjs decodeStandardMay True
+ where
+  sql = [qc|
+      WITH {sourceCTEName} AS ({mutateQuery} RETURNING {fromQi qi}.*)
+      SELECT {cols}
+      FROM ({selectQuery}) t |]
+  cols = intercalate ", " [
+      "'' AS total_result_set", -- when updateing it does not make sense
+      "pg_catalog.count(t) AS page_total",
+      if isSingle then locationF pKeys else noLocationF <> " AS header",
+      bodyF <> " AS body"
+    ]
+  bodyF
+    | asCsv = asCsvF
+    | isSingle = asJsonSingleF
+    | otherwise = asJsonF
 
 addRelations :: Schema -> [Relation] -> Maybe ReadRequest -> ReadRequest -> Either Text ReadRequest
-addRelations schema allRelations parentNode node@(Node n@(query, (table, _)) forest) =
+addRelations schema allRelations parentNode node@(Node readNode@(query, (name, _, alias)) forest) =
   case parentNode of
-    Nothing -> Node (query, (table, Nothing)) <$> updatedForest
-    (Just (Node (_, (parentTable, _)) _)) -> Node <$> (addRel n <$> rel) <*> updatedForest
+    (Just (Node (Select{from=[parentTable]}, (_, _, _)) _)) -> Node <$> (addRel readNode <$> rel) <*> updatedForest
       where
-        rel = note ("no relation between " <> table <> " and " <> parentTable)
-            $  findRelation schema table parentTable
-           <|> findRelation schema parentTable table
-        addRel :: (ReadQuery, (NodeName, Maybe Relation)) -> Relation -> (ReadQuery, (NodeName, Maybe Relation))
-        addRel (q, (t, _)) r = (q, (t, Just r))
+        rel = note ("no relation between " <> parentTable <> " and " <> name)
+            $  findRelationByTable schema name parentTable
+           <|> findRelationByColumn schema parentTable name
+        addRel :: (ReadQuery, (NodeName, Maybe Relation, Maybe Alias)) -> Relation -> (ReadQuery, (NodeName, Maybe Relation, Maybe Alias))
+        addRel (query', (n, _, a)) r = (query' {from=fromRelation}, (n, Just r, a))
+          where fromRelation = map (\t -> if t == n then tableName (relTable r) else t) (from query')
+
+    _ -> Node (query, (name, Nothing, alias)) <$> updatedForest
   where
     updatedForest = mapM (addRelations schema allRelations (Just node)) forest
-    findRelation s t1 t2 =
-      find (\r -> s == (tableSchema . relTable) r && t1 == (tableName . relTable) r && t2 == (tableName . relFTable) r) allRelations
+    -- Searches through all the relations and returns a match given the parameter conditions.
+    -- Will only find a relation where both schemas are in the PostgREST schema.
+    -- `findRelationByColumn` also does a ducktype check to see if the column name has any variation of `id` or `fk`. If so then the relation is returned as a match.
+    findRelationByTable s t1 t2 =
+      find (\r -> s == tableSchema (relTable r) && s == tableSchema (relFTable r) && t1 == tableName (relTable r) && t2 == tableName (relFTable r)) allRelations
+    findRelationByColumn s t c =
+      find (\r -> s == tableSchema (relTable r) && s == tableSchema (relFTable r) && t == tableName (relFTable r) && length (relFColumns r) == 1 && c `colMatches` (colName . head . relFColumns) r) allRelations
+      where n `colMatches` rc = (cs ("^" <> rc <> "_?(?:|[iI][dD]|[fF][kK])$") :: BS.ByteString) =~ (cs n :: BS.ByteString)
 
 addJoinConditions :: Schema -> ReadRequest -> Either Text ReadRequest
-addJoinConditions schema (Node (query, (n, r)) forest) =
+addJoinConditions schema (Node nn@(query, (n, r, a)) forest) =
   case r of
-    Nothing -> Node (updatedQuery, (n,r))  <$> updatedForest -- this is the root node
-    Just rel@(Relation{relType=Child}) -> Node (addCond updatedQuery (getJoinConditions rel),(n,r)) <$> updatedForest
-    Just (Relation{relType=Parent}) -> Node (updatedQuery, (n,r)) <$> updatedForest
-    Just rel@(Relation{relType=Many, relLTable=(Just linkTable)}) ->
-      Node (qq, (n, r)) <$> updatedForest
+    Nothing -> Node nn  <$> updatedForest -- this is the root node
+    Just rel@Relation{relType=Child} -> Node (addCond query (getJoinConditions rel),(n,r,a)) <$> updatedForest
+    Just Relation{relType=Parent} -> Node nn <$> updatedForest
+    Just rel@Relation{relType=Many, relLTable=(Just linkTable)} ->
+      Node (qq, (n, r, a)) <$> updatedForest
       where
-         q = addCond updatedQuery (getJoinConditions rel)
-         qq = q{from=tableName linkTable : from q}
+         query' = addCond query (getJoinConditions rel)
+         qq = query'{from=tableName linkTable : from query'}
     _ -> Left "unknown relation"
   where
-    -- add parentTable and parentJoinConditions to the query
-    updatedQuery = foldr (flip addCond) (query{from = parentTables ++ from query}) parentJoinConditions
-      where
-        parentJoinConditions = map (getJoinConditions . snd) parents
-        parentTables = map fst parents
-        parents = mapMaybe (getParents . rootLabel) forest
-        getParents (_, (tbl, Just rel@(Relation{relType=Parent}))) = Just (tbl, rel)
-        getParents _ = Nothing
     updatedForest = mapM (addJoinConditions schema) forest
-    addCond q con = q{flt_=con ++ flt_ q}
+    addCond query' con = query'{flt_=con ++ flt_ query'}
 
-asJson :: StatementT
-asJson s = s {
-  B.stmtTemplate =
-    "array_to_json(coalesce(array_agg(row_to_json(t)), '{}'))::character varying from ("
-    <> B.stmtTemplate s <> ") t" }
-
-callProc :: QualifiedIdentifier -> JSON.Object -> PStmt
-callProc qi params = do
-  let args = intercalate "," $ map assignment (HM.toList params)
-  B.Stmt ("select * from " <> fromQi qi <> "(" <> args <> ")") empty True
+type ProcResults = (Maybe Int64, Int64, JSON.Value)
+callProc :: QualifiedIdentifier -> JSON.Object -> NonnegRange -> Bool -> H.Query () (Maybe ProcResults)
+callProc qi params range countTotal =
+  unicodeStatement sql HE.unit decodeProc True
   where
-    assignment (n,v) = pgFmtIdent n <> ":=" <> insertableValue v
+    sql = [qc|
+            WITH t AS (select * {_callSql})
+            SELECT
+              {_countExpr} as countTotal,
+              pg_catalog.count(1) as countResult,
+              array_to_json(
+                coalesce(array_agg(row_to_json(r)), '\{}')
+              )::character varying
+            FROM (select * from t {limitF range}) r;
+          |]
+    _args = intercalate "," $ map _assignment (HM.toList params)
+    _assignment (n,v) = pgFmtIdent n <> ":=" <> insertableValue v
+    _callSql = [qc| from {fromQi qi}({_args}) |] :: Text
+    _countExpr = if countTotal
+                   then "(select pg_catalog.count(1) from t)"
+                   else "null::bigint" :: Text
+    decodeProc = HD.maybeRow procRow
+    procRow = (,,) <$> HD.nullableValue HD.int8 <*> HD.value HD.int8
+                   <*> HD.value HD.json
 
 operators :: [(Text, SqlFragment)]
 operators = [
@@ -170,76 +247,104 @@ operators = [
   ]
 
 pgFmtIdent :: SqlFragment -> SqlFragment
-pgFmtIdent x =
- let escaped = replace "\"" "\"\"" (trimNullChars $ cs x) in
- if (cs escaped :: BS.ByteString) =~ danger
-   then "\"" <> escaped <> "\""
-   else escaped
- where danger = "^$|^[^a-z_]|[^a-z_0-9]" :: BS.ByteString
+pgFmtIdent x = "\"" <> replace "\"" "\"\"" (trimNullChars $ cs x) <> "\""
 
 pgFmtLit :: SqlFragment -> SqlFragment
 pgFmtLit x =
  let trimmed = trimNullChars x
      escaped = "'" <> replace "'" "''" trimmed <> "'"
      slashed = replace "\\" "\\\\" escaped in
- if "\\\\" `isInfixOf` escaped
+ if "\\" `isInfixOf` escaped
    then "E" <> slashed
    else slashed
 
+requestToCountQuery :: Schema -> DbRequest -> SqlQuery
+requestToCountQuery _ (DbMutate _) = undefined
+requestToCountQuery schema (DbRead (Node (Select _ _ conditions _ _, (mainTbl, _, _)) _)) =
+ unwords [
+   "SELECT pg_catalog.count(1)",
+   "FROM ", fromQi $ QualifiedIdentifier schema mainTbl,
+   ("WHERE " <> intercalate " AND " ( map (pgFmtCondition (QualifiedIdentifier schema mainTbl)) localConditions )) `emptyOnNull` localConditions
+   ]
+ where
+   fn Filter{value=VText _} = True
+   fn Filter{value=VForeignKey _ _} = False
+   localConditions = filter fn conditions
+
 requestToQuery :: Schema -> DbRequest -> SqlQuery
 requestToQuery _ (DbMutate (Insert _ (PayloadParseError _))) = undefined
 requestToQuery _ (DbMutate (Update _ (PayloadParseError _) _)) = undefined
-requestToQuery schema (DbRead (Node (Select colSelects tbls conditions ord, (mainTbl, _)) forest)) =
+requestToQuery schema (DbRead (Node (Select colSelects tbls conditions ord range, (nodeName, maybeRelation, _)) forest)) =
   query
   where
     -- TODO! the folloing helper functions are just to remove the "schema" part when the table is "source" which is the name
     -- of our WITH query part
-    tblSchema tbl = if tbl == sourceSubqueryName then "" else schema
+    mainTbl = fromMaybe nodeName (tableName . relTable <$> maybeRelation)
+    tblSchema tbl = if tbl == sourceCTEName then "" else schema
     qi = QualifiedIdentifier (tblSchema mainTbl) mainTbl
     toQi t = QualifiedIdentifier (tblSchema t) t
     query = unwords [
-      ("WITH " <> intercalate ", " withs) `emptyOnNull` withs,
       "SELECT ", intercalate ", " (map (pgFmtSelectItem qi) colSelects ++ selects),
       "FROM ", intercalate ", " (map (fromQi . toQi) tbls),
+      unwords joins,
       ("WHERE " <> intercalate " AND " ( map (pgFmtCondition qi ) conditions )) `emptyOnNull` conditions,
-      orderF (fromMaybe [] ord)
+      orderF (fromMaybe [] ord),
+      limitF range
       ]
-    (withs, selects) = foldr getQueryParts ([],[]) forest
+    orderF ts =
+        if null ts
+            then ""
+            else "ORDER BY " <> clause
+        where
+            clause = intercalate "," (map queryTerm ts)
+            queryTerm :: OrderTerm -> Text
+            queryTerm t = " "
+                <> cs (pgFmtColumn qi $ otTerm t) <> " "
+                <> (cs.show) (otDirection t) <> " "
+                <> maybe "" (cs.show) (otNullOrder t) <> " "
+    (joins, selects) = foldr getQueryParts ([],[]) forest
+
     getQueryParts :: Tree ReadNode -> ([SqlFragment], [SqlFragment]) -> ([SqlFragment], [SqlFragment])
-    getQueryParts (Node n@(_, (table, Just (Relation {relType=Child}))) forst) (w,s) = (w,sel:s)
+    getQueryParts (Node n@(_, (name, Just Relation{relType=Child,relTable=Table{tableName=table}}, alias)) forst) (j,s) = (j,sel:s)
       where
-        sel = "("
-           <> "SELECT array_to_json(array_agg(row_to_json("<>table<>"))) "
-           <> "FROM (" <> subquery <> ") " <> table
-           <> ") AS " <> table
+        sel = "COALESCE(("
+           <> "SELECT array_to_json(array_agg(row_to_json("<>pgFmtIdent table<>"))) "
+           <> "FROM (" <> subquery <> ") " <> pgFmtIdent table
+           <> "), '[]') AS " <> pgFmtIdent (fromMaybe name alias)
            where subquery = requestToQuery schema (DbRead (Node n forst))
-    getQueryParts (Node n@(_, (table, Just (Relation {relType=Parent}))) forst) (w,s) = (wit:w,sel:s)
+    getQueryParts (Node n@(_, (name, Just r@Relation{relType=Parent,relTable=Table{tableName=table}}, alias)) forst) (j,s) = (joi:j,sel:s)
       where
-        sel = "row_to_json(" <> table <> ".*) AS "<>table --TODO must be singular
-        wit = table <> " AS ( " <> subquery <> " )"
+        node_name = fromMaybe name alias
+        local_table_name = table <> "_" <> node_name
+        replaceTableName localTableName (Filter a b (VForeignKey (QualifiedIdentifier "" _) c)) = Filter a b (VForeignKey (QualifiedIdentifier "" localTableName) c)
+        replaceTableName _ x = x
+        sel = "row_to_json(" <> pgFmtIdent local_table_name <> ".*) AS " <> pgFmtIdent node_name
+        joi = " LEFT OUTER JOIN ( " <> subquery <> " ) AS " <> pgFmtIdent local_table_name  <>
+              " ON " <> intercalate " AND " ( map (pgFmtCondition qi . replaceTableName local_table_name) (getJoinConditions r) )
           where subquery = requestToQuery schema (DbRead (Node n forst))
-    getQueryParts (Node n@(_, (table, Just (Relation {relType=Many}))) forst) (w,s) = (w,sel:s)
+    getQueryParts (Node n@(_, (name, Just Relation{relType=Many,relTable=Table{tableName=table}}, alias)) forst) (j,s) = (j,sel:s)
       where
-        sel = "("
-           <> "SELECT array_to_json(array_agg(row_to_json("<>table<>"))) "
-           <> "FROM (" <> subquery <> ") " <> table
-           <> ") AS " <> table
+        sel = "COALESCE (("
+           <> "SELECT array_to_json(array_agg(row_to_json("<>pgFmtIdent table<>"))) "
+           <> "FROM (" <> subquery <> ") " <> pgFmtIdent table
+           <> "), '[]') AS " <> pgFmtIdent (fromMaybe name alias)
            where subquery = requestToQuery schema (DbRead (Node n forst))
     --the following is just to remove the warning
     --getQueryParts is not total but requestToQuery is called only after addJoinConditions which ensures the only
     --posible relations are Child Parent Many
-    getQueryParts (Node (_,(_,Nothing)) _) _ = undefined
+    getQueryParts (Node (_,(_,Nothing,_)) _) _ = undefined
 requestToQuery schema (DbMutate (Insert mainTbl (PayloadJSON (UniformObjects rows)))) =
   let qi = QualifiedIdentifier schema mainTbl
       cols = map pgFmtIdent $ fromMaybe [] (HM.keys <$> (rows V.!? 0))
-      colsString = intercalate ", " cols in
-  unwords [
-    "INSERT INTO ", fromQi qi,
-    " (" <> colsString <> ")" <>
-    " SELECT " <> colsString <>
-    " FROM json_populate_recordset(null::" , fromQi qi, ", ?)",
-    " RETURNING " <> fromQi qi <> ".*"
-    ]
+      colsString = intercalate ", " cols
+      insInto = unwords [ "INSERT INTO" , fromQi qi,
+          if T.null colsString then "" else "(" <> colsString <> ")"
+        ]
+      vals = unwords $ if T.null colsString
+                then ["DEFAULT VALUES"]
+                else ["SELECT", colsString, "FROM json_populate_recordset(null::" , fromQi qi, ", $1)"] in
+  insInto <> vals
+
 requestToQuery schema (DbMutate (Update mainTbl (PayloadJSON (UniformObjects rows)) conditions)) =
   case rows V.!? 0 of
     Just obj ->
@@ -248,25 +353,22 @@ requestToQuery schema (DbMutate (Update mainTbl (PayloadJSON (UniformObjects row
       unwords [
         "UPDATE ", fromQi qi,
         " SET " <> intercalate "," assignments <> " ",
-        ("WHERE " <> intercalate " AND " ( map (pgFmtCondition qi ) conditions )) `emptyOnNull` conditions,
-        "RETURNING " <> fromQi qi <> ".*"
+        ("WHERE " <> intercalate " AND " ( map (pgFmtCondition qi ) conditions )) `emptyOnNull` conditions
         ]
     Nothing -> undefined
   where
     qi = QualifiedIdentifier schema mainTbl
-
 requestToQuery schema (DbMutate (Delete mainTbl conditions)) =
   query
   where
     qi = QualifiedIdentifier schema mainTbl
     query = unwords [
       "DELETE FROM ", fromQi qi,
-      ("WHERE " <> intercalate " AND " ( map (pgFmtCondition qi ) conditions )) `emptyOnNull` conditions,
-      "RETURNING " <> fromQi qi <> ".*"
+      ("WHERE " <> intercalate " AND " ( map (pgFmtCondition qi ) conditions )) `emptyOnNull` conditions
       ]
 
-sourceSubqueryName :: SqlFragment
-sourceSubqueryName = "pg_source"
+sourceCTEName :: SqlFragment
+sourceCTEName = "pg_source"
 
 unquoted :: JSON.Value -> Text
 unquoted (JSON.String t) = t
@@ -280,43 +382,41 @@ asCsvF :: SqlFragment
 asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
   where
     asCsvHeaderF =
-      "(SELECT string_agg(a.k, ',')" <>
+      "(SELECT coalesce(string_agg(a.k, ','), '')" <>
       "  FROM (" <>
       "    SELECT json_object_keys(r)::TEXT as k" <>
       "    FROM ( " <>
-      "      SELECT row_to_json(hh) as r from " <> sourceSubqueryName <> " as hh limit 1" <>
+      "      SELECT row_to_json(hh) as r from " <> sourceCTEName <> " as hh limit 1" <>
       "    ) s" <>
       "  ) a" <>
       ")"
     asCsvBodyF = "coalesce(string_agg(substring(t::text, 2, length(t::text) - 2), '\n'), '')"
 
 asJsonF :: SqlFragment
-asJsonF = "array_to_json(array_agg(row_to_json(t)))::character varying"
+asJsonF = "coalesce(array_to_json(array_agg(row_to_json(t))), '[]')::character varying"
 
 asJsonSingleF :: SqlFragment --TODO! unsafe when the query actually returns multiple rows, used only on inserting and returning single element
-asJsonSingleF = "string_agg(row_to_json(t)::text, ',')::character varying "
-
-countAllF :: SqlFragment
-countAllF = "(SELECT pg_catalog.count(1) FROM (SELECT * FROM " <> sourceSubqueryName <> ") a )"
-
-countF :: SqlFragment
-countF = "pg_catalog.count(t)"
-
-countNoneF :: SqlFragment
-countNoneF = "null"
+asJsonSingleF = "coalesce(string_agg(row_to_json(t)::text, ','), '')::character varying "
 
 locationF :: [Text] -> SqlFragment
 locationF pKeys =
     "(" <>
-    " WITH s AS (SELECT row_to_json(ss) as r from " <> sourceSubqueryName <> " as ss  limit 1)" <>
-    " SELECT string_agg(json_data.key || '=' || coalesce( 'eq.' || json_data.value, 'is.null'), '&')" <>
+    " WITH s AS (SELECT row_to_json(ss) as r from " <> sourceCTEName <> " as ss  limit 1)" <>
+    " SELECT array_agg(json_data.key || '=' || coalesce('eq.' || json_data.value, 'is.null'))" <>
     " FROM s, json_each_text(s.r) AS json_data" <>
     (
       if null pKeys
       then ""
       else " WHERE json_data.key IN ('" <> intercalate "','" pKeys <> "')"
-    ) <>
-    ")"
+    ) <> ")"
+
+limitF :: NonnegRange -> SqlFragment
+limitF r  = if r == allRange
+  then ""
+  else "LIMIT " <> limit <> " OFFSET " <> offset
+  where
+    limit  = maybe "ALL" (cs . show) $ rangeLimit r
+    offset = cs . show $ rangeOffset r
 
 fromQi :: QualifiedIdentifier -> SqlFragment
 fromQi t = (if s == "" then "" else pgFmtIdent s <> ".") <> pgFmtIdent n
@@ -331,29 +431,19 @@ getJoinConditions (Relation t cols ft fcs typ lt lc1 lc2) =
     Parent -> zipWith (toFilter tN ftN) cols fcs
     Many   -> zipWith (toFilter tN ltN) cols (fromMaybe [] lc1) ++ zipWith (toFilter ftN ltN) fcs (fromMaybe [] lc2)
   where
-    s = tableSchema t
+    s = if typ == Parent then "" else tableSchema t
     tN = tableName t
     ftN = tableName ft
     ltN = fromMaybe "" (tableName <$> lt)
     toFilter :: Text -> Text -> Column -> Column -> Filter
     toFilter tb ftb c fc = Filter (colName c, Nothing) "=" (VForeignKey (QualifiedIdentifier s tb) (ForeignKey fc{colTable=(colTable fc){tableName=ftb}}))
 
+unicodeStatement :: Text -> HE.Params a -> HD.Result b -> Bool -> H.Query a b
+unicodeStatement = H.statement . T.encodeUtf8
+
 emptyOnNull :: Text -> [a] -> Text
 emptyOnNull val x = if null x then "" else val
 
-orderF :: [OrderTerm] -> SqlFragment
-orderF ts =
-  if null ts
-    then ""
-    else "ORDER BY " <> clause
-  where
-    clause = intercalate "," (map queryTerm ts)
-    queryTerm :: OrderTerm -> Text
-    queryTerm t = " "
-           <> cs (pgFmtIdent $ otTerm t) <> " "
-           <> (cs.show) (otDirection t) <> " "
-           <> maybe "" (cs.show) (otNullOrder t) <> " "
-
 insertableValue :: JSON.Value -> SqlFragment
 insertableValue JSON.Null = "null"
 insertableValue v = (<> "::unknown") . pgFmtLit $ unquoted v
@@ -371,8 +461,8 @@ pgFmtField :: QualifiedIdentifier -> Field -> SqlFragment
 pgFmtField table (c, jp) = pgFmtColumn table c <> pgFmtJsonPath jp
 
 pgFmtSelectItem :: QualifiedIdentifier -> SelectItem -> SqlFragment
-pgFmtSelectItem table (f@(_, jp), Nothing) = pgFmtField table f <> pgFmtAsJsonPath jp
-pgFmtSelectItem table (f@(_, jp), Just cast ) = "CAST (" <> pgFmtField table f <> " AS " <> cast <> " )" <> pgFmtAsJsonPath jp
+pgFmtSelectItem table (f@(_, jp), Nothing, alias) = pgFmtField table f <> pgFmtAs jp alias
+pgFmtSelectItem table (f@(_, jp), Just cast, alias) = "CAST (" <> pgFmtField table f <> " AS " <> cast <> " )" <> pgFmtAs jp alias
 
 pgFmtCondition :: QualifiedIdentifier -> Filter -> SqlFragment
 pgFmtCondition table (Filter (col,jp) ops val) =
@@ -393,7 +483,7 @@ pgFmtCondition table (Filter (col,jp) ops val) =
     valToStr v = case v of
       VText s -> pgFmtValue opCode s
       VForeignKey (QualifiedIdentifier s _) (ForeignKey Column{colTable=Table{tableName=ft}, colName=fc}) -> pgFmtColumn qi fc
-        where qi = QualifiedIdentifier (if ft == sourceSubqueryName then "" else s) ft
+        where qi = QualifiedIdentifier (if ft == sourceCTEName then "" else s) ft
       _ -> ""
 
 pgFmtValue :: Text -> Text -> SqlFragment
@@ -419,32 +509,10 @@ pgFmtJsonPath (Just [x]) = "->>" <> pgFmtLit x
 pgFmtJsonPath (Just (x:xs)) = "->" <> pgFmtLit x <> pgFmtJsonPath ( Just xs )
 pgFmtJsonPath _ = ""
 
-pgFmtAsJsonPath :: Maybe JsonPath -> SqlFragment
-pgFmtAsJsonPath Nothing = ""
-pgFmtAsJsonPath (Just xx) = " AS " <> last xx
+pgFmtAs :: Maybe JsonPath -> Maybe Alias -> SqlFragment
+pgFmtAs Nothing Nothing = ""
+pgFmtAs (Just xx) Nothing = " AS " <> pgFmtIdent (last xx)
+pgFmtAs _ (Just alias) = " AS " <> pgFmtIdent alias
 
 trimNullChars :: Text -> Text
 trimNullChars = T.takeWhile (/= '\x0')
-
-withSourceF :: SqlFragment -> SqlFragment
-withSourceF s = "WITH " <> sourceSubqueryName <> " AS (" <> s <>")"
-
-fromF :: SqlFragment -> SqlFragment -> SqlFragment
-fromF sel limit = "FROM (" <> sel <> " " <> limit <> ") t"
-
-limitF :: Maybe NonnegRange -> SqlFragment
-limitF r  = "LIMIT " <> limit <> " OFFSET " <> offset
-  where
-    limit  = maybe "ALL" (cs . show) $ join $ rangeLimit <$> r
-    offset = cs . show $ fromMaybe 0 $ rangeOffset <$> r
-
-selectStarF :: SqlFragment
-selectStarF = "SELECT * FROM " <> sourceSubqueryName
-
-wrapQuery :: SqlQuery -> [Text] -> Text ->  Maybe NonnegRange -> SqlQuery
-wrapQuery source selectColumns returnSelect range =
-  withSourceF source <>
-  " SELECT " <>
-  intercalate ", " selectColumns <>
-  " " <>
-  fromF returnSelect ( limitF range )

src/PostgREST/RangeQuery.hs

@@ -3,13 +3,15 @@ module PostgREST.RangeQuery (
 , rangeRequested
 , rangeLimit
 , rangeOffset
+, restrictRange
+, rangeGeq
+, allRange
 , NonnegRange
 ) where
 
 
 import           Control.Applicative
 import           Network.HTTP.Types.Header
-import           PostgREST.Types           ()
 
 import qualified Data.ByteString.Char8     as BS
 import           Data.Ranged.Boundaries
@@ -23,39 +25,49 @@ import           Data.Maybe                (fromMaybe, listToMaybe)
 
 import           Prelude
 
-type NonnegRange = Range Int
+type NonnegRange = Range Integer
 
-rangeParse :: BS.ByteString -> Maybe NonnegRange
+rangeParse :: BS.ByteString -> NonnegRange
 rangeParse range = do
   let rangeRegex = "^([0-9]+)-([0-9]*)$" :: BS.ByteString
 
-  parsedRange <- listToMaybe (range =~ rangeRegex :: [[BS.ByteString]])
+  case listToMaybe (range =~ rangeRegex :: [[BS.ByteString]]) of
+    Just parsedRange ->
+      let [_, from, to] = readMaybe . cs <$> parsedRange
+          lower         = fromMaybe emptyRange   (rangeGeq <$> from)
+          upper         = fromMaybe allRange (rangeLeq <$> to) in
+      rangeIntersection lower upper
+    Nothing -> allRange
 
-  let [_, from, to] = readMaybe . cs <$> parsedRange
-  let lower = fromMaybe emptyRange   (rangeGeq <$> from)
-  let upper = fromMaybe (rangeGeq 0) (rangeLeq <$> to)
+rangeRequested :: RequestHeaders -> NonnegRange
+rangeRequested headers = fromMaybe allRange $
+  rangeParse <$> lookup hRange headers
 
-  return $ rangeIntersection lower upper
+restrictRange :: Maybe Integer -> NonnegRange -> NonnegRange
+restrictRange Nothing r = r
+restrictRange (Just limit) r =
+   rangeIntersection r $
+     Range BoundaryBelowAll (BoundaryAbove $ rangeOffset r + limit - 1)
 
-rangeRequested :: RequestHeaders -> Maybe NonnegRange
-rangeRequested = (rangeParse =<<) . lookup hRange
-
-rangeLimit :: NonnegRange -> Maybe Int
+rangeLimit :: NonnegRange -> Maybe Integer
 rangeLimit range =
   case [rangeLower range, rangeUpper range] of
     [BoundaryBelow from, BoundaryAbove to] -> Just (1 + to - from)
     _ -> Nothing
 
-rangeOffset :: NonnegRange -> Int
+rangeOffset :: NonnegRange -> Integer
 rangeOffset range =
   case rangeLower range of
     BoundaryBelow from -> from
     _ -> error "range without lower bound" -- should never happen
 
-rangeGeq :: Int -> NonnegRange
+rangeGeq :: Integer -> NonnegRange
 rangeGeq n =
   Range (BoundaryBelow n) BoundaryAboveAll
 
-rangeLeq :: Int -> NonnegRange
+allRange :: NonnegRange
+allRange = rangeGeq 0
+
+rangeLeq :: Integer -> NonnegRange
 rangeLeq n =
   Range BoundaryBelowAll (BoundaryAbove n)

src/PostgREST/Types.hs

@@ -1,15 +1,17 @@
 module PostgREST.Types where
-import Data.Text
-import Data.Tree
-import qualified Data.ByteString.Lazy as BL
+import           Data.Aeson
 import qualified Data.ByteString      as BS
+import qualified Data.ByteString.Lazy as BL
+import           Data.Int             (Int32)
+import           Data.Text
+import           Data.Tree
 import qualified Data.Vector          as V
-import Data.Aeson
+import           PostgREST.RangeQuery (NonnegRange)
 
 data DbStructure = DbStructure {
-  dbTables :: [Table]
-, dbColumns :: [Column]
-, dbRelations :: [Relation]
+  dbTables      :: [Table]
+, dbColumns     :: [Column]
+, dbRelations   :: [Relation]
 , dbPrimaryKeys :: [PrimaryKey]
 } deriving (Show, Eq)
 
@@ -31,12 +33,12 @@ data Column =
     Column {
       colTable     :: Table
     , colName      :: Text
-    , colPosition  :: Int
+    , colPosition  :: Int32
     , colNullable  :: Bool
     , colType      :: Text
     , colUpdatable :: Bool
-    , colMaxLen    :: Maybe Int
-    , colPrecision :: Maybe Int
+    , colMaxLen    :: Maybe Int32
+    , colPrecision :: Maybe Int32
     , colDefault   :: Maybe Text
     , colEnum      :: [Text]
     , colFK        :: Maybe ForeignKey
@@ -90,6 +92,9 @@ data Relation = Relation {
 newtype UniformObjects = UniformObjects (V.Vector Object)
   deriving (Show, Eq)
 
+unUniformObjects :: UniformObjects -> V.Vector Object
+unUniformObjects (UniformObjects objs) = objs
+
 -- | When Hasql supports the COPY command then we can
 -- have a special payload just for CSV, but until
 -- then CSV is converted to a JSON array.
@@ -102,16 +107,17 @@ data FValue = VText Text | VForeignKey QualifiedIdentifier ForeignKey deriving (
 type FieldName = Text
 type JsonPath = [Text]
 type Field = (FieldName, Maybe JsonPath)
+type Alias = Text
 type Cast = Text
 type NodeName = Text
-type SelectItem = (Field, Maybe Cast)
+type SelectItem = (Field, Maybe Cast, Maybe Alias)
 type Path = [Text]
-data ReadQuery = Select { select::[SelectItem], from::[Text], flt_::[Filter], order::Maybe [OrderTerm] }  deriving (Show, Eq)
-data MutateQuery = Insert { in_::Text, qPayload::Payload }
-                 | Delete { in_::Text, where_::[Filter] }
-                 | Update { in_::Text, qPayload::Payload, where_::[Filter] } deriving (Show, Eq)
+data ReadQuery = Select { select::[SelectItem], from::[TableName], flt_::[Filter], order::Maybe [OrderTerm], range_::NonnegRange } deriving (Show, Eq)
+data MutateQuery = Insert { in_::TableName, qPayload::Payload }
+                 | Delete { in_::TableName, where_::[Filter] }
+                 | Update { in_::TableName, qPayload::Payload, where_::[Filter] } deriving (Show, Eq)
 data Filter = Filter {field::Field, operator::Operator, value::FValue} deriving (Show, Eq)
-type ReadNode = (ReadQuery, (NodeName, Maybe Relation))
+type ReadNode = (ReadQuery, (NodeName, Maybe Relation, Maybe Alias))
 type ReadRequest = Tree ReadNode
 type MutateRequest = MutateQuery
 data DbRequest = DbRead ReadRequest | DbMutate MutateRequest

stack.yaml

@@ -1,7 +1,24 @@
-flags: {}
-packages:
-- '.'
+resolver: lts-6.2
 extra-deps:
   - Ranged-sets-0.3.0
-  - packdeps-0.4.1
-resolver: nightly-2015-10-27
+  - bytestring-tree-builder-0.2.7
+  - hasql-0.19.12
+  - hasql-pool-0.4.1
+  - hasql-transaction-0.4.5
+  - jwt-0.7.2
+  - postgresql-binary-0.9.0.1
+  - binary-parser-0.5.2
+  - contravariant-extras-0.3.2
+  - placeholders-0.1
+  - postgresql-error-codes-1
+  - success-0.2.6
+  - tuple-th-0.2.5
+  - wai-cors-0.2.5
+  - cryptohash-sha256-0.11.100.0
+  - hackage-security-0.5.2.1
+
+ghc-options:
+  postgrest: -O2 -Werror -Wall -fwarn-identities
+
+packages:
+- .

test/Feature/AuthSpec.hs

@@ -7,25 +7,60 @@ import Test.Hspec.Wai.JSON
 import Network.HTTP.Types
 
 import SpecHelper
+import Network.Wai (Application)
 -- }}}
 
-spec :: Spec
-spec = beforeAll
-  (clearTable "postgrest.auth") . afterAll_ (clearTable "postgrest.auth")
-  $ around withApp
-  $ describe "authorization" $ do
+spec :: SpecWith Application
+spec = describe "authorization" $ do
 
-  it "hides tables that anonymous does not own" $
-    get "/authors_only" `shouldRespondWith` 404
+  it "denies access to tables that anonymous does not own" $
+    get "/authors_only" `shouldRespondWith` ResponseMatcher {
+        matchBody = Just [json| {
+          "hint":null,
+          "details":null,
+          "code":"42501",
+          "message":"permission denied for relation authors_only"} |]
+      , matchStatus = 401
+      , matchHeaders = ["WWW-Authenticate" <:> "Bearer"]
+      }
+
+  it "denies access to tables that postgrest_test_author does not own" $
+    let auth = authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.y4vZuu1dDdwAl0-S00MCRWRYMlJ5YAMSir6Es6WtWx0" in
+    request methodGet "/private_table" [auth] ""
+      `shouldRespondWith` ResponseMatcher {
+        matchBody = Just [json| {
+          "hint":null,
+          "details":null,
+          "code":"42501",
+          "message":"permission denied for relation private_table"} |]
+      , matchStatus = 403
+      , matchHeaders = []
+      }
 
   it "returns jwt functions as jwt tokens" $
     post "/rpc/login" [json| { "id": "jdoe", "pass": "1234" } |]
       `shouldRespondWith` ResponseMatcher {
           matchBody = Just [json| {"token":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.y4vZuu1dDdwAl0-S00MCRWRYMlJ5YAMSir6Es6WtWx0"} |]
         , matchStatus = 200
-        , matchHeaders = ["Content-Type" <:> "application/json"]
+        , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
         }
 
+  it "sql functions can encode custom and standard claims" $
+    post "/rpc/jwt_test" "{}"
+      `shouldRespondWith` ResponseMatcher {
+          matchBody = Just [json| {"token":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJmdW4iLCJqdGkiOiJmb28iLCJuYmYiOjEzMDA4MTkzODAsImV4cCI6MTMwMDgxOTM4MCwiaHR0cDovL3Bvc3RncmVzdC5jb20vZm9vIjp0cnVlLCJpc3MiOiJqb2UiLCJyb2xlIjoicG9zdGdyZXN0X3Rlc3QiLCJpYXQiOjEzMDA4MTkzODAsImF1ZCI6ImV2ZXJ5b25lIn0._tQCF79-ZZGMlLktd3csM_bVaiMg7A8YvIb6K2hcu5w"} |]
+        , matchStatus = 200
+        , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
+        }
+
+  it "sql functions can read custom and standard claims variables" $ do
+    let auth = authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJmdW4iLCJqdGkiOiJmb28iLCJuYmYiOjEzMDA4MTkzODAsImV4cCI6OTk5OTk5OTk5OSwiaHR0cDovL3Bvc3RncmVzdC5jb20vZm9vIjp0cnVlLCJpc3MiOiJqb2UiLCJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWF0IjoxMzAwODE5MzgwLCJhdWQiOiJldmVyeW9uZSJ9.AQmCA7CMScvfaDRMqRPeUY6eNf--69gpW-kxaWfq9X0"
+    request methodPost "/rpc/reveal_big_jwt" [auth] "{}"
+      `shouldRespondWith` [json| [
+          {"sub":"fun", "jti":"foo", "nbf":1300819380, "exp":9999999999,
+          "http://postgrest.com/foo":true, "iss":"joe", "iat":1300819380,
+          "aud":"everyone"}] |]
+
   it "allows users with permissions to see their tables" $ do
     let auth = authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.y4vZuu1dDdwAl0-S00MCRWRYMlJ5YAMSir6Es6WtWx0"
     request methodGet "/authors_only" [auth] ""

test/Feature/ConcurrentSpec.hs

@@ -0,0 +1,51 @@
+{-# LANGUAGE MultiParamTypeClasses, TypeFamilies, UndecidableInstances #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+module Feature.ConcurrentSpec where
+
+import Control.Monad (void)
+import Control.Monad.Base
+
+import Control.Monad.Trans.Control
+import Control.Concurrent.Async (mapConcurrently)
+
+import Test.Hspec hiding (pendingWith)
+import Test.Hspec.Wai.Internal
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+import Network.Wai.Test (Session)
+
+import Network.Wai (Application)
+
+spec :: SpecWith Application
+spec =
+  describe "Queryiny in parallel" $
+    it "should not raise 'transaction in progress' error" $
+      raceTest 10 $
+        get "/fakefake"
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [json|
+              { "hint": null,
+                "details":null,
+                "code":"42P01",
+                "message":"relation \"test.fakefake\" does not exist"
+              } |]
+          , matchStatus  = 404
+          , matchHeaders = []
+          }
+
+raceTest :: Int -> WaiExpectation -> WaiExpectation
+raceTest times = liftBaseDiscard go
+ where
+  go test = void $ mapConcurrently (const test) [1..times]
+
+instance MonadBaseControl IO WaiSession where
+  type StM WaiSession a = StM Session a
+  liftBaseWith f = WaiSession $
+    liftBaseWith $ \runInBase ->
+      f $ \k -> runInBase (unWaiSession k)
+  restoreM = WaiSession . restoreM
+  {-# INLINE liftBaseWith #-}
+  {-# INLINE restoreM #-}
+
+instance MonadBase IO WaiSession where
+  liftBase = liftIO

test/Feature/CorsSpec.hs

@@ -9,10 +9,12 @@ import qualified Data.ByteString.Lazy as BL
 import SpecHelper
 
 import Network.HTTP.Types
+import Network.Wai (Application)
 -- }}}
 
-spec :: Spec
-spec = around withApp $ describe "CORS" $ do
+spec :: SpecWith Application
+spec =
+  describe "CORS" $ do
     let preflightHeaders = [
           ("Accept", "*/*"),
           ("Origin", "http://example.com"),

test/Feature/DeleteSpec.hs

@@ -2,13 +2,13 @@ module Feature.DeleteSpec where
 
 import Test.Hspec
 import Test.Hspec.Wai
-import SpecHelper
+import Text.Heredoc
 
 import Network.HTTP.Types
+import Network.Wai (Application)
 
-spec :: Spec
-spec = beforeAll (clearTable "items" >> createItems 15) . afterAll_ (clearTable "items")
-  . around withApp $
+spec :: SpecWith Application
+spec =
   describe "Deleting" $ do
     context "existing record" $ do
       it "succeeds with 204 and deletion count" $
@@ -19,11 +19,33 @@ spec = beforeAll (clearTable "items" >> createItems 15) . afterAll_ (clearTable
           , matchHeaders = ["Content-Range" <:> "*/1"]
           }
 
+      it "returns the deleted item" $
+        request methodDelete "/items?id=eq.2" [("Prefer", "return=representation")] ""
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|[{"id":2}]|]
+          , matchStatus  = 200
+          , matchHeaders = ["Content-Range" <:> "*/1"]
+          }
+      it "returns the deleted item and shapes the response" $
+        request methodDelete "/complex_items?id=eq.2&select=id,name" [("Prefer", "return=representation")] ""
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|[{"id":2,"name":"Two"}]|]
+          , matchStatus  = 200
+          , matchHeaders = ["Content-Range" <:> "*/1"]
+          }
+      it "can embed (parent) entities" $
+        request methodDelete "/tasks?id=eq.8&select=id,name,project{id}" [("Prefer", "return=representation")] ""
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|[{"id":8,"name":"Code OSX","project":{"id":4}}]|]
+          , matchStatus  = 200
+          , matchHeaders = ["Content-Range" <:> "*/1"]
+          }
+
       it "actually clears items ouf the db" $ do
         _ <- request methodDelete "/items?id=lt.15" [] ""
         get "/items"
           `shouldRespondWith` ResponseMatcher {
-            matchBody    = Just "[{\"id\":15}]"
+            matchBody    = Just [str|[{"id":15}]|]
           , matchStatus  = 200
           , matchHeaders = ["Content-Range" <:> "0-0/1"]
           }

test/Feature/InsertSpec.hs

@@ -9,17 +9,19 @@ import SpecHelper
 
 import qualified Data.Aeson as JSON
 import Data.Maybe (fromJust)
+import Data.Monoid ((<>))
 import Text.Heredoc
 import Network.HTTP.Types.Header
 import Network.HTTP.Types
-import Control.Monad (replicateM_)
+import Control.Monad (replicateM_, void)
 
 import TestTypes(IncPK(..), CompoundPK(..))
+import Network.Wai (Application)
 
-spec :: Spec
-spec = afterAll_ resetDb $ around withApp $ do
+spec :: SpecWith Application
+spec = do
   describe "Posting new record" $ do
-    after_ (clearTable "menagerie") . context "disparate csv types" $ do
+    context "disparate json types" $ do
       it "accepts disparate json types" $ do
         p <- post "/menagerie"
           [json| {
@@ -30,6 +32,8 @@ spec = afterAll_ resetDb $ around withApp $ do
         liftIO $ do
           simpleBody p `shouldBe` ""
           simpleStatus p `shouldBe` created201
+          -- should not have content type set when body is empty
+          lookup hContentType (simpleHeaders p) `shouldBe` Nothing
 
       it "filters columns in result using &select" $
         request methodPost "/menagerie?select=integer,varchar" [("Prefer", "return=representation")]
@@ -40,20 +44,29 @@ spec = afterAll_ resetDb $ around withApp $ do
           } |] `shouldRespondWith` ResponseMatcher {
             matchBody    = Just [str|{"integer":14,"varchar":"testing!"}|]
           , matchStatus  = 201
-          , matchHeaders = ["Content-Type" <:> "application/json"]
+          , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
           }
 
       it "includes related data after insert" $
         request methodPost "/projects?select=id,name,clients{id,name}" [("Prefer", "return=representation")]
-          [str|{"id":5,"name":"New Project","client_id":2}|] `shouldRespondWith` ResponseMatcher {
-            matchBody    = Just [str|{"id":5,"name":"New Project","clients":{"id":2,"name":"Apple"}}|]
+          [str|{"id":6,"name":"New Project","client_id":2}|] `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|{"id":6,"name":"New Project","clients":{"id":2,"name":"Apple"}}|]
           , matchStatus  = 201
-          , matchHeaders = ["Content-Type" <:> "application/json", "Location" <:> "/projects?id=eq.5"]
+          , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8", "Location" <:> "/projects?id=eq.6"]
           }
 
+    context "from an html form" $
+      it "accepts disparate json types" $ do
+        p <- request methodPost "/menagerie"
+               [("Content-Type", "application/x-www-form-urlencoded")]
+               ("integer=7&double=2.71828&varchar=forms+are+fun&" <>
+                "boolean=false&date=1900-01-01&money=$3.99&enum=foo")
+        liftIO $ do
+          simpleBody p `shouldBe` ""
+          simpleStatus p `shouldBe` created201
 
     context "with no pk supplied" $ do
-      context "into a table with auto-incrementing pk" . after_ (clearTable "auto_incrementing_pk") $
+      context "into a table with auto-incrementing pk" $
         it "succeeds with 201 and link" $ do
           p <- post "/auto_incrementing_pk" [json| { "non_nullable_string":"not null"} |]
           liftIO $ do
@@ -72,7 +85,7 @@ spec = afterAll_ resetDb $ around withApp $ do
           post "/simple_pk" [json| { "extra":"foo"} |]
             `shouldRespondWith` 400
 
-      context "into a table with no pk" . after_ (clearTable "no_pk") $ do
+      context "into a table with no pk" $ do
         it "succeeds with 201 and a link including all fields" $ do
           p <- post "/no_pk" [json| { "a":"foo", "b":"bar" } |]
           liftIO $ do
@@ -89,6 +102,15 @@ spec = afterAll_ resetDb $ around withApp $ do
             simpleHeaders p `shouldSatisfy` matchHeader hLocation "/no_pk\\?a=eq.bar&b=eq.baz"
             simpleStatus p `shouldBe` created201
 
+        it "can insert in tables with no select privileges" $ do
+          p <- request methodPost "/insertonly"
+                       [("Prefer", "return=minimal")]
+                       [json| { "v":"some value" } |]
+          liftIO $ do
+            simpleBody p `shouldBe` ""
+            simpleStatus p `shouldBe` created201
+
+
         it "can post nulls" $ do
           p <- request methodPost "/no_pk"
                        [("Prefer", "return=representation")]
@@ -98,58 +120,86 @@ spec = afterAll_ resetDb $ around withApp $ do
             simpleHeaders p `shouldSatisfy` matchHeader hLocation "/no_pk\\?a=is.null&b=eq.foo"
             simpleStatus p `shouldBe` created201
 
-    context "with compound pk supplied" . after_ (clearTable "compound_pk") $
-      it "builds response location header appropriately" $
-        post "/compound_pk" [json| { "k1":12, "k2":42 } |]
-          `shouldRespondWith` ResponseMatcher {
-            matchBody    = Nothing,
-            matchStatus  = 201,
-            matchHeaders = ["Location" <:> "/compound_pk?k1=eq.12&k2=eq.42"]
-          }
+    context "with compound pk supplied" $
+      it "builds response location header appropriately" $ do
+        let inserted    = [json| { "k1":12, "k2":"Rock & R+ll" } |]
+            expectedObj = CompoundPK 12 "Rock & R+ll" Nothing
+            expectedLoc = "/compound_pk?k1=eq.12&k2=eq.Rock%20%26%20R%2Bll"
+        p <- request methodPost "/compound_pk"
+                     [("Prefer", "return=representation")]
+                     inserted
+        liftIO $ do
+          JSON.decode (simpleBody p) `shouldBe` Just expectedObj
+          simpleStatus p `shouldBe` created201
+          lookup hLocation (simpleHeaders p) `shouldBe` Just expectedLoc
+
+        r <- get expectedLoc
+        liftIO $ do
+          JSON.decode (simpleBody r) `shouldBe` Just [expectedObj]
+          simpleStatus r `shouldBe` ok200
+
+    context "with bulk insert" $
+      it "returns 201 but no location header" $ do
+        let bulkData = [json| [ {"k1":21, "k2":"hello world"}
+                              , {"k1":22, "k2":"bye for now"}]
+                            |]
+        p <- request methodPost "/compound_pk" [] bulkData
+        liftIO $ do
+          simpleStatus p `shouldBe` created201
+          lookup hLocation (simpleHeaders p) `shouldBe` Nothing
 
     context "with invalid json payload" $
       it "fails with 400 and error" $
         post "/simple_pk" "}{ x = 2" `shouldRespondWith` 400
 
-    context "jsonb" . after_ (clearTable "json") $ do
+    context "with valid json payload" $
+      it "succeeds and returns 201 created" $
+        post "/simple_pk" [json| { "k":"k1", "extra":"e1" } |] `shouldRespondWith` 201
+
+    context "attempting to insert a row with the same primary key" $
+      it "fails returning a 409 Conflict" $
+        post "/simple_pk" [json| { "k":"k1", "extra":"e1" } |] `shouldRespondWith` 409
+
+    context "attempting to insert a row with conflicting unique constraint" $
+      it "fails returning a 409 Conflict" $
+        post "/withUnique"  [json| { "uni":"nodup", "extra":"e2" } |] `shouldRespondWith` 409
+
+    context "jsonb" $ do
       it "serializes nested object" $ do
         let inserted = [json| { "data": { "foo":"bar" } } |]
+            location = "/json?data=eq.%7B%22foo%22%3A%22bar%22%7D"
         request methodPost "/json"
                      [("Prefer", "return=representation")]
                      inserted
           `shouldRespondWith` ResponseMatcher {
             matchBody    = Just inserted
           , matchStatus  = 201
-          , matchHeaders = ["Location" <:> [str|/json?data=eq.{"foo":"bar"}|]]
+          , matchHeaders = ["Location" <:> location]
           }
 
-        -- TODO! the test above seems right, why was the one below working before and not now
-        -- p <- request methodPost "/json" [("Prefer", "return=representation")] inserted
-        -- liftIO $ do
-        --   simpleBody p `shouldBe` inserted
-        --   simpleHeaders p `shouldSatisfy` matchHeader hLocation "/json\\?data=eq\\.%7B%22foo%22%3A%22bar%22%7D"
-        --   simpleStatus p `shouldBe` created201
-
       it "serializes nested array" $ do
         let inserted = [json| { "data": [1,2,3] } |]
+            location = "/json?data=eq.%5B1%2C2%2C3%5D"
         request methodPost "/json"
                      [("Prefer", "return=representation")]
                      inserted
           `shouldRespondWith` ResponseMatcher {
             matchBody    = Just inserted
           , matchStatus  = 201
-          , matchHeaders = ["Location" <:> [str|/json?data=eq.[1,2,3]|]]
+          , matchHeaders = ["Location" <:> location]
+          }
+
+    context "empty object" $
+      it "successfully populates table with all-default columns" $
+        post "/items" "{}" `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just ""
+          , matchStatus  = 201
+          , matchHeaders = []
           }
-        -- TODO! the test above seems right, why was the one below working before and not now
-        -- p <- request methodPost "/json" [("Prefer", "return=representation")] inserted
-        -- liftIO $ do
-        --   simpleBody p `shouldBe` inserted
-        --   simpleHeaders p `shouldSatisfy` matchHeader hLocation "/json\\?data=eq\\.%5B1%2C2%2C3%5D"
-        --   simpleStatus p `shouldBe` created201
 
   describe "CSV insert" $ do
 
-    after_ (clearTable "menagerie") . context "disparate csv types" $
+    context "disparate csv types" $
       it "succeeds with multipart response" $ do
         pendingWith "Decide on what to do with CSV insert"
         let inserted = [str|integer,double,varchar,boolean,date,money,enum
@@ -161,18 +211,10 @@ spec = afterAll_ resetDb $ around withApp $ do
            `shouldRespondWith` ResponseMatcher {
              matchBody    = Just inserted
            , matchStatus  = 201
-           , matchHeaders = ["Content-Type" <:> "text/csv"]
+           , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8"]
            }
-        -- p <- request methodPost "/menagerie" [("Content-Type", "text/csv")]
-        --        [str|integer,double,varchar,boolean,date,money,enum
-        --            |13,3.14159,testing!,false,1900-01-01,$3.99,foo
-        --            |12,0.1,a string,true,1929-10-01,12,bar
-        --            |]
-        -- liftIO $ do
-        --   simpleBody p `shouldBe` "Content-Type: application/json\nLocation: /menagerie?integer=eq.13\n\n\n--postgrest_boundary\nContent-Type: application/json\nLocation: /menagerie?integer=eq.12\n\n"
-        --   simpleStatus p `shouldBe` created201
 
-    after_ (clearTable "no_pk") . context "requesting full representation" $ do
+    context "requesting full representation" $ do
       it "returns full details of inserted record" $
         request methodPost "/no_pk"
                      [("Content-Type", "text/csv"), ("Accept", "text/csv"),  ("Prefer", "return=representation")]
@@ -180,21 +222,10 @@ spec = afterAll_ resetDb $ around withApp $ do
           `shouldRespondWith` ResponseMatcher {
             matchBody    = Just "a,b\nbar,baz"
           , matchStatus  = 201
-          , matchHeaders = ["Content-Type" <:> "text/csv",
+          , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8",
                             "Location" <:> "/no_pk?a=eq.bar&b=eq.baz"]
           }
 
-      -- it "can post nulls (old way)" $ do
-      --   pendingWith "changed the response when in csv mode"
-      --   request methodPost "/no_pk"
-      --                [("Content-Type", "text/csv"), ("Prefer", "return=representation")]
-      --                "a,b\nNULL,foo"
-      --     `shouldRespondWith` ResponseMatcher {
-      --       matchBody    = Just [json| { "a":null, "b":"foo" } |]
-      --     , matchStatus  = 201
-      --     , matchHeaders = ["Content-Type" <:> "application/json",
-      --                       "Location" <:> "/no_pk?a=is.null&b=eq.foo"]
-      --     }
       it "can post nulls" $
         request methodPost "/no_pk"
                      [("Content-Type", "text/csv"), ("Accept", "text/csv"), ("Prefer", "return=representation")]
@@ -202,23 +233,34 @@ spec = afterAll_ resetDb $ around withApp $ do
           `shouldRespondWith` ResponseMatcher {
             matchBody    = Just "a,b\n,foo"
           , matchStatus  = 201
-          , matchHeaders = ["Content-Type" <:> "text/csv",
+          , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8",
                             "Location" <:> "/no_pk?a=is.null&b=eq.foo"]
           }
 
 
-    after_ (clearTable "no_pk") . context "with wrong number of columns" $
+    context "with wrong number of columns" $
       it "fails for too few" $ do
         p <- request methodPost "/no_pk" [("Content-Type", "text/csv")] "a,b\nfoo,bar\nbaz"
         liftIO $ simpleStatus p `shouldBe` badRequest400
-      -- it does not fail because the extra columns are ignored
-      -- it "fails for too many" $ do
-      --   p <- request methodPost "/no_pk" [("Content-Type", "text/csv")] "a,b\nfoo,bar\nbaz,bat,bad"
-      --   liftIO $ simpleStatus p `shouldBe` badRequest400
+
+    context "with unicode values" $
+      it "succeeds and returns usable location header" $ do
+        let payload = [json| { "a":"圍棋", "b":"￥" } |]
+        p <- request methodPost "/no_pk"
+                     [("Prefer", "return=representation")]
+                     payload
+        liftIO $ do
+          simpleBody p `shouldBe` payload
+          simpleStatus p `shouldBe` created201
+
+        let Just location = lookup hLocation $ simpleHeaders p
+        r <- get location
+        liftIO $ simpleBody r `shouldBe` "["<>payload<>"]"
+
 
   describe "Putting record" $ do
 
-    context "to unkonwn uri" $
+    context "to unknown uri" $
       it "gives a 404" $ do
         pendingWith "Decide on PUT usefullness"
         request methodPut "/fake" []
@@ -242,7 +284,7 @@ spec = afterAll_ resetDb $ around withApp $ do
               [json| { "k1":12, "k2":42 } |]
                 `shouldRespondWith` 400
 
-        context "specifying every column in the table" . after_ (clearTable "compound_pk") $ do
+        context "specifying every column in the table" $ do
           it "can create a new record" $ do
             pendingWith "Decide on PUT usefullness"
             p <- request methodPut "/compound_pk?k1=eq.12&k2=eq.42" []
@@ -257,7 +299,7 @@ spec = afterAll_ resetDb $ around withApp $ do
               length rows `shouldBe` 1
               let record = head rows
               compoundK1 record `shouldBe` 12
-              compoundK2 record `shouldBe` 42
+              compoundK2 record `shouldBe` "42"
               compoundExtra record `shouldBe` Just 3
 
           it "can update an existing record" $ do
@@ -274,7 +316,7 @@ spec = afterAll_ resetDb $ around withApp $ do
               let record = head rows
               compoundExtra record `shouldBe` Just 5
 
-      context "with an auto-incrementing primary key" . after_ (clearTable "auto_incrementing_pk") $
+      context "with an auto-incrementing primary key"$
 
         it "succeeds with 204" $ do
           pendingWith "Decide on PUT usefullness"
@@ -293,7 +335,7 @@ spec = afterAll_ resetDb $ around withApp $ do
 
   describe "Patching record" $ do
 
-    context "to unkonwn uri" $
+    context "to unknown uri" $
       it "gives a 404" $
         request methodPatch "/fake" []
           [json| { "real": false } |]
@@ -301,23 +343,24 @@ spec = afterAll_ resetDb $ around withApp $ do
 
     context "on an empty table" $
       it "indicates no records found to update" $
-        request methodPatch "/simple_pk" []
+        request methodPatch "/empty_table" []
           [json| { "extra":20 } |]
             `shouldRespondWith` 404
 
-    context "in a nonempty table" . before_ (clearTable "items" >> createItems 15) .
-      after_ (clearTable "items") $ do
+    context "in a nonempty table" $ do
       it "can update a single item" $ do
         g <- get "/items?id=eq.42"
         liftIO $ simpleHeaders g
           `shouldSatisfy` matchHeader "Content-Range" "\\*/0"
-        request methodPatch "/items?id=eq.1" []
-          [json| { "id":42 } |]
-            `shouldRespondWith` ResponseMatcher {
-              matchBody    = Nothing,
-              matchStatus  = 204,
-              matchHeaders = ["Content-Range" <:> "0-0/1"]
-            }
+        p <- request methodPatch "/items?id=eq.2" [] [json| { "id":42 } |]
+        pure p `shouldRespondWith` ResponseMatcher {
+            matchBody    = Nothing,
+            matchStatus  = 204,
+            matchHeaders = ["Content-Range" <:> "0-0/1"]
+          }
+        liftIO $
+          lookup hContentType (simpleHeaders p) `shouldBe` Nothing
+
         g' <- get "/items?id=eq.42"
         liftIO $ simpleHeaders g'
           `shouldSatisfy` matchHeader "Content-Range" "0-0/1"
@@ -355,6 +398,28 @@ spec = afterAll_ resetDb $ around withApp $ do
           [json| { id: 99 } |]
           `shouldRespondWith` [json| [{id:99}] |]
 
+      it "can set a json column to escaped value" $ do
+        _ <- post "/json" [json| { data: {"escaped":"bar"} } |]
+        request methodPatch "/json?data->>escaped=eq.bar"
+                     [("Prefer", "return=representation")]
+                     [json| { "data": { "escaped":" \"bar" } } |]
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [json| [{ "data": { "escaped":" \"bar" } }] |]
+          , matchStatus  = 200
+          , matchHeaders = []
+          }
+
+    context "with unicode values" $
+      it "succeeds and returns values intact" $ do
+        void $ request methodPost "/no_pk" []
+          [json| { "a":"patchme", "b":"patchme" } |]
+        let payload = [json| { "a":"圍棋", "b":"￥" } |]
+        p <- request methodPatch "/no_pk?a=eq.patchme&b=eq.patchme"
+          [("Prefer", "return=representation")] payload
+        liftIO $ do
+          simpleBody p `shouldBe` "["<>payload<>"]"
+          simpleStatus p `shouldBe` ok200
+
   describe "Row level permission" $
     it "set user_id when inserting rows" $ do
       let auth = authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.y4vZuu1dDdwAl0-S00MCRWRYMlJ5YAMSir6Es6WtWx0"

test/Feature/QueryLimitedSpec.hs

@@ -0,0 +1,37 @@
+module Feature.QueryLimitedSpec where
+
+import Test.Hspec hiding (pendingWith)
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+import Network.HTTP.Types
+import Network.Wai.Test (SResponse(simpleHeaders, simpleStatus))
+import Text.Heredoc
+import SpecHelper
+import Network.Wai (Application)
+
+spec :: SpecWith Application
+spec =
+  describe "Requesting many items with server limits enabled" $ do
+    it "restricts results" $
+      get "/items"
+        `shouldRespondWith` ResponseMatcher {
+          matchBody    = Just [json| [{"id":1},{"id":2}] |]
+        , matchStatus  = 206
+        , matchHeaders = ["Content-Range" <:> "0-1/15"]
+        }
+
+    it "respects additional client limiting" $ do
+      r <- request methodGet  "/items"
+                   (rangeHdrs $ ByteRangeFromTo 0 0) ""
+      liftIO $ do
+        simpleHeaders r `shouldSatisfy`
+          matchHeader "Content-Range" "0-0/15"
+        simpleStatus r `shouldBe` partialContent206
+
+    it "limit works on all levels" $
+      get "/users?select=id,tasks{id}&order=id.asc&tasks.order=id.asc"
+        `shouldRespondWith` ResponseMatcher {
+          matchBody    = Just [str|[{"id":1,"tasks":[{"id":1},{"id":2}]},{"id":2,"tasks":[{"id":5},{"id":6}]}]|]
+        , matchStatus  = 206
+        , matchHeaders = ["Content-Range" <:> "0-1/3"]
+        }

test/Feature/QuerySpec.hs

@@ -8,21 +8,10 @@ import Network.Wai.Test (SResponse(simpleHeaders))
 
 import SpecHelper
 import Text.Heredoc
+import Network.Wai (Application)
 
-
-spec :: Spec
-spec =
-  beforeAll (clearTable "items" >> createItems 15)
-   . beforeAll clearProjectsTable
-   . beforeAll (clearTable "complex_items" >> createComplexItems)
-   . beforeAll (clearTable "nullable_integer" >> createNullInteger)
-   . beforeAll (
-       clearTable "no_pk" >>
-       createNulls 2 >>
-       createLikableStrings >>
-       createJsonData)
-   . afterAll_ (clearTable "items" >> clearTable "complex_items" >> clearTable "no_pk" >> clearTable "simple_pk")
-   . around withApp $ do
+spec :: SpecWith Application
+spec = do
 
   describe "Querying a table with a column called count" $
     it "should not confuse count column with pg_catalog.count aggregate" $
@@ -98,25 +87,25 @@ spec =
       get "/no_pk?a=is.null" `shouldRespondWith`
         [json| [{"a": null, "b": null}] |]
 
-      get "/nullable_integer?a=is.null" `shouldRespondWith` "[{\"a\":null}]"
+      get "/nullable_integer?a=is.null" `shouldRespondWith` [str|[{"a":null}]|]
 
     it "matches with like" $ do
       get "/simple_pk?k=like.*yx" `shouldRespondWith`
-        "[{\"k\":\"xyyx\",\"extra\":\"u\"}]"
+        [str|[{"k":"xyyx","extra":"u"}]|]
       get "/simple_pk?k=like.xy*" `shouldRespondWith`
-        "[{\"k\":\"xyyx\",\"extra\":\"u\"}]"
+        [str|[{"k":"xyyx","extra":"u"}]|]
       get "/simple_pk?k=like.*YY*" `shouldRespondWith`
-        "[{\"k\":\"xYYx\",\"extra\":\"v\"}]"
+        [str|[{"k":"xYYx","extra":"v"}]|]
 
     it "matches with like using not operator" $
       get "/simple_pk?k=not.like.*yx" `shouldRespondWith`
-        "[{\"k\":\"xYYx\",\"extra\":\"v\"}]"
+        [str|[{"k":"xYYx","extra":"v"}]|]
 
     it "matches with ilike" $ do
       get "/simple_pk?k=ilike.xy*&order=extra.asc" `shouldRespondWith`
-        "[{\"k\":\"xyyx\",\"extra\":\"u\"},{\"k\":\"xYYx\",\"extra\":\"v\"}]"
+        [str|[{"k":"xyyx","extra":"u"},{"k":"xYYx","extra":"v"}]|]
       get "/simple_pk?k=ilike.*YY*&order=extra.asc" `shouldRespondWith`
-        "[{\"k\":\"xyyx\",\"extra\":\"u\"},{\"k\":\"xYYx\",\"extra\":\"v\"}]"
+        [str|[{"k":"xyyx","extra":"u"},{"k":"xYYx","extra":"v"}]|]
 
     it "matches with ilike using not operator" $
       get "/simple_pk?k=not.ilike.xy*&order=extra.asc" `shouldRespondWith` "[]"
@@ -130,12 +119,16 @@ spec =
         [json| [{"text_search_vector":"'baz':1 'qux':2"}] |]
 
     it "matches with computed column" $
-      get "/items?always_true=eq.true" `shouldRespondWith`
+      get "/items?always_true=eq.true&order=id.asc" `shouldRespondWith`
+        [json| [{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13},{"id":14},{"id":15}] |]
+
+    it "order by computed column" $
+      get "/items?order=anti_id.desc" `shouldRespondWith`
         [json| [{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13},{"id":14},{"id":15}] |]
 
     it "matches filtering nested items" $
       get "/clients?select=id,projects{id,tasks{id,name}}&projects.tasks.name=like.Design*" `shouldRespondWith`
-        "[{\"id\":1,\"projects\":[{\"id\":1,\"tasks\":[{\"id\":1,\"name\":\"Design w7\"}]},{\"id\":2,\"tasks\":[{\"id\":3,\"name\":\"Design w10\"}]}]},{\"id\":2,\"projects\":[{\"id\":3,\"tasks\":[{\"id\":5,\"name\":\"Design IOS\"}]},{\"id\":4,\"tasks\":[{\"id\":7,\"name\":\"Design OSX\"}]}]}]"
+        [str|[{"id":1,"projects":[{"id":1,"tasks":[{"id":1,"name":"Design w7"}]},{"id":2,"tasks":[{"id":3,"name":"Design w10"}]}]},{"id":2,"projects":[{"id":3,"tasks":[{"id":5,"name":"Design IOS"}]},{"id":4,"tasks":[{"id":7,"name":"Design OSX"}]}]}]|]
 
     it "matches with @> operator" $
       get "/complex_items?select=id&arr_data=@>.{2}" `shouldRespondWith`
@@ -150,16 +143,29 @@ spec =
 
     it "selectStar works in absense of parameter" $
       get "/complex_items?id=eq.3" `shouldRespondWith`
-        [str|[{"id":3,"name":"Three","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3]}]|]
+        [str|[{"id":3,"name":"Three","settings":{"foo":{"int":1,"bar":"baz"}},"arr_data":[1,2,3],"field-with_sep":1}]|]
+
+    it "dash `-` in column names is accepted" $
+      get "/complex_items?id=eq.3&select=id,field-with_sep" `shouldRespondWith`
+        [str|[{"id":3,"field-with_sep":1}]|]
 
     it "one simple column" $
       get "/complex_items?select=id" `shouldRespondWith`
         [json| [{"id":1},{"id":2},{"id":3}] |]
 
+    it "rename simple column" $
+      get "/complex_items?id=eq.1&select=myId:id" `shouldRespondWith`
+        [json| [{"myId":1}] |]
+
+
     it "one simple column with casting (text)" $
       get "/complex_items?select=id::text" `shouldRespondWith`
         [json| [{"id":"1"},{"id":"2"},{"id":"3"}] |]
 
+    it "rename simple column with casting" $
+      get "/complex_items?id=eq.1&select=myId:id::text" `shouldRespondWith`
+        [json| [{"myId":"1"}] |]
+
     it "json column" $
       get "/complex_items?id=eq.1&select=settings" `shouldRespondWith`
         [json| [{"settings":{"foo":{"int":1,"bar":"baz"}}}] |]
@@ -168,6 +174,10 @@ spec =
       get "/complex_items?id=eq.1&select=settings->>foo::json" `shouldRespondWith`
         [json| [{"foo":{"int":1,"bar":"baz"}}] |] -- the value of foo here is of type "text"
 
+    it "rename json subfield one level with casting (json)" $
+      get "/complex_items?id=eq.1&select=myFoo:settings->>foo::json" `shouldRespondWith`
+        [json| [{"myFoo":{"int":1,"bar":"baz"}}] |] -- the value of foo here is of type "text"
+
     it "fails on bad casting (data of the wrong format)" $
       get "/complex_items?select=settings->foo->>bar::integer"
         `shouldRespondWith` ResponseMatcher {
@@ -189,30 +199,82 @@ spec =
       get "/complex_items?id=eq.1&select=settings->foo->>bar" `shouldRespondWith`
         [json| [{"bar":"baz"}] |]
 
+    it "rename json subfield two levels (string)" $
+      get "/complex_items?id=eq.1&select=myBar:settings->foo->>bar" `shouldRespondWith`
+        [json| [{"myBar":"baz"}] |]
+
 
     it "json subfield two levels with casting (int)" $
       get "/complex_items?id=eq.1&select=settings->foo->>int::integer" `shouldRespondWith`
         [json| [{"int":1}] |] -- the value in the db is an int, but here we expect a string for now
 
+    it "rename json subfield two levels with casting (int)" $
+      get "/complex_items?id=eq.1&select=myInt:settings->foo->>int::integer" `shouldRespondWith`
+        [json| [{"myInt":1}] |] -- the value in the db is an int, but here we expect a string for now
+
     it "requesting parents and children" $
       get "/projects?id=eq.1&select=id, name, clients{*}, tasks{id, name}" `shouldRespondWith`
-        "[{\"id\":1,\"name\":\"Windows 7\",\"clients\":{\"id\":1,\"name\":\"Microsoft\"},\"tasks\":[{\"id\":1,\"name\":\"Design w7\"},{\"id\":2,\"name\":\"Code w7\"}]}]"
+        [str|[{"id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"},"tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}]|]
+
+    it "embed data with two fk pointing to the same table" $
+      get "/orders?id=eq.1&select=id, name, billing_address_id{id}, shipping_address_id{id}" `shouldRespondWith`
+        [str|[{"id":1,"name":"order 1","billing_address_id":{"id":1},"shipping_address_id":{"id":2}}]|]
+
+
+    it "requesting parents and children while renaming them" $
+      get "/projects?id=eq.1&select=myId:id, name, project_client:client_id{*}, project_tasks:tasks{id, name}" `shouldRespondWith`
+        [str|[{"myId":1,"name":"Windows 7","project_client":{"id":1,"name":"Microsoft"},"project_tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}]|]
+
+
+    it "requesting parents and filtering parent columns" $
+      get "/projects?id=eq.1&select=id, name, clients{id}" `shouldRespondWith`
+        [str|[{"id":1,"name":"Windows 7","clients":{"id":1}}]|]
+
+    it "rows with missing parents are included" $
+      get "/projects?id=in.1,5&select=id,clients{id}" `shouldRespondWith`
+        [str|[{"id":1,"clients":{"id":1}},{"id":5,"clients":null}]|]
+
+    it "rows with no children return [] instead of null" $
+      get "/projects?id=in.5&select=id,tasks{id}" `shouldRespondWith`
+        [str|[{"id":5,"tasks":[]}]|]
 
     it "requesting children 2 levels" $
       get "/clients?id=eq.1&select=id,projects{id,tasks{id}}" `shouldRespondWith`
-        "[{\"id\":1,\"projects\":[{\"id\":1,\"tasks\":[{\"id\":1},{\"id\":2}]},{\"id\":2,\"tasks\":[{\"id\":3},{\"id\":4}]}]}]"
+        [str|[{"id":1,"projects":[{"id":1,"tasks":[{"id":1},{"id":2}]},{"id":2,"tasks":[{"id":3},{"id":4}]}]}]|]
 
     it "requesting many<->many relation" $
       get "/tasks?select=id,users{id}" `shouldRespondWith`
-        "[{\"id\":1,\"users\":[{\"id\":1},{\"id\":3}]},{\"id\":2,\"users\":[{\"id\":1}]},{\"id\":3,\"users\":[{\"id\":1}]},{\"id\":4,\"users\":[{\"id\":1}]},{\"id\":5,\"users\":[{\"id\":2},{\"id\":3}]},{\"id\":6,\"users\":[{\"id\":2}]},{\"id\":7,\"users\":[{\"id\":2}]},{\"id\":8,\"users\":null}]"
+        [str|[{"id":1,"users":[{"id":1},{"id":3}]},{"id":2,"users":[{"id":1}]},{"id":3,"users":[{"id":1}]},{"id":4,"users":[{"id":1}]},{"id":5,"users":[{"id":2},{"id":3}]},{"id":6,"users":[{"id":2}]},{"id":7,"users":[{"id":2}]},{"id":8,"users":[]}]|]
+
+    it "requesting many<->many relation with rename" $
+      get "/tasks?id=eq.1&select=id,theUsers:users{id}" `shouldRespondWith`
+        [str|[{"id":1,"theUsers":[{"id":1},{"id":3}]}]|]
+
+
+    it "requesting many<->many relation reverse" $
+      get "/users?select=id,tasks{id}" `shouldRespondWith`
+        [str|[{"id":1,"tasks":[{"id":1},{"id":2},{"id":3},{"id":4}]},{"id":2,"tasks":[{"id":5},{"id":6},{"id":7}]},{"id":3,"tasks":[{"id":1},{"id":5}]}]|]
 
     it "requesting parents and children on views" $
       get "/projects_view?id=eq.1&select=id, name, clients{*}, tasks{id, name}" `shouldRespondWith`
-        "[{\"id\":1,\"name\":\"Windows 7\",\"clients\":{\"id\":1,\"name\":\"Microsoft\"},\"tasks\":[{\"id\":1,\"name\":\"Design w7\"},{\"id\":2,\"name\":\"Code w7\"}]}]"
+        [str|[{"id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"},"tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}]|]
 
     it "requesting children with composite key" $
       get "/users_tasks?user_id=eq.2&task_id=eq.6&select=*, comments{content}" `shouldRespondWith`
-        "[{\"user_id\":2,\"task_id\":6,\"comments\":[{\"content\":\"Needs to be delivered ASAP\"}]}]"
+        [str|[{"user_id":2,"task_id":6,"comments":[{"content":"Needs to be delivered ASAP"}]}]|]
+
+    it "detect relations in views from exposed schema that are based on tables in private schema and have columns renames" $
+      get "/articles?id=eq.1&select=id,articleStars{users{*}}" `shouldRespondWith`
+        [str|[{"id":1,"articleStars":[{"users":{"id":1,"name":"Angela Martin"}},{"users":{"id":2,"name":"Michael Scott"}},{"users":{"id":3,"name":"Dwight Schrute"}}]}]|]
+
+    it "can select by column name" $
+      get "/projects?id=in.1,3&select=id,name,client_id,client_id{id,name}" `shouldRespondWith`
+        [str|[{"id":1,"name":"Windows 7","client_id":1,"client_id":{"id":1,"name":"Microsoft"}},{"id":3,"name":"IOS","client_id":2,"client_id":{"id":2,"name":"Apple"}}]|]
+
+    it "can select by column name sans id" $
+      get "/projects?id=in.1,3&select=id,name,client_id,client{id,name}" `shouldRespondWith`
+        [str|[{"id":1,"name":"Windows 7","client_id":1,"client":{"id":1,"name":"Microsoft"}},{"id":3,"name":"IOS","client_id":2,"client":{"id":2,"name":"Apple"}}]|]
+
 
   describe "Plurality singular" $ do
     it "will select an existing object" $
@@ -223,6 +285,14 @@ spec =
         , matchHeaders = []
         }
 
+    it "can combine multiple prefer values" $
+      request methodGet "/items?id=eq.5" [("Prefer","plurality=singular ; future=new; count=none")] ""
+        `shouldRespondWith` ResponseMatcher {
+          matchBody    = Just [json| {"id":5} |]
+        , matchStatus  = 200
+        , matchHeaders = []
+        }
+
     it "works in the presence of a range header" $
       let headers = ("Prefer","plurality=singular") :
             rangeHdrs (ByteRangeFromTo 0 9) in
@@ -240,7 +310,7 @@ spec =
     it "can shape plurality singular object routes" $
       request methodGet "/projects_view?id=eq.1&select=id,name,clients{*},tasks{id,name}" [("Prefer","plurality=singular")] ""
         `shouldRespondWith`
-          "{\"id\":1,\"name\":\"Windows 7\",\"clients\":{\"id\":1,\"name\":\"Microsoft\"},\"tasks\":[{\"id\":1,\"name\":\"Design w7\"},{\"id\":2,\"name\":\"Code w7\"}]}"
+          [str|{"id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"},"tasks":[{"id":1,"name":"Design w7"},{"id":2,"name":"Code w7"}]}|]
 
 
   describe "ordering response" $ do
@@ -292,6 +362,28 @@ spec =
     it "without other constraints" $
       get "/items?order=id.asc" `shouldRespondWith` 200
 
+    it "ordering embeded entities" $
+      get "/projects?id=eq.1&select=id, name, tasks{id, name}&tasks.order=name.asc" `shouldRespondWith`
+        [str|[{"id":1,"name":"Windows 7","tasks":[{"id":2,"name":"Code w7"},{"id":1,"name":"Design w7"}]}]|]
+
+    it "ordering embeded entities with alias" $
+      get "/projects?id=eq.1&select=id, name, the_tasks:tasks{id, name}&tasks.order=name.asc" `shouldRespondWith`
+        [str|[{"id":1,"name":"Windows 7","the_tasks":[{"id":2,"name":"Code w7"},{"id":1,"name":"Design w7"}]}]|]
+
+    it "ordering embeded entities, two levels" $
+      get "/projects?id=eq.1&select=id, name, tasks{id, name, users{id, name}}&tasks.order=name.asc&tasks.users.order=name.desc" `shouldRespondWith`
+        [str|[{"id":1,"name":"Windows 7","tasks":[{"id":2,"name":"Code w7","users":[{"id":1,"name":"Angela Martin"}]},{"id":1,"name":"Design w7","users":[{"id":3,"name":"Dwight Schrute"},{"id":1,"name":"Angela Martin"}]}]}]|]
+
+    it "ordering embeded parents does not break things" $
+      get "/projects?id=eq.1&select=id, name, clients{id, name}&clients.order=name.asc" `shouldRespondWith`
+        [str|[{"id":1,"name":"Windows 7","clients":{"id":1,"name":"Microsoft"}}]|]
+
+    it "ordering embeded parents does not break things when using ducktape names" $
+      get "/projects?id=eq.1&select=id, name, client{id, name}&client.order=name.asc" `shouldRespondWith`
+        [str|[{"id":1,"name":"Windows 7","client":{"id":1,"name":"Microsoft"}}]|]
+
+
+
   describe "Accept headers" $ do
     it "should respond an unknown accept type with 415" $
       request methodGet "/simple_pk"
@@ -314,7 +406,7 @@ spec =
         `shouldRespondWith` ResponseMatcher {
           matchBody    = Just "k,extra\nxyyx,u\nxYYx,v"
         , matchStatus  = 200
-        , matchHeaders = ["Content-Type" <:> "text/csv"]
+        , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8"]
         }
 
   describe "Canonical location" $ do
@@ -347,8 +439,17 @@ spec =
         [json| [{"data": {"id": 1, "foo": {"bar": "baz"}}}] |]
 
   describe "remote procedure call" $ do
-    context "a proc that returns a set" . before_ (clearTable "items" >> createItems 10) .
-      after_ (clearTable "items") $
+    context "a proc that returns a set" $ do
+      it "returns paginated results" $
+        request methodPost "/rpc/getitemrange"
+                (rangeHdrs (ByteRangeFromTo 0 0))  [json| { "min": 2, "max": 4 } |]
+           `shouldRespondWith` ResponseMatcher {
+              matchBody    = Just [json| [{"id":3}] |]
+            , matchStatus = 206
+            , matchHeaders = ["Content-Range" <:> "0-0/2"]
+            }
+
+
       it "returns proper json" $
         post "/rpc/getitemrange" [json| { "min": 2, "max": 4 } |] `shouldRespondWith`
           [json| [ {"id": 3}, {"id":4} ] |]
@@ -358,7 +459,59 @@ spec =
         post "/rpc/test_empty_rowset" [json| {} |] `shouldRespondWith`
           [json| [] |]
 
-    context "a proc that returns plain text" $
+    context "a proc that returns plain text" $ do
       it "returns proper json" $
         post "/rpc/sayhello" [json| { "name": "world" } |] `shouldRespondWith`
           [json| [{"sayhello":"Hello, world"}] |]
+
+      it "can handle unicode" $
+        post "/rpc/sayhello" [json| { "name": "￥" } |] `shouldRespondWith`
+          [json| [{"sayhello":"Hello, ￥"}] |]
+
+    context "improper input" $ do
+      it "rejects unknown content type even if payload is good" $
+        request methodPost "/rpc/sayhello"
+          (acceptHdrs "audio/mpeg3") [json| { "name": "world" } |]
+            `shouldRespondWith` 415
+      it "rejects malformed json payload" $
+        request methodPost "/rpc/sayhello"
+          (acceptHdrs "application/json") "sdfsdf"
+            `shouldRespondWith` 400
+
+    context "unsupported verbs" $ do
+      it "DELETE fails" $
+        request methodDelete "/rpc/sayhello" [] ""
+          `shouldRespondWith` 405
+      it "PATCH fails" $
+        request methodPatch "/rpc/sayhello" [] ""
+          `shouldRespondWith` 405
+      it "OPTIONS fails" $
+        -- TODO: should return info about the function
+        request methodOptions "/rpc/sayhello" [] ""
+          `shouldRespondWith` 405
+      it "GET fails with 405 on unknown procs" $
+        -- TODO: should this be 404?
+        get "/rpc/fake" `shouldRespondWith` 405
+      it "GET with 405 on known procs" $
+        get "/rpc/sayhello" `shouldRespondWith` 405
+
+    it "executes the proc exactly once per request" $ do
+      post "/rpc/callcounter" [json| {} |] `shouldRespondWith`
+        [json| [{"callcounter":1}] |]
+      post "/rpc/callcounter" [json| {} |] `shouldRespondWith`
+        [json| [{"callcounter":2}] |]
+
+  describe "weird requests" $ do
+    it "can query as normal" $ do
+      get "/Escap3e;" `shouldRespondWith`
+        [json| [{"so6meIdColumn":1},{"so6meIdColumn":2},{"so6meIdColumn":3},{"so6meIdColumn":4},{"so6meIdColumn":5}] |]
+      get "/ghostBusters" `shouldRespondWith`
+        [json| [{"escapeId":1},{"escapeId":3},{"escapeId":5}] |]
+
+    it "will embed a collection" $
+      get "/Escap3e;?select=ghostBusters{*}" `shouldRespondWith`
+        [json| [{"ghostBusters":[{"escapeId":1}]},{"ghostBusters":[]},{"ghostBusters":[{"escapeId":3}]},{"ghostBusters":[]},{"ghostBusters":[{"escapeId":5}]}] |]
+
+    it "will embed using a column" $
+      get "/ghostBusters?select=escapeId{*}" `shouldRespondWith`
+        [json| [{"escapeId":{"so6meIdColumn":1}},{"escapeId":{"so6meIdColumn":3}},{"escapeId":{"so6meIdColumn":5}}] |]

test/Feature/RangeSpec.hs

@@ -6,13 +6,110 @@ import Test.Hspec.Wai.JSON
 import Network.HTTP.Types
 import Network.Wai.Test (SResponse(simpleHeaders,simpleStatus))
 
+import qualified Data.ByteString.Lazy         as BL
+
 import SpecHelper
+import Text.Heredoc
+import Network.Wai (Application)
 
-spec :: Spec
-spec = beforeAll (clearTable "items" >> createItems 15) . afterAll_ (clearTable "items")
-  . around withApp $
+defaultRange :: BL.ByteString
+defaultRange = [json| { "min": 0, "max": 15 } |]
+
+emptyRange :: BL.ByteString
+emptyRange = [json| { "min": 2, "max": 2 } |]
+
+spec :: SpecWith Application
+spec = do
+  describe "POST /rpc/getitemrange" $ do
+    context "without range headers" $ do
+      context "with response under server size limit" $
+        it "returns whole range with status 200" $
+           post "/rpc/getitemrange" defaultRange `shouldRespondWith` 200
+
+      context "when I don't want the count" $ do
+        it "returns range Content-Range with */* for empty range" $
+          request methodPost "/rpc/getitemrange"
+                  [("Prefer", "count=none")] emptyRange
+            `shouldRespondWith` ResponseMatcher {
+              matchBody    = Just [json| [] |]
+            , matchStatus  = 200
+            , matchHeaders = ["Content-Range" <:> "*/*"]
+            }
+
+        it "returns range Content-Range with range/*" $
+          request methodPost "/rpc/getitemrange"
+                  [("Prefer", "count=none")] defaultRange
+            `shouldRespondWith` ResponseMatcher {
+              matchBody    = Just [json| [{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13},{"id":14},{"id":15}] |]
+            , matchStatus  = 200
+            , matchHeaders = ["Content-Range" <:> "0-14/*"]
+            }
+
+    context "with range headers" $ do
+
+      context "of acceptable range" $ do
+        it "succeeds with partial content" $ do
+          r <- request methodPost  "/rpc/getitemrange"
+                       (rangeHdrs $ ByteRangeFromTo 0 1) defaultRange
+          liftIO $ do
+            simpleHeaders r `shouldSatisfy`
+              matchHeader "Content-Range" "0-1/15"
+            simpleStatus r `shouldBe` partialContent206
+
+        it "understands open-ended ranges" $
+          request methodPost "/rpc/getitemrange"
+                  (rangeHdrs $ ByteRangeFrom 0) defaultRange
+            `shouldRespondWith` 200
+
+        it "returns an empty body when there are no results" $
+          request methodPost "/rpc/getitemrange"
+                  (rangeHdrs $ ByteRangeFromTo 0 1) emptyRange
+            `shouldRespondWith` ResponseMatcher {
+              matchBody    = Just "[]"
+            , matchStatus  = 200
+            , matchHeaders = ["Content-Range" <:> "*/0"]
+            }
+
+        it "allows one-item requests" $ do
+          r <- request methodPost  "/rpc/getitemrange"
+                       (rangeHdrs $ ByteRangeFromTo 0 0) defaultRange
+          liftIO $ do
+            simpleHeaders r `shouldSatisfy`
+              matchHeader "Content-Range" "0-0/15"
+            simpleStatus r `shouldBe` partialContent206
+
+        it "handles ranges beyond collection length via truncation" $ do
+          r <- request methodPost  "/rpc/getitemrange"
+                       (rangeHdrs $ ByteRangeFromTo 10 100) defaultRange
+          liftIO $ do
+            simpleHeaders r `shouldSatisfy`
+              matchHeader "Content-Range" "10-14/15"
+            simpleStatus r `shouldBe` partialContent206
+
+      context "of invalid range" $ do
+        it "fails with 416 for offside range" $
+          request methodPost  "/rpc/getitemrange"
+                  (rangeHdrs $ ByteRangeFromTo 1 0) emptyRange
+            `shouldRespondWith` 416
+
+        it "refuses a range with nonzero start when there are no items" $
+          request methodPost "/rpc/getitemrange"
+                  (rangeHdrs $ ByteRangeFromTo 1 2) emptyRange
+            `shouldRespondWith` ResponseMatcher {
+              matchBody    = Nothing
+            , matchStatus  = 416
+            , matchHeaders = ["Content-Range" <:> "*/0"]
+            }
+
+        it "refuses a range requesting start past last item" $
+          request methodPost "/rpc/getitemrange"
+                  (rangeHdrs $ ByteRangeFromTo 100 199) defaultRange
+            `shouldRespondWith` ResponseMatcher {
+              matchBody    = Nothing
+            , matchStatus  = 416
+            , matchHeaders = ["Content-Range" <:> "*/15"]
+            }
   describe "GET /items" $ do
-
     context "without range headers" $ do
       context "with response under server size limit" $
         it "returns whole range with status 200" $
@@ -46,6 +143,50 @@ spec = beforeAll (clearTable "items" >> createItems 15) . afterAll_ (clearTable
             , matchHeaders = ["Content-Range" <:> "0-0/*"]
             }
 
+    context "with limit/offset parameters" $ do
+      it "no parameters return everything" $
+        get "/items?select=id&order=id.asc"
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|[{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13},{"id":14},{"id":15}]|]
+          , matchStatus  = 200
+          , matchHeaders = ["Content-Range" <:> "0-14/15"]
+          }
+      it "top level limit with parameter" $
+        get "/items?select=id&order=id.asc&limit=3"
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|[{"id":1},{"id":2},{"id":3}]|]
+          , matchStatus  = 206
+          , matchHeaders = ["Content-Range" <:> "0-2/15"]
+          }
+      it "headers override get parameters" $
+        request methodGet  "/items?select=id&order=id.asc&limit=3"
+                     (rangeHdrs $ ByteRangeFromTo 0 1) ""
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|[{"id":1},{"id":2}]|]
+          , matchStatus  = 206
+          , matchHeaders = ["Content-Range" <:> "0-1/15"]
+          }
+
+      it "limit works on all levels" $
+        get "/clients?select=id,projects{id,tasks{id}}&order=id.asc&limit=1&projects.order=id.asc&projects.limit=1&projects.tasks.order=id.asc&projects.tasks.limit=2"
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|[{"id":1,"projects":[{"id":1,"tasks":[{"id":1},{"id":2}]}]}]|]
+          , matchStatus  = 206
+          , matchHeaders = ["Content-Range" <:> "0-0/2"]
+          }
+
+      it "fails on offset specified below level 1" $
+        get "/clients?select=id,projects{id,tasks{id}}&projects.offset=2&projects.limit=1"
+          `shouldRespondWith` 400
+
+      it "limit and offset works on first level" $
+        get "/items?select=id&order=id.asc&limit=3&offset=2"
+          `shouldRespondWith` ResponseMatcher {
+            matchBody    = Just [str|[{"id":3},{"id":4},{"id":5}]|]
+          , matchStatus  = 206
+          , matchHeaders = ["Content-Range" <:> "2-4/15"]
+          }
+
     context "with range headers" $ do
 
       context "of acceptable range" $ do

test/Feature/StructureSpec.hs

@@ -7,29 +7,39 @@ import Test.Hspec.Wai.JSON
 import SpecHelper
 
 import Network.HTTP.Types
+import Network.Wai (Application)
+import Network.Wai.Test (SResponse(simpleHeaders))
+
+spec :: SpecWith Application
+spec = do
 
-spec :: Spec
-spec = around withApp $ do
   describe "GET /" $ do
     it "lists views in schema" $
       request methodGet "/" [] ""
         `shouldRespondWith` [json| [
-          {"schema":"test","name":"articleStars","insertable":true}
+          {"schema":"test","name":"Escap3e;","insertable":true}
+        , {"schema":"test","name":"addresses","insertable":true}
+        , {"schema":"test","name":"articleStars","insertable":true}
         , {"schema":"test","name":"articles","insertable":true}
         , {"schema":"test","name":"auto_incrementing_pk","insertable":true}
         , {"schema":"test","name":"clients","insertable":true}
         , {"schema":"test","name":"comments","insertable":true}
         , {"schema":"test","name":"complex_items","insertable":true}
         , {"schema":"test","name":"compound_pk","insertable":true}
+        , {"schema":"test","name":"empty_table","insertable":true}
+        , {"schema":"test","name":"filtered_tasks","insertable":true}
+        , {"schema":"test","name":"ghostBusters","insertable":true}
         , {"schema":"test","name":"has_count_column","insertable":false}
         , {"schema":"test","name":"has_fk","insertable":true}
         , {"schema":"test","name":"insertable_view_with_join","insertable":true}
+        , {"schema":"test","name":"insertonly","insertable":true}
         , {"schema":"test","name":"items","insertable":true}
         , {"schema":"test","name":"json","insertable":true}
         , {"schema":"test","name":"materialized_view","insertable":false}
         , {"schema":"test","name":"menagerie","insertable":true}
         , {"schema":"test","name":"no_pk","insertable":true}
         , {"schema":"test","name":"nullable_integer","insertable":true}
+        , {"schema":"test","name":"orders","insertable":true}
         , {"schema":"test","name":"projects","insertable":true}
         , {"schema":"test","name":"projects_view","insertable":true}
         , {"schema":"test","name":"simple_pk","insertable":true}
@@ -38,6 +48,7 @@ spec = around withApp $ do
         , {"schema":"test","name":"users","insertable":true}
         , {"schema":"test","name":"users_projects","insertable":true}
         , {"schema":"test","name":"users_tasks","insertable":true}
+        , {"schema":"test","name":"withUnique","insertable":true}
         ] |]
         {matchStatus = 200}
 
@@ -51,6 +62,61 @@ spec = around withApp $ do
         {matchStatus = 200}
 
   describe "Table info" $ do
+    it "The structure of complex views is correctly detected" $
+      request methodOptions "/filtered_tasks" [] "" `shouldRespondWith`
+      [json|
+      {
+        "pkey": [
+          "myId"
+        ],
+        "columns": [
+          {
+            "references": null,
+            "default": null,
+            "precision": 32,
+            "updatable": true,
+            "schema": "test",
+            "name": "myId",
+            "type": "integer",
+            "maxLen": null,
+            "enum": [],
+            "nullable": true,
+            "position": 1
+          },
+          {
+            "references": null,
+            "default": null,
+            "precision": null,
+            "updatable": true,
+            "schema": "test",
+            "name": "name",
+            "type": "text",
+            "maxLen": null,
+            "enum": [],
+            "nullable": true,
+            "position": 2
+          },
+          {
+            "references": {
+              "schema": "test",
+              "column": "id",
+              "table": "projects"
+            },
+            "default": null,
+            "precision": 32,
+            "updatable": true,
+            "schema": "test",
+            "name": "projectID",
+            "type": "integer",
+            "maxLen": null,
+            "enum": [],
+            "nullable": true,
+            "position": 3
+          }
+        ]
+      }
+      |]
+
     it "is available with OPTIONS verb" $
       request methodOptions "/menagerie" [] "" `shouldRespondWith`
       [json|
@@ -208,16 +274,14 @@ spec = around withApp $ do
       }
       |]
 
-    it "includes foreign key data" $ do
-      pendingWith "have to resolve issue #107"
-
+    it "includes foreign key data" $
       request methodOptions "/has_fk" [] ""
         `shouldRespondWith` [json|
       {
         "pkey": ["id"],
         "columns":[
           {
-            "default": "nextval('\"1\".has_fk_id_seq'::regclass)",
+            "default": "nextval('test.has_fk_id_seq'::regclass)",
             "precision": 64,
             "updatable": true,
             "schema": "test",
@@ -239,7 +303,7 @@ spec = around withApp $ do
             "nullable": true,
             "position": 2,
             "enum": [],
-            "references": {"table": "auto_incrementing_pk", "column": "id"}
+            "references": {"schema":"test", "table": "auto_incrementing_pk", "column": "id"}
           }, {
             "default": null,
             "precision": null,
@@ -251,7 +315,7 @@ spec = around withApp $ do
             "nullable": true,
             "position": 3,
             "enum": [],
-            "references": {"table": "simple_pk", "column": "k"}
+            "references": {"schema":"test", "table": "simple_pk", "column": "k"}
           }
         ]
       }
@@ -316,3 +380,20 @@ spec = around withApp $ do
             ]
           }
         |]
+
+    it "errors for non existant tables" $
+      request methodOptions "/dne" [] "" `shouldRespondWith` 404
+
+  describe "Allow header" $ do
+
+    it "includes read/write verbs for writeable table" $ do
+      r <- request methodOptions "/items" [] ""
+      liftIO $
+        simpleHeaders r `shouldSatisfy`
+          matchHeader "Allow" "GET,POST,PATCH,DELETE"
+
+    it "includes read verbs for read-only table" $ do
+      r <- request methodOptions "/has_count_column" [] ""
+      liftIO $
+        simpleHeaders r `shouldSatisfy`
+          matchHeader "Allow" "GET"

test/Feature/UnicodeSpec.hs

@@ -0,0 +1,20 @@
+module Feature.UnicodeSpec where
+
+import Test.Hspec
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+import Network.Wai (Application)
+import Control.Monad (void)
+
+spec :: SpecWith Application
+spec =
+  describe "Reading and writing to unicode schema and table names" $
+    it "Can read and write values" $ do
+      get "/%D9%85%D9%88%D8%A7%D8%B1%D8%AF"
+        `shouldRespondWith` "[]"
+
+      void $ post "/%D9%85%D9%88%D8%A7%D8%B1%D8%AF"
+        [json| { "هویت": 1 } |]
+
+      get "/%D9%85%D9%88%D8%A7%D8%B1%D8%AF"
+        `shouldRespondWith` [json| [{ "هویت": 1 }] |]

test/Main.hs

@@ -2,7 +2,56 @@ module Main where
 
 import Test.Hspec
 import SpecHelper
-import Spec
+
+import qualified Hasql.Pool as P
+
+import PostgREST.DbStructure (getDbStructure)
+import PostgREST.App (postgrest)
+import Data.IORef
+import Data.String.Conversions (cs)
+
+import qualified Feature.AuthSpec
+import qualified Feature.ConcurrentSpec
+import qualified Feature.CorsSpec
+import qualified Feature.DeleteSpec
+import qualified Feature.InsertSpec
+import qualified Feature.QueryLimitedSpec
+import qualified Feature.QuerySpec
+import qualified Feature.RangeSpec
+import qualified Feature.StructureSpec
+import qualified Feature.UnicodeSpec
 
 main :: IO ()
-main = resetDb >> hspec spec
+main = do
+  setupDb
+
+  pool <- P.acquire (3, 10, cs testDbConn)
+
+  result <- P.use pool $ getDbStructure "test"
+  refDbStructure <- newIORef $ either (error.show) id result
+  let withApp = return $ postgrest testCfg refDbStructure pool
+      ltdApp  = return $ postgrest testLtdRowsCfg refDbStructure pool
+      unicodeApp = return $ postgrest testUnicodeCfg refDbStructure pool
+
+  hspec $ do
+    mapM_ (beforeAll_ resetDb . before withApp) specs
+
+    -- this test runs with a different server flag
+    beforeAll_ resetDb . before ltdApp $
+      describe "Feature.QueryLimitedSpec" Feature.QueryLimitedSpec.spec
+
+    -- this test runs with a different schema
+    beforeAll_ resetDb . before unicodeApp $
+      describe "Feature.UnicodeSpec" Feature.UnicodeSpec.spec
+
+ where
+  specs = map (uncurry describe) [
+      ("Feature.AuthSpec"         , Feature.AuthSpec.spec)
+    , ("Feature.ConcurrentSpec"   , Feature.ConcurrentSpec.spec)
+    , ("Feature.CorsSpec"         , Feature.CorsSpec.spec)
+    , ("Feature.DeleteSpec"       , Feature.DeleteSpec.spec)
+    , ("Feature.InsertSpec"       , Feature.InsertSpec.spec)
+    , ("Feature.QuerySpec"        , Feature.QuerySpec.spec)
+    , ("Feature.RangeSpec"        , Feature.RangeSpec.spec)
+    , ("Feature.StructureSpec"    , Feature.StructureSpec.spec)
+    ]

test/Spec.hs

@@ -1 +0,0 @@
-{-# OPTIONS_GHC -F -pgmF hspec-discover -optF --no-main #-}

test/SpecHelper.hs

@@ -1,91 +1,49 @@
 module SpecHelper where
 
-import Network.Wai
-import Test.Hspec
-import Test.Hspec.Wai
-
-import Hasql as H
-import Hasql.Backend as B
-import Hasql.Postgres as P
-
 import Data.String.Conversions (cs)
-import Data.Monoid
-import Data.Text hiding (map)
-import qualified Data.Vector as V
 import Control.Monad (void)
-import Control.Applicative
 
 import Network.HTTP.Types.Header (Header, ByteRange, renderByteRange,
                                   hRange, hAuthorization, hAccept)
 import Codec.Binary.Base64.String (encode)
 import Data.CaseInsensitive (CI(..))
-import Data.Maybe (fromMaybe)
 import Text.Regex.TDFA ((=~))
 import qualified Data.ByteString.Char8 as BS
 import System.Process (readProcess)
 import Web.JWT (secret)
 
-import qualified Data.Aeson.Types as J
-
-import PostgREST.App (app)
 import PostgREST.Config (AppConfig(..))
-import PostgREST.Middleware
-import PostgREST.Error(pgErrResponse)
-import PostgREST.DbStructure
 
-dbString :: String
-dbString = "postgres://postgrest_test@localhost:5432/postgrest_test"
+testDbConn :: String
+testDbConn = "postgres://postgrest_test_authenticator@localhost:5432/postgrest_test"
 
-isLeft :: Either a b -> Bool
-isLeft (Left _ ) = True
-isLeft _ = False
+testCfg :: AppConfig
+testCfg =
+  AppConfig testDbConn "postgrest_test_anonymous" "test" 3000 (secret "safe") 10 Nothing True
 
-cfg :: AppConfig
-cfg = AppConfig dbString 3000 "postgrest_anonymous" "test" (secret "safe") 10
+testUnicodeCfg :: AppConfig
+testUnicodeCfg =
+  AppConfig testDbConn "postgrest_test_anonymous" "تست" 3000 (secret "safe") 10 Nothing True
 
-testPoolOpts :: PoolSettings
-testPoolOpts = fromMaybe (error "bad settings") $ H.poolSettings 1 30
-
-pgSettings :: P.Settings
-pgSettings = P.StringSettings $ cs dbString
-
-withApp :: ActionWith Application -> IO ()
-withApp perform = do
-  pool :: H.Pool P.Postgres
-    <- H.acquirePool pgSettings testPoolOpts
-
-  let txSettings = Just (H.ReadCommitted, Just True)
-  dbOrError <- H.session pool $ H.tx txSettings $ getDbStructure (cs $ configSchema cfg)
-  db <- either (fail . show) return dbOrError
-
-  perform $ middle $ \req resp -> do
-    body <- strictRequestBody req
-    result <- liftIO $ H.session pool $ H.tx txSettings
-      $ runWithClaims cfg (app db cfg body) req
-    either (resp . pgErrResponse) resp result
-
-  where middle = defaultMiddle
-
-
-resetDb :: IO ()
-resetDb = do
-  pool :: H.Pool P.Postgres
-    <- H.acquirePool pgSettings testPoolOpts
-  void . liftIO $ H.session pool $
-    H.tx Nothing $ do
-      H.unitEx [H.stmt| drop schema if exists test cascade |]
-      H.unitEx [H.stmt| drop schema if exists private cascade |]
-      H.unitEx [H.stmt| drop schema if exists postgrest cascade |]
+testLtdRowsCfg :: AppConfig
+testLtdRowsCfg =
+  AppConfig testDbConn "postgrest_test_anonymous" "test" 3000 (secret "safe") 10 (Just 2) True
 
+setupDb :: IO ()
+setupDb = do
+  void $ readProcess "psql" ["-d", "postgres", "-a", "-f", "test/fixtures/database.sql"] []
   loadFixture "roles"
   loadFixture "schema"
+  loadFixture "privileges"
+  resetDb
 
+resetDb :: IO ()
+resetDb = loadFixture "data"
 
 loadFixture :: FilePath -> IO()
 loadFixture name =
   void $ readProcess "psql" ["-U", "postgrest_test", "-d", "postgrest_test", "-a", "-f", "test/fixtures/" ++ name ++ ".sql"] []
 
-
 rangeHdrs :: ByteRange -> [Header]
 rangeHdrs r = [rangeUnit, (hRange, renderByteRange r)]
 
@@ -106,77 +64,3 @@ authHeaderBasic u p =
 authHeaderJWT :: String -> Header
 authHeaderJWT token =
   (hAuthorization, cs $ "Bearer " ++ token)
-
-testPool :: IO(H.Pool P.Postgres)
-testPool = H.acquirePool pgSettings testPoolOpts
-
-clearTable :: Text -> IO ()
-clearTable table = do
-  pool <- testPool
-  void . liftIO $ H.session pool $ H.tx Nothing $
-    H.unitEx $ B.Stmt ("delete from test."<>table) V.empty True
-
-clearProjectsTable :: IO ()
-clearProjectsTable = do
-  pool <- testPool
-  void . liftIO $ H.session pool $ H.tx Nothing $
-    H.unitEx $ B.Stmt "delete from test.projects where id > 4" V.empty True
-
-
-createItems :: Int -> IO ()
-createItems n = do
-  pool <- testPool
-  void . liftIO $ H.session pool $ H.tx Nothing txn
-  where
-    txn = mapM_ H.unitEx stmts
-    stmts = map [H.stmt|insert into test.items (id) values (?)|] [1..n]
-
-createComplexItems :: IO ()
-createComplexItems = do
-  pool <- testPool
-  void . liftIO $ H.session pool $ H.tx Nothing txn
-  where
-    txn = mapM_ H.unitEx stmts
-    stmts = getZipList $ [H.stmt|insert into test.complex_items (id, name, settings, arr_data) values (?,?,?,?)|]
-        <$> ZipList ([1..3]::[Int])
-        <*> ZipList (["One", "Two", "Three"]::[Text])
-        <*> ZipList [jobj,jobj,jobj]
-        <*> ZipList ([[1], [1,2], [1,2,3]]::[[Int]])
-    jobj = J.object [("foo", J.object [("int", J.Number 1),("bar", J.String "baz")])]
-
-createNulls :: Int -> IO ()
-createNulls n = do
-  pool <- testPool
-  void . liftIO $ H.session pool $ H.tx Nothing txn
-  where
-    txn = mapM_ H.unitEx (stmt':stmts)
-    stmt' = [H.stmt|insert into test.no_pk (a,b) values (null,null)|]
-    stmts = map [H.stmt|insert into test.no_pk (a,b) values (?,0)|] [1..n]
-
-createNullInteger :: IO ()
-createNullInteger = do
-  pool <- testPool
-  void . liftIO $ H.session pool $ H.tx Nothing $
-    H.unitEx $ [H.stmt| insert into "test".nullable_integer (a) values (null) |]
-
-createLikableStrings :: IO ()
-createLikableStrings = do
-  pool <- testPool
-  void . liftIO $ H.session pool $ H.tx Nothing $ do
-    H.unitEx $ insertSimplePk "xyyx" "u"
-    H.unitEx $ insertSimplePk "xYYx" "v"
-  where
-    insertSimplePk :: Text -> Text -> H.Stmt P.Postgres
-    insertSimplePk = [H.stmt|insert into test.simple_pk (k, extra) values (?,?)|]
-
-createJsonData :: IO ()
-createJsonData = do
-  pool <- testPool
-  void . liftIO $ H.session pool $ H.tx Nothing $
-    H.unitEx $
-      [H.stmt|
-        insert into test.json (data) values (?)
-      |]
-      (J.object [("id", J.Number 1)
-                ,("foo", J.object [("bar", J.String "baz")])
-                ])

test/TestTypes.hs

@@ -37,9 +37,9 @@ instance JSON.FromJSON IncPK where
 
 data CompoundPK = CompoundPK {
   compoundK1 :: Int
-, compoundK2 :: Int
+, compoundK2 :: String
 , compoundExtra :: Maybe Int
-}
+} deriving (Eq, Show)
 
 instance JSON.FromJSON CompoundPK where
   parseJSON (JSON.Object r) = CompoundPK <$>

test/Unit/PgQuerySpec.hx

@@ -60,7 +60,7 @@ spec = around dbWithSchema $ do
         quickALQuery c "select * from \"1\".items where id = ?" [snd row]
           `shouldReturn` [[row]]
 
-  let {user = "jdoe"; pass = "secret"; role = "test_default_role"}
+  let {user = "jdoe"; pass = "secret"; role = "postgrest_test_default_role"}
   describe "addUser" $ do
     it "adds a correct user to the right table" $ \conn -> do
       addUser user pass role conn

test/fixtures/data.sql

@@ -0,0 +1,286 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 9.5beta1
+-- Dumped by pg_dump version 9.5beta1
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+
+SET search_path = postgrest, pg_catalog;
+
+--
+-- Data for Name: auth; Type: TABLE DATA; Schema: postgrest; Owner: -
+--
+
+TRUNCATE TABLE auth CASCADE;
+INSERT INTO auth VALUES ('jdoe', 'postgrest_test_author', '1234                                                        ');
+
+
+SET search_path = private, pg_catalog;
+
+--
+-- Data for Name: articles; Type: TABLE DATA; Schema: private; Owner: -
+--
+
+TRUNCATE TABLE articles CASCADE;
+INSERT INTO articles VALUES (1, 'No… It''s a thing; it''s like a plan, but with more greatness.', 'diogo');
+INSERT INTO articles VALUES (2, 'Stop talking, brain thinking. Hush.', 'diogo');
+INSERT INTO articles VALUES (3, 'It''s a fez. I wear a fez now. Fezes are cool.', 'diogo');
+
+
+SET search_path = test, pg_catalog;
+
+--
+-- Data for Name: users; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE users CASCADE;
+INSERT INTO users VALUES (1, 'Angela Martin');
+INSERT INTO users VALUES (2, 'Michael Scott');
+INSERT INTO users VALUES (3, 'Dwight Schrute');
+
+
+SET search_path = private, pg_catalog;
+
+--
+-- Data for Name: article_stars; Type: TABLE DATA; Schema: private; Owner: -
+--
+
+TRUNCATE TABLE article_stars CASCADE;
+INSERT INTO article_stars VALUES (1, 1, '2015-12-08 04:22:57.472738');
+INSERT INTO article_stars VALUES (1, 2, '2015-12-08 04:22:57.472738');
+INSERT INTO article_stars VALUES (2, 3, '2015-12-08 04:22:57.472738');
+INSERT INTO article_stars VALUES (3, 2, '2015-12-08 04:22:57.472738');
+INSERT INTO article_stars VALUES (1, 3, '2015-12-08 04:22:57.472738');
+
+
+SET search_path = test, pg_catalog;
+
+--
+-- Data for Name: authors_only; Type: TABLE DATA; Schema: test; Owner: -
+--
+TRUNCATE TABLE authors_only CASCADE;
+
+
+--
+-- Data for Name: auto_incrementing_pk; Type: TABLE DATA; Schema: test; Owner: -
+--
+TRUNCATE TABLE auto_incrementing_pk CASCADE;
+
+
+--
+-- Name: auto_incrementing_pk_id_seq; Type: SEQUENCE SET; Schema: test; Owner: -
+--
+
+SELECT pg_catalog.setval('auto_incrementing_pk_id_seq', 1, true);
+
+
+--
+-- Data for Name: clients; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE clients CASCADE;
+INSERT INTO clients VALUES (1, 'Microsoft');
+INSERT INTO clients VALUES (2, 'Apple');
+
+
+--
+-- Data for Name: projects; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE projects CASCADE;
+INSERT INTO projects VALUES (1, 'Windows 7', 1);
+INSERT INTO projects VALUES (2, 'Windows 10', 1);
+INSERT INTO projects VALUES (3, 'IOS', 2);
+INSERT INTO projects VALUES (4, 'OSX', 2);
+INSERT INTO projects VALUES (5, 'Orphan', NULL);
+
+
+--
+-- Data for Name: tasks; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE tasks CASCADE;
+INSERT INTO tasks VALUES (1, 'Design w7', 1);
+INSERT INTO tasks VALUES (2, 'Code w7', 1);
+INSERT INTO tasks VALUES (3, 'Design w10', 2);
+INSERT INTO tasks VALUES (4, 'Code w10', 2);
+INSERT INTO tasks VALUES (5, 'Design IOS', 3);
+INSERT INTO tasks VALUES (6, 'Code IOS', 3);
+INSERT INTO tasks VALUES (7, 'Design OSX', 4);
+INSERT INTO tasks VALUES (8, 'Code OSX', 4);
+
+
+--
+-- Data for Name: users_tasks; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE users_tasks CASCADE;
+INSERT INTO users_tasks VALUES (1, 1);
+INSERT INTO users_tasks VALUES (1, 2);
+INSERT INTO users_tasks VALUES (1, 3);
+INSERT INTO users_tasks VALUES (1, 4);
+INSERT INTO users_tasks VALUES (2, 5);
+INSERT INTO users_tasks VALUES (2, 6);
+INSERT INTO users_tasks VALUES (2, 7);
+INSERT INTO users_tasks VALUES (3, 1);
+INSERT INTO users_tasks VALUES (3, 5);
+
+
+--
+-- Data for Name: comments; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE comments CASCADE;
+INSERT INTO comments VALUES (1, 1, 2, 6, 'Needs to be delivered ASAP');
+
+
+--
+-- Data for Name: complex_items; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE complex_items CASCADE;
+INSERT INTO complex_items VALUES (1, 'One', '{"foo":{"int":1,"bar":"baz"}}', '{1}');
+INSERT INTO complex_items VALUES (2, 'Two', '{"foo":{"int":1,"bar":"baz"}}', '{1,2}');
+INSERT INTO complex_items VALUES (3, 'Three', '{"foo":{"int":1,"bar":"baz"}}', '{1,2,3}');
+
+
+--
+-- Data for Name: compound_pk; Type: TABLE DATA; Schema: test; Owner: -
+--
+TRUNCATE TABLE compound_pk CASCADE;
+
+
+--
+-- Data for Name: simple_pk; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE simple_pk CASCADE;
+INSERT INTO simple_pk VALUES ('xyyx', 'u');
+INSERT INTO simple_pk VALUES ('xYYx', 'v');
+
+--
+-- Data for Name: has_fk; Type: TABLE DATA; Schema: test; Owner: -
+--
+TRUNCATE TABLE has_fk CASCADE;
+
+
+--
+-- Name: has_fk_id_seq; Type: SEQUENCE SET; Schema: test; Owner: -
+--
+
+SELECT pg_catalog.setval('has_fk_id_seq', 1, false);
+
+
+--
+-- Data for Name: items; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE items CASCADE;
+INSERT INTO items VALUES (1);
+INSERT INTO items VALUES (2);
+INSERT INTO items VALUES (3);
+INSERT INTO items VALUES (4);
+INSERT INTO items VALUES (5);
+INSERT INTO items VALUES (6);
+INSERT INTO items VALUES (7);
+INSERT INTO items VALUES (8);
+INSERT INTO items VALUES (9);
+INSERT INTO items VALUES (10);
+INSERT INTO items VALUES (11);
+INSERT INTO items VALUES (12);
+INSERT INTO items VALUES (13);
+INSERT INTO items VALUES (14);
+INSERT INTO items VALUES (15);
+
+
+--
+-- Name: items_id_seq; Type: SEQUENCE SET; Schema: test; Owner: -
+--
+
+SELECT pg_catalog.setval('items_id_seq', 15, true);
+
+
+--
+-- Data for Name: json; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE json CASCADE;
+INSERT INTO json VALUES ('{"foo":{"bar":"baz"},"id":1}');
+
+
+--
+-- Data for Name: menagerie; Type: TABLE DATA; Schema: test; Owner: -
+--
+TRUNCATE TABLE menagerie CASCADE;
+
+
+--
+-- Data for Name: no_pk; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE no_pk CASCADE;
+INSERT INTO no_pk VALUES (NULL, NULL);
+INSERT INTO no_pk VALUES ('1', '0');
+INSERT INTO no_pk VALUES ('2', '0');
+
+
+--
+-- Data for Name: nullable_integer; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE nullable_integer CASCADE;
+INSERT INTO nullable_integer VALUES (NULL);
+
+
+--
+-- Data for Name: tsearch; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE tsearch CASCADE;
+INSERT INTO tsearch VALUES ('''bar'':2 ''foo'':1');
+INSERT INTO tsearch VALUES ('''baz'':1 ''qux'':2');
+
+
+--
+-- Data for Name: users_projects; Type: TABLE DATA; Schema: test; Owner: -
+--
+
+TRUNCATE TABLE users_projects CASCADE;
+INSERT INTO users_projects VALUES (1, 1);
+INSERT INTO users_projects VALUES (1, 2);
+INSERT INTO users_projects VALUES (2, 3);
+INSERT INTO users_projects VALUES (2, 4);
+INSERT INTO users_projects VALUES (3, 1);
+INSERT INTO users_projects VALUES (3, 3);
+
+TRUNCATE TABLE "Escap3e;" CASCADE;
+INSERT INTO "Escap3e;" VALUES (1), (2), (3), (4), (5);
+
+TRUNCATE TABLE "ghostBusters" CASCADE;
+INSERT INTO "ghostBusters" VALUES (1), (3), (5);
+
+TRUNCATE TABLE "withUnique" CASCADE;
+INSERT INTO "withUnique" VALUES ('nodup', 'blah');
+
+
+
+TRUNCATE TABLE addresses CASCADE;
+INSERT INTO addresses VALUES (1, 'address 1');
+INSERT INTO addresses VALUES (2, 'address 2');
+INSERT INTO addresses VALUES (3, 'address 3');
+INSERT INTO addresses VALUES (4, 'address 4');
+
+TRUNCATE TABLE orders CASCADE;
+INSERT INTO orders VALUES (1, 'order 1', 1, 2);
+INSERT INTO orders VALUES (2, 'order 2', 3, 4);
+
+--
+-- PostgreSQL database dump complete
+--

test/fixtures/database.sql

@@ -0,0 +1,4 @@
+DROP DATABASE IF EXISTS postgrest_test;
+DROP ROLE IF EXISTS postgrest_test;
+CREATE USER postgrest_test createdb createrole;
+CREATE DATABASE postgrest_test OWNER postgrest_test;

test/fixtures/privileges.sql

@@ -0,0 +1,56 @@
+-- Privileges for anonymous
+GRANT USAGE ON SCHEMA
+      postgrest
+    , test
+    , "تست"
+TO postgrest_test_anonymous;
+
+-- Schema test objects
+SET search_path = test, "تست", pg_catalog;
+
+GRANT ALL ON TABLE
+    items
+    , "articleStars"
+    , articles
+    , auto_incrementing_pk
+    , clients
+    , comments
+    , complex_items
+    , compound_pk
+    , empty_table
+    , has_count_column
+    , has_fk
+    , insertable_view_with_join
+    , json
+    , materialized_view
+    , menagerie
+    , no_pk
+    , nullable_integer
+    , projects
+    , projects_view
+    , simple_pk
+    , tasks
+    , filtered_tasks
+    , tsearch
+    , users
+    , users_projects
+    , users_tasks
+    , "Escap3e;"
+    , "ghostBusters"
+    , "withUnique"
+    , "موارد"
+    , addresses
+    , orders
+TO postgrest_test_anonymous;
+
+GRANT INSERT ON TABLE insertonly TO postgrest_test_anonymous;
+
+GRANT USAGE ON SEQUENCE
+      auto_incrementing_pk_id_seq
+    , items_id_seq
+    , callcounter_count
+TO postgrest_test_anonymous;
+
+-- Privileges for non anonymous users
+GRANT USAGE ON SCHEMA test TO postgrest_test_author;
+GRANT ALL ON TABLE authors_only TO postgrest_test_author;

test/fixtures/roles.sql

@@ -1,16 +1,7 @@
-create function pg_temp.create_role_if_not_exists(rolename name, opts character varying) RETURNS text
-  LANGUAGE plpgsql
-  AS $$
-BEGIN
-  IF NOT EXISTS (SELECT * FROM pg_roles WHERE rolname = rolename) THEN
-    EXECUTE format('CREATE ROLE %I %s', rolename, opts);
-    RETURN 'CREATE ROLE';
-  ELSE
-    RETURN format('ROLE ''%I'' ALREADY EXISTS', rolename);
-  END IF;
-END;
-$$;
+DROP ROLE IF EXISTS postgrest_test_authenticator, postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author;
+CREATE ROLE postgrest_test_authenticator WITH login noinherit;
+CREATE ROLE postgrest_test_anonymous;
+CREATE ROLE postgrest_test_default_role;
+CREATE ROLE postgrest_test_author;
 
-select pg_temp.create_role_if_not_exists('postgrest_anonymous', 'with nologin') as a
-     , pg_temp.create_role_if_not_exists('test_default_role', 'with nologin') as b
-     , pg_temp.create_role_if_not_exists('postgrest_test_author', 'with nologin') into temp shh;
+GRANT postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author TO postgrest_test_authenticator;