feat: Support accessing array items and fields of composite types through json operators

This is supported in select=, in filters and in order=.

Resolves #1543

Resolves #2075

Signed-off-by: Wolfgang Walther <walther@technowledgy.de>

CHANGELOG.md

@@ -16,6 +16,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
    + #1991, Add the ability to run without `db-uri` using libpq's PG environment variables to connect. - @wolfgangwalther
    + #1769, Add the ability to run without `db-schemas`, defaulting to `db-schemas=public`. - @wolfgangwalther
    + #1689, Add the ability to run without `db-anon-role` disabling anonymous access. - @wolfgangwalther
+ - #1543, Allow access to fields of composite types in select=, order= and filters through JSON operators -> and ->>. - @wolfgangwalther
+ - #2075, Allow access to array items in ?select=, ?order= and filters through JSON operators -> and ->>. - @wolfgangwalther
 
 ### Fixed
 
@@ -29,6 +31,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
  - #2120, Fix reading database configuration properly when `=` is present in value - @wolfgangwalther
  - #1771, Fix silently ignoring filter on a non-existent embedded resource - @steve-chavez
  - #2135, Remove trigger functions from schema cache and OpenAPI output, because they can't be called directly anyway. - @wolfgangwalther
+ - #2145, Fix accessing json array fields with -> and ->> in ?select= and ?order=. - @wolfgangwalther
 
 ## [9.0.0] - 2021-11-25
 

src/PostgREST/Query/SqlFragment.hs

@@ -197,7 +197,10 @@ pgFmtColumn table "*" = fromQi table <> ".*"
 pgFmtColumn table c   = fromQi table <> "." <> pgFmtIdent c
 
 pgFmtField :: QualifiedIdentifier -> Field -> SQL.Snippet
-pgFmtField table (c, jp) = SQL.sql (pgFmtColumn table c) <> pgFmtJsonPath jp
+pgFmtField table (c, []) = SQL.sql (pgFmtColumn table c)
+-- Using to_jsonb instead of to_json to avoid missing operator errors when filtering:
+-- "operator does not exist: json = unknown"
+pgFmtField table (c, jp) = SQL.sql ("to_jsonb(" <> pgFmtColumn table c <> ")") <> pgFmtJsonPath jp
 
 pgFmtSelectItem :: QualifiedIdentifier -> SelectItem -> SQL.Snippet
 pgFmtSelectItem table (f@(fName, jp), Nothing, alias, _, _) = pgFmtField table f <> SQL.sql (pgFmtAs fName jp alias)

src/PostgREST/Request/QueryParams.hs

@@ -318,6 +318,26 @@ pFieldName =
     dash :: Parser Char
     dash = isDash $> '-'
 
+-- |
+-- Parse json operators in select, order and filters
+--
+-- >>> P.parse pJsonPath "" "->text"
+-- Right [JArrow {jOp = JKey {jVal = "text"}}]
+--
+-- >>> P.parse pJsonPath "" "->1"
+-- Right [JArrow {jOp = JIdx {jVal = "+1"}}]
+--
+-- >>> P.parse pJsonPath "" "->>text"
+-- Right [J2Arrow {jOp = JKey {jVal = "text"}}]
+--
+-- >>> P.parse pJsonPath "" "->>1"
+-- Right [J2Arrow {jOp = JIdx {jVal = "+1"}}]
+--
+-- >>> P.parse pJsonPath "" "->0,other"
+-- Right [JArrow {jOp = JIdx {jVal = "+0"}}]
+--
+-- >>> P.parse pJsonPath "" "->0.desc"
+-- Right [JArrow {jOp = JIdx {jVal = "+0"}}]
 pJsonPath :: Parser JsonPath
 pJsonPath = many pJsonOperation
   where
@@ -333,6 +353,8 @@ pJsonPath = many pJsonOperation
           pJIdx = JIdx . toS <$> ((:) <$> P.option '+' (char '-') <*> many1 digit) <* pEnd
           pEnd = try (void $ lookAhead (string "->")) <|>
                  try (void $ lookAhead (string "::")) <|>
+                 try (void $ lookAhead (string ".")) <|>
+                 try (void $ lookAhead (string ",")) <|>
                  try eof in
       try pJIdx <|> try pJKey
 

test/spec/Feature/Query/JsonOperatorSpec.hs

@@ -103,7 +103,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
 
       it "can get array of objects" $ do
         get "/json_arr?select=data->0->>a&id=in.(5,6)" `shouldRespondWith`
-          [json| [{"a":"A"}, {"a":"[1,2,3]"}] |]
+          [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
           { matchHeaders = [matchContentTypeJson] }
         get "/json_arr?select=data->0->a->>2&id=in.(5,6)" `shouldRespondWith`
           [json| [{"a":null}, {"a":"3"}] |]
@@ -139,6 +139,22 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
           [json| [{"d":[4,5,6,7,8]}] |]
           { matchHeaders = [matchContentTypeJson] }
 
+    it "obtains a composite type field" $ do
+      get "/fav_numbers?select=num->i"
+        `shouldRespondWith`
+          [json| [{"i":0.5},{"i":0.6}] |]
+      get "/fav_numbers?select=num->>i"
+        `shouldRespondWith`
+          [json| [{"i":"0.5"},{"i":"0.6"}] |]
+
+    it "obtains an array item" $ do
+      get "/arrays?select=a:numbers->0,b:numbers->1,c:numbers_mult->0->0,d:numbers_mult->1->2"
+        `shouldRespondWith`
+          [json| [{"a":1,"b":2,"c":1,"d":6},{"a":11,"b":12,"c":11,"d":16}] |]
+      get "/arrays?select=a:numbers->>0,b:numbers->>1,c:numbers_mult->0->>0,d:numbers_mult->1->>2"
+        `shouldRespondWith`
+          [json| [{"a":"1","b":"2","c":"1","d":"6"},{"a":"11","b":"12","c":"11","d":"16"}] |]
+
   context "filtering response" $ do
     it "can filter by properties inside json column" $ do
       get "/json_table?data->foo->>bar=eq.baz" `shouldRespondWith`
@@ -190,6 +206,25 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         [json| [{"id":3,"data":[{"d": "test"}]}] |]
         { matchHeaders = [matchContentTypeJson] }
 
+    it "can filter composite type field" $
+      get "/fav_numbers?num->>i=gt.0.5"
+        `shouldRespondWith`
+          [json| [{"num":{"r":0.6,"i":0.6},"person":"B"}] |]
+
+    it "can filter array item" $ do
+      get "/arrays?select=id&numbers->0=eq.1"
+        `shouldRespondWith`
+          [json| [{"id":0}] |]
+      get "/arrays?select=id&numbers->>0=eq.11"
+        `shouldRespondWith`
+          [json| [{"id":1}] |]
+      get "/arrays?select=id&numbers_mult->1->1=eq.5"
+        `shouldRespondWith`
+          [json| [{"id":0}] |]
+      get "/arrays?select=id&numbers_mult->2->>2=eq.19"
+        `shouldRespondWith`
+          [json| [{"id":1}] |]
+
   context "ordering response" $ do
     it "orders by a json column property asc" $
       get "/json_table?order=data->>id.asc" `shouldRespondWith`
@@ -201,6 +236,28 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         [json| [{"data": {"id": 3}}, {"data": {"id": 0}}, {"data": {"id": 1, "foo": {"bar": "baz"}}}] |]
         { matchHeaders = [matchContentTypeJson] }
 
+    it "orders by composite type field" $ do
+      get "/fav_numbers?order=num->i.asc"
+        `shouldRespondWith`
+          [json| [{"num":{"r":0.5,"i":0.5},"person":"A"}, {"num":{"r":0.6,"i":0.6},"person":"B"}] |]
+      get "/fav_numbers?order=num->>i.desc"
+        `shouldRespondWith`
+          [json| [{"num":{"r":0.6,"i":0.6},"person":"B"}, {"num":{"r":0.5,"i":0.5},"person":"A"}] |]
+
+    it "orders by array item" $ do
+      get "/arrays?select=id&order=numbers->0.desc"
+        `shouldRespondWith`
+          [json| [{"id":1},{"id":0}] |]
+      get "/arrays?select=id&order=numbers->1.asc"
+        `shouldRespondWith`
+          [json| [{"id":0},{"id":1}] |]
+      get "/arrays?select=id&order=numbers_mult->0->0.desc"
+        `shouldRespondWith`
+          [json| [{"id":1},{"id":0}] |]
+      get "/arrays?select=id&order=numbers_mult->2->2.asc"
+        `shouldRespondWith`
+          [json| [{"id":0},{"id":1}] |]
+
   context "Patching record, in a nonempty table" $
     it "can set a json column to escaped value" $ do
       request methodPatch "/json_table?data->>id=eq.3"
@@ -218,7 +275,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
         [json| [{"data":8}, {"data":7}] |]
         { matchHeaders = [matchContentTypeJson] }
       get "/json_arr?select=data->-2->>a&id=in.(5,6)" `shouldRespondWith`
-        [json| [{"a":"A"}, {"a":"[1,2,3]"}] |]
+        [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |]
         { matchHeaders = [matchContentTypeJson] }
 
     it "can filter with negative indexes" $ do
@@ -238,7 +295,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
     it "should fail on badly formed negatives" $ do
       get "/json_arr?select=data->>-78xy" `shouldRespondWith`
         [json|
-          {"details": "unexpected 'x' expecting digit, \"->\", \"::\" or end of input",
+          {"details": "unexpected 'x' expecting digit, \"->\", \"::\", \".\", \",\" or end of input",
            "message": "\"failed to parse select parameter (data->>-78xy)\" (line 1, column 11)"} |]
         { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
       get "/json_arr?select=data->>--34" `shouldRespondWith`

test/spec/fixtures/data.sql

@@ -730,3 +730,9 @@ INSERT INTO test.clientinfo (id,clientid, other) values (1,1,'123 Main St'),(2,2
 
 TRUNCATE TABLE test.chores CASCADE;
 INSERT INTO test.chores (id, name, done) values (1, 'take out the garbage', true), (2, 'do the laundry', false), (3, 'wash the dishes', null);
+
+TRUNCATE TABLE test.fav_numbers CASCADE;
+INSERT INTO test.fav_numbers VALUES (ROW(0.5, 0.5), 'A'),  (ROW(0.6, 0.6), 'B');
+
+TRUNCATE TABLE test.arrays CASCADE;
+INSERT INTO test.arrays VALUES (0, '{1,2,3}', '{{1,2,3},{4,5,6},{7,8,9}}'), (1, '{11,12,13}', '{{11,12,13},{14,15,16},{17,18,19}}');

test/spec/fixtures/privileges.sql

@@ -19,6 +19,7 @@ GRANT ALL ON TABLE
     , items3
     , "articleStars"
     , articles
+    , arrays
     , auto_incrementing_pk
     , clients
     , comments
@@ -27,6 +28,7 @@ GRANT ALL ON TABLE
     , compound_pk_view
     , deferrable_unique_constraint
     , empty_table
+    , fav_numbers
     , has_count_column
     , has_fk
     , insertable_view_with_join

test/spec/fixtures/schema.sql

@@ -2427,3 +2427,21 @@ BEGIN
   END IF;
 END
 $do$;
+
+-- https://github.com/PostgREST/postgrest/issues/1543
+CREATE TYPE complex AS (
+ r double precision,
+ i double precision
+);
+
+CREATE TABLE test.fav_numbers (
+ num    complex,
+ person text
+);
+
+-- https://github.com/PostgREST/postgrest/issues/2075
+create table test.arrays (
+  id int primary key,
+  numbers int[],
+  numbers_mult int[][]
+);