From 9f1b5c0a81a97646fb76a4e01322290ce5370c5e Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 2 Feb 2022 20:57:16 +0100 Subject: [PATCH] feat: Support accessing array items and fields of composite types through json operators This is supported in select=, in filters and in order=. Resolves #1543 Resolves #2075 Signed-off-by: Wolfgang Walther --- CHANGELOG.md | 3 + src/PostgREST/Query/SqlFragment.hs | 5 +- src/PostgREST/Request/QueryParams.hs | 22 +++++++ test/spec/Feature/Query/JsonOperatorSpec.hs | 63 ++++++++++++++++++++- test/spec/fixtures/data.sql | 6 ++ test/spec/fixtures/privileges.sql | 2 + test/spec/fixtures/schema.sql | 18 ++++++ 7 files changed, 115 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ffefaf0..26f18da78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). + #1991, Add the ability to run without `db-uri` using libpq's PG environment variables to connect. - @wolfgangwalther + #1769, Add the ability to run without `db-schemas`, defaulting to `db-schemas=public`. - @wolfgangwalther + #1689, Add the ability to run without `db-anon-role` disabling anonymous access. - @wolfgangwalther + - #1543, Allow access to fields of composite types in select=, order= and filters through JSON operators -> and ->>. - @wolfgangwalther + - #2075, Allow access to array items in ?select=, ?order= and filters through JSON operators -> and ->>. - @wolfgangwalther ### Fixed @@ -29,6 +31,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #2120, Fix reading database configuration properly when `=` is present in value - @wolfgangwalther - #1771, Fix silently ignoring filter on a non-existent embedded resource - @steve-chavez - #2135, Remove trigger functions from schema cache and OpenAPI output, because they can't be called directly anyway. - @wolfgangwalther + - #2145, Fix accessing json array fields with -> and ->> in ?select= and ?order=. - @wolfgangwalther ## [9.0.0] - 2021-11-25 diff --git a/src/PostgREST/Query/SqlFragment.hs b/src/PostgREST/Query/SqlFragment.hs index 68e342d4e..32eda78bf 100644 --- a/src/PostgREST/Query/SqlFragment.hs +++ b/src/PostgREST/Query/SqlFragment.hs @@ -197,7 +197,10 @@ pgFmtColumn table "*" = fromQi table <> ".*" pgFmtColumn table c = fromQi table <> "." <> pgFmtIdent c pgFmtField :: QualifiedIdentifier -> Field -> SQL.Snippet -pgFmtField table (c, jp) = SQL.sql (pgFmtColumn table c) <> pgFmtJsonPath jp +pgFmtField table (c, []) = SQL.sql (pgFmtColumn table c) +-- Using to_jsonb instead of to_json to avoid missing operator errors when filtering: +-- "operator does not exist: json = unknown" +pgFmtField table (c, jp) = SQL.sql ("to_jsonb(" <> pgFmtColumn table c <> ")") <> pgFmtJsonPath jp pgFmtSelectItem :: QualifiedIdentifier -> SelectItem -> SQL.Snippet pgFmtSelectItem table (f@(fName, jp), Nothing, alias, _, _) = pgFmtField table f <> SQL.sql (pgFmtAs fName jp alias) diff --git a/src/PostgREST/Request/QueryParams.hs b/src/PostgREST/Request/QueryParams.hs index d364e65dd..71e2c9202 100644 --- a/src/PostgREST/Request/QueryParams.hs +++ b/src/PostgREST/Request/QueryParams.hs @@ -318,6 +318,26 @@ pFieldName = dash :: Parser Char dash = isDash $> '-' +-- | +-- Parse json operators in select, order and filters +-- +-- >>> P.parse pJsonPath "" "->text" +-- Right [JArrow {jOp = JKey {jVal = "text"}}] +-- +-- >>> P.parse pJsonPath "" "->1" +-- Right [JArrow {jOp = JIdx {jVal = "+1"}}] +-- +-- >>> P.parse pJsonPath "" "->>text" +-- Right [J2Arrow {jOp = JKey {jVal = "text"}}] +-- +-- >>> P.parse pJsonPath "" "->>1" +-- Right [J2Arrow {jOp = JIdx {jVal = "+1"}}] +-- +-- >>> P.parse pJsonPath "" "->0,other" +-- Right [JArrow {jOp = JIdx {jVal = "+0"}}] +-- +-- >>> P.parse pJsonPath "" "->0.desc" +-- Right [JArrow {jOp = JIdx {jVal = "+0"}}] pJsonPath :: Parser JsonPath pJsonPath = many pJsonOperation where @@ -333,6 +353,8 @@ pJsonPath = many pJsonOperation pJIdx = JIdx . toS <$> ((:) <$> P.option '+' (char '-') <*> many1 digit) <* pEnd pEnd = try (void $ lookAhead (string "->")) <|> try (void $ lookAhead (string "::")) <|> + try (void $ lookAhead (string ".")) <|> + try (void $ lookAhead (string ",")) <|> try eof in try pJIdx <|> try pJKey diff --git a/test/spec/Feature/Query/JsonOperatorSpec.hs b/test/spec/Feature/Query/JsonOperatorSpec.hs index 0ca44da35..08596505d 100644 --- a/test/spec/Feature/Query/JsonOperatorSpec.hs +++ b/test/spec/Feature/Query/JsonOperatorSpec.hs @@ -103,7 +103,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do it "can get array of objects" $ do get "/json_arr?select=data->0->>a&id=in.(5,6)" `shouldRespondWith` - [json| [{"a":"A"}, {"a":"[1,2,3]"}] |] + [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |] { matchHeaders = [matchContentTypeJson] } get "/json_arr?select=data->0->a->>2&id=in.(5,6)" `shouldRespondWith` [json| [{"a":null}, {"a":"3"}] |] @@ -139,6 +139,22 @@ spec actualPgVersion = describe "json and jsonb operators" $ do [json| [{"d":[4,5,6,7,8]}] |] { matchHeaders = [matchContentTypeJson] } + it "obtains a composite type field" $ do + get "/fav_numbers?select=num->i" + `shouldRespondWith` + [json| [{"i":0.5},{"i":0.6}] |] + get "/fav_numbers?select=num->>i" + `shouldRespondWith` + [json| [{"i":"0.5"},{"i":"0.6"}] |] + + it "obtains an array item" $ do + get "/arrays?select=a:numbers->0,b:numbers->1,c:numbers_mult->0->0,d:numbers_mult->1->2" + `shouldRespondWith` + [json| [{"a":1,"b":2,"c":1,"d":6},{"a":11,"b":12,"c":11,"d":16}] |] + get "/arrays?select=a:numbers->>0,b:numbers->>1,c:numbers_mult->0->>0,d:numbers_mult->1->>2" + `shouldRespondWith` + [json| [{"a":"1","b":"2","c":"1","d":"6"},{"a":"11","b":"12","c":"11","d":"16"}] |] + context "filtering response" $ do it "can filter by properties inside json column" $ do get "/json_table?data->foo->>bar=eq.baz" `shouldRespondWith` @@ -190,6 +206,25 @@ spec actualPgVersion = describe "json and jsonb operators" $ do [json| [{"id":3,"data":[{"d": "test"}]}] |] { matchHeaders = [matchContentTypeJson] } + it "can filter composite type field" $ + get "/fav_numbers?num->>i=gt.0.5" + `shouldRespondWith` + [json| [{"num":{"r":0.6,"i":0.6},"person":"B"}] |] + + it "can filter array item" $ do + get "/arrays?select=id&numbers->0=eq.1" + `shouldRespondWith` + [json| [{"id":0}] |] + get "/arrays?select=id&numbers->>0=eq.11" + `shouldRespondWith` + [json| [{"id":1}] |] + get "/arrays?select=id&numbers_mult->1->1=eq.5" + `shouldRespondWith` + [json| [{"id":0}] |] + get "/arrays?select=id&numbers_mult->2->>2=eq.19" + `shouldRespondWith` + [json| [{"id":1}] |] + context "ordering response" $ do it "orders by a json column property asc" $ get "/json_table?order=data->>id.asc" `shouldRespondWith` @@ -201,6 +236,28 @@ spec actualPgVersion = describe "json and jsonb operators" $ do [json| [{"data": {"id": 3}}, {"data": {"id": 0}}, {"data": {"id": 1, "foo": {"bar": "baz"}}}] |] { matchHeaders = [matchContentTypeJson] } + it "orders by composite type field" $ do + get "/fav_numbers?order=num->i.asc" + `shouldRespondWith` + [json| [{"num":{"r":0.5,"i":0.5},"person":"A"}, {"num":{"r":0.6,"i":0.6},"person":"B"}] |] + get "/fav_numbers?order=num->>i.desc" + `shouldRespondWith` + [json| [{"num":{"r":0.6,"i":0.6},"person":"B"}, {"num":{"r":0.5,"i":0.5},"person":"A"}] |] + + it "orders by array item" $ do + get "/arrays?select=id&order=numbers->0.desc" + `shouldRespondWith` + [json| [{"id":1},{"id":0}] |] + get "/arrays?select=id&order=numbers->1.asc" + `shouldRespondWith` + [json| [{"id":0},{"id":1}] |] + get "/arrays?select=id&order=numbers_mult->0->0.desc" + `shouldRespondWith` + [json| [{"id":1},{"id":0}] |] + get "/arrays?select=id&order=numbers_mult->2->2.asc" + `shouldRespondWith` + [json| [{"id":0},{"id":1}] |] + context "Patching record, in a nonempty table" $ it "can set a json column to escaped value" $ do request methodPatch "/json_table?data->>id=eq.3" @@ -218,7 +275,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do [json| [{"data":8}, {"data":7}] |] { matchHeaders = [matchContentTypeJson] } get "/json_arr?select=data->-2->>a&id=in.(5,6)" `shouldRespondWith` - [json| [{"a":"A"}, {"a":"[1,2,3]"}] |] + [json| [{"a":"A"}, {"a":"[1, 2, 3]"}] |] { matchHeaders = [matchContentTypeJson] } it "can filter with negative indexes" $ do @@ -238,7 +295,7 @@ spec actualPgVersion = describe "json and jsonb operators" $ do it "should fail on badly formed negatives" $ do get "/json_arr?select=data->>-78xy" `shouldRespondWith` [json| - {"details": "unexpected 'x' expecting digit, \"->\", \"::\" or end of input", + {"details": "unexpected 'x' expecting digit, \"->\", \"::\", \".\", \",\" or end of input", "message": "\"failed to parse select parameter (data->>-78xy)\" (line 1, column 11)"} |] { matchStatus = 400, matchHeaders = [matchContentTypeJson] } get "/json_arr?select=data->>--34" `shouldRespondWith` diff --git a/test/spec/fixtures/data.sql b/test/spec/fixtures/data.sql index 71a0efe29..b6af72797 100644 --- a/test/spec/fixtures/data.sql +++ b/test/spec/fixtures/data.sql @@ -730,3 +730,9 @@ INSERT INTO test.clientinfo (id,clientid, other) values (1,1,'123 Main St'),(2,2 TRUNCATE TABLE test.chores CASCADE; INSERT INTO test.chores (id, name, done) values (1, 'take out the garbage', true), (2, 'do the laundry', false), (3, 'wash the dishes', null); + +TRUNCATE TABLE test.fav_numbers CASCADE; +INSERT INTO test.fav_numbers VALUES (ROW(0.5, 0.5), 'A'), (ROW(0.6, 0.6), 'B'); + +TRUNCATE TABLE test.arrays CASCADE; +INSERT INTO test.arrays VALUES (0, '{1,2,3}', '{{1,2,3},{4,5,6},{7,8,9}}'), (1, '{11,12,13}', '{{11,12,13},{14,15,16},{17,18,19}}'); diff --git a/test/spec/fixtures/privileges.sql b/test/spec/fixtures/privileges.sql index b6899e012..3d4a3b585 100644 --- a/test/spec/fixtures/privileges.sql +++ b/test/spec/fixtures/privileges.sql @@ -19,6 +19,7 @@ GRANT ALL ON TABLE , items3 , "articleStars" , articles + , arrays , auto_incrementing_pk , clients , comments @@ -27,6 +28,7 @@ GRANT ALL ON TABLE , compound_pk_view , deferrable_unique_constraint , empty_table + , fav_numbers , has_count_column , has_fk , insertable_view_with_join diff --git a/test/spec/fixtures/schema.sql b/test/spec/fixtures/schema.sql index eb25c6cc0..3c9428e54 100644 --- a/test/spec/fixtures/schema.sql +++ b/test/spec/fixtures/schema.sql @@ -2427,3 +2427,21 @@ BEGIN END IF; END $do$; + +-- https://github.com/PostgREST/postgrest/issues/1543 +CREATE TYPE complex AS ( + r double precision, + i double precision +); + +CREATE TABLE test.fav_numbers ( + num complex, + person text +); + +-- https://github.com/PostgREST/postgrest/issues/2075 +create table test.arrays ( + id int primary key, + numbers int[], + numbers_mult int[][] +);