diff --git a/CHANGELOG.md b/CHANGELOG.md index c65dddf4b..f345af02f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,16 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Added +- #1099, Add support for getting json/jsonb by array index - @steve-chavez + ### Fixed - #1113, Fix UPSERT failing when having a camel case PK column - @steve-chavez ### Changed +- #1099, Numbers in json path `?select=data->1->>key` now get treated as json array indexes instead of keys - @steve-chavez + ## [0.5.0.0] - 2018-05-14 ### Added diff --git a/postgrest.cabal b/postgrest.cabal index 81efcf4db..781cfd811 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -124,6 +124,7 @@ Test-Suite spec , Feature.InsertSpec , Feature.JsonOperatorSpec , Feature.NoJwtSpec + , Feature.PgVersion95Spec , Feature.PgVersion96Spec , Feature.ProxySpec , Feature.QueryLimitedSpec diff --git a/src/PostgREST/Parsers.hs b/src/PostgREST/Parsers.hs index a4bdb1ace..724d0b5a5 100644 --- a/src/PostgREST/Parsers.hs +++ b/src/PostgREST/Parsers.hs @@ -80,11 +80,16 @@ pFieldName = do dash :: Parser Char dash = isDash *> pure '-' -pJsonPathStep :: Parser Text -pJsonPathStep = toS <$> try (string "->" *> pFieldName) - -pJsonPath :: Parser [Text] -pJsonPath = (<>) <$> many pJsonPathStep <*> ( (:[]) <$> (string "->>" *> pFieldName) ) +pJsonPath :: Parser JsonPath +pJsonPath = (<>) <$> many pJsonPathOp <*> ( (:[]) <$> (string "->>" *> (try pJIdx <|> pJKey)) ) + where + pJsonPathOp :: Parser JsonPathOp + pJsonPathOp = try (string "->" *> pJIdx) <|> try (string "->" *> pJKey) + pJKey = JKey . toS <$> pFieldName + pJIdx = JIdx . toS <$> ((:) <$> option '+' (char '-') <*> many1 digit) <* pEnd + pEnd = try (void $ lookAhead (string "->")) <|> + try (void $ lookAhead (string "::")) <|> + try eof pField :: Parser Field pField = lexeme $ (,) <$> pFieldName <*> option [] pJsonPath diff --git a/src/PostgREST/QueryBuilder.hs b/src/PostgREST/QueryBuilder.hs index 15f58abc7..91fdf2a2c 100644 --- a/src/PostgREST/QueryBuilder.hs +++ b/src/PostgREST/QueryBuilder.hs @@ -1,7 +1,8 @@ -{-# LANGUAGE FlexibleInstances #-} -{-# LANGUAGE FlexibleContexts #-} -{-# LANGUAGE DuplicateRecordFields #-} -{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE DuplicateRecordFields #-} +{-# LANGUAGE LambdaCase #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} {-| Module : PostgREST.QueryBuilder Description : PostgREST SQL generating functions. @@ -394,8 +395,8 @@ pgFmtField :: QualifiedIdentifier -> Field -> SqlFragment pgFmtField table (c, jp) = pgFmtColumn table c <> pgFmtJsonPath jp pgFmtSelectItem :: QualifiedIdentifier -> SelectItem -> SqlFragment -pgFmtSelectItem table (f@(_, jp), Nothing, alias, _) = pgFmtField table f <> pgFmtAs jp alias -pgFmtSelectItem table (f@(_, jp), Just cast, alias, _) = "CAST (" <> pgFmtField table f <> " AS " <> cast <> " )" <> pgFmtAs jp alias +pgFmtSelectItem table (f@(fName, jp), Nothing, alias, _) = pgFmtField table f <> pgFmtAs fName jp alias +pgFmtSelectItem table (f@(fName, jp), Just cast, alias, _) = "CAST (" <> pgFmtField table f <> " AS " <> cast <> " )" <> pgFmtAs fName jp alias pgFmtOrderTerm :: QualifiedIdentifier -> OrderTerm -> SqlFragment pgFmtOrderTerm qi ot = unwords [ @@ -448,16 +449,25 @@ pgFmtLogicTree qi (Expr hasNot op forest) = notOp <> " (" <> intercalate (" " <> pgFmtLogicTree qi (Stmnt flt) = pgFmtFilter qi flt pgFmtJsonPath :: JsonPath -> SqlFragment -pgFmtJsonPath [] = "" -pgFmtJsonPath [x] = "->>" <> pgFmtLit x -pgFmtJsonPath (x:xs) = "->" <> pgFmtLit x <> pgFmtJsonPath xs +pgFmtJsonPath = \case + [] -> "" + [x] -> "->>" <> pgFmtJsonPathOp x + (x:xs) -> "->" <> pgFmtJsonPathOp x <> pgFmtJsonPath xs + where + pgFmtJsonPathOp (JKey k) = pgFmtLit k + pgFmtJsonPathOp (JIdx i) = pgFmtLit i <> "::int" -pgFmtAs :: JsonPath -> Maybe Alias -> SqlFragment -pgFmtAs [] Nothing = "" -pgFmtAs jp Nothing = case lastMay jp of - Just alias -> " AS " <> pgFmtIdent alias +pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SqlFragment +pgFmtAs _ [] Nothing = "" +pgFmtAs fName jp Nothing = case lastMay jp of + Just (JKey key) -> " AS " <> pgFmtIdent key + Just (JIdx _) -> " AS " <> pgFmtIdent (fromMaybe fName lastKey) + -- We get the lastKey because on: + -- `select=data->1->mycol->>2`, we need to show the result as [ {"mycol": ..}, {"mycol": ..} ] + -- `select=data->3`, we need to show the result as [ {"data": ..}, {"data": ..} ] + where lastKey = jpOp <$> find (\case JKey{} -> True; _ -> False) (reverse jp) Nothing -> "" -pgFmtAs _ (Just alias) = " AS " <> pgFmtIdent alias +pgFmtAs _ _ (Just alias) = " AS " <> pgFmtIdent alias pgFmtEnvVar :: Text -> (Text, Text) -> SqlFragment pgFmtEnvVar prefix (k, v) = diff --git a/src/PostgREST/Types.hs b/src/PostgREST/Types.hs index 7e0e17756..375cd4f00 100644 --- a/src/PostgREST/Types.hs +++ b/src/PostgREST/Types.hs @@ -238,7 +238,12 @@ instance Show LogicOperator where data LogicTree = Expr Bool LogicOperator [LogicTree] | Stmnt Filter deriving (Show, Eq) type FieldName = Text -type JsonPath = [Text] +type JsonPath = [JsonPathOp] +{-| + Json path operands as specified in https://www.postgresql.org/docs/9.5/static/functions-json.html + the array index is Text because we reuse our escaping functons and let pg do the casting with '1'::int +-} +data JsonPathOp = JKey{jpOp :: Text} | JIdx{jpOp :: Text} deriving (Show, Eq) type Field = (FieldName, JsonPath) type Alias = Text type Cast = Text diff --git a/test/Feature/JsonOperatorSpec.hs b/test/Feature/JsonOperatorSpec.hs index 00ff92c5d..20328eec0 100644 --- a/test/Feature/JsonOperatorSpec.hs +++ b/test/Feature/JsonOperatorSpec.hs @@ -48,6 +48,44 @@ spec = describe "json and jsonb operators" $ do [json| [{"myInt":1}] |] -- the value in the db is an int, but here we expect a string for now { matchHeaders = [matchContentTypeJson] } + context "with array index" $ do + it "can get array of ints and alias/cast it" $ do + get "/json_arr?select=data->>0::int&id=in.(1,2)" `shouldRespondWith` + [json| [{"data":1}, {"data":4}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=idx0:data->>0::int,idx1:data->>1::int&id=in.(1,2)" `shouldRespondWith` + [json| [{"idx0":1,"idx1":2}, {"idx0":4,"idx1":5}] |] + { matchHeaders = [matchContentTypeJson] } + + it "can get nested array of ints" $ do + get "/json_arr?select=data->0->>1::int&id=in.(3,4)" `shouldRespondWith` + [json| [{"data":8}, {"data":7}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data->0->0->>1::int&id=in.(3,4)" `shouldRespondWith` + [json| [{"data":null}, {"data":6}] |] + { matchHeaders = [matchContentTypeJson] } + + it "can get array of objects" $ do + get "/json_arr?select=data->0->>a&id=in.(5,6)" `shouldRespondWith` + [json| [{"a":"A"}, {"a":"[1,2,3]"}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data->0->a->>2&id=in.(5,6)" `shouldRespondWith` + [json| [{"a":null}, {"a":"3"}] |] + { matchHeaders = [matchContentTypeJson] } + + it "can get array in object keys" $ do + get "/json_arr?select=data->c->>0::json&id=in.(7,8)" `shouldRespondWith` + [json| [{"c":1}, {"c":{"d": [4,5,6,7,8]}}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data->c->0->d->>4::int&id=in.(7,8)" `shouldRespondWith` + [json| [{"d":null}, {"d":8}] |] + { matchHeaders = [matchContentTypeJson] } + + it "only treats well formed numbers as indexes" $ + get "/json_arr?select=data->0->0xy1->1->23-xy-45->1->xy-6->>0::int&id=eq.9" `shouldRespondWith` + [json| [{"xy-6":3}] |] + { matchHeaders = [matchContentTypeJson] } + context "filtering response" $ do it "can filter by properties inside json column" $ do get "/json?data->foo->>bar=eq.baz" `shouldRespondWith` @@ -71,6 +109,20 @@ spec = describe "json and jsonb operators" $ do get "/grandchild_entities?or=(jsonb_col->a->>b.eq.foo, jsonb_col->>b.eq.bar)&select=id" `shouldRespondWith` [json|[{id: 4}, {id: 5}]|] { matchStatus = 200, matchHeaders = [matchContentTypeJson] } + it "can filter by array indexes" $ do + get "/json_arr?select=data&data->>0=eq.1" `shouldRespondWith` + [json| [{"data":[1, 2, 3]}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data&data->1->>2=eq.13" `shouldRespondWith` + [json| [{"data":[[9, 8, 7], [11, 12, 13]]}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data&data->1->>b=eq.B" `shouldRespondWith` + [json| [{"data":[{"a": "A"}, {"b": "B"}]}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data&data->1->b->>1=eq.5" `shouldRespondWith` + [json| [{"data":[{"a": [1,2,3]}, {"b": [4,5]}]}] |] + { matchHeaders = [matchContentTypeJson] } + context "ordering response" $ do it "orders by a json column property asc" $ get "/json?order=data->>id.asc" `shouldRespondWith` @@ -82,7 +134,7 @@ spec = describe "json and jsonb operators" $ do [json| [{"data": {"id": 3}}, {"data": {"id": 0}}, {"data": {"id": 1, "foo": {"bar": "baz"}}}] |] { matchHeaders = [matchContentTypeJson] } - context "Patching record, in a nonempty table" $ do + context "Patching record, in a nonempty table" $ it "can set a json column to escaped value" $ do _ <- post "/json" [json| { data: {"escaped":"bar"} } |] request methodPatch "/json?data->>escaped=eq.bar" diff --git a/test/Feature/PgVersion95Spec.hs b/test/Feature/PgVersion95Spec.hs new file mode 100644 index 000000000..62c06ca8a --- /dev/null +++ b/test/Feature/PgVersion95Spec.hs @@ -0,0 +1,55 @@ +module Feature.PgVersion95Spec where + +import Test.Hspec hiding (pendingWith) +import Test.Hspec.Wai +import Test.Hspec.Wai.JSON + +import SpecHelper +import Network.Wai (Application) + +import Protolude hiding (get) + +spec :: SpecWith Application +spec = describe "features supported on PostgreSQL 9.5" $ + context "json array negative index" $ do + it "can select with negative indexes" $ do + get "/json_arr?select=data->>-1::int&id=in.(1,2)" `shouldRespondWith` + [json| [{"data":3}, {"data":6}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data->0->>-2::int&id=in.(3,4)" `shouldRespondWith` + [json| [{"data":8}, {"data":7}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data->-2->>a&id=in.(5,6)" `shouldRespondWith` + [json| [{"a":"A"}, {"a":"[1,2,3]"}] |] + { matchHeaders = [matchContentTypeJson] } + + it "can filter with negative indexes" $ do + get "/json_arr?select=data&data->>-3=eq.1" `shouldRespondWith` + [json| [{"data":[1, 2, 3]}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data&data->-1->>-3=eq.11" `shouldRespondWith` + [json| [{"data":[[9, 8, 7], [11, 12, 13]]}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data&data->-1->>b=eq.B" `shouldRespondWith` + [json| [{"data":[{"a": "A"}, {"b": "B"}]}] |] + { matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data&data->-1->b->>-1=eq.5" `shouldRespondWith` + [json| [{"data":[{"a": [1,2,3]}, {"b": [4,5]}]}] |] + { matchHeaders = [matchContentTypeJson] } + + it "should fail on badly formed negatives" $ do + get "/json_arr?select=data->>-78xy" `shouldRespondWith` + [json| + {"details": "unexpected 'x' expecting digit, \"->\", \"::\" or end of input", + "message": "\"failed to parse select parameter (data->>-78xy)\" (line 1, column 11)"} |] + { matchStatus = 400, matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data->>--34" `shouldRespondWith` + [json| + {"details": "unexpected \"-\" expecting digit", + "message": "\"failed to parse select parameter (data->>--34)\" (line 1, column 9)"} |] + { matchStatus = 400, matchHeaders = [matchContentTypeJson] } + get "/json_arr?select=data->>-xy-4" `shouldRespondWith` + [json| + {"details":"unexpected \"x\" expecting digit", + "message":"\"failed to parse select parameter (data->>-xy-4)\" (line 1, column 9)"} |] + { matchStatus = 400, matchHeaders = [matchContentTypeJson] } diff --git a/test/Main.hs b/test/Main.hs index f382bb3cc..d52dd683c 100644 --- a/test/Main.hs +++ b/test/Main.hs @@ -34,6 +34,7 @@ import qualified Feature.ProxySpec import qualified Feature.AndOrParamsSpec import qualified Feature.RpcSpec import qualified Feature.NonexistentSchemaSpec +import qualified Feature.PgVersion95Spec import qualified Feature.PgVersion96Spec import qualified Feature.UpsertSpec @@ -70,6 +71,7 @@ main = do actualPgVersion = pgVersion dbStructure extraSpecs = [("Feature.UpsertSpec", Feature.UpsertSpec.spec) | actualPgVersion >= pgVersion95] ++ + [("Feature.PgVersion95Spec", Feature.PgVersion95Spec.spec) | actualPgVersion >= pgVersion95] ++ [("Feature.PgVersion96Spec", Feature.PgVersion96Spec.spec) | actualPgVersion >= pgVersion96] specs = uncurry describe <$> [ diff --git a/test/fixtures/data.sql b/test/fixtures/data.sql index b8d9e6436..fd5d842fa 100644 --- a/test/fixtures/data.sql +++ b/test/fixtures/data.sql @@ -414,3 +414,14 @@ copy (select id, name, client_id from projects) to '/tmp/projects_dump.csv' with TRUNCATE TABLE "UnitTest" CASCADE; INSERT INTO "UnitTest" VALUES (1, 'unit test 1'); + +TRUNCATE TABLE json_arr CASCADE; +INSERT INTO json_arr VALUES (1, '[1, 2, 3]'); +INSERT INTO json_arr VALUES (2, '[4, 5, 6]'); +INSERT INTO json_arr VALUES (3, '[[9, 8, 7], [11, 12, 13]]'); +INSERT INTO json_arr VALUES (4, '[[[5, 6], 7, 8]]'); +INSERT INTO json_arr VALUES (5, '[{"a": "A"}, {"b": "B"}]'); +INSERT INTO json_arr VALUES (6, '[{"a": [1,2,3]}, {"b": [4,5]}]'); +INSERT INTO json_arr VALUES (7, '{"c": [1,2,3], "d": [4,5]}'); +INSERT INTO json_arr VALUES (8, '{"c": [{"d": [4,5,6,7,8]}]}'); +INSERT INTO json_arr VALUES (9, '[{"0xy1": [1,{"23-xy-45": [2, {"xy-6": [3]}]}]}]'); diff --git a/test/fixtures/privileges.sql b/test/fixtures/privileges.sql index beced9875..574dad22a 100644 --- a/test/fixtures/privileges.sql +++ b/test/fixtures/privileges.sql @@ -80,6 +80,7 @@ GRANT ALL ON TABLE , zone , projects_dump , "UnitTest" + , json_arr TO postgrest_test_anonymous; GRANT INSERT ON TABLE insertonly TO postgrest_test_anonymous; diff --git a/test/fixtures/schema.sql b/test/fixtures/schema.sql index 05df3f9d4..e55a26771 100755 --- a/test/fixtures/schema.sql +++ b/test/fixtures/schema.sql @@ -1456,3 +1456,8 @@ create table "UnitTest"( "idUnitTest" integer primary key, "nameUnitTest" text ); + +create table json_arr( + id integer primary key, + data pg_catalog.json +);