Add support for getting json by array index

* Also support json negative array index

CHANGELOG.md

@@ -7,12 +7,16 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- #1099, Add support for getting json/jsonb by array index - @steve-chavez
+
 ### Fixed
 
 - #1113, Fix UPSERT failing when having a camel case PK column - @steve-chavez
 
 ### Changed
 
+- #1099, Numbers in json path `?select=data->1->>key` now get treated as json array indexes instead of keys - @steve-chavez
+
 ## [0.5.0.0] - 2018-05-14
 
 ### Added

postgrest.cabal

@@ -124,6 +124,7 @@ Test-Suite spec
                      , Feature.InsertSpec
                      , Feature.JsonOperatorSpec
                      , Feature.NoJwtSpec
+                     , Feature.PgVersion95Spec
                      , Feature.PgVersion96Spec
                      , Feature.ProxySpec
                      , Feature.QueryLimitedSpec

src/PostgREST/Parsers.hs

@@ -80,11 +80,16 @@ pFieldName = do
     dash :: Parser Char
     dash = isDash *> pure '-'
 
-pJsonPathStep :: Parser Text
+pJsonPath :: Parser JsonPath
-pJsonPathStep = toS <$> try (string "->" *> pFieldName)
+pJsonPath = (<>) <$> many pJsonPathOp <*> ( (:[]) <$> (string "->>" *> (try pJIdx <|> pJKey)) )
-
+  where
-pJsonPath :: Parser [Text]
+    pJsonPathOp :: Parser JsonPathOp
-pJsonPath = (<>) <$> many pJsonPathStep <*> ( (:[]) <$> (string "->>" *> pFieldName) )
+    pJsonPathOp = try (string "->" *> pJIdx) <|> try (string "->" *> pJKey)
+    pJKey = JKey . toS <$> pFieldName
+    pJIdx = JIdx . toS <$> ((:) <$> option '+' (char '-') <*> many1 digit) <* pEnd
+    pEnd = try (void $ lookAhead (string "->")) <|>
+           try (void $ lookAhead (string "::")) <|>
+           try eof
 
 pField :: Parser Field
 pField = lexeme $ (,) <$> pFieldName <*> option [] pJsonPath

src/PostgREST/QueryBuilder.hs

@@ -1,7 +1,8 @@
-{-# LANGUAGE FlexibleInstances    #-}
+{-# LANGUAGE FlexibleInstances     #-}
-{-# LANGUAGE FlexibleContexts    #-}
+{-# LANGUAGE FlexibleContexts      #-}
-{-# LANGUAGE DuplicateRecordFields    #-}
+{-# LANGUAGE DuplicateRecordFields #-}
-{-# OPTIONS_GHC -fno-warn-orphans #-}
+{-# LANGUAGE LambdaCase            #-}
+{-# OPTIONS_GHC -fno-warn-orphans  #-}
 {-|
 Module      : PostgREST.QueryBuilder
 Description : PostgREST SQL generating functions.
@@ -394,8 +395,8 @@ pgFmtField :: QualifiedIdentifier -> Field -> SqlFragment
 pgFmtField table (c, jp) = pgFmtColumn table c <> pgFmtJsonPath jp
 
 pgFmtSelectItem :: QualifiedIdentifier -> SelectItem -> SqlFragment
-pgFmtSelectItem table (f@(_, jp), Nothing, alias, _) = pgFmtField table f <> pgFmtAs jp alias
+pgFmtSelectItem table (f@(fName, jp), Nothing, alias, _) = pgFmtField table f <> pgFmtAs fName jp alias
-pgFmtSelectItem table (f@(_, jp), Just cast, alias, _) = "CAST (" <> pgFmtField table f <> " AS " <> cast <> " )" <> pgFmtAs jp alias
+pgFmtSelectItem table (f@(fName, jp), Just cast, alias, _) = "CAST (" <> pgFmtField table f <> " AS " <> cast <> " )" <> pgFmtAs fName jp alias
 
 pgFmtOrderTerm :: QualifiedIdentifier -> OrderTerm -> SqlFragment
 pgFmtOrderTerm qi ot = unwords [
@@ -448,16 +449,25 @@ pgFmtLogicTree qi (Expr hasNot op forest) = notOp <> " (" <> intercalate (" " <>
 pgFmtLogicTree qi (Stmnt flt) = pgFmtFilter qi flt
 
 pgFmtJsonPath :: JsonPath -> SqlFragment
-pgFmtJsonPath [] = ""
+pgFmtJsonPath = \case
-pgFmtJsonPath [x] = "->>" <> pgFmtLit x
+  []     -> ""
-pgFmtJsonPath (x:xs) = "->" <> pgFmtLit x <> pgFmtJsonPath xs
+  [x]    -> "->>" <> pgFmtJsonPathOp x
+  (x:xs) -> "->" <> pgFmtJsonPathOp x <> pgFmtJsonPath xs
+  where
+    pgFmtJsonPathOp (JKey k) = pgFmtLit k
+    pgFmtJsonPathOp (JIdx i) = pgFmtLit i <> "::int"
 
-pgFmtAs :: JsonPath -> Maybe Alias -> SqlFragment
+pgFmtAs :: FieldName -> JsonPath -> Maybe Alias -> SqlFragment
-pgFmtAs [] Nothing = ""
+pgFmtAs _ [] Nothing = ""
-pgFmtAs jp Nothing = case lastMay jp of
+pgFmtAs fName jp Nothing = case lastMay jp of
-  Just alias -> " AS " <> pgFmtIdent alias
+  Just (JKey key) -> " AS " <> pgFmtIdent key
+  Just (JIdx _)   -> " AS " <> pgFmtIdent (fromMaybe fName lastKey)
+    -- We get the lastKey because on:
+    -- `select=data->1->mycol->>2`, we need to show the result as [ {"mycol": ..}, {"mycol": ..} ]
+    -- `select=data->3`, we need to show the result as [ {"data": ..}, {"data": ..} ]
+    where lastKey = jpOp <$> find (\case JKey{} -> True; _ -> False) (reverse jp)
   Nothing -> ""
-pgFmtAs _ (Just alias) = " AS " <> pgFmtIdent alias
+pgFmtAs _ _ (Just alias) = " AS " <> pgFmtIdent alias
 
 pgFmtEnvVar :: Text -> (Text, Text) -> SqlFragment
 pgFmtEnvVar prefix (k, v) =

src/PostgREST/Types.hs

@@ -238,7 +238,12 @@ instance Show LogicOperator where
 data LogicTree = Expr Bool LogicOperator [LogicTree] | Stmnt Filter deriving (Show, Eq)
 
 type FieldName = Text
-type JsonPath = [Text]
+type JsonPath = [JsonPathOp]
+{-|
+  Json path operands as specified in https://www.postgresql.org/docs/9.5/static/functions-json.html
+  the array index is Text because we reuse our escaping functons and let pg do the casting with '1'::int
+-}
+data JsonPathOp = JKey{jpOp :: Text} | JIdx{jpOp :: Text} deriving (Show, Eq)
 type Field = (FieldName, JsonPath)
 type Alias = Text
 type Cast = Text

test/Feature/JsonOperatorSpec.hs

@@ -48,6 +48,44 @@ spec = describe "json and jsonb operators" $ do
         [json| [{"myInt":1}] |] -- the value in the db is an int, but here we expect a string for now
         { matchHeaders = [matchContentTypeJson] }
 
+    context "with array index" $ do
+      it "can get array of ints and alias/cast it" $ do
+        get "/json_arr?select=data->>0::int&id=in.(1,2)" `shouldRespondWith`
+          [json| [{"data":1}, {"data":4}] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/json_arr?select=idx0:data->>0::int,idx1:data->>1::int&id=in.(1,2)" `shouldRespondWith`
+          [json| [{"idx0":1,"idx1":2}, {"idx0":4,"idx1":5}] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "can get nested array of ints" $ do
+        get "/json_arr?select=data->0->>1::int&id=in.(3,4)" `shouldRespondWith`
+          [json| [{"data":8}, {"data":7}] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/json_arr?select=data->0->0->>1::int&id=in.(3,4)" `shouldRespondWith`
+          [json| [{"data":null}, {"data":6}] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "can get array of objects" $ do
+        get "/json_arr?select=data->0->>a&id=in.(5,6)" `shouldRespondWith`
+          [json| [{"a":"A"}, {"a":"[1,2,3]"}] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/json_arr?select=data->0->a->>2&id=in.(5,6)" `shouldRespondWith`
+          [json| [{"a":null}, {"a":"3"}] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "can get array in object keys" $ do
+        get "/json_arr?select=data->c->>0::json&id=in.(7,8)" `shouldRespondWith`
+          [json| [{"c":1}, {"c":{"d": [4,5,6,7,8]}}] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/json_arr?select=data->c->0->d->>4::int&id=in.(7,8)" `shouldRespondWith`
+          [json| [{"d":null}, {"d":8}] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "only treats well formed numbers as indexes" $
+        get "/json_arr?select=data->0->0xy1->1->23-xy-45->1->xy-6->>0::int&id=eq.9" `shouldRespondWith`
+          [json| [{"xy-6":3}] |]
+          { matchHeaders = [matchContentTypeJson] }
+
   context "filtering response" $ do
     it "can filter by properties inside json column" $ do
       get "/json?data->foo->>bar=eq.baz" `shouldRespondWith`
@@ -71,6 +109,20 @@ spec = describe "json and jsonb operators" $ do
       get "/grandchild_entities?or=(jsonb_col->a->>b.eq.foo, jsonb_col->>b.eq.bar)&select=id" `shouldRespondWith`
         [json|[{id: 4}, {id: 5}]|] { matchStatus = 200, matchHeaders = [matchContentTypeJson] }
 
+    it "can filter by array indexes" $ do
+      get "/json_arr?select=data&data->>0=eq.1" `shouldRespondWith`
+        [json| [{"data":[1, 2, 3]}] |]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data&data->1->>2=eq.13" `shouldRespondWith`
+        [json| [{"data":[[9, 8, 7], [11, 12, 13]]}] |]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data&data->1->>b=eq.B" `shouldRespondWith`
+        [json| [{"data":[{"a": "A"}, {"b": "B"}]}] |]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data&data->1->b->>1=eq.5" `shouldRespondWith`
+        [json| [{"data":[{"a": [1,2,3]}, {"b": [4,5]}]}] |]
+        { matchHeaders = [matchContentTypeJson] }
+
   context "ordering response" $ do
     it "orders by a json column property asc" $
       get "/json?order=data->>id.asc" `shouldRespondWith`
@@ -82,7 +134,7 @@ spec = describe "json and jsonb operators" $ do
         [json| [{"data": {"id": 3}}, {"data": {"id": 0}}, {"data": {"id": 1, "foo": {"bar": "baz"}}}] |]
         { matchHeaders = [matchContentTypeJson] }
 
-  context "Patching record, in a nonempty table" $ do
+  context "Patching record, in a nonempty table" $
     it "can set a json column to escaped value" $ do
       _ <- post "/json" [json| { data: {"escaped":"bar"} } |]
       request methodPatch "/json?data->>escaped=eq.bar"

test/Feature/PgVersion95Spec.hs

@@ -0,0 +1,55 @@
+module Feature.PgVersion95Spec where
+
+import Test.Hspec hiding (pendingWith)
+import Test.Hspec.Wai
+import Test.Hspec.Wai.JSON
+
+import SpecHelper
+import Network.Wai (Application)
+
+import Protolude hiding (get)
+
+spec :: SpecWith Application
+spec = describe "features supported on PostgreSQL 9.5" $
+  context "json array negative index" $ do
+    it "can select with negative indexes" $ do
+      get "/json_arr?select=data->>-1::int&id=in.(1,2)" `shouldRespondWith`
+        [json| [{"data":3}, {"data":6}] |]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data->0->>-2::int&id=in.(3,4)" `shouldRespondWith`
+        [json| [{"data":8}, {"data":7}] |]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data->-2->>a&id=in.(5,6)" `shouldRespondWith`
+        [json| [{"a":"A"}, {"a":"[1,2,3]"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "can filter with negative indexes" $ do
+      get "/json_arr?select=data&data->>-3=eq.1" `shouldRespondWith`
+        [json| [{"data":[1, 2, 3]}] |]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data&data->-1->>-3=eq.11" `shouldRespondWith`
+        [json| [{"data":[[9, 8, 7], [11, 12, 13]]}] |]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data&data->-1->>b=eq.B" `shouldRespondWith`
+        [json| [{"data":[{"a": "A"}, {"b": "B"}]}] |]
+        { matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data&data->-1->b->>-1=eq.5" `shouldRespondWith`
+        [json| [{"data":[{"a": [1,2,3]}, {"b": [4,5]}]}] |]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "should fail on badly formed negatives" $ do
+      get "/json_arr?select=data->>-78xy" `shouldRespondWith`
+        [json|
+          {"details": "unexpected 'x' expecting digit, \"->\", \"::\" or end of input",
+           "message": "\"failed to parse select parameter (data->>-78xy)\" (line 1, column 11)"} |]
+        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data->>--34" `shouldRespondWith`
+        [json|
+          {"details": "unexpected \"-\" expecting digit",
+           "message": "\"failed to parse select parameter (data->>--34)\" (line 1, column 9)"} |]
+        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }
+      get "/json_arr?select=data->>-xy-4" `shouldRespondWith`
+        [json|
+          {"details":"unexpected \"x\" expecting digit",
+           "message":"\"failed to parse select parameter (data->>-xy-4)\" (line 1, column 9)"} |]
+        { matchStatus = 400, matchHeaders = [matchContentTypeJson] }

test/Main.hs

@@ -34,6 +34,7 @@ import qualified Feature.ProxySpec
 import qualified Feature.AndOrParamsSpec
 import qualified Feature.RpcSpec
 import qualified Feature.NonexistentSchemaSpec
+import qualified Feature.PgVersion95Spec
 import qualified Feature.PgVersion96Spec
 import qualified Feature.UpsertSpec
 
@@ -70,6 +71,7 @@ main = do
       actualPgVersion = pgVersion dbStructure
       extraSpecs =
         [("Feature.UpsertSpec", Feature.UpsertSpec.spec) | actualPgVersion >= pgVersion95] ++
+        [("Feature.PgVersion95Spec", Feature.PgVersion95Spec.spec) | actualPgVersion >= pgVersion95] ++
         [("Feature.PgVersion96Spec", Feature.PgVersion96Spec.spec) | actualPgVersion >= pgVersion96]
 
       specs = uncurry describe <$> [

test/fixtures/data.sql

@@ -414,3 +414,14 @@ copy (select id, name, client_id from projects) to '/tmp/projects_dump.csv' with
 
 TRUNCATE TABLE "UnitTest" CASCADE;
 INSERT INTO "UnitTest" VALUES (1, 'unit test 1');
+
+TRUNCATE TABLE json_arr CASCADE;
+INSERT INTO json_arr VALUES (1, '[1, 2, 3]');
+INSERT INTO json_arr VALUES (2, '[4, 5, 6]');
+INSERT INTO json_arr VALUES (3, '[[9, 8, 7], [11, 12, 13]]');
+INSERT INTO json_arr VALUES (4, '[[[5, 6], 7, 8]]');
+INSERT INTO json_arr VALUES (5, '[{"a": "A"}, {"b": "B"}]');
+INSERT INTO json_arr VALUES (6, '[{"a": [1,2,3]}, {"b": [4,5]}]');
+INSERT INTO json_arr VALUES (7, '{"c": [1,2,3], "d": [4,5]}');
+INSERT INTO json_arr VALUES (8, '{"c": [{"d": [4,5,6,7,8]}]}');
+INSERT INTO json_arr VALUES (9, '[{"0xy1": [1,{"23-xy-45": [2, {"xy-6": [3]}]}]}]');

test/fixtures/privileges.sql

@@ -80,6 +80,7 @@ GRANT ALL ON TABLE
     , zone
     , projects_dump
     , "UnitTest"
+    , json_arr
 TO postgrest_test_anonymous;
 
 GRANT INSERT ON TABLE insertonly TO postgrest_test_anonymous;

test/fixtures/schema.sql

@@ -1456,3 +1456,8 @@ create table "UnitTest"(
   "idUnitTest" integer primary key,
   "nameUnitTest" text
 );
+
+create table json_arr(
+  id integer primary key,
+  data pg_catalog.json
+);