Merge pull request #895 from steve-chavez/binary-proc

Binary output support for RPC

CHANGELOG.md

@@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 
 - #889, Allow more than two conditions in a single and/or - @steve-chavez
+- #883, Binary output support for RPC - @steve-chavez
 
 ### Fixed
 

src/PostgREST/App.hs

@@ -232,15 +232,25 @@ app dbStructure conf apiRequest =
               return $ responseLBS status200 [allOrigins, acceptH] ""
 
         (ActionInvoke, TargetProc qi, Just (PayloadJSON payload)) ->
-          case readSqlParts of
+          let proc = M.lookup (qiName qi) allProcs
+              returnsScalar = case proc of
+                Just ProcDescription{pdReturnType = (Single (Scalar _))} -> True
+                _ -> False
+              rpcBinaryField = if returnsScalar
+                                 then Right Nothing
+                                 else binaryField contentType =<< fldNames
+              partsField = (,) <$> readSqlParts <*> rpcBinaryField in
+          case partsField of
             Left errorResponse -> return errorResponse
-            Right (q, cq) -> do
+            Right ((q, cq), bField) -> do
               let p = V.head payload
                   singular = contentType == CTSingularJSON
                   paramsAsSingleObject = iPreferSingleObjectParameter apiRequest
               row <- H.query () $
-                callProc qi p q cq topLevelRange shouldCount singular
+                callProc qi p returnsScalar q cq topLevelRange shouldCount
-                         paramsAsSingleObject (contentType == CTTextCSV)
+                         singular paramsAsSingleObject
+                         (contentType == CTTextCSV)
+                         (contentType == CTOctetStream) bField
               let (tableTotal, queryTotal, body) =
                     fromMaybe (Just 0, 0, "[]") row
                   (status, contentRange) = rangeHeader queryTotal tableTotal
@@ -257,7 +267,7 @@ app dbStructure conf apiRequest =
               uri Nothing = ("http", host, port, "/")
               uri (Just Proxy { proxyScheme = s, proxyHost = h, proxyPort = p, proxyPath = b }) = (s, h, p, b)
               uri' = uri proxy
-              encodeApi ti = encodeOpenAPI (M.elems $ dbProcs dbStructure) ti uri'
+              encodeApi ti = encodeOpenAPI (M.elems allProcs) ti uri'
           body <- encodeApi . toTableInfo <$> H.query schema accessibleTables
           return $ responseLBS status200 [toHeader CTOpenAPI] $ toS body
 
@@ -276,6 +286,7 @@ app dbStructure conf apiRequest =
       filterCol :: Schema -> TableName -> Column -> Bool
       filterCol sc tb Column{colTable=Table{tableSchema=s, tableName=t}} = s==sc && t==tb
       allPrKeys = dbPrimaryKeys dbStructure
+      allProcs = dbProcs dbStructure
       allOrigins = ("Access-Control-Allow-Origin", "*") :: Header
       shouldCount = iPreferCount apiRequest
       schema = toS $ configSchema conf
@@ -287,7 +298,7 @@ app dbStructure conf apiRequest =
             status = rangeStatus lower upper (toInteger <$> tableTotal)
         in (status, contentRange)
 
-      readReq = readRequest (configMaxRows conf) (dbRelations dbStructure) (dbProcs dbStructure) apiRequest
+      readReq = readRequest (configMaxRows conf) (dbRelations dbStructure) allProcs apiRequest
       fldNames = fieldNames <$> readReq
       readDbRequest = DbRead <$> readReq
       mutateDbRequest = DbMutate <$> (mutateRequest apiRequest =<< fldNames)
@@ -306,7 +317,7 @@ responseContentTypeOrError accepts action = serves contentTypesForRequest accept
         ActionCreate ->  [CTApplicationJSON, CTSingularJSON, CTTextCSV]
         ActionUpdate ->  [CTApplicationJSON, CTSingularJSON, CTTextCSV]
         ActionDelete ->  [CTApplicationJSON, CTSingularJSON, CTTextCSV]
-        ActionInvoke ->  [CTApplicationJSON, CTSingularJSON, CTTextCSV]
+        ActionInvoke ->  [CTApplicationJSON, CTSingularJSON, CTTextCSV, CTOctetStream]
         ActionInspect -> [CTOpenAPI, CTApplicationJSON]
         ActionInfo ->    [CTTextCSV]
     serves sProduces cAccepts =

src/PostgREST/DbStructure.hs

@@ -137,7 +137,9 @@ accessibleProcs =
       qi = QualifiedIdentifier schema name
       pgType = case typ of
         'c' -> Composite qi
-        'p' -> Pseudo name
+        'p' -> if name == "record" -- Only pg pseudo type that is a row type is 'record'
+                 then Composite qi
+                 else Scalar qi
         _   -> Scalar qi -- 'b'ase, 'd'omain, 'e'num, 'r'ange
 
   parseVolatility :: Char -> ProcVolatility

src/PostgREST/QueryBuilder.hs

@@ -144,42 +144,45 @@ createWriteStatement selectQuery mutateQuery wantSingle wantHdrs asCsv rep pKeys
     | otherwise = asJsonF
 
 type ProcResults = (Maybe Int64, Int64, ByteString)
-callProc :: QualifiedIdentifier -> JSON.Object -> SqlQuery -> SqlQuery -> NonnegRange ->
+callProc :: QualifiedIdentifier -> JSON.Object -> Bool -> SqlQuery -> SqlQuery -> NonnegRange ->
-  Bool -> Bool -> Bool -> Bool -> H.Query () (Maybe ProcResults)
+  Bool -> Bool -> Bool -> Bool -> Bool -> Maybe FieldName -> H.Query () (Maybe ProcResults)
-callProc qi params selectQuery countQuery _ countTotal isSingle paramsAsJson asCsv =
+callProc qi params returnsScalar selectQuery countQuery _ countTotal isSingle paramsAsJson asCsv asBinary binaryField =
   unicodeStatement sql HE.unit decodeProc True
   where
-    sql = [qc|
+    sql = 
-            WITH {sourceCTEName} AS ({_callSql})
+     if returnsScalar then [qc|
-            SELECT
+       WITH {sourceCTEName} AS ({_callSql})
-              {countResultF} AS total_result_set,
+       SELECT
-              pg_catalog.count(_postgrest_t) AS page_total,
+         {countResultF} AS total_result_set,
-              case
+         1 AS page_total,
-                when pg_catalog.count(*) > 1 then
+         {scalarBodyF} as body
-                  {bodyF}
+       FROM ({selectQuery}) _postgrest_t;|]
-                else
+     else [qc| 
-                  coalesce(((array_agg(row_to_json(_postgrest_t)))[1]->{_procName})::character varying, {bodyF})
+       WITH {sourceCTEName} AS ({_callSql})
+       SELECT
+         {countResultF} AS total_result_set,
+         pg_catalog.count(_postgrest_t) AS page_total,
+         {bodyF} as body
+       FROM ({selectQuery}) _postgrest_t;|]
 
-              end as body
+    countResultF = if countTotal then "( "<> countQuery <> ")" else "null::bigint" :: Text
-            FROM ({selectQuery}) _postgrest_t;
-          |]
-          -- FROM (select * from {sourceCTEName} {limitF range}) t;
-    countResultF = if countTotal then "("<>countQuery<>")" else "null::bigint" :: Text
     _args = if paramsAsJson
                 then insertableValueWithType "json" $ JSON.Object params
                 else intercalate "," $ map _assignment (HM.toList params)
-    _procName = pgFmtLit $ qiName qi
+    _procName = qiName qi
     _assignment (n,v) = pgFmtIdent n <> ":=" <> insertableValue v
     _callSql = [qc|select * from {fromQi qi}({_args}) |] :: Text
-    _countExpr = if countTotal
-                   then [qc|(select pg_catalog.count(*) from {sourceCTEName})|]
-                   else "null::bigint" :: Text
     decodeProc = HD.maybeRow procRow
     procRow = (,,) <$> HD.nullableValue HD.int8 <*> HD.value HD.int8
                    <*> HD.value HD.bytea
+    scalarBodyF
+     | asBinary = asBinaryF _procName
+     | otherwise = "(row_to_json(_postgrest_t)->" <> pgFmtLit _procName <> ")::character varying"
+
     bodyF
      | isSingle = asJsonSingleF
      | asCsv = asCsvF
+     | isJust binaryField = asBinaryF $ fromJust binaryField
      | otherwise = asJsonF
 
 pgFmtIdent :: SqlFragment -> SqlFragment

src/PostgREST/Types.hs

@@ -38,7 +38,7 @@ data PgArg = PgArg {
 , pgaReq  :: Bool
 } deriving (Show, Eq)
 
-data PgType = Scalar QualifiedIdentifier | Composite QualifiedIdentifier | Pseudo Text deriving (Eq, Show)
+data PgType = Scalar QualifiedIdentifier | Composite QualifiedIdentifier deriving (Eq, Show)
 
 data RetType = Single PgType | SetOf PgType deriving (Eq, Show)
 

test/Feature/QuerySpec.hs

@@ -553,6 +553,18 @@ spec = do
             [json|"Hello, ￥"|]
             { matchHeaders = [matchContentTypeJson] }
 
+      it "returns array" $
+        post "/rpc/ret_array" [json|{}|] `shouldRespondWith`
+          [json|[1, 2, 3]|]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "returns setof integers" $
+        post "/rpc/ret_setof_integers" [json|{}|] `shouldRespondWith`
+          [json|[{ "ret_setof_integers": 1 },
+                 { "ret_setof_integers": 2 },
+                 { "ret_setof_integers": 3 }]|]
+          { matchHeaders = [matchContentTypeJson] }
+
       it "returns enum value" $
         post "/rpc/ret_enum" [json|{ "val": "foo" }|] `shouldRespondWith`
           [json|"foo"|]
@@ -655,6 +667,10 @@ spec = do
           [json| "Return value of no parameters procedure." |]
           { matchHeaders = [matchContentTypeJson] }
 
+    it "returns proper output when having the same return col name as the proc name" $
+      post "/rpc/test" [json|{}|] `shouldRespondWith`
+        [json|[{"test":"hello","value":1}]|] { matchHeaders = [matchContentTypeJson] }
+
   describe "weird requests" $ do
     it "can query as normal" $ do
       get "/Escap3e;" `shouldRespondWith`
@@ -681,27 +697,50 @@ spec = do
         { matchHeaders = [matchContentTypeJson] }
 
   describe "binary output" $ do
-    it "can query if a single column is selected" $
+    context "on GET" $ do
-      request methodGet "/images_base64?select=img&name=eq.A.png" (acceptHdrs "application/octet-stream") ""
+      it "can query if a single column is selected" $
-        `shouldRespondWith` "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEUAAAD/AAAb/40iAAAAP0lEQVQI12NgwAbYG2AE/wEYwQMiZB4ACQkQYZEAIgqAhAGIKLCAEQ8kgMT/P1CCEUwc4IMSzA3sUIIdCHECAGSQEkeOTUyCAAAAAElFTkSuQmCC"
+        request methodGet "/images_base64?select=img&name=eq.A.png" (acceptHdrs "application/octet-stream") ""
-        { matchStatus = 200
+          `shouldRespondWith` "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEUAAAD/AAAb/40iAAAAP0lEQVQI12NgwAbYG2AE/wEYwQMiZB4ACQkQYZEAIgqAhAGIKLCAEQ8kgMT/P1CCEUwc4IMSzA3sUIIdCHECAGSQEkeOTUyCAAAAAElFTkSuQmCC"
-        , matchHeaders = ["Content-Type" <:> "application/octet-stream; charset=utf-8"]
+          { matchStatus = 200
-        }
+          , matchHeaders = ["Content-Type" <:> "application/octet-stream; charset=utf-8"]
+          }
 
-    it "fails if a single column is not selected" $ do
+      it "fails if a single column is not selected" $ do
-      request methodGet "/images?select=img,name&name=eq.A.png" (acceptHdrs "application/octet-stream") ""
+        request methodGet "/images?select=img,name&name=eq.A.png" (acceptHdrs "application/octet-stream") ""
-        `shouldRespondWith` 406
+          `shouldRespondWith` 406
-      request methodGet "/images?select=*&name=eq.A.png" (acceptHdrs "application/octet-stream") ""
+        request methodGet "/images?select=*&name=eq.A.png" (acceptHdrs "application/octet-stream") ""
-        `shouldRespondWith` 406
+          `shouldRespondWith` 406
-      request methodGet "/images?name=eq.A.png" (acceptHdrs "application/octet-stream") ""
+        request methodGet "/images?name=eq.A.png" (acceptHdrs "application/octet-stream") ""
-        `shouldRespondWith` 406
+          `shouldRespondWith` 406
+
+      it "concatenates results if more than one row is returned" $
+        request methodGet "/images_base64?select=img&name=in.A.png,B.png" (acceptHdrs "application/octet-stream") ""
+          `shouldRespondWith` "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEUAAAD/AAAb/40iAAAAP0lEQVQI12NgwAbYG2AE/wEYwQMiZB4ACQkQYZEAIgqAhAGIKLCAEQ8kgMT/P1CCEUwc4IMSzA3sUIIdCHECAGSQEkeOTUyCAAAAAElFTkSuQmCCiVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEX///8AAP94wDzzAAAAL0lEQVQIW2NgwAb+HwARH0DEDyDxwAZEyGAhLODqHmBRzAcn5GAS///A1IF14AAA5/Adbiiz/0gAAAAASUVORK5CYII="
+          { matchStatus = 200
+          , matchHeaders = ["Content-Type" <:> "application/octet-stream; charset=utf-8"]
+          }
+
+    context "on RPC" $ do
+      context "Proc that returns scalar" $
+        it "can query without selecting column" $
+          request methodPost "/rpc/ret_base64_bin" (acceptHdrs "application/octet-stream") ""
+            `shouldRespondWith` "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEUAAAD/AAAb/40iAAAAP0lEQVQI12NgwAbYG2AE/wEYwQMiZB4ACQkQYZEAIgqAhAGIKLCAEQ8kgMT/P1CCEUwc4IMSzA3sUIIdCHECAGSQEkeOTUyCAAAAAElFTkSuQmCC"
+            { matchStatus = 200
+            , matchHeaders = ["Content-Type" <:> "application/octet-stream; charset=utf-8"]
+            }
+
+      context "Proc that returns rows" $ do
+        it "can query if a single column is selected" $
+          request methodPost "/rpc/ret_rows_with_base64_bin?select=img" (acceptHdrs "application/octet-stream") ""
+            `shouldRespondWith` "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEUAAAD/AAAb/40iAAAAP0lEQVQI12NgwAbYG2AE/wEYwQMiZB4ACQkQYZEAIgqAhAGIKLCAEQ8kgMT/P1CCEUwc4IMSzA3sUIIdCHECAGSQEkeOTUyCAAAAAElFTkSuQmCCiVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEX///8AAP94wDzzAAAAL0lEQVQIW2NgwAb+HwARH0DEDyDxwAZEyGAhLODqHmBRzAcn5GAS///A1IF14AAA5/Adbiiz/0gAAAAASUVORK5CYII="
+            { matchStatus = 200
+            , matchHeaders = ["Content-Type" <:> "application/octet-stream; charset=utf-8"]
+            }
+
+        it "fails if a single column is not selected" $
+          request methodPost "/rpc/ret_rows_with_base64_bin" (acceptHdrs "application/octet-stream") ""
+            `shouldRespondWith` 406
 
-    it "concatenates results if more than one row is returned" $
-      request methodGet "/images_base64?select=img&name=in.A.png,B.png" (acceptHdrs "application/octet-stream") ""
-        `shouldRespondWith` "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEUAAAD/AAAb/40iAAAAP0lEQVQI12NgwAbYG2AE/wEYwQMiZB4ACQkQYZEAIgqAhAGIKLCAEQ8kgMT/P1CCEUwc4IMSzA3sUIIdCHECAGSQEkeOTUyCAAAAAElFTkSuQmCCiVBORw0KGgoAAAANSUhEUgAAAB4AAAAeAQMAAAAB/jzhAAAABlBMVEX///8AAP94wDzzAAAAL0lEQVQIW2NgwAb+HwARH0DEDyDxwAZEyGAhLODqHmBRzAcn5GAS///A1IF14AAA5/Adbiiz/0gAAAAASUVORK5CYII="
-        { matchStatus = 200
-        , matchHeaders = ["Content-Type" <:> "application/octet-stream; charset=utf-8"]
-        }
   describe "HTTP request env vars" $ do
     it "custom header is set" $
       request methodPost "/rpc/get_guc_value"

test/fixtures/schema.sql

@@ -1115,6 +1115,7 @@ create table images (
 );
 
 create view images_base64 as (
+  -- encoding in base64 puts a '\n' after every 76 character due to legacy reasons, this is isn't necessary here so it's removed
   select name, replace(encode(img, 'base64'), E'\n', '') as img from images
 );
 
@@ -1124,6 +1125,10 @@ $$ language sql;
 
 create domain one_nine as integer check (value >= 1 and value <= 9);
 
+create function test.ret_array() returns integer[] as $$ 
+  select '{1,2,3}'::integer[];
+$$ language sql;
+
 create function test.ret_domain(val integer) returns test.one_nine as $$ 
   select val::test.one_nine;
 $$ language sql;
@@ -1132,6 +1137,10 @@ create function test.ret_range(low integer, up integer) returns int4range as $$
   select int4range(low, up);
 $$ language sql;
 
+create function test.ret_setof_integers() returns setof integer as $$
+  values (1), (2), (3);
+$$ language sql;
+
 create function test.ret_scalars() returns table(
   a text, b test.enum_menagerie_type, c test.one_nine, d int4range
 ) as $$ 
@@ -1153,6 +1162,14 @@ $$ language sql;
 
 create function test.ret_void() returns void as '' language sql;
 
+create function test.ret_base64_bin() returns text as $$
+  select i.img from test.images_base64 i where i.name = 'A.png';
+$$ language sql;
+
+create function test.ret_rows_with_base64_bin() returns setof test.images_base64 as $$
+  select i.name, i.img from test.images_base64 i;
+$$ language sql;
+
 create function test.single_article(id integer) returns test.articles as $$ 
   select a.* from test.articles a where a.id = $1;
 $$ language sql;
@@ -1198,6 +1215,12 @@ create table grandchild_entities (
   jsonb_col jsonb
 );
 
+-- Used for testing that having the same return column name as the proc name
+-- doesn't conflict with the required output, details in #901
+create function test.test() returns table(test text, value int) as $$
+  values ('hello', 1);
+$$ language sql;
+
 --
 -- PostgreSQL database dump complete
 --