Change callProc query to a parametrized query

CHANGELOG.md

@@ -10,10 +10,12 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Fixed
 
 - #828, Fix computed column only working in public schema - @steve-chavez
+- #925, Avoid RPC high memory usage by using parametrized query - @steve-chavez
 
 ### Changed
 
 - Computed columns now only work if they belong to the db-schema - @steve-chavez
+- To use RPC now the `json_to_record/json_to_recordset` functions are needed, these are available starting from PostgreSQL 9.4 - @steve-chavez
 
 ## [0.4.4.0] - 2018-01-08
 

src/PostgREST/App.hs

@@ -241,11 +241,12 @@ app dbStructure conf apiRequest =
             Right ((q, cq), bField, params) -> do
               let prms = case payload of
                           Just (PayloadJSON pld) -> V.head pld
-                          Nothing -> M.fromList $ second toJSON <$> params -- toJSON is just for reusing the callProc function
+                          Nothing -> M.fromList $ second toJSON <$> params
                   singular = contentType == CTSingularJSON
                   paramsAsSingleObject = iPreferSingleObjectParameter apiRequest
-              row <- H.query () $
+                  specifiedPgArgs = filter (flip M.member prms . pgaName) $ fromMaybe [] (pdArgs <$> proc)
-                callProc qi prms returnsScalar q cq shouldCount
+              row <- H.query (toJSON prms) $
+                callProc qi specifiedPgArgs returnsScalar q cq shouldCount
                          singular paramsAsSingleObject
                          (contentType == CTTextCSV)
                          (contentType == CTOctetStream) _isReadOnly bField

src/PostgREST/QueryBuilder.hs

@@ -142,14 +142,20 @@ createWriteStatement selectQuery mutateQuery wantSingle wantHdrs asCsv rep pKeys
     | otherwise = asJsonF
 
 type ProcResults = (Maybe Int64, Int64, ByteString, ByteString)
-callProc :: QualifiedIdentifier -> JSON.Object -> Bool -> SqlQuery -> SqlQuery -> Bool ->
+callProc :: QualifiedIdentifier -> [PgArg] -> Bool -> SqlQuery -> SqlQuery -> Bool ->
-  Bool -> Bool -> Bool -> Bool -> Bool -> Maybe FieldName -> PgVersion -> H.Query () (Maybe ProcResults)
+            Bool -> Bool -> Bool -> Bool -> Bool -> Maybe FieldName -> PgVersion ->
-callProc qi params returnsScalar selectQuery countQuery countTotal isSingle paramsAsJson asCsv asBinary isReadOnly binaryField pgVer =
+            H.Query JSON.Value (Maybe ProcResults)
-  unicodeStatement sql HE.unit decodeProc True
+callProc qi pgArgs returnsScalar selectQuery countQuery countTotal isSingle paramsAsJson asCsv asBinary isReadOnly binaryField pgVer =
+  unicodeStatement sql (HE.value HE.json) decodeProc True
   where
     sql =
      if returnsScalar then [qc|
-       WITH {sourceCTEName} AS (select {fromQi qi}({_args}))
+       WITH _args_record AS (
+         {argsRecord}
+       ),
+       {sourceCTEName} AS (
+         SELECT {fromQi qi}({args})
+       )
        SELECT
          {countResultF} AS total_result_set,
          1 AS page_total,
@@ -157,7 +163,12 @@ callProc qi params returnsScalar selectQuery countQuery countTotal isSingle para
          {responseHeaders} AS response_headers
        FROM ({selectQuery}) _postgrest_t;|]
      else [qc|
-       WITH {sourceCTEName} AS (select * from {fromQi qi}({_args}))
+       WITH _args_record AS (
+         {argsRecord}
+       ),
+       {sourceCTEName} AS (
+         SELECT * FROM {fromQi qi}({args})
+       )
        SELECT
          {countResultF} AS total_result_set,
          pg_catalog.count(_postgrest_t) AS page_total,
@@ -165,12 +176,14 @@ callProc qi params returnsScalar selectQuery countQuery countTotal isSingle para
          {responseHeaders} AS response_headers
        FROM ({selectQuery}) _postgrest_t;|]
 
+    (argsRecord, args) | paramsAsJson && not isReadOnly  = ("SELECT NULL", "$1")
+                       | null pgArgs = ("SELECT NULL", "")
+                       | otherwise = (
+                           "SELECT * FROM json_to_record($1) AS _(" <> intercalate ", " ((\a -> pgaName a <> " " <> pgaType a) <$> pgArgs) <> ")",
+                           intercalate ", " ((\a -> pgaName a <> " := (SELECT " <> pgaName a <> " FROM _args_record)") <$> pgArgs)
+                         )
     countResultF = if countTotal then "( "<> countQuery <> ")" else "null::bigint" :: Text
-    _args = if paramsAsJson && not isReadOnly
-                then insertableValueWithType "json" $ JSON.Object params
-                else intercalate "," $ map _assignment (HM.toList params)
     _procName = qiName qi
-    _assignment (n,v) = pgFmtIdent n <> ":=" <> insertableValue v
     responseHeaders =
       if pgVer >= pgVersion96
         then "coalesce(nullif(current_setting('response.headers', true), ''), '[]')" :: Text -- nullif is used because of https://gist.github.com/steve-chavez/8d7033ea5655096903f3b52f8ed09a15
@@ -380,10 +393,6 @@ insertableValue :: JSON.Value -> SqlFragment
 insertableValue JSON.Null = "null"
 insertableValue v = (<> "::unknown") . pgFmtLit $ unquoted v
 
-insertableValueWithType :: Text -> JSON.Value -> SqlFragment
-insertableValueWithType t v =
-  pgFmtLit (unquoted v) <> "::" <> t
-
 pgFmtColumn :: QualifiedIdentifier -> Text -> SqlFragment
 pgFmtColumn table "*" = fromQi table <> ".*"
 pgFmtColumn table c = fromQi table <> "." <> pgFmtIdent c

test/Feature/RpcSpec.hs

@@ -288,23 +288,23 @@ spec =
     it "defaults to status 500 if RAISE code is PT not followed by a number" $
       get "/rpc/raise_bad_pt" `shouldRespondWith` 500
 
-    context "only for POST rpc" $ do
+    context "expects a single json object" $ do
-      context "expects a single json object" $ do
+      it "does not expand posted json into parameters" $
-        it "does not expand posted json into parameters" $
+        request methodPost "/rpc/singlejsonparam"
-          request methodPost "/rpc/singlejsonparam"
+          [("Prefer","params=single-object")] [json| { "p1": 1, "p2": "text", "p3" : {"obj":"text"} } |] `shouldRespondWith`
-            [("Prefer","params=single-object")] [json| { "p1": 1, "p2": "text", "p3" : {"obj":"text"} } |] `shouldRespondWith`
+          [json| { "p1": 1, "p2": "text", "p3" : {"obj":"text"} } |]
-            [json| { "p1": 1, "p2": "text", "p3" : {"obj":"text"} } |]
+          { matchHeaders = [matchContentTypeJson] }
-            { matchHeaders = [matchContentTypeJson] }
 
-        it "accepts parameters from an html form" $
+      it "accepts parameters from an html form" $
-          request methodPost "/rpc/singlejsonparam"
+        request methodPost "/rpc/singlejsonparam"
-            [("Prefer","params=single-object"),("Content-Type", "application/x-www-form-urlencoded")]
+          [("Prefer","params=single-object"),("Content-Type", "application/x-www-form-urlencoded")]
-            ("integer=7&double=2.71828&varchar=forms+are+fun&" <>
+          ("integer=7&double=2.71828&varchar=forms+are+fun&" <>
-             "boolean=false&date=1900-01-01&money=$3.99&enum=foo") `shouldRespondWith`
+           "boolean=false&date=1900-01-01&money=$3.99&enum=foo") `shouldRespondWith`
-            [json| { "integer": "7", "double": "2.71828", "varchar" : "forms are fun"
+          [json| { "integer": "7", "double": "2.71828", "varchar" : "forms are fun"
-                   , "boolean":"false", "date":"1900-01-01", "money":"$3.99", "enum":"foo" } |]
+                 , "boolean":"false", "date":"1900-01-01", "money":"$3.99", "enum":"foo" } |]
-                   { matchHeaders = [matchContentTypeJson] }
+                 { matchHeaders = [matchContentTypeJson] }
 
+    context "only for POST rpc" $
       it "gives a parse filter error if GET style proc args are specified" $
         post "/rpc/sayhello?name=John" [json|{}|] `shouldRespondWith` 400