diff --git a/CHANGELOG.md b/CHANGELOG.md index cd23774ff..5b2687970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,12 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed - #828, Fix computed column only working in public schema - @steve-chavez +- #925, Avoid RPC high memory usage by using parametrized query - @steve-chavez ### Changed - Computed columns now only work if they belong to the db-schema - @steve-chavez +- To use RPC now the `json_to_record/json_to_recordset` functions are needed, these are available starting from PostgreSQL 9.4 - @steve-chavez ## [0.4.4.0] - 2018-01-08 diff --git a/src/PostgREST/App.hs b/src/PostgREST/App.hs index 9bb2e0003..2ecb88d8f 100644 --- a/src/PostgREST/App.hs +++ b/src/PostgREST/App.hs @@ -241,11 +241,12 @@ app dbStructure conf apiRequest = Right ((q, cq), bField, params) -> do let prms = case payload of Just (PayloadJSON pld) -> V.head pld - Nothing -> M.fromList $ second toJSON <$> params -- toJSON is just for reusing the callProc function + Nothing -> M.fromList $ second toJSON <$> params singular = contentType == CTSingularJSON paramsAsSingleObject = iPreferSingleObjectParameter apiRequest - row <- H.query () $ - callProc qi prms returnsScalar q cq shouldCount + specifiedPgArgs = filter (flip M.member prms . pgaName) $ fromMaybe [] (pdArgs <$> proc) + row <- H.query (toJSON prms) $ + callProc qi specifiedPgArgs returnsScalar q cq shouldCount singular paramsAsSingleObject (contentType == CTTextCSV) (contentType == CTOctetStream) _isReadOnly bField diff --git a/src/PostgREST/QueryBuilder.hs b/src/PostgREST/QueryBuilder.hs index 2b2e7e0b1..157e1c7c3 100644 --- a/src/PostgREST/QueryBuilder.hs +++ b/src/PostgREST/QueryBuilder.hs @@ -142,14 +142,20 @@ createWriteStatement selectQuery mutateQuery wantSingle wantHdrs asCsv rep pKeys | otherwise = asJsonF type ProcResults = (Maybe Int64, Int64, ByteString, ByteString) -callProc :: QualifiedIdentifier -> JSON.Object -> Bool -> SqlQuery -> SqlQuery -> Bool -> - Bool -> Bool -> Bool -> Bool -> Bool -> Maybe FieldName -> PgVersion -> H.Query () (Maybe ProcResults) -callProc qi params returnsScalar selectQuery countQuery countTotal isSingle paramsAsJson asCsv asBinary isReadOnly binaryField pgVer = - unicodeStatement sql HE.unit decodeProc True +callProc :: QualifiedIdentifier -> [PgArg] -> Bool -> SqlQuery -> SqlQuery -> Bool -> + Bool -> Bool -> Bool -> Bool -> Bool -> Maybe FieldName -> PgVersion -> + H.Query JSON.Value (Maybe ProcResults) +callProc qi pgArgs returnsScalar selectQuery countQuery countTotal isSingle paramsAsJson asCsv asBinary isReadOnly binaryField pgVer = + unicodeStatement sql (HE.value HE.json) decodeProc True where sql = if returnsScalar then [qc| - WITH {sourceCTEName} AS (select {fromQi qi}({_args})) + WITH _args_record AS ( + {argsRecord} + ), + {sourceCTEName} AS ( + SELECT {fromQi qi}({args}) + ) SELECT {countResultF} AS total_result_set, 1 AS page_total, @@ -157,7 +163,12 @@ callProc qi params returnsScalar selectQuery countQuery countTotal isSingle para {responseHeaders} AS response_headers FROM ({selectQuery}) _postgrest_t;|] else [qc| - WITH {sourceCTEName} AS (select * from {fromQi qi}({_args})) + WITH _args_record AS ( + {argsRecord} + ), + {sourceCTEName} AS ( + SELECT * FROM {fromQi qi}({args}) + ) SELECT {countResultF} AS total_result_set, pg_catalog.count(_postgrest_t) AS page_total, @@ -165,12 +176,14 @@ callProc qi params returnsScalar selectQuery countQuery countTotal isSingle para {responseHeaders} AS response_headers FROM ({selectQuery}) _postgrest_t;|] + (argsRecord, args) | paramsAsJson && not isReadOnly = ("SELECT NULL", "$1") + | null pgArgs = ("SELECT NULL", "") + | otherwise = ( + "SELECT * FROM json_to_record($1) AS _(" <> intercalate ", " ((\a -> pgaName a <> " " <> pgaType a) <$> pgArgs) <> ")", + intercalate ", " ((\a -> pgaName a <> " := (SELECT " <> pgaName a <> " FROM _args_record)") <$> pgArgs) + ) countResultF = if countTotal then "( "<> countQuery <> ")" else "null::bigint" :: Text - _args = if paramsAsJson && not isReadOnly - then insertableValueWithType "json" $ JSON.Object params - else intercalate "," $ map _assignment (HM.toList params) _procName = qiName qi - _assignment (n,v) = pgFmtIdent n <> ":=" <> insertableValue v responseHeaders = if pgVer >= pgVersion96 then "coalesce(nullif(current_setting('response.headers', true), ''), '[]')" :: Text -- nullif is used because of https://gist.github.com/steve-chavez/8d7033ea5655096903f3b52f8ed09a15 @@ -380,10 +393,6 @@ insertableValue :: JSON.Value -> SqlFragment insertableValue JSON.Null = "null" insertableValue v = (<> "::unknown") . pgFmtLit $ unquoted v -insertableValueWithType :: Text -> JSON.Value -> SqlFragment -insertableValueWithType t v = - pgFmtLit (unquoted v) <> "::" <> t - pgFmtColumn :: QualifiedIdentifier -> Text -> SqlFragment pgFmtColumn table "*" = fromQi table <> ".*" pgFmtColumn table c = fromQi table <> "." <> pgFmtIdent c diff --git a/test/Feature/RpcSpec.hs b/test/Feature/RpcSpec.hs index f051fc07a..02e60e07e 100644 --- a/test/Feature/RpcSpec.hs +++ b/test/Feature/RpcSpec.hs @@ -288,23 +288,23 @@ spec = it "defaults to status 500 if RAISE code is PT not followed by a number" $ get "/rpc/raise_bad_pt" `shouldRespondWith` 500 - context "only for POST rpc" $ do - context "expects a single json object" $ do - it "does not expand posted json into parameters" $ - request methodPost "/rpc/singlejsonparam" - [("Prefer","params=single-object")] [json| { "p1": 1, "p2": "text", "p3" : {"obj":"text"} } |] `shouldRespondWith` - [json| { "p1": 1, "p2": "text", "p3" : {"obj":"text"} } |] - { matchHeaders = [matchContentTypeJson] } + context "expects a single json object" $ do + it "does not expand posted json into parameters" $ + request methodPost "/rpc/singlejsonparam" + [("Prefer","params=single-object")] [json| { "p1": 1, "p2": "text", "p3" : {"obj":"text"} } |] `shouldRespondWith` + [json| { "p1": 1, "p2": "text", "p3" : {"obj":"text"} } |] + { matchHeaders = [matchContentTypeJson] } - it "accepts parameters from an html form" $ - request methodPost "/rpc/singlejsonparam" - [("Prefer","params=single-object"),("Content-Type", "application/x-www-form-urlencoded")] - ("integer=7&double=2.71828&varchar=forms+are+fun&" <> - "boolean=false&date=1900-01-01&money=$3.99&enum=foo") `shouldRespondWith` - [json| { "integer": "7", "double": "2.71828", "varchar" : "forms are fun" - , "boolean":"false", "date":"1900-01-01", "money":"$3.99", "enum":"foo" } |] - { matchHeaders = [matchContentTypeJson] } + it "accepts parameters from an html form" $ + request methodPost "/rpc/singlejsonparam" + [("Prefer","params=single-object"),("Content-Type", "application/x-www-form-urlencoded")] + ("integer=7&double=2.71828&varchar=forms+are+fun&" <> + "boolean=false&date=1900-01-01&money=$3.99&enum=foo") `shouldRespondWith` + [json| { "integer": "7", "double": "2.71828", "varchar" : "forms are fun" + , "boolean":"false", "date":"1900-01-01", "money":"$3.99", "enum":"foo" } |] + { matchHeaders = [matchContentTypeJson] } + context "only for POST rpc" $ it "gives a parse filter error if GET style proc args are specified" $ post "/rpc/sayhello?name=John" [json|{}|] `shouldRespondWith` 400