jimmy/postgrest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix #690, add columns query arg for RPC/POST/PATCH

Browse Source 
* Refactor normalizing json CTE
* Refactor CTE to use CASE instead of UNION
This commit is contained in:
steve-chavez
2019-02-26 11:38:21 -05:00
committed by Steve Chávez
parent 3946dfbc64
commit c9b2830e52
10 changed files with 157 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2
View File
@@ -7,6 +7,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Added ### Added
- #690, Add `?columns` query parameter for faster bulk inserts, also ignores unspecified json keys in a payload - @steve-chavez
### Fixed ### Fixed
- #1223, Fix incorrect OpenAPI externalDocs url - @steve-chavez - #1223, Fix incorrect OpenAPI externalDocs url - @steve-chavez
src/PostgREST/ApiRequest.hs
+23 -12
View File
@@ -19,6 +19,7 @@ import qualified Data.ByteString as BS
import qualified Data.ByteString.Internal as BS (c2w) import qualified Data.ByteString.Internal as BS (c2w)
import qualified Data.ByteString.Lazy as BL import qualified Data.ByteString.Lazy as BL
import qualified Data.Csv as CSV import qualified Data.Csv as CSV
import Data.Either.Combinators (mapLeft)
import qualified Data.List as L import qualified Data.List as L
import Data.List (lookup, last, partition) import Data.List (lookup, last, partition)
import qualified Data.HashMap.Strict as M import qualified Data.HashMap.Strict as M
@@ -32,6 +33,7 @@ import Network.HTTP.Types.Header (hAuthorization, hCookie)
import Network.HTTP.Types.URI (parseSimpleQuery) import Network.HTTP.Types.URI (parseSimpleQuery)
import Network.Wai (Request (..)) import Network.Wai (Request (..))
import Network.Wai.Parse (parseHttpAccept) import Network.Wai.Parse (parseHttpAccept)
import PostgREST.Parsers (pRequestColumns)
import PostgREST.RangeQuery (NonnegRange, rangeRequested, restrictRange, rangeGeq, allRange, rangeLimit, rangeOffset) import PostgREST.RangeQuery (NonnegRange, rangeRequested, restrictRange, rangeGeq, allRange, rangeLimit, rangeOffset)
import Data.Ranged.Boundaries import Data.Ranged.Boundaries
import PostgREST.Types import PostgREST.Types
@@ -120,7 +122,7 @@ userApiRequest schema req reqBody
else Nothing else Nothing
, iFilters = filters , iFilters = filters
, iLogic = [(toS k, toS $ fromJust v) | (k,v) <- qParams, isJust v, endingIn ["and", "or"] k ] , iLogic = [(toS k, toS $ fromJust v) | (k,v) <- qParams, isJust v, endingIn ["and", "or"] k ]
, iSelect = toS $ fromMaybe "*" $ fromMaybe (Just "*") $ lookup "select" qParams , iSelect = toS $ fromMaybe "*" $ join $ lookup "select" qParams
, iOrder = [(toS k, toS $ fromJust v) | (k,v) <- qParams, isJust v, endingIn ["order"] k ] , iOrder = [(toS k, toS $ fromJust v) | (k,v) <- qParams, isJust v, endingIn ["order"] k ]
, iCanonicalQS = toS $ urlEncodeVars , iCanonicalQS = toS $ urlEncodeVars
. L.sortBy (comparing fst) . L.sortBy (comparing fst)
@@ -137,28 +139,37 @@ userApiRequest schema req reqBody
case action of case action of
ActionInvoke{isReadOnly=True} -> partition (liftM2 (||) (isEmbedPath . fst) (hasOperator . snd)) flts ActionInvoke{isReadOnly=True} -> partition (liftM2 (||) (isEmbedPath . fst) (hasOperator . snd)) flts
_ -> (flts, []) _ -> (flts, [])
flts = [ (toS k, toS $ fromJust v) | (k,v) <- qParams, isJust v, k /= "select", not (endingIn ["order", "limit", "offset", "and", "or"] k) ] flts =
[ (toS k, toS $ fromJust v) |
(k,v) <- qParams, isJust v,
k `notElem` ["select", "columns"],
not (endingIn ["order", "limit", "offset", "and", "or"] k) ]
hasOperator val = any (`T.isPrefixOf` val) $ hasOperator val = any (`T.isPrefixOf` val) $
((<> ".") <$> "not":M.keys operators) ++ ((<> ".") <$> "not":M.keys operators) ++
((<> "(") <$> M.keys ftsOperators) ((<> "(") <$> M.keys ftsOperators)
isEmbedPath = T.isInfixOf "." isEmbedPath = T.isInfixOf "."
isTargetingProc = (== Just "rpc") $ listToMaybe path isTargetingProc = (== Just "rpc") $ listToMaybe path
contentType = decodeContentType . fromMaybe "application/json" $ lookupHeader "content-type"
payload = payload =
case (decodeContentType . fromMaybe "application/json" $ lookupHeader "content-type", action) of case (contentType, action) of
(_, ActionInvoke{isReadOnly=True}) -> (_, ActionInvoke{isReadOnly=True}) ->
Right $ PayloadJSON (JSON.encode $ M.fromList $ second JSON.toJSON <$> rpcQParams) PJObject (S.fromList $ fst <$> rpcQParams) Right $ ProcessedJSON (JSON.encode $ M.fromList $ second JSON.toJSON <$> rpcQParams) PJObject (S.fromList $ fst <$> rpcQParams)
(CTApplicationJSON, _) -> (CTApplicationJSON, _) ->
note "All object keys must match" . payloadAttributes reqBody let columns | action `elem` [ActionCreate, ActionUpdate, ActionInvoke{isReadOnly=False}] = toS <$> join (lookup "columns" qParams)
=<< if BL.null reqBody && isTargetingProc | otherwise = Nothing in
then Right emptyObject case columns of
else JSON.eitherDecode reqBody Just cols -> RawJSON reqBody <$> mapLeft show (pRequestColumns cols)
Nothing -> note "All object keys must match" . payloadAttributes reqBody
=<< if BL.null reqBody && isTargetingProc
then Right emptyObject
else JSON.eitherDecode reqBody
(CTTextCSV, _) -> do (CTTextCSV, _) -> do
json <- csvToJson <$> CSV.decodeByName reqBody json <- csvToJson <$> CSV.decodeByName reqBody
note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json
(CTOther "application/x-www-form-urlencoded", _) -> (CTOther "application/x-www-form-urlencoded", _) ->
let json = M.fromList . map (toS *** JSON.String . toS) . parseSimpleQuery $ toS reqBody let json = M.fromList . map (toS *** JSON.String . toS) . parseSimpleQuery $ toS reqBody
keys = S.fromList $ M.keys json in keys = S.fromList $ M.keys json in
Right $ PayloadJSON (JSON.encode json) PJObject keys Right $ ProcessedJSON (JSON.encode json) PJObject keys
(ct, _) -> (ct, _) ->
Left $ toS $ "Content-Type not acceptable: " <> toMime ct Left $ toS $ "Content-Type not acceptable: " <> toMime ct
topLevelRange = fromMaybe allRange $ M.lookup "limit" ranges topLevelRange = fromMaybe allRange $ M.lookup "limit" ranges
@@ -291,14 +302,14 @@ payloadAttributes raw json =
JSON.Object x -> S.fromList (M.keys x) == canonicalKeys JSON.Object x -> S.fromList (M.keys x) == canonicalKeys
_ -> False) arr in _ -> False) arr in
if areKeysUniform if areKeysUniform
then Just $ PayloadJSON raw (PJArray $ V.length arr) canonicalKeys then Just $ ProcessedJSON raw (PJArray $ V.length arr) canonicalKeys
else Nothing else Nothing
Just _ -> Nothing Just _ -> Nothing
Nothing -> Just emptyPJArray Nothing -> Just emptyPJArray
JSON.Object o -> Just $ PayloadJSON raw PJObject (S.fromList $ M.keys o) JSON.Object o -> Just $ ProcessedJSON raw PJObject (S.fromList $ M.keys o)
-- truncate everything else to an empty array. -- truncate everything else to an empty array.
_ -> Just emptyPJArray _ -> Just emptyPJArray
where where
emptyPJArray = PayloadJSON (JSON.encode emptyArray) (PJArray 0) S.empty emptyPJArray = ProcessedJSON (JSON.encode emptyArray) (PJArray 0) S.empty
src/PostgREST/App.hs
+10 -11
View File
@@ -78,10 +78,9 @@ postgrest conf refDbStructure pool getTime worker =
Left err -> return $ apiRequestError err Left err -> return $ apiRequestError err
Right apiRequest -> do Right apiRequest -> do
eClaims <- jwtClaims jwtSecret (configJwtAudience conf) (toS $ iJWT apiRequest) time (rightToMaybe $ configRoleClaimKey conf) eClaims <- jwtClaims jwtSecret (configJwtAudience conf) (toS $ iJWT apiRequest) time (rightToMaybe $ configRoleClaimKey conf)
let authed = containsRole eClaims let authed = containsRole eClaims
proc = case (iTarget apiRequest, iPayload apiRequest, iPreferSingleObjectParameter apiRequest) of proc = case (iTarget apiRequest, iPayload apiRequest, iPreferSingleObjectParameter apiRequest) of
(TargetProc qi, Just PayloadJSON{pjKeys}, s) -> findProc qi pjKeys s $ dbProcs dbStructure (TargetProc qi, Just pJson, s) -> findProc qi (pjKeys pJson) s $ dbProcs dbStructure
_ -> Nothing _ -> Nothing
handleReq = runWithClaims conf eClaims (app dbStructure proc conf) apiRequest handleReq = runWithClaims conf eClaims (app dbStructure proc conf) apiRequest
txMode = transactionMode proc (iAction apiRequest) txMode = transactionMode proc (iAction apiRequest)
@@ -109,7 +108,7 @@ transactionMode proc action =
ActionInfo -> HT.Read ActionInfo -> HT.Read
ActionInspect -> HT.Read ActionInspect -> HT.Read
ActionInvoke{isReadOnly=False} -> ActionInvoke{isReadOnly=False} ->
let v = maybe Volatile pdVolatility proc in let v = maybe Volatile pdVolatility proc in
if v == Stable || v == Immutable if v == Stable || v == Immutable
then HT.Read then HT.Read
else HT.Write else HT.Write
@@ -146,7 +145,7 @@ app dbStructure proc conf apiRequest =
) )
] (toS body) ] (toS body)
(ActionCreate, TargetIdent (QualifiedIdentifier tSchema tName), Just PayloadJSON{pjRaw}) -> (ActionCreate, TargetIdent (QualifiedIdentifier tSchema tName), Just pJson) ->
case mutateSqlParts tSchema tName of case mutateSqlParts tSchema tName of
Left errorResponse -> return errorResponse Left errorResponse -> return errorResponse
Right (sq, mq) -> do Right (sq, mq) -> do
@@ -154,7 +153,7 @@ app dbStructure proc conf apiRequest =
stm = createWriteStatement sq mq stm = createWriteStatement sq mq
(contentType == CTSingularJSON) True (contentType == CTSingularJSON) True
(contentType == CTTextCSV) (iPreferRepresentation apiRequest) pkCols (contentType == CTTextCSV) (iPreferRepresentation apiRequest) pkCols
row <- H.statement (toS pjRaw) stm row <- H.statement (toS $ pjRaw pJson) stm
let (_, queryTotal, fs, body) = extractQueryResult row let (_, queryTotal, fs, body) = extractQueryResult row
headers = catMaybes [ headers = catMaybes [
if null fs if null fs
@@ -180,14 +179,14 @@ app dbStructure proc conf apiRequest =
if iPreferRepresentation apiRequest == Full if iPreferRepresentation apiRequest == Full
then toS body else "" then toS body else ""
(ActionUpdate, TargetIdent (QualifiedIdentifier tSchema tName), Just PayloadJSON{pjRaw}) -> (ActionUpdate, TargetIdent (QualifiedIdentifier tSchema tName), Just pJson) ->
case mutateSqlParts tSchema tName of case mutateSqlParts tSchema tName of
Left errorResponse -> return errorResponse Left errorResponse -> return errorResponse
Right (sq, mq) -> do Right (sq, mq) -> do
let stm = createWriteStatement sq mq let stm = createWriteStatement sq mq
(contentType == CTSingularJSON) False (contentType == CTTextCSV) (contentType == CTSingularJSON) False (contentType == CTTextCSV)
(iPreferRepresentation apiRequest) [] (iPreferRepresentation apiRequest) []
row <- H.statement (toS pjRaw) stm row <- H.statement (toS $ pjRaw pJson) stm
let (_, queryTotal, _, body) = extractQueryResult row let (_, queryTotal, _, body) = extractQueryResult row
if contentType == CTSingularJSON if contentType == CTSingularJSON
&& queryTotal /= 1 && queryTotal /= 1
@@ -205,7 +204,7 @@ app dbStructure proc conf apiRequest =
then responseLBS s [toHeader contentType, r] (toS body) then responseLBS s [toHeader contentType, r] (toS body)
else responseLBS s [r] "" else responseLBS s [r] ""
(ActionSingleUpsert, TargetIdent (QualifiedIdentifier tSchema tName), Just PayloadJSON{pjRaw, pjType, pjKeys}) -> (ActionSingleUpsert, TargetIdent (QualifiedIdentifier tSchema tName), Just ProcessedJSON{pjRaw, pjType, pjKeys}) ->
case mutateSqlParts tSchema tName of case mutateSqlParts tSchema tName of
Left errorResponse -> return errorResponse Left errorResponse -> return errorResponse
Right (sq, mq) -> do Right (sq, mq) -> do
@@ -267,7 +266,7 @@ app dbStructure proc conf apiRequest =
let acceptH = (hAllow, if tableInsertable table then "GET,POST,PATCH,DELETE" else "GET") in let acceptH = (hAllow, if tableInsertable table then "GET,POST,PATCH,DELETE" else "GET") in
return $ responseLBS status200 [allOrigins, acceptH] "" return $ responseLBS status200 [allOrigins, acceptH] ""
(ActionInvoke _, TargetProc qi, Just PayloadJSON{pjRaw, pjKeys}) -> (ActionInvoke _, TargetProc qi, Just pJson) ->
let returnsScalar = case proc of let returnsScalar = case proc of
Just ProcDescription{pdReturnType = (Single (Scalar _))} -> True Just ProcDescription{pdReturnType = (Single (Scalar _))} -> True
_ -> False _ -> False
@@ -279,8 +278,8 @@ app dbStructure proc conf apiRequest =
Left errorResponse -> return errorResponse Left errorResponse -> return errorResponse
Right ((q, cq), bField) -> do Right ((q, cq), bField) -> do
let singular = contentType == CTSingularJSON let singular = contentType == CTSingularJSON
specifiedPgArgs = filter ((`S.member` pjKeys) . pgaName) $ maybe [] pdArgs proc specifiedPgArgs = filter ((`S.member` pjKeys pJson) . pgaName) $ maybe [] pdArgs proc
row <- H.statement (toS pjRaw) $ row <- H.statement (toS $ pjRaw pJson) $
callProc qi specifiedPgArgs returnsScalar q cq shouldCount callProc qi specifiedPgArgs returnsScalar q cq shouldCount
singular (iPreferSingleObjectParameter apiRequest) singular (iPreferSingleObjectParameter apiRequest)
(contentType == CTTextCSV) (contentType == CTTextCSV)
src/PostgREST/DbRequestBuilder.hs
+4 -4
View File
@@ -321,8 +321,8 @@ addProperty f (targetNodeName:remainingPath, a) (Node rn forest) =
mutateRequest :: ApiRequest -> TableName -> [Text] -> [FieldName] -> Either Response MutateRequest mutateRequest :: ApiRequest -> TableName -> [Text] -> [FieldName] -> Either Response MutateRequest
mutateRequest apiRequest tName pkCols fldNames = mapLeft apiRequestError $ mutateRequest apiRequest tName pkCols fldNames = mapLeft apiRequestError $
case action of case action of
ActionCreate -> Right $ Insert tName (pjKeys payload) ((,) <$> iPreferResolution apiRequest <*> Just pkCols) [] returnings ActionCreate -> Right $ Insert tName pjCols ((,) <$> iPreferResolution apiRequest <*> Just pkCols) [] returnings
ActionUpdate -> Update tName (pjKeys payload) <$> combinedLogic <*> pure returnings ActionUpdate -> Update tName pjCols <$> combinedLogic <*> pure returnings
ActionSingleUpsert -> ActionSingleUpsert ->
(\flts -> (\flts ->
if null (iLogic apiRequest) && if null (iLogic apiRequest) &&
@@ -331,14 +331,14 @@ mutateRequest apiRequest tName pkCols fldNames = mapLeft apiRequestError $
all (\case all (\case
Filter _ (OpExpr False (Op "eq" _)) -> True Filter _ (OpExpr False (Op "eq" _)) -> True
_ -> False) flts _ -> False) flts
then Insert tName (pjKeys payload) (Just (MergeDuplicates, pkCols)) <$> combinedLogic <*> pure returnings then Insert tName pjCols (Just (MergeDuplicates, pkCols)) <$> combinedLogic <*> pure returnings
else else
Left InvalidFilters) =<< filters Left InvalidFilters) =<< filters
ActionDelete -> Delete tName <$> combinedLogic <*> pure returnings ActionDelete -> Delete tName <$> combinedLogic <*> pure returnings
_ -> Left UnsupportedVerb _ -> Left UnsupportedVerb
where where
action = iAction apiRequest action = iAction apiRequest
payload = fromJust $ iPayload apiRequest pjCols = pjKeys $ fromJust $ iPayload apiRequest
returnings = if iPreferRepresentation apiRequest == None then [] else fldNames returnings = if iPreferRepresentation apiRequest == None then [] else fldNames
filters = map snd <$> mapM pRequestFilter mutateFilters filters = map snd <$> mapM pRequestFilter mutateFilters
logic = map snd <$> mapM pRequestLogicTree logicFilters logic = map snd <$> mapM pRequestLogicTree logicFilters
src/PostgREST/Parsers.hs
+8
View File
@@ -13,6 +13,7 @@ import Data.Functor (($>))
import qualified Data.HashMap.Strict as M import qualified Data.HashMap.Strict as M
import Data.Text (intercalate, replace, strip) import Data.Text (intercalate, replace, strip)
import Data.List (init, last) import Data.List (init, last)
import qualified Data.Set as S
import Data.Tree import Data.Tree
import Data.Either.Combinators (mapLeft) import Data.Either.Combinators (mapLeft)
import PostgREST.RangeQuery (NonnegRange) import PostgREST.RangeQuery (NonnegRange)
@@ -217,6 +218,13 @@ pLogicPath = do
notOp = "not." <> op notOp = "not." <> op
return (filter (/= "not") (init path), if "not" `elem` path then notOp else op) return (filter (/= "not") (init path), if "not" `elem` path then notOp else op)
pRequestColumns :: Text -> Either ParseError (S.Set FieldName)
pRequestColumns colStr =
S.fromList <$> parse pColumns ("failed to parse columns parameter (" <> toS colStr <> ")") (toS colStr)
pColumns :: Parser [FieldName]
pColumns = pFieldName `sepBy1` lexeme (char ',')
mapError :: Either ParseError a -> Either ApiRequestError a mapError :: Either ParseError a -> Either ApiRequestError a
mapError = mapLeft translateError mapError = mapLeft translateError
where where
src/PostgREST/QueryBuilder.hs
+25 -19
View File
@@ -177,13 +177,9 @@ callProc qi pgArgs returnsScalar selectQuery countQuery countTotal isSingle para
| null pgArgs = (ignoredBody, "") | null pgArgs = (ignoredBody, "")
| otherwise = ( | otherwise = (
unwords [ unwords [
"payload AS (SELECT $1::json AS json_data),", normalizedBody <> ",",
"vals AS (",
"SELECT json_data AS val FROM payload WHERE json_typeof(json_data) = 'array'",
"UNION ALL",
"SELECT json_build_array(json_data) AS val FROM payload WHERE json_typeof(json_data) = 'object'),",
"_args_record AS (", "_args_record AS (",
"SELECT * FROM json_to_recordset((SELECT val FROM vals)) AS _(" <> "SELECT * FROM json_to_recordset(" <> selectBody <> ") AS _(" <>
intercalate ", " ((\a -> pgFmtIdent (pgaName a) <> " " <> pgaType a) <$> pgArgs) <> ")", intercalate ", " ((\a -> pgFmtIdent (pgaName a) <> " " <> pgaType a) <$> pgArgs) <> ")",
")"] ")"]
, intercalate ", " ((\a -> pgFmtIdent (pgaName a) <> " := (SELECT " <> pgFmtIdent (pgaName a) <> " FROM _args_record)") <$> pgArgs)) , intercalate ", " ((\a -> pgFmtIdent (pgaName a) <> " := (SELECT " <> pgFmtIdent (pgaName a) <> " FROM _args_record)") <$> pgArgs))
@@ -276,16 +272,11 @@ requestToQuery schema isParent (DbRead (Node (Select colSelects tbl tblAlias imp
getQueryParts _ _ = witness getQueryParts _ _ = witness
requestToQuery schema _ (DbMutate (Insert mainTbl iCols onConflct putConditions returnings)) = requestToQuery schema _ (DbMutate (Insert mainTbl iCols onConflct putConditions returnings)) =
unwords [ unwords [
"WITH payload AS (SELECT $1::json AS json_data),", "WITH " <> normalizedBody,
"vals AS (",
unwords [
"SELECT json_data AS val FROM payload WHERE json_typeof(json_data) = 'array'",
"UNION ALL",
"SELECT json_build_array(json_data) AS val FROM payload WHERE json_typeof(json_data) = 'object')"],
"INSERT INTO ", fromQi qi, if S.null iCols then " " else "(" <> cols <> ")", "INSERT INTO ", fromQi qi, if S.null iCols then " " else "(" <> cols <> ")",
unwords [ unwords [
"SELECT " <> cols <> " FROM", "SELECT " <> cols <> " FROM",
"json_populate_recordset", "(null::", fromQi qi, ", (SELECT val FROM vals)) _", "json_populate_recordset", "(null::", fromQi qi, ", " <> selectBody <> ") _",
-- Only used for PUT -- Only used for PUT
("WHERE " <> intercalate " AND " (pgFmtLogicTree (QualifiedIdentifier "" "_") <$> putConditions)) `emptyOnFalse` null putConditions], ("WHERE " <> intercalate " AND " (pgFmtLogicTree (QualifiedIdentifier "" "_") <$> putConditions)) `emptyOnFalse` null putConditions],
maybe "" (\(oncDo, oncCols) -> ( maybe "" (\(oncDo, oncCols) -> (
@@ -304,13 +295,9 @@ requestToQuery schema _ (DbMutate (Update mainTbl uCols logicForest returnings))
then "WITH " <> ignoredBody <> "SELECT null WHERE false" -- if there are no columns we cannot do UPDATE table SET {empty}, it'd be invalid syntax then "WITH " <> ignoredBody <> "SELECT null WHERE false" -- if there are no columns we cannot do UPDATE table SET {empty}, it'd be invalid syntax
else else
unwords [ unwords [
"WITH payload AS (SELECT $1::json AS json_data),", "WITH " <> normalizedBody,
"vals AS (",
"SELECT json_data AS val FROM payload WHERE json_typeof(json_data) = 'array'",
"UNION ALL",
"SELECT json_build_array(json_data) AS val FROM payload WHERE json_typeof(json_data) = 'object')",
"UPDATE " <> fromQi qi <> " SET " <> cols, "UPDATE " <> fromQi qi <> " SET " <> cols,
"FROM (SELECT * FROM json_populate_recordset", "(null::", fromQi qi, ", (SELECT val FROM vals))) _ ", "FROM (SELECT * FROM json_populate_recordset", "(null::", fromQi qi, ", " <> selectBody <> ")) _ ",
("WHERE " <> intercalate " AND " (pgFmtLogicTree qi <$> logicForest)) `emptyOnFalse` null logicForest, ("WHERE " <> intercalate " AND " (pgFmtLogicTree qi <$> logicForest)) `emptyOnFalse` null logicForest,
("RETURNING " <> intercalate ", " (pgFmtColumn qi <$> returnings)) `emptyOnFalse` null returnings ("RETURNING " <> intercalate ", " (pgFmtColumn qi <$> returnings)) `emptyOnFalse` null returnings
] ]
@@ -334,6 +321,25 @@ requestToQuery schema _ (DbMutate (Delete mainTbl logicForest returnings)) =
ignoredBody :: SqlFragment ignoredBody :: SqlFragment
ignoredBody = "ignored_body AS (SELECT $1::text) " ignoredBody = "ignored_body AS (SELECT $1::text) "
-- |
-- These CTEs convert a json object into a json array, this way we can use json_populate_recordset for all json payloads
-- Otherwise we'd have to use json_populate_record for json objects and json_populate_recordset for json arrays
-- We do this in SQL to avoid processing the JSON in application code
normalizedBody :: SqlFragment
normalizedBody =
unwords [
"pgrst_payload AS (SELECT $1::json AS json_data),",
"pgrst_body AS (",
"SELECT",
"CASE WHEN json_typeof(json_data) = 'array'",
"THEN json_data",
"ELSE json_build_array(json_data)",
"END AS val",
"FROM pgrst_payload)"]
selectBody :: SqlFragment
selectBody = "(SELECT val FROM pgrst_body)"
removeSourceCTESchema :: Schema -> TableName -> QualifiedIdentifier removeSourceCTESchema :: Schema -> TableName -> QualifiedIdentifier
removeSourceCTESchema schema tbl = QualifiedIdentifier (if tbl == sourceCTEName then "" else schema) tbl removeSourceCTESchema schema tbl = QualifiedIdentifier (if tbl == sourceCTEName then "" else schema) tbl
src/PostgREST/Types.hs
+15 -10
View File
@@ -187,16 +187,21 @@ data Relation = Relation {
isSelfJoin :: Relation -> Bool isSelfJoin :: Relation -> Bool
isSelfJoin r = relType r /= Root && relTable r == relFTable r isSelfJoin r = relType r /= Root && relTable r == relFTable r
-- | Cached attributes of a JSON payload data PayloadJSON =
data PayloadJSON = PayloadJSON { -- | Cached attributes of a JSON payload
-- | This is the raw ByteString that comes from the request body. ProcessedJSON {
-- We cache this instead of an Aeson Value because it was detected that for large payloads the encoding -- | This is the raw ByteString that comes from the request body.
-- had high memory usage, see #1005 for more details -- We cache this instead of an Aeson Value because it was detected that for large payloads the encoding
pjRaw :: BL.ByteString -- had high memory usage, see #1005 for more details
, pjType :: PJType pjRaw :: BL.ByteString
-- | Keys of the object or if it's an array these keys are guaranteed to be the same across all its objects , pjType :: PJType
, pjKeys :: S.Set Text -- | Keys of the object or if it's an array these keys are guaranteed to be the same across all its objects
} deriving (Show, Eq) , pjKeys :: S.Set Text
}|
RawJSON {
pjRaw :: BL.ByteString
, pjKeys :: S.Set Text
} deriving (Show, Eq)
data PJType = PJArray { pjaLength :: Int } | PJObject deriving (Show, Eq) data PJType = PJArray { pjaLength :: Int } | PJObject deriving (Show, Eq)
test/Feature/InsertSpec.hs
+50
View File
@@ -260,6 +260,48 @@ spec = do
, matchHeaders = [] , matchHeaders = []
} }
context "POST with ?columns parameter" $ do
it "ignores json keys not included in ?columns" $ do
request methodPost "/articles?columns=id,body" [("Prefer", "return=representation")]
[json| {"id": 200, "body": "xxx", "smth": "here", "other": "stuff", "fake_id": 13} |] `shouldRespondWith`
[json|[{"id": 200, "body": "xxx", "owner": "postgrest_test_anonymous"}]|]
{ matchStatus = 201
, matchHeaders = [] }
request methodPost "/articles?columns=id,body&select=id,body" [("Prefer", "return=representation")]
[json| [
{"id": 201, "body": "yyy", "smth": "here", "other": "stuff", "fake_id": 13},
{"id": 202, "body": "zzz", "garbage": "%%$&", "kkk": "jjj"},
{"id": 203, "body": "aaa", "hey": "ho"} ]|] `shouldRespondWith`
[json|[
{"id": 201, "body": "yyy"},
{"id": 202, "body": "zzz"},
{"id": 203, "body": "aaa"} ]|]
{ matchStatus = 201
, matchHeaders = [] }
-- TODO parse columns error message needs to be improved
it "disallows blank ?columns" $
post "/articles?columns="
[json|[
{"id": 204, "body": "yyy"},
{"id": 205, "body": "zzz"}]|] `shouldRespondWith` 400
it "disallows array elements that are not json objects" $
post "/articles?columns=id,body"
[json|[
{"id": 204, "body": "yyy"},
333,
"asdf",
{"id": 205, "body": "zzz"}]|] `shouldRespondWith`
[json|{
"code": "22023",
"details": null,
"hint": null,
"message": "argument of json_populate_recordset must be an array of objects"}|]
{ matchStatus = 400
, matchHeaders = []
}
describe "CSV insert" $ do describe "CSV insert" $ do
context "disparate csv types" $ context "disparate csv types" $
@@ -464,6 +506,14 @@ spec = do
simpleBody p `shouldBe` "["<>payload<>"]" simpleBody p `shouldBe` "["<>payload<>"]"
simpleStatus p `shouldBe` ok200 simpleStatus p `shouldBe` ok200
context "PATCH with ?columns parameter" $
it "ignores json keys not included in ?columns" $
request methodPatch "/articles?id=eq.200&columns=body" [("Prefer", "return=representation")]
[json| {"body": "Some real content", "smth": "here", "other": "stuff", "fake_id": 13} |] `shouldRespondWith`
[json|[{"id": 200, "body": "Some real content", "owner": "postgrest_test_anonymous"}]|]
{ matchStatus = 200
, matchHeaders = [] }
describe "Row level permission" $ describe "Row level permission" $
it "set user_id when inserting rows" $ do it "set user_id when inserting rows" $ do
let auth = authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.B-lReuGNDwAlU1GOC476MlO0vAt9JNoHIlxg2vwMaO0" let auth = authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.B-lReuGNDwAlU1GOC476MlO0vAt9JNoHIlxg2vwMaO0"
test/Feature/RpcSpec.hs
+7 -1
View File
@@ -339,10 +339,16 @@ spec =
get "/rpc/overloaded?a=1&b=2" `shouldRespondWith` [str|3|] get "/rpc/overloaded?a=1&b=2" `shouldRespondWith` [str|3|]
get "/rpc/overloaded?a=1&b=2&c=3" `shouldRespondWith` [str|"123"|] get "/rpc/overloaded?a=1&b=2&c=3" `shouldRespondWith` [str|"123"|]
context "only for POST rpc" $ context "only for POST rpc" $ do
it "gives a parse filter error if GET style proc args are specified" $ it "gives a parse filter error if GET style proc args are specified" $
post "/rpc/sayhello?name=John" [json|{}|] `shouldRespondWith` 400 post "/rpc/sayhello?name=John" [json|{}|] `shouldRespondWith` 400
it "ignores json keys not included in ?columns" $
post "/rpc/sayhello?columns=name"
[json|{"name": "John", "smth": "here", "other": "stuff", "fake_id": 13}|] `shouldRespondWith`
[json|"Hello, John"|]
{ matchHeaders = [matchContentTypeJson] }
context "only for GET rpc" $ do context "only for GET rpc" $ do
it "should fail on mutating procs" $ do it "should fail on mutating procs" $ do
get "/rpc/callcounter" `shouldRespondWith` 500 get "/rpc/callcounter" `shouldRespondWith` 500
test/memory-tests.sh
+12 -12
View File
@@ -96,21 +96,21 @@ setUp
echo "Running memory usage tests.." echo "Running memory usage tests.."
jsonKeyTest "1M" "POST" "/rpc/leak" "20M" jsonKeyTest "1M" "POST" "/rpc/leak?columns=blob" "12M"
jsonKeyTest "1M" "POST" "/leak" "20M" jsonKeyTest "1M" "POST" "/leak?columns=blob" "12M"
jsonKeyTest "1M" "PATCH" "/leak?id=eq.1" "20M" jsonKeyTest "1M" "PATCH" "/leak?id=eq.1&columns=blob" "12M"
jsonKeyTest "10M" "POST" "/rpc/leak" "105M" jsonKeyTest "10M" "POST" "/rpc/leak?columns=blob" "40M"
jsonKeyTest "10M" "POST" "/leak" "105M" jsonKeyTest "10M" "POST" "/leak?columns=blob" "40M"
jsonKeyTest "10M" "PATCH" "/leak?id=eq.1" "105M" jsonKeyTest "10M" "PATCH" "/leak?id=eq.1&columns=blob" "40M"
jsonKeyTest "50M" "POST" "/rpc/leak" "500M" jsonKeyTest "50M" "POST" "/rpc/leak?columns=blob" "170M"
jsonKeyTest "50M" "POST" "/leak" "500M" jsonKeyTest "50M" "POST" "/leak?columns=blob" "170M"
jsonKeyTest "50M" "PATCH" "/leak?id=eq.1" "500M" jsonKeyTest "50M" "PATCH" "/leak?id=eq.1&columns=blob" "170M"
postJsonArrayTest "1000" "/perf_articles" "20M" postJsonArrayTest "1000" "/perf_articles?columns=id,body" "9M"
postJsonArrayTest "10000" "/perf_articles" "150M" postJsonArrayTest "10000" "/perf_articles?columns=id,body" "10M"
postJsonArrayTest "100000" "/perf_articles" "1.15G" postJsonArrayTest "100000" "/perf_articles?columns=id,body" "20M"
cleanUp cleanUp