jimmy/postgrest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for Prefer: count=planned/estimated on GET /table (#1386)

Browse Source
This commit is contained in:
Steve Chávez
2019-09-18 10:13:48 -05:00
committed by GitHub
parent e044488f73
commit 186381bab2
14 changed files with 225 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Added
- #1383, Add support for HEAD request - @steve-chavez
- #1378, Add support for `Prefer: count=planned` and `Prefer: count=estimated` on GET /table - @steve-chavez
### Changed
src/PostgREST/ApiRequest.hs
+7 -3
View File
@@ -70,6 +70,7 @@ data Target = TargetIdent QualifiedIdentifier
-- | How to return the inserted data
data PreferRepresentation = Full | HeadersOnly | None deriving Eq
{-|
Describes what the user wants to do. This data type is a
translation of the raw elements of an HTTP request into domain
@@ -94,8 +95,8 @@ data ApiRequest = ApiRequest {
, iPreferRepresentation :: PreferRepresentation
-- | How to pass parameters to a stored procedure
, iPreferParameters :: Maybe PreferParameters
-- | Whether the client wants a result count (slower)
, iPreferCount :: Bool
-- | Whether the client wants a result count
, iPreferCount :: Maybe PreferCount
-- | Whether the client wants to UPSERT or ignore records on PK conflict
, iPreferResolution :: Maybe PreferResolution
-- | Filters on the result ("id", "eq.10")
@@ -135,7 +136,10 @@ userApiRequest schema rootSpec req reqBody
, iPreferParameters = if | hasPrefer (show SingleObject) -> Just SingleObject
| hasPrefer (show MultipleObjects) -> Just MultipleObjects
| otherwise -> Nothing
, iPreferCount = hasPrefer "count=exact"
, iPreferCount = if | hasPrefer (show ExactCount) -> Just ExactCount
| hasPrefer (show PlannedCount) -> Just PlannedCount
| hasPrefer (show EstimatedCount) -> Just EstimatedCount
| otherwise -> Nothing
, iPreferResolution = if | hasPrefer (show MergeDuplicates) -> Just MergeDuplicates
| hasPrefer (show IgnoreDuplicates) -> Just IgnoreDuplicates
| otherwise -> Nothing
src/PostgREST/App.hs
+24 -7
View File
@@ -10,6 +10,7 @@ Some of its functionality includes:
- Content Negotiation
-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE MultiWayIf #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE ScopedTypeVariables #-}
@@ -55,12 +56,14 @@ import PostgREST.Error (PgError (..), SimpleError (..),
import PostgREST.Middleware
import PostgREST.OpenAPI
import PostgREST.Parsers (pRequestColumns)
import PostgREST.QueryBuilder (requestToCallProcQuery,
import PostgREST.QueryBuilder (limitedQuery,
requestToCallProcQuery,
requestToCountQuery,
requestToQuery)
import PostgREST.RangeQuery (allRange, contentRangeH,
rangeStatusHeader)
import PostgREST.Statements (callProcStatement,
createExplainStatement,
createReadStatement,
createWriteStatement)
import PostgREST.Types
@@ -128,11 +131,21 @@ app dbStructure proc cols conf apiRequest =
case partsField of
Left errorResponse -> return errorResponse
Right ((q, cq), bField) -> do
let stm = createReadStatement q cq (contentType == CTSingularJSON) shouldCount
(contentType == CTTextCSV) bField
let cQuery = if estimatedCount
then limitedQuery cq ((+ 1) <$> maxRows) -- LIMIT maxRows + 1 so we can determine below that maxRows was surpassed
else cq
stm = createReadStatement q cQuery (contentType == CTSingularJSON) shouldCount
(contentType == CTTextCSV) bField
explStm = createExplainStatement cq
row <- H.statement () stm
let (tableTotal, queryTotal, _ , body) = row
(status, contentRange) = rangeStatusHeader topLevelRange queryTotal tableTotal
total <- if | plannedCount -> H.statement () explStm
| estimatedCount -> if tableTotal > (fromIntegral <$> maxRows)
then do estTotal <- H.statement () explStm
pure $ if estTotal > tableTotal then estTotal else tableTotal
else pure tableTotal
| otherwise -> pure tableTotal
let (status, contentRange) = rangeStatusHeader topLevelRange queryTotal total
return $
if contentType == CTSingularJSON && queryTotal /= 1
then errorResponseFor . singularityError $ queryTotal
@@ -317,10 +330,14 @@ app dbStructure proc cols conf apiRequest =
where
notFound = responseLBS status404 [] ""
shouldCount = iPreferCount apiRequest
topLevelRange = iTopLevelRange apiRequest
schema = toS $ configSchema conf
readReq = readRequest (configMaxRows conf) (dbRelations dbStructure) proc apiRequest
maxRows = configMaxRows conf
exactCount = iPreferCount apiRequest == Just ExactCount
estimatedCount = iPreferCount apiRequest == Just EstimatedCount
plannedCount = iPreferCount apiRequest == Just PlannedCount
shouldCount = exactCount || estimatedCount
topLevelRange = iTopLevelRange apiRequest
readReq = readRequest maxRows (dbRelations dbStructure) proc apiRequest
fldNames = fieldNames <$> readReq
readDbRequest = DbRead <$> readReq
selectQuery = requestToQuery schema False <$> readDbRequest
src/PostgREST/QueryBuilder.hs
+20 -11
View File
@@ -14,6 +14,7 @@ module PostgREST.QueryBuilder (
requestToQuery
, requestToCountQuery
, requestToCallProcQuery
, limitedQuery
, setLocalQuery
, setLocalSearchPathQuery
) where
@@ -32,17 +33,6 @@ import PostgREST.Types
import Protolude hiding (cast, intercalate,
replace)
requestToCountQuery :: Schema -> DbRequest -> SqlQuery
requestToCountQuery _ (DbMutate _) = witness
requestToCountQuery schema (DbRead (Node (Select{where_=logicForest}, (mainTbl, _, _, _, _)) _)) =
unwords [
"SELECT pg_catalog.count(*)",
"FROM ", fromQi qi,
("WHERE " <> intercalate " AND " (map (pgFmtLogicTree qi) logicForest)) `emptyOnFalse` null logicForest
]
where
qi = removeSourceCTESchema schema mainTbl
requestToQuery :: Schema -> Bool -> DbRequest -> SqlQuery
requestToQuery schema isParent (DbRead (Node (Select colSelects tbl tblAlias implJoins logicForest joinConditions_ ordts range, _) forest)) =
unwords [
@@ -177,6 +167,25 @@ requestToCallProcQuery qi pgArgs returnsScalar preferParams =
callIt :: SqlFragment
callIt = fromQi qi <> "(" <> args <> ")"
-- | SQL query meant for COUNTing the root node of the DbRead Tree.
-- It only takes WHERE into account and doesn't include LIMIT/OFFSET because it would reduce the COUNT.
-- SELECT 1 is done instead of SELECT * to prevent doing expensive operations(like functions based on the columns)
-- inside the FROM target.
requestToCountQuery :: Schema -> DbRequest -> SqlQuery
requestToCountQuery _ (DbMutate _) = witness
requestToCountQuery schema (DbRead (Node (Select{where_=logicForest}, (mainTbl, _, _, _, _)) _)) =
unwords [
"SELECT 1",
"FROM " <> fromQi qi,
("WHERE " <> intercalate " AND " (map (pgFmtLogicTree qi) logicForest)) `emptyOnFalse` null logicForest
]
where
qi = removeSourceCTESchema schema mainTbl
limitedQuery :: SqlQuery -> Maybe Integer -> SqlQuery
limitedQuery query maxRows = query <> maybe mempty (\x -> " LIMIT " <> show x) maxRows
setLocalQuery :: Text -> (Text, Text) -> SqlQuery
setLocalQuery prefix (k, v) =
"SET LOCAL " <> pgFmtIdent (prefix <> k) <> " = " <> pgFmtLit v <> ";"
src/PostgREST/QueryBuilder/Private.hs
+10
View File
@@ -185,3 +185,13 @@ trimNullChars = T.takeWhile (/= '\x0')
removeSourceCTESchema :: Schema -> TableName -> QualifiedIdentifier
removeSourceCTESchema schema tbl = QualifiedIdentifier (if tbl == sourceCTEName then "" else schema) tbl
countF :: SqlQuery -> Bool -> (SqlFragment, SqlFragment)
countF countQuery shouldCount =
if shouldCount
then (
", pg_source_count AS (" <> countQuery <> ")"
, "(SELECT pg_catalog.count(*) FROM pg_source_count)" )
else (
mempty
, "null::bigint")
src/PostgREST/Statements.hs
+36 -10
View File
@@ -13,9 +13,13 @@ module PostgREST.Statements (
createWriteStatement
, createReadStatement
, callProcStatement
, createExplainStatement
) where
import Control.Lens ((^?))
import Data.Aeson as JSON
import qualified Data.Aeson.Lens as L
import qualified Data.ByteString.Char8 as BS
import Data.Maybe
import Data.Text (intercalate, unwords)
@@ -86,15 +90,18 @@ createReadStatement selectQuery countQuery isSingle countTotal asCsv binaryField
unicodeStatement sql HE.noParams decodeStandard False
where
sql = [qc|
WITH {sourceCTEName} AS ({selectQuery}) SELECT {cols}
WITH
{sourceCTEName} AS ({selectQuery})
{countCTEF}
SELECT
{countResultF} AS total_result_set,
pg_catalog.count(_postgrest_t) AS page_total,
{noLocationF} AS header,
{bodyF} AS body
FROM ( SELECT * FROM {sourceCTEName}) _postgrest_t |]
countResultF = if countTotal then "("<>countQuery<>")" else "null"
cols = intercalate ", " [
countResultF <> " AS total_result_set",
"pg_catalog.count(_postgrest_t) AS page_total",
noLocationF <> " AS header",
bodyF <> " AS body"
]
(countCTEF, countResultF) = countF countQuery countTotal
bodyF
| asCsv = asCsvF
| isSingle = asJsonSingleF
@@ -125,6 +132,7 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
where
sql = [qc|
WITH {sourceCTEName} AS ({callProcQuery})
{countCTEF}
SELECT
{countResultF} AS total_result_set,
pg_catalog.count(_postgrest_t) AS page_total,
@@ -132,6 +140,8 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
{responseHeaders} AS response_headers
FROM ({selectQuery}) _postgrest_t;|]
(countCTEF, countResultF) = countF countQuery countTotal
bodyF
| returnsScalar = scalarBodyF
| isSingle = asJsonSingleF
@@ -144,8 +154,6 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
| multObjects = "json_agg(_postgrest_t.pgrst_scalar)::character varying"
| otherwise = "(json_agg(_postgrest_t.pgrst_scalar)->0)::character varying"
countResultF = if countTotal then "( "<> countQuery <> ")" else "null::bigint" :: Text
responseHeaders =
if pgVer >= pgVersion96
then "coalesce(nullif(current_setting('response.headers', true), ''), '[]')" :: Text -- nullif is used because of https://gist.github.com/steve-chavez/8d7033ea5655096903f3b52f8ed09a15
@@ -159,6 +167,24 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
procRow = (,,,) <$> nullableColumn HD.int8 <*> column HD.int8
<*> column HD.bytea <*> column HD.bytea
createExplainStatement :: SqlQuery -> H.Statement () (Maybe Int64)
createExplainStatement countQuery =
unicodeStatement sql HE.noParams decodeExplain False
where
sql = [qc| EXPLAIN (FORMAT JSON) {countQuery} |]
-- |
-- An `EXPLAIN (FORMAT JSON) select * from items;` output looks like this:
-- [{
-- "Plan": {
-- "Node Type": "Seq Scan", "Parallel Aware": false, "Relation Name": "items",
-- "Alias": "items", "Startup Cost": 0.00, "Total Cost": 32.60,
-- "Plan Rows": 2260,"Plan Width": 8} }]
-- We only obtain the Plan Rows here.
decodeExplain :: HD.Result (Maybe Int64)
decodeExplain =
let row = HD.singleRow $ column HD.bytea in
(^? L.nth 0 . L.key "Plan" . L.key "Plan Rows" . L._Integral) <$> row
unicodeStatement :: Text -> HE.Params a -> HD.Result b -> Bool -> H.Statement a b
unicodeStatement = H.Statement . encodeUtf8
src/PostgREST/Types.hs
+11
View File
@@ -76,6 +76,17 @@ instance Show PreferParameters where
show SingleObject = "params=single-object"
show MultipleObjects = "params=multiple-objects"
data PreferCount
= ExactCount -- ^ exact count(slower)
| PlannedCount -- ^ PostgreSQL query planner rows count guess. Done by using EXPLAIN {query}.
| EstimatedCount -- ^ use the query planner rows if the count is superior to max-rows, otherwise get the exact count.
deriving Eq
instance Show PreferCount where
show ExactCount = "count=exact"
show PlannedCount = "count=planned"
show EstimatedCount = "count=estimated"
data DbStructure = DbStructure {
dbTables :: [Table]
, dbColumns :: [Column]
test/Feature/QueryLimitedSpec.hs
+25 -3
View File
@@ -13,7 +13,7 @@ import SpecHelper
spec :: SpecWith Application
spec =
describe "Requesting many items with server limits enabled" $ do
describe "Requesting many items with server limits(max-rows) enabled" $ do
it "restricts results" $
get "/items"
`shouldRespondWith` [json| [{"id":1},{"id":2}] |]
@@ -29,16 +29,38 @@ spec =
matchHeader "Content-Range" "0-0/*"
simpleStatus r `shouldBe` ok200
it "limit works on all levels" $
it "works on all levels" $
get "/users?select=id,tasks(id)&order=id.asc&tasks.order=id.asc"
`shouldRespondWith` [json|[{"id":1,"tasks":[{"id":1},{"id":2}]},{"id":2,"tasks":[{"id":5},{"id":6}]}]|]
{ matchStatus = 200
, matchHeaders = ["Content-Range" <:> "0-1/*"]
}
it "limit is not applied to parent embeds" $
it "is not applied to parent embeds" $
get "/tasks?select=id,project(id)&id=gt.5"
`shouldRespondWith` [json|[{"id":6,"project":{"id":3}},{"id":7,"project":{"id":4}}]|]
{ matchStatus = 200
, matchHeaders = ["Content-Range" <:> "0-1/*"]
}
context "count=estimated" $ do
it "uses the query planner guess when query rows > maxRows" $
request methodHead "/getallprojects_view" [("Prefer", "count=estimated")] ""
`shouldRespondWith` ""
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "0-1/2019"]
}
it "gives exact count when query rows <= maxRows" $
request methodHead "/getallprojects_view?id=lt.3" [("Prefer", "count=estimated")] ""
`shouldRespondWith` ""
{ matchStatus = 200
, matchHeaders = ["Content-Range" <:> "0-1/2"]
}
it "only uses the query planner guess if it's indeed greater than the exact count" $
request methodHead "/get_projects_above_view" [("Prefer", "count=estimated")] ""
`shouldRespondWith` ""
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "0-1/3"]
}
test/Feature/RangeSpec.hs
+59
View File
@@ -197,6 +197,65 @@ spec = do
, matchHeaders = [matchContentTypeJson]
}
context "when count=planned" $ do
it "obtains a filtered range" $ do
request methodGet "/items?select=id&id=gt.8" [("Prefer", "count=planned")] ""
`shouldRespondWith` [json|[{"id":9}, {"id":10}, {"id":11}, {"id":12}, {"id":13}, {"id":14}, {"id":15}]|]
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "0-6/8"]
}
request methodGet "/child_entities?select=id&id=gt.3" [("Prefer", "count=planned")] ""
`shouldRespondWith` [json|[{"id":4}, {"id":5}, {"id":6}]|]
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "0-2/4"]
}
request methodGet "/getallprojects_view?select=id&id=lt.3" [("Prefer", "count=planned")] ""
`shouldRespondWith` [json|[{"id":1}, {"id":2}]|]
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "0-1/673"]
}
it "obtains the full range" $ do
request methodHead "/items" [("Prefer", "count=planned")] ""
`shouldRespondWith` ""
{ matchStatus = 200
, matchHeaders = ["Content-Range" <:> "0-14/15"]
}
request methodHead "/child_entities" [("Prefer", "count=planned")] ""
`shouldRespondWith` ""
{ matchStatus = 200
, matchHeaders = ["Content-Range" <:> "0-5/6"]
}
request methodHead "/getallprojects_view" [("Prefer", "count=planned")] ""
`shouldRespondWith` ""
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "0-4/2019"]
}
it "ignores limit/offset on the planned count" $ do
request methodHead "/items?limit=2&offset=3" [("Prefer", "count=planned")] ""
`shouldRespondWith` ""
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "3-4/15"]
}
request methodHead "/child_entities?limit=2" [("Prefer", "count=planned")] ""
`shouldRespondWith` ""
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "0-1/6"]
}
request methodHead "/getallprojects_view?limit=2" [("Prefer", "count=planned")] ""
`shouldRespondWith` ""
{ matchStatus = 206
, matchHeaders = ["Content-Range" <:> "0-1/2019"]
}
it "works with two levels" $
request methodHead "/child_entities?select=*,entities(*)" [("Prefer", "count=planned")] ""
`shouldRespondWith` ""
{ matchStatus = 200
, matchHeaders = ["Content-Range" <:> "0-5/6"]
}
context "with range headers" $ do
context "of acceptable range" $ do
it "succeeds with partial content" $ do
test/Main.hs
+10 -4
View File
@@ -65,7 +65,7 @@ main = do
refDbStructure <- newIORef $ Just dbStructure
let withApp = return $ postgrest (testCfg testDbConn) refDbStructure pool getTime $ pure ()
ltdApp = return $ postgrest (testLtdRowsCfg testDbConn) refDbStructure pool getTime $ pure ()
maxRowsApp = return $ postgrest (testMaxRowsCfg testDbConn) refDbStructure pool getTime $ pure ()
unicodeApp = return $ postgrest (testUnicodeCfg testDbConn) refDbStructure pool getTime $ pure ()
proxyApp = return $ postgrest (testProxyCfg testDbConn) refDbStructure pool getTime $ pure ()
noJwtApp = return $ postgrest (testCfgNoJWT testDbConn) refDbStructure pool getTime $ pure ()
@@ -78,8 +78,11 @@ main = do
rootSpecApp = return $ postgrest (testCfgRootSpec testDbConn) refDbStructure pool getTime $ pure ()
htmlRawOutputApp = return $ postgrest (testCfgHtmlRawOutput testDbConn) refDbStructure pool getTime $ pure ()
let reset :: IO ()
let reset, analyze :: IO ()
reset = resetDb testDbConn
analyze = do
analyzeTable testDbConn "items"
analyzeTable testDbConn "child_entities"
actualPgVersion = pgVersion dbStructure
extraSpecs =
@@ -95,7 +98,6 @@ main = do
, ("Feature.JsonOperatorSpec" , Feature.JsonOperatorSpec.spec actualPgVersion)
, ("Feature.QuerySpec" , Feature.QuerySpec.spec actualPgVersion)
, ("Feature.RpcSpec" , Feature.RpcSpec.spec actualPgVersion)
, ("Feature.RangeSpec" , Feature.RangeSpec.spec)
, ("Feature.StructureSpec" , Feature.StructureSpec.spec)
, ("Feature.AndOrParamsSpec" , Feature.AndOrParamsSpec.spec actualPgVersion)
] ++ extraSpecs
@@ -112,12 +114,16 @@ main = do
mapM_ (before withApp) specs
-- we analyze to get accurate results from EXPLAIN
beforeAll_ analyze . before withApp $
describe "Feature.RangeSpec" Feature.RangeSpec.spec
-- this test runs with a raw-output-media-types set to text/html
before htmlRawOutputApp $
describe "Feature.HtmlRawOutputSpec" Feature.HtmlRawOutputSpec.spec
-- this test runs with a different server flag
before ltdApp $
before maxRowsApp $
describe "Feature.QueryLimitedSpec" Feature.QueryLimitedSpec.spec
-- this test runs with a different schema
test/QueryCost.hs
+1 -1
View File
@@ -37,7 +37,7 @@ main = do
cost <- exec pool mempty $
requestToCallProcQuery (QualifiedIdentifier "test" "getallprojects") [] False Nothing
liftIO $
cost `shouldSatisfy` (< Just 20)
cost `shouldSatisfy` (< Just 30)
it "should not exceed cost when calling scalar proc" $ do
cost <- exec pool [str| {"a": 3, "b": 4} |] $
test/SpecHelper.hs
+6 -2
View File
@@ -93,8 +93,8 @@ testCfgNoJWT testDbConn = (testCfg testDbConn) { configJwtSecret = Nothing }
testUnicodeCfg :: Text -> AppConfig
testUnicodeCfg testDbConn = (testCfg testDbConn) { configSchema = "تست" }
testLtdRowsCfg :: Text -> AppConfig
testLtdRowsCfg testDbConn = (testCfg testDbConn) { configMaxRows = Just 2 }
testMaxRowsCfg :: Text -> AppConfig
testMaxRowsCfg testDbConn = (testCfg testDbConn) { configMaxRows = Just 2 }
testProxyCfg :: Text -> AppConfig
testProxyCfg testDbConn = (testCfg testDbConn) { configProxyUri = Just "https://postgrest.com/openapi.json" }
@@ -149,6 +149,10 @@ setupDb dbConn = do
resetDb :: Text -> IO ()
resetDb dbConn = loadFixture dbConn "data"
analyzeTable :: Text -> Text -> IO ()
analyzeTable dbConn tableName =
void $ readProcess "psql" ["--set", "ON_ERROR_STOP=1", toS dbConn, "-a", "-c", toS $ "ANALYZE test.\"" <> tableName <> "\""] []
loadFixture :: Text -> FilePath -> IO()
loadFixture dbConn name =
void $ readProcess "psql" ["--set", "ON_ERROR_STOP=1", toS dbConn, "-a", "-f", "test/fixtures/" ++ name ++ ".sql"] []
test/fixtures/privileges.sql
Vendored
+2
View File
@@ -101,6 +101,8 @@ GRANT ALL ON TABLE
, pgrst_reserved_chars
, authors_w_entities
, openapi_types
, getallprojects_view
, get_projects_above_view
TO postgrest_test_anonymous;
GRANT INSERT ON TABLE insertonly TO postgrest_test_anonymous;
test/fixtures/schema.sql
Vendored
+13 -1
View File
@@ -1096,11 +1096,17 @@ CREATE FUNCTION get_projects_below(id int) RETURNS SETOF projects
SELECT * FROM test.projects WHERE id < $1;
$_$;
CREATE FUNCTION get_projects_above(id int) RETURNS SETOF projects
LANGUAGE sql
AS $_$
SELECT * FROM test.projects WHERE id > $1;
$_$ ROWS 1;
CREATE FUNCTION getallprojects() RETURNS SETOF projects
LANGUAGE sql
AS $_$
SELECT * FROM test.projects;
$_$;
$_$ ROWS 2019;
CREATE FUNCTION setprojects(id_l int, id_h int, name text) RETURNS SETOF projects
LANGUAGE sql
@@ -1730,3 +1736,9 @@ select $$
</html>
$$::text;
$_$ language sql;
create view getallprojects_view as
select * from getallprojects();
create view get_projects_above_view as
select * from get_projects_above(1);