jimmy/postgrest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for Prefer: count=planned/estimated on GET /table (#1386)

Browse Source
This commit is contained in:
Steve Chávez
2019-09-18 10:13:48 -05:00
committed by GitHub
parent e044488f73
commit 186381bab2
14 changed files with 225 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/PostgREST/ApiRequest.hs
+7 -3
View File
@@ -70,6 +70,7 @@ data Target = TargetIdent QualifiedIdentifier
-- | How to return the inserted data
data PreferRepresentation = Full | HeadersOnly | None deriving Eq
{-|
Describes what the user wants to do. This data type is a
translation of the raw elements of an HTTP request into domain
@@ -94,8 +95,8 @@ data ApiRequest = ApiRequest {
, iPreferRepresentation :: PreferRepresentation
-- | How to pass parameters to a stored procedure
, iPreferParameters :: Maybe PreferParameters
-- | Whether the client wants a result count (slower)
, iPreferCount :: Bool
-- | Whether the client wants a result count
, iPreferCount :: Maybe PreferCount
-- | Whether the client wants to UPSERT or ignore records on PK conflict
, iPreferResolution :: Maybe PreferResolution
-- | Filters on the result ("id", "eq.10")
@@ -135,7 +136,10 @@ userApiRequest schema rootSpec req reqBody
, iPreferParameters = if | hasPrefer (show SingleObject) -> Just SingleObject
| hasPrefer (show MultipleObjects) -> Just MultipleObjects
| otherwise -> Nothing
, iPreferCount = hasPrefer "count=exact"
, iPreferCount = if | hasPrefer (show ExactCount) -> Just ExactCount
| hasPrefer (show PlannedCount) -> Just PlannedCount
| hasPrefer (show EstimatedCount) -> Just EstimatedCount
| otherwise -> Nothing
, iPreferResolution = if | hasPrefer (show MergeDuplicates) -> Just MergeDuplicates
| hasPrefer (show IgnoreDuplicates) -> Just IgnoreDuplicates
| otherwise -> Nothing
src/PostgREST/App.hs
+24 -7
View File
@@ -10,6 +10,7 @@ Some of its functionality includes:
- Content Negotiation
-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE MultiWayIf #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE ScopedTypeVariables #-}
@@ -55,12 +56,14 @@ import PostgREST.Error (PgError (..), SimpleError (..),
import PostgREST.Middleware
import PostgREST.OpenAPI
import PostgREST.Parsers (pRequestColumns)
import PostgREST.QueryBuilder (requestToCallProcQuery,
import PostgREST.QueryBuilder (limitedQuery,
requestToCallProcQuery,
requestToCountQuery,
requestToQuery)
import PostgREST.RangeQuery (allRange, contentRangeH,
rangeStatusHeader)
import PostgREST.Statements (callProcStatement,
createExplainStatement,
createReadStatement,
createWriteStatement)
import PostgREST.Types
@@ -128,11 +131,21 @@ app dbStructure proc cols conf apiRequest =
case partsField of
Left errorResponse -> return errorResponse
Right ((q, cq), bField) -> do
let stm = createReadStatement q cq (contentType == CTSingularJSON) shouldCount
(contentType == CTTextCSV) bField
let cQuery = if estimatedCount
then limitedQuery cq ((+ 1) <$> maxRows) -- LIMIT maxRows + 1 so we can determine below that maxRows was surpassed
else cq
stm = createReadStatement q cQuery (contentType == CTSingularJSON) shouldCount
(contentType == CTTextCSV) bField
explStm = createExplainStatement cq
row <- H.statement () stm
let (tableTotal, queryTotal, _ , body) = row
(status, contentRange) = rangeStatusHeader topLevelRange queryTotal tableTotal
total <- if | plannedCount -> H.statement () explStm
| estimatedCount -> if tableTotal > (fromIntegral <$> maxRows)
then do estTotal <- H.statement () explStm
pure $ if estTotal > tableTotal then estTotal else tableTotal
else pure tableTotal
| otherwise -> pure tableTotal
let (status, contentRange) = rangeStatusHeader topLevelRange queryTotal total
return $
if contentType == CTSingularJSON && queryTotal /= 1
then errorResponseFor . singularityError $ queryTotal
@@ -317,10 +330,14 @@ app dbStructure proc cols conf apiRequest =
where
notFound = responseLBS status404 [] ""
shouldCount = iPreferCount apiRequest
topLevelRange = iTopLevelRange apiRequest
schema = toS $ configSchema conf
readReq = readRequest (configMaxRows conf) (dbRelations dbStructure) proc apiRequest
maxRows = configMaxRows conf
exactCount = iPreferCount apiRequest == Just ExactCount
estimatedCount = iPreferCount apiRequest == Just EstimatedCount
plannedCount = iPreferCount apiRequest == Just PlannedCount
shouldCount = exactCount || estimatedCount
topLevelRange = iTopLevelRange apiRequest
readReq = readRequest maxRows (dbRelations dbStructure) proc apiRequest
fldNames = fieldNames <$> readReq
readDbRequest = DbRead <$> readReq
selectQuery = requestToQuery schema False <$> readDbRequest
src/PostgREST/QueryBuilder.hs
+20 -11
View File
@@ -14,6 +14,7 @@ module PostgREST.QueryBuilder (
requestToQuery
, requestToCountQuery
, requestToCallProcQuery
, limitedQuery
, setLocalQuery
, setLocalSearchPathQuery
) where
@@ -32,17 +33,6 @@ import PostgREST.Types
import Protolude hiding (cast, intercalate,
replace)
requestToCountQuery :: Schema -> DbRequest -> SqlQuery
requestToCountQuery _ (DbMutate _) = witness
requestToCountQuery schema (DbRead (Node (Select{where_=logicForest}, (mainTbl, _, _, _, _)) _)) =
unwords [
"SELECT pg_catalog.count(*)",
"FROM ", fromQi qi,
("WHERE " <> intercalate " AND " (map (pgFmtLogicTree qi) logicForest)) `emptyOnFalse` null logicForest
]
where
qi = removeSourceCTESchema schema mainTbl
requestToQuery :: Schema -> Bool -> DbRequest -> SqlQuery
requestToQuery schema isParent (DbRead (Node (Select colSelects tbl tblAlias implJoins logicForest joinConditions_ ordts range, _) forest)) =
unwords [
@@ -177,6 +167,25 @@ requestToCallProcQuery qi pgArgs returnsScalar preferParams =
callIt :: SqlFragment
callIt = fromQi qi <> "(" <> args <> ")"
-- | SQL query meant for COUNTing the root node of the DbRead Tree.
-- It only takes WHERE into account and doesn't include LIMIT/OFFSET because it would reduce the COUNT.
-- SELECT 1 is done instead of SELECT * to prevent doing expensive operations(like functions based on the columns)
-- inside the FROM target.
requestToCountQuery :: Schema -> DbRequest -> SqlQuery
requestToCountQuery _ (DbMutate _) = witness
requestToCountQuery schema (DbRead (Node (Select{where_=logicForest}, (mainTbl, _, _, _, _)) _)) =
unwords [
"SELECT 1",
"FROM " <> fromQi qi,
("WHERE " <> intercalate " AND " (map (pgFmtLogicTree qi) logicForest)) `emptyOnFalse` null logicForest
]
where
qi = removeSourceCTESchema schema mainTbl
limitedQuery :: SqlQuery -> Maybe Integer -> SqlQuery
limitedQuery query maxRows = query <> maybe mempty (\x -> " LIMIT " <> show x) maxRows
setLocalQuery :: Text -> (Text, Text) -> SqlQuery
setLocalQuery prefix (k, v) =
"SET LOCAL " <> pgFmtIdent (prefix <> k) <> " = " <> pgFmtLit v <> ";"
src/PostgREST/QueryBuilder/Private.hs
+10
View File
@@ -185,3 +185,13 @@ trimNullChars = T.takeWhile (/= '\x0')
removeSourceCTESchema :: Schema -> TableName -> QualifiedIdentifier
removeSourceCTESchema schema tbl = QualifiedIdentifier (if tbl == sourceCTEName then "" else schema) tbl
countF :: SqlQuery -> Bool -> (SqlFragment, SqlFragment)
countF countQuery shouldCount =
if shouldCount
then (
", pg_source_count AS (" <> countQuery <> ")"
, "(SELECT pg_catalog.count(*) FROM pg_source_count)" )
else (
mempty
, "null::bigint")
src/PostgREST/Statements.hs
+36 -10
View File
@@ -13,9 +13,13 @@ module PostgREST.Statements (
createWriteStatement
, createReadStatement
, callProcStatement
, createExplainStatement
) where
import Control.Lens ((^?))
import Data.Aeson as JSON
import qualified Data.Aeson.Lens as L
import qualified Data.ByteString.Char8 as BS
import Data.Maybe
import Data.Text (intercalate, unwords)
@@ -86,15 +90,18 @@ createReadStatement selectQuery countQuery isSingle countTotal asCsv binaryField
unicodeStatement sql HE.noParams decodeStandard False
where
sql = [qc|
WITH {sourceCTEName} AS ({selectQuery}) SELECT {cols}
WITH
{sourceCTEName} AS ({selectQuery})
{countCTEF}
SELECT
{countResultF} AS total_result_set,
pg_catalog.count(_postgrest_t) AS page_total,
{noLocationF} AS header,
{bodyF} AS body
FROM ( SELECT * FROM {sourceCTEName}) _postgrest_t |]
countResultF = if countTotal then "("<>countQuery<>")" else "null"
cols = intercalate ", " [
countResultF <> " AS total_result_set",
"pg_catalog.count(_postgrest_t) AS page_total",
noLocationF <> " AS header",
bodyF <> " AS body"
]
(countCTEF, countResultF) = countF countQuery countTotal
bodyF
| asCsv = asCsvF
| isSingle = asJsonSingleF
@@ -125,6 +132,7 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
where
sql = [qc|
WITH {sourceCTEName} AS ({callProcQuery})
{countCTEF}
SELECT
{countResultF} AS total_result_set,
pg_catalog.count(_postgrest_t) AS page_total,
@@ -132,6 +140,8 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
{responseHeaders} AS response_headers
FROM ({selectQuery}) _postgrest_t;|]
(countCTEF, countResultF) = countF countQuery countTotal
bodyF
| returnsScalar = scalarBodyF
| isSingle = asJsonSingleF
@@ -144,8 +154,6 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
| multObjects = "json_agg(_postgrest_t.pgrst_scalar)::character varying"
| otherwise = "(json_agg(_postgrest_t.pgrst_scalar)->0)::character varying"
countResultF = if countTotal then "( "<> countQuery <> ")" else "null::bigint" :: Text
responseHeaders =
if pgVer >= pgVersion96
then "coalesce(nullif(current_setting('response.headers', true), ''), '[]')" :: Text -- nullif is used because of https://gist.github.com/steve-chavez/8d7033ea5655096903f3b52f8ed09a15
@@ -159,6 +167,24 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
procRow = (,,,) <$> nullableColumn HD.int8 <*> column HD.int8
<*> column HD.bytea <*> column HD.bytea
createExplainStatement :: SqlQuery -> H.Statement () (Maybe Int64)
createExplainStatement countQuery =
unicodeStatement sql HE.noParams decodeExplain False
where
sql = [qc| EXPLAIN (FORMAT JSON) {countQuery} |]
-- |
-- An `EXPLAIN (FORMAT JSON) select * from items;` output looks like this:
-- [{
-- "Plan": {
-- "Node Type": "Seq Scan", "Parallel Aware": false, "Relation Name": "items",
-- "Alias": "items", "Startup Cost": 0.00, "Total Cost": 32.60,
-- "Plan Rows": 2260,"Plan Width": 8} }]
-- We only obtain the Plan Rows here.
decodeExplain :: HD.Result (Maybe Int64)
decodeExplain =
let row = HD.singleRow $ column HD.bytea in
(^? L.nth 0 . L.key "Plan" . L.key "Plan Rows" . L._Integral) <$> row
unicodeStatement :: Text -> HE.Params a -> HD.Result b -> Bool -> H.Statement a b
unicodeStatement = H.Statement . encodeUtf8
src/PostgREST/Types.hs
+11
View File
@@ -76,6 +76,17 @@ instance Show PreferParameters where
show SingleObject = "params=single-object"
show MultipleObjects = "params=multiple-objects"
data PreferCount
= ExactCount -- ^ exact count(slower)
| PlannedCount -- ^ PostgreSQL query planner rows count guess. Done by using EXPLAIN {query}.
| EstimatedCount -- ^ use the query planner rows if the count is superior to max-rows, otherwise get the exact count.
deriving Eq
instance Show PreferCount where
show ExactCount = "count=exact"
show PlannedCount = "count=planned"
show EstimatedCount = "count=estimated"
data DbStructure = DbStructure {
dbTables :: [Table]
, dbColumns :: [Column]