Compare commits

..
9 Commits
Author SHA1 Message Date
jimmy 7989108b0b feat: expose row-level can_edit/can_delete on select *
Compute per-row editability and deletability from a table's row-level
security policies and return them as synthetic columns so clients can
hide edit/delete affordances for rows the user cannot change.

- Introspect pg_policies and relrowsecurity at schema-cache load and
  combine the UPDATE/DELETE USING qualifiers per table (permissive OR,
  restrictive AND).
- Store the combined qualifiers on Table and inject can_edit/can_delete
  as computed select fields when expanding `select *`, only for
  RLS-enabled tables with a matching policy (COALESCE'd to a boolean).
- Keep the computed columns out of the OpenAPI spec so they are not
  rendered as regular fields.
- Add a cfExpression field to CoercibleField to carry raw SQL
  expressions through the planner to SqlFragment.
2026-08-29 11:30:21 +02:00
jimmy 77ab8f83ac feat: expose unique columns and many-to-many markers in OpenAPI
Add unique constraint and many-to-many relationship metadata to the
generated OpenAPI spec so clients can render them.

- Store unique constraints on Table as tableUniqueCols (mirroring
  tablePKCols) instead of denormalizing them onto each Column.
- Compute unique constraints via a per-table tbl_unique_cols CTE in
  tablesSqlQuery.
- Annotate unique columns and composite unique constraints in property
  descriptions, and emit m2m markers in table descriptions.
2026-08-20 18:03:43 +02:00
jimmy ce7ea53a57 instead of select * inspect schema to get selectable columns 2026-08-16 14:41:13 +02:00
jimmy 4a5d626112 restrict openapi spec based on sql grants 2026-08-15 21:41:45 +02:00
Taimoor Zaeem a8feaadc01 test(io): move remaining tests in test_io.py to their modules
We had just 3 tests remaining in test_io.py. This commit moves them to
their modules. So we have:

* test_graceful_shutdown.py

* test_zero_downtime.py

* test_pg_internal.py

Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>
(cherry picked from commit 02d83d1c01)
2026-08-13 13:16:57 +05:00
Taimoor Zaeem e0b9023677 test(io): move config related behavior tests to test_config.py
Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>
(cherry picked from commit e89e0bc255)
2026-08-13 13:16:57 +05:00
Taimoor Zaeem 8f93a0ed2e test(io): move logs and observations tests to test_log.py
Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>
(cherry picked from commit 9e20e5df90)
2026-08-13 13:16:57 +05:00
Taimoor Zaeem a1b01335dc test(io): move connection related tests to test_connection.py
Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>
(cherry picked from commit 2122dcef97)
2026-08-13 13:16:57 +05:00
Taimoor Zaeem 2348cb3f84 test(io): move reloading related tests to test_reloading.py
Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>
(cherry picked from commit d84d00be8e)
2026-08-13 13:16:57 +05:00
33 changed files with 2644 additions and 1883 deletions
+4
View File
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. From versio
## Unreleased
### Fixed
- The OpenAPI output now reflects table privileges: only the granted HTTP methods are exposed (e.g. `SELECT` grants `GET`, `INSERT` grants `POST`) and column-level grants filter the columns shown on table definitions and row filters.
## [16.1] - 2026-08-10
### Fixed
+2
View File
@@ -9,6 +9,8 @@ PostgREST automatically serves a full `OpenAPI <https://www.openapis.org/>`_ des
By default, this output depends on the permissions of the role that is contained in the JWT role claim (or the :ref:`db-anon-role` if no JWT is sent). If you need to show all the endpoints disregarding the role's permissions, set the :ref:`openapi-mode` config to :code:`ignore-privileges`.
When following privileges, the output reflects both the granted HTTP methods and columns: a relation with only ``SELECT`` will only expose ``GET``, a relation with only ``INSERT`` will only expose ``POST``, and column-level grants limit the columns shown on the table definitions and row filters.
For extra customization, the OpenAPI output contains a "description" field for every `SQL comment <https://www.postgresql.org/docs/current/sql-comment.html>`_ on any database object. For instance,
.. code-block:: postgres
+2 -1
View File
@@ -78,6 +78,7 @@ library
PostgREST.Network
PostgREST.Observation
PostgREST.Query
PostgREST.Query.OpenApi
PostgREST.Query.PreQuery
PostgREST.Query.QueryBuilder
PostgREST.Query.SqlFragment
@@ -158,7 +159,7 @@ library
, wai-extra >= 3.1.8 && < 3.2
-- We already depend on wai-logger >= 2.3.7 indirectly via wai-extra,
-- but we want to depend on 2.4.0 which fixes 'unknownSocket' log output
-- for unix sockets; this is tested in test/io/test_io.py. See
-- for unix sockets; this is tested in test/io/test_log.py. See
-- https://github.com/kazu-yamamoto/logger/commit/3a71ca70afdbb93d4ecf0083eeba1fbbbcab3fc3
, wai-logger >= 2.4.0
, warp >= 3.4.14 && < 3.5
+34 -1
View File
@@ -31,6 +31,10 @@ import Network.Wai.Handler.Warp (defaultSettings, setBeforeMainLoop, setHost,
setOnException, setPort, setServerName)
import qualified Data.Text.Encoding as T
import qualified Hasql.Decoders as HD
import qualified Hasql.DynamicStatements.Statement as SQL
import qualified Hasql.Transaction as SQL
import qualified Hasql.Transaction.Sessions as SQL
import qualified Network.Wai as Wai
import qualified Network.Wai.Handler.Warp as Warp
import qualified Network.Wai.Header as WaiHeader
@@ -48,6 +52,7 @@ import qualified PostgREST.Response as Response
import qualified PostgREST.Unix as Unix (installSignalHandlers)
import PostgREST.ApiRequest (ApiRequest (..))
import PostgREST.ApiRequest.Types (Action (..), DbAction (..))
import PostgREST.AppState (AppState)
import PostgREST.AppState.Reload (runListener)
import PostgREST.Auth.Types (AuthResult (..))
@@ -55,6 +60,8 @@ import PostgREST.Config (AppConfig (..))
import PostgREST.Error (Error)
import PostgREST.Network (resolveSocketToAddress)
import PostgREST.Observation (Observation (..))
import PostgREST.Query.OpenApi (TablesAccess, tablesAccessStatement)
import PostgREST.Query.SqlFragment (setConfigWithConstantName)
import PostgREST.Response.Performance (ServerTiming (..), serverTimingHeader)
import PostgREST.SchemaCache (SchemaCache (..))
import PostgREST.TimeIt (timeItT)
@@ -207,7 +214,8 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jwtTime authResul
body <- liftIO $ Wai.strictRequestBody req
(parseTime, apiReq@ApiRequest{..}) <- withTiming conf $ liftEither . mapLeft Error.ApiRequestErr $ ApiRequest.userApiRequest conf prefs req body
(planTime, plan) <- withTiming conf $ liftEither $ Plan.actionPlan iAction conf apiReq sCache
tableAccess <- liftIO $ getTablesAccess appState apiReq authResult
(planTime, plan) <- withTiming conf $ liftEither $ Plan.actionPlan iAction conf apiReq tableAccess sCache
let warnings = Plan.legacyWarnings plan
legacyWarnMsg = "Embedded resource was referenced by relation name even though it has an alias. This is deprecated and will stop working in a future release."
@@ -269,6 +277,31 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jwtTime authResul
in
[(hWarning, "299 " <> pgrstVer <> " \"" <> encodeUtf8 warnMsg <> "\"")]
-- | Fetch the privileges the request role has on the tables of the requested
-- schema, so that the planner can restrict the default "select *" to the
-- columns the role can actually read. Returns an empty map when the request
-- doesn't need it or when the query fails (degrading to the previous behavior).
getTablesAccess :: AppState -> ApiRequest -> AuthResult -> IO TablesAccess
getTablesAccess appState ApiRequest{iAction, iSchema} AuthResult{authRole} =
case iAction of
ActDb ActRelationRead{} -> query
ActDb ActRelationMut{} -> query
ActDb ActRoutine{} -> query
_ -> pure mempty
where
query = do
result <- AppState.usePool appState $
SQL.transactionNoRetry SQL.ReadCommitted SQL.Read $ do
SQL.statement mempty (roleStatement authRole)
SQL.statement mempty (tablesAccessStatement iSchema)
pure $ fromRight mempty result
roleStatement role =
SQL.dynamicallyParameterized
("select " <> setConfigWithConstantName ("role", role))
HD.noResult
False
withTiming :: (MonadError e m, MonadIO m) => AppConfig -> m a -> m (Maybe Double, a)
withTiming AppConfig{configServerTimingEnabled} f = if configServerTimingEnabled
then do
+6 -15
View File
@@ -19,7 +19,6 @@ import qualified Data.Aeson.Lens as L
import qualified Data.ByteString as BS hiding (break)
import qualified Data.ByteString.Char8 as BS
import qualified Data.HashMap.Strict as HM
import qualified Data.Set as S
import qualified Hasql.Decoders as HD
import qualified Hasql.DynamicStatements.Statement as SQL
import qualified Hasql.Session as SQL (Session)
@@ -44,6 +43,7 @@ import PostgREST.Plan (ActionPlan (..), CrudPlan (..),
DbActionPlan (..), InfoPlan (..),
InspectPlan (..))
import PostgREST.Query (MainQuery (..))
import PostgREST.Query.OpenApi (TablesAccess, decodeTablesAccess)
import PostgREST.SchemaCache (SchemaCache (..))
import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..))
import PostgREST.SchemaCache.Routine (Routine (..), RoutineMap)
@@ -60,7 +60,7 @@ data MainTx
data DbResult
= DbCrudResult CrudPlan ResultSet
| DbPlanResult MediaType BS.ByteString
| MaybeDbResult InspectPlan (Maybe (TablesMap, RoutineMap, Maybe Text))
| MaybeDbResult InspectPlan (Maybe (TablesMap, TablesAccess, RoutineMap, Maybe Text))
| NoDbResult InfoPlan
-- | Standard result set format used for the mqMain query
@@ -174,34 +174,25 @@ actionResult MainQuery{mqOpenAPI=(tblsQ, funcsQ, schQ)} (MayUseDb plan@InspectPl
mainActionQuery = lift $
case configOpenApiMode of
OAFollowPriv -> do
tableAccess <- SQL.statement mempty $ SQL.dynamicallyParameterized tblsQ decodeAccessibleIdentifiers configDbPreparedStatements
tableAccess <- SQL.statement mempty $ SQL.dynamicallyParameterized tblsQ decodeTablesAccess configDbPreparedStatements
accFuncs <- SQL.statement mempty $ SQL.dynamicallyParameterized funcsQ SchemaCache.decodeFuncs configDbPreparedStatements
schDesc <- SQL.statement mempty $ SQL.dynamicallyParameterized schQ decodeSchemaDesc configDbPreparedStatements
let tbls = HM.filterWithKey (\qi _ -> S.member qi tableAccess) $ SchemaCache.dbTables sCache
let tbls = HM.filterWithKey (\qi _ -> HM.member qi tableAccess) $ SchemaCache.dbTables sCache
pure $ MaybeDbResult plan (Just (tbls, accFuncs, schDesc))
pure $ MaybeDbResult plan (Just (tbls, tableAccess, accFuncs, schDesc))
OAIgnorePriv -> do
schDesc <- SQL.statement mempty (SQL.dynamicallyParameterized schQ decodeSchemaDesc configDbPreparedStatements)
let tbls = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch == tSchema) (SchemaCache.dbTables sCache)
routs = HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch == tSchema) (SchemaCache.dbRoutines sCache)
pure $ MaybeDbResult plan (Just (tbls, routs, schDesc))
pure $ MaybeDbResult plan (Just (tbls, mempty, routs, schDesc))
OADisabled ->
pure $ MaybeDbResult plan Nothing
decodeSchemaDesc :: HD.Result (Maybe Text)
decodeSchemaDesc = join <$> HD.rowMaybe (nullableColumn HD.text)
decodeAccessibleIdentifiers :: HD.Result (S.Set QualifiedIdentifier)
decodeAccessibleIdentifiers =
let
row = QualifiedIdentifier
<$> column HD.text
<*> column HD.text
in
S.fromList <$> HD.rowList row
-- Makes sure the querystring pk matches the payload pk
-- e.g. PUT /items?id=eq.1 { "id" : 1, .. } is accepted,
-- PUT /items?id=eq.14 { "id" : 2, .. } is rejected.
+69 -31
View File
@@ -45,6 +45,7 @@ import PostgREST.Error (ApiRequestError (..), Error (..),
SchemaCacheError (..))
import PostgREST.MediaType (MediaType (..))
import PostgREST.Plan.Negotiate (negotiateContent)
import PostgREST.Query.OpenApi (TableAccess (..), TablesAccess)
import PostgREST.Query.SqlFragment (sourceCTEName)
import PostgREST.RangeQuery (NonnegRange, allRange,
convertToLimitZeroRange,
@@ -167,23 +168,23 @@ readPlanWarning :: ReadPlan -> Maybe (Text, Text)
readPlanWarning ReadPlan{relName, relAlias = Just alias, relIsLegacyTargetNameMatch = True} = Just (relName, alias)
readPlanWarning _ = Nothing
actionPlan :: Action -> AppConfig -> ApiRequest -> SchemaCache -> Either Error ActionPlan
actionPlan act conf apiReq sCache = case act of
ActDb dbAct -> Db <$> dbActionPlan dbAct conf apiReq sCache
actionPlan :: Action -> AppConfig -> ApiRequest -> TablesAccess -> SchemaCache -> Either Error ActionPlan
actionPlan act conf apiReq tAccess sCache = case act of
ActDb dbAct -> Db <$> dbActionPlan dbAct conf apiReq tAccess sCache
ActRelationInfo ident -> pure . NoDb $ RelInfoPlan ident
ActRoutineInfo ident inv ->
let crPln = callReadPlan ident conf sCache apiReq inv in
let crPln = callReadPlan ident conf tAccess sCache apiReq inv in
NoDb . RoutineInfoPlan . crProc <$> crPln
ActSchemaInfo -> pure $ NoDb SchemaInfoPlan
dbActionPlan :: DbAction -> AppConfig -> ApiRequest -> SchemaCache -> Either Error DbActionPlan
dbActionPlan dbAct conf apiReq sCache = case dbAct of
dbActionPlan :: DbAction -> AppConfig -> ApiRequest -> TablesAccess -> SchemaCache -> Either Error DbActionPlan
dbActionPlan dbAct conf apiReq tAccess sCache = case dbAct of
ActRelationRead identifier headersOnly ->
toDbActPlan <$> wrappedReadPlan identifier conf sCache apiReq headersOnly
toDbActPlan <$> wrappedReadPlan identifier conf tAccess sCache apiReq headersOnly
ActRelationMut identifier mut ->
toDbActPlan <$> mutateReadPlan mut apiReq identifier conf sCache
toDbActPlan <$> mutateReadPlan mut apiReq identifier conf tAccess sCache
ActRoutine identifier invMethod ->
toDbActPlan <$> callReadPlan identifier conf sCache apiReq invMethod
toDbActPlan <$> callReadPlan identifier conf tAccess sCache apiReq invMethod
ActSchemaRead tSchema headersOnly ->
MayUseDb <$> inspectPlan apiReq headersOnly tSchema
where
@@ -191,32 +192,32 @@ dbActionPlan dbAct conf apiReq sCache = case dbAct of
MTVndPlan{} -> DbCrud True pl
_ -> DbCrud False pl
wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Bool -> Either Error CrudPlan
wrappedReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} headersOnly = do
wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> TablesAccess -> SchemaCache -> ApiRequest -> Bool -> Either Error CrudPlan
wrappedReadPlan identifier conf tAccess sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} headersOnly = do
qi <- findTable identifier sCache
rPlan <- readPlan qi conf sCache apiRequest
rPlan <- readPlan qi conf tAccess sCache apiRequest
(handler, mediaType) <- mapLeft ApiRequestErr $ negotiateContent conf apiRequest qi iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan)
if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestErr $ InvalidPreferences invalidPrefs else Right ()
return $ WrappedReadPlan rPlan SQL.Read handler mediaType headersOnly qi
mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error CrudPlan
mutateReadPlan mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf sCache = do
mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> TablesAccess -> SchemaCache -> Either Error CrudPlan
mutateReadPlan mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf tAccess sCache = do
qi <- findTable identifier sCache
rPlan <- readPlan qi conf sCache apiRequest
rPlan <- readPlan qi conf tAccess sCache apiRequest
mPlan <- mutatePlan mutation qi apiRequest sCache rPlan
if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestErr $ InvalidPreferences invalidPrefs else Right ()
(handler, mediaType) <- mapLeft ApiRequestErr $ negotiateContent conf apiRequest qi iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan)
return $ MutateReadPlan rPlan mPlan SQL.Write handler mediaType mutation qi
callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CrudPlan
callReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{preferHandling, invalidPrefs, preferMaxAffected},..} invMethod = do
callReadPlan :: QualifiedIdentifier -> AppConfig -> TablesAccess -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CrudPlan
callReadPlan identifier conf tAccess sCache apiRequest@ApiRequest{iPreferences=Preferences{preferHandling, invalidPrefs, preferMaxAffected},..} invMethod = do
let paramKeys = case invMethod of
InvRead _ -> S.fromList $ fst <$> qsParams'
Inv -> iColumns
proc@Function{..} <- mapLeft SchemaCacheErr $
findProc identifier paramKeys (dbRoutines sCache) iContentMediaType (invMethod == Inv)
let relIdentifier = QualifiedIdentifier pdSchema (fromMaybe pdName $ Routine.funcTableName proc) -- done so a set returning function can embed other relations
rPlan <- readPlan relIdentifier conf sCache apiRequest
rPlan <- readPlan relIdentifier conf tAccess sCache apiRequest
let args = case (invMethod, iContentMediaType) of
(InvRead _, _) -> DirectArgs $ toRpcParams proc qsParams'
(Inv, MTUrlEncoded) -> DirectArgs $ maybe mempty (toRpcParams proc . payArray) iPayload
@@ -313,10 +314,11 @@ data ResolverContext = ResolverContext
, representations :: RepresentationsMap
, qi :: QualifiedIdentifier -- ^ The table we're currently attending; changes as we recurse into joins etc.
, outputType :: Text -- ^ The output type for the response payload; e.g. "csv", "json", "binary".
, tablesAccess :: TablesAccess -- ^ Privileges the request role has on the exposed tables.
}
resolveColumnField :: Column -> Maybe ToTsVector -> CoercibleField
resolveColumnField col toTsV = CoercibleField (colName col) mempty False toTsV (colNominalType col) (colType col) Nothing (colDefault col) False
resolveColumnField col toTsV = CoercibleField (colName col) mempty False toTsV (colNominalType col) (colType col) Nothing (colDefault col) False Nothing
resolveTableFieldName :: Table -> FieldName -> Maybe ToTsVector -> CoercibleField
resolveTableFieldName table fieldName toTsV=
@@ -376,11 +378,11 @@ resolveQueryInputField ctx field opExpr = withTextParse ctx $ resolveTypeOrUnkno
-- | Builds the ReadPlan tree on a number of stages.
-- | Adds filters, order, limits on its respective nodes.
-- | Adds joins conditions obtained from resource embedding.
readPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error ReadPlanTree
readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows, configDbAggregates, configUrlUseLegacyTargetNames} SchemaCache{dbTables, dbRelationships, dbRepresentations} apiRequest =
readPlan :: QualifiedIdentifier -> AppConfig -> TablesAccess -> SchemaCache -> ApiRequest -> Either Error ReadPlanTree
readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows, configDbAggregates, configUrlUseLegacyTargetNames} tAccess SchemaCache{dbTables, dbRelationships, dbRepresentations} apiRequest =
let
-- JSON output format hardcoded for now. In the future we might want to support other output mappings such as CSV.
ctx = ResolverContext dbTables dbRepresentations qi "json"
ctx = ResolverContext dbTables dbRepresentations qi "json" tAccess
in
treeRestrictRange configDbMaxRows (iAction apiRequest) =<<
addToManyOrderSelects =<<
@@ -476,7 +478,8 @@ knownColumnsInContext ResolverContext{..} =
-- | Expand "select *" into explicit field names of the table in the following situations:
-- * When there are data representations present.
-- * When there is an aggregate function in a given ReadPlan or its parent.
-- * When the ReadPlan is a to-many spread relationship
-- * When the ReadPlan is a to-many spread relationship.
-- * When the default select(when no "select" is given) would include columns the request role cannot read.
expandStars :: ResolverContext -> ReadPlanTree -> Either Error ReadPlanTree
expandStars ctx rPlanTree = Right $ expandStarsForReadPlan False rPlanTree
where
@@ -496,13 +499,16 @@ expandStars ctx rPlanTree = Right $ expandStarsForReadPlan False rPlanTree
adjustContext context fromQI _ = context{qi=fromQI}
expandStarsForTable :: ResolverContext -> Bool -> ReadPlan -> ReadPlan
expandStarsForTable ctx@ResolverContext{representations, outputType} hasAgg rp@ReadPlan{select=selectFields, relSpread=spread}
-- We expand if either of the below are true:
-- * We have a '*' select AND there is an aggregate function in this ReadPlan's sub-tree.
-- * We have a '*' select AND the target table has at least one data representation.
expandStarsForTable ctx@ResolverContext{representations, outputType, tables, qi} hasAgg rp@ReadPlan{select=selectFields, relSpread=spread}
-- We expand the '*' select if either of the below are true:
-- * The target table has columns the request role cannot read.
-- * There is an aggregate function in this ReadPlan's sub-tree.
-- * The target table has at least one data representation.
-- We ignore '*' selects that have an aggregate function attached, unless it's a `COUNT(*)` for a Spread Embed,
-- we tag it as "full row" in that case.
| hasStarSelect && (hasAgg || hasDataRepresentation) = rp{select = concatMap (expandStarSelectField (isJust spread) knownColumns) selectFields}
| hasStarSelect && hasLimitedPrivileges = rp{select = concatMap (expandStarSelectField (isJust spread) accessibleColumns) selectFields <> rlsFields}
| hasStarSelect && (hasAgg || hasDataRepresentation) = rp{select = concatMap (expandStarSelectField (isJust spread) knownColumns) selectFields <> rlsFields}
| hasStarSelect = rp{select = selectFields <> rlsFields}
| otherwise = rp
where
hasStarSelect = "*" `elem` map (cfName . csField) filteredSelectFields
@@ -510,6 +516,11 @@ expandStarsForTable ctx@ResolverContext{representations, outputType} hasAgg rp@R
shouldExpandOrTag aggFunc = isNothing aggFunc || (isJust spread && aggFunc == Just Count)
hasDataRepresentation = any hasOutputRep knownColumns
knownColumns = knownColumnsInContext ctx
hasLimitedPrivileges = accessibleColumns /= knownColumns
accessibleColumns = accessibleColumnsInContext ctx
rlsFields = case HM.lookup qi tables of
Just tbl -> rlsSelectFields tbl
Nothing -> []
hasOutputRep :: Column -> Bool
hasOutputRep col = HM.member (colNominalType col, outputType) representations
@@ -521,6 +532,33 @@ expandStarsForTable ctx@ResolverContext{representations, outputType} hasAgg rp@R
[sel { csField = fld { cfFullRow = True } }]
expandStarSelectField _ _ selectField = [selectField]
-- | Synthetic can_edit/can_delete select fields for a table that has row-level
-- security policies restricting UPDATE/DELETE. These carry the raw qualifier
-- expression and are rendered as computed columns on `select *`.
rlsSelectFields :: Table -> [CoercibleSelectField]
rlsSelectFields tbl = catMaybes [rlsField "can_edit" (tableRlsEditQual tbl), rlsField "can_delete" (tableRlsDeleteQual tbl)]
where
rlsField :: FieldName -> Maybe Text -> Maybe CoercibleSelectField
rlsField name qual = do
expr <- qual
pure CoercibleSelectField
{ csField = (unknownField name []) { cfIRType = "boolean", cfBaseType = "boolean", cfExpression = Just expr }
, csAggFunction = Nothing
, csAggCast = Nothing
, csCast = Nothing
, csAlias = Just name
}
-- | The columns of the current table that the request role can SELECT. Falls
-- back to all known columns when no access info is available or the role has
-- no SELECT privilege on any column.
accessibleColumnsInContext :: ResolverContext -> [Column]
accessibleColumnsInContext ctx@ResolverContext{qi=tblQi, tablesAccess} =
case HM.lookup tblQi tablesAccess of
Just (TableAccess selCols _ _ _) | not (null selCols) ->
filter (\col -> colName col `elem` selCols) (knownColumnsInContext ctx)
_ -> knownColumnsInContext ctx
-- | Enforces the `max-rows` config on the result
treeRestrictRange :: Maybe Integer -> Action -> ReadPlanTree -> Either Error ReadPlanTree
treeRestrictRange _ (ActDb (ActRelationMut _ _)) request = Right request
@@ -930,7 +968,7 @@ addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
-- where_ = [
-- CoercibleStmnt (
-- CoercibleFilter {
-- field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson=False, cfToTsVector = Nothing, cfIRType = "", cfBaseType = "", cfTransform = Nothing, cfDefault = Nothing, cfFullRow = False},
-- field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson=False, cfToTsVector = Nothing, cfIRType = "", cfBaseType = "", cfTransform = Nothing, cfDefault = Nothing, cfFullRow = False, cfExpression = Nothing},
-- opExpr = op
-- }
-- )
@@ -947,7 +985,7 @@ addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
-- Don't do anything to the filter if there's no embedding (a subtree) on projects. Assume it's a normal filter.
--
-- >>> ReadPlan.where_ . rootLabel <$> addNullEmbedFilters (readPlanTree nullOp [])
-- Right [CoercibleStmnt (CoercibleFilter {field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson = False, cfToTsVector = Nothing, cfIRType = "", cfBaseType = "", cfTransform = Nothing, cfDefault = Nothing, cfFullRow = False}, opExpr = OpExpr True (Is IsNull)})]
-- Right [CoercibleStmnt (CoercibleFilter {field = CoercibleField {cfName = "projects", cfJsonPath = [], cfToJson = False, cfToTsVector = Nothing, cfIRType = "", cfBaseType = "", cfTransform = Nothing, cfDefault = Nothing, cfFullRow = False, cfExpression = Nothing}, opExpr = OpExpr True (Is IsNull)})]
--
-- If there's an embedding on projects, then change the filter to use the internal aggregate name (`clients_projects_1`) so the filter can succeed later.
--
@@ -1059,7 +1097,7 @@ mutatePlan mutation qi ApiRequest{iPreferences=Preferences{..}, ..} SchemaCache{
Left $ ApiRequestErr InvalidFilters
MutationDelete -> Right $ Delete qi combinedLogic returnings
where
ctx = ResolverContext dbTables dbRepresentations qi "json"
ctx = ResolverContext dbTables dbRepresentations qi "json" mempty
confCols = fromMaybe pkCols qsOnConflict
QueryParams.QueryParams{..} = iQueryParams
returnings =
+2 -1
View File
@@ -47,10 +47,11 @@ data CoercibleField = CoercibleField
, cfTransform :: Maybe TransformerProc -- ^ The optional mapping from irType -> targetType.
, cfDefault :: Maybe Text
, cfFullRow :: Bool -- ^ True if the field represents the whole selected row. Used in spread rels: instead of COUNT(*), it does a COUNT(<row>) in order to not mix with other spread resources.
, cfExpression :: Maybe Text -- ^ Raw SQL expression for a computed field (e.g. RLS-derived can_edit/can_delete). When present the field is rendered as this expression instead of a table column.
} deriving (Eq, Show)
unknownField :: FieldName -> JsonPath -> CoercibleField
unknownField name path = CoercibleField name path False Nothing "" "" Nothing Nothing False
unknownField name path = CoercibleField name path False Nothing "" "" Nothing Nothing False Nothing
-- | Like an API request LogicTree, but with coercible field information.
data CoercibleLogicTree
+60
View File
@@ -0,0 +1,60 @@
{-|
Module : PostgREST.Query.OpenApi
Description : Types for reflecting the role privileges on the OpenAPI output.
-}
module PostgREST.Query.OpenApi
( TableAccess (..)
, TablesAccess
, tablesAccessStatement
, decodeTablesAccess
) where
import qualified Data.HashMap.Strict as HM
import qualified Hasql.Decoders as HD
import qualified Hasql.DynamicStatements.Statement as SQL
import qualified Hasql.Statement as SQL
import qualified PostgREST.Query.SqlFragment as SqlFragment
import PostgREST.SchemaCache.Identifiers (FieldName, QualifiedIdentifier (..))
import Protolude
-- | Privileges that a role has on a relation, used to reflect them on the OpenAPI output.
data TableAccess = TableAccess
{ taSelectCols :: [FieldName]
-- ^ columns the role can SELECT
, taInsertCols :: [FieldName]
-- ^ columns the role can INSERT into
, taUpdateCols :: [FieldName]
-- ^ columns the role can UPDATE
, taDelete :: Bool
-- ^ whether the role can DELETE rows
}
deriving (Show, Eq)
type TablesAccess = HM.HashMap QualifiedIdentifier TableAccess
-- | Statement that returns the privileges the current role has on each
-- accessible relation of the given schema.
tablesAccessStatement :: Text -> SQL.Statement () TablesAccess
tablesAccessStatement schema =
SQL.dynamicallyParameterized (SqlFragment.accessibleTables schema) decodeTablesAccess False
decodeTablesAccess :: HD.Result TablesAccess
decodeTablesAccess =
let
row = (,) <$> (QualifiedIdentifier <$> column HD.text <*> column HD.text)
<*> (TableAccess
<$> arrayColumn HD.text
<*> arrayColumn HD.text
<*> arrayColumn HD.text
<*> column HD.bool)
in
HM.fromList <$> HD.rowList row
column :: HD.Value a -> HD.Row a
column = HD.column . HD.nonNullable
arrayColumn :: HD.Value a -> HD.Row [a]
arrayColumn = column . HD.listArray . HD.nonNullable
+1 -1
View File
@@ -180,7 +180,7 @@ callPlanToQuery (FunctionCall qi params arguments returnsScalar returnsSetOfScal
KeyParams [] -> "FROM " <> callIt mempty
KeyParams prms -> case arguments of
DirectArgs args -> "FROM " <> callIt (fmtArgs prms args)
JsonArgs json -> fromJsonBodyF json ((\p -> CoercibleField (ppName p) mempty False Nothing (ppTypeMaxLength p) mempty Nothing Nothing False) <$> prms) False True False <> ", " <>
JsonArgs json -> fromJsonBodyF json ((\p -> CoercibleField (ppName p) mempty False Nothing (ppTypeMaxLength p) mempty Nothing Nothing False Nothing) <$> prms) False True False <> ", " <>
"LATERAL " <> callIt (fmtParams prms)
callIt :: SQL.Snippet -> SQL.Snippet
+27 -1
View File
@@ -249,6 +249,7 @@ pgFmtField table cf = case cfToTsVector cf of
_ -> fmtFld
where
fmtFld = case cf of
CoercibleField{cfExpression=Just expr} -> SQL.sql (encodeUtf8 expr)
CoercibleField{cfFullRow=True} -> pgFmtIdent (qiName table)
CoercibleField{cfName=fn, cfJsonPath=[]} -> pgFmtColumn table fn
CoercibleField{cfName=fn, cfToJson=doToJson, cfJsonPath=jp} | doToJson -> "to_jsonb(" <> pgFmtColumn table fn <> ")" <> pgFmtJsonPath jp
@@ -598,7 +599,32 @@ accessibleTables :: Text -> SQL.Snippet
accessibleTables schema = SQL.sql (encodeUtf8 [trimming|
SELECT
n.nspname AS table_schema,
c.relname AS table_name
c.relname AS table_name,
COALESCE((
SELECT array_agg(a.attname ORDER BY a.attnum)
FROM pg_attribute a
WHERE a.attrelid = c.oid
AND a.attnum > 0
AND NOT a.attisdropped
AND has_column_privilege(c.oid, a.attnum, 'SELECT')
), '{}') AS select_cols,
COALESCE((
SELECT array_agg(a.attname ORDER BY a.attnum)
FROM pg_attribute a
WHERE a.attrelid = c.oid
AND a.attnum > 0
AND NOT a.attisdropped
AND has_column_privilege(c.oid, a.attnum, 'INSERT')
), '{}') AS insert_cols,
COALESCE((
SELECT array_agg(a.attname ORDER BY a.attnum)
FROM pg_attribute a
WHERE a.attrelid = c.oid
AND a.attnum > 0
AND NOT a.attisdropped
AND has_column_privilege(c.oid, a.attnum, 'UPDATE')
), '{}') AS update_cols,
has_table_privilege(c.oid, 'DELETE') AS has_delete
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('v','r','m','f','p')
+1 -1
View File
@@ -202,7 +202,7 @@ actionResponse (DbPlanResult media plan) ctxApiRequest _ _ _ =
actionResponse (MaybeDbResult InspectPlan{ipHdrsOnly=headersOnly} body) ApiRequest{..} versions conf sCache =
let
rsBody = maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache x y z) body
rsBody = maybe mempty (\(tbls, tblAccess, procs, schDesc) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache tbls tblAccess procs schDesc) body
cLHeader = if headersOnly then mempty else [contentLengthHeader rsBody]
in
Right $ PgrstResponse HTTP.status200 (MediaType.toContentType MTOpenAPI : cLHeader ++ maybeToList (profileHeader iSchema iNegotiatedByProfile)) rsBody
+98 -38
View File
@@ -27,10 +27,11 @@ import PostgREST.Config (AppConfig (..), Proxy (..),
isMalformedProxyUri, toURI)
import PostgREST.MediaType
import PostgREST.Network (escapeHostName)
import PostgREST.Query.OpenApi (TableAccess (..), TablesAccess)
import PostgREST.SchemaCache (SchemaCache (..))
import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..))
import PostgREST.SchemaCache.Relationship (Cardinality (..), Relationship (..),
RelationshipsMap)
import PostgREST.SchemaCache.Identifiers (FieldName, QualifiedIdentifier (..))
import PostgREST.SchemaCache.Relationship (Cardinality (..), Junction (..),
Relationship (..), RelationshipsMap)
import PostgREST.SchemaCache.Routine (FuncVolatility (..), Routine (..),
RoutineParam (..))
import PostgREST.SchemaCache.Table (Column (..), Table (..), TablesMap,
@@ -38,18 +39,27 @@ import PostgREST.SchemaCache.Table (Column (..), Table (..), TablesMap,
import Protolude hiding (Proxy, get)
encode :: (Text, Text) -> AppConfig -> SchemaCache -> TablesMap -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
encode versions conf sCache tables procs schemaDescription =
encode :: (Text, Text) -> AppConfig -> SchemaCache -> TablesMap -> TablesAccess -> HM.HashMap k [Routine] -> Maybe Text -> LBS.ByteString
encode versions conf sCache tables access procs schemaDescription =
JSON.encode $
postgrestSpec
versions
(dbRelationships sCache)
(concat $ HM.elems procs)
(snd <$> HM.toList tables)
(fmap (\(_, t) -> (t, accessFor access t)) (HM.toList tables))
(proxyUri conf)
schemaDescription
(configOpenApiSecurityActive conf)
-- | Get the access privileges for a table. When the table is not present in the
-- map(ignore-privileges mode), assume the role has full access to it.
accessFor :: TablesAccess -> Table -> TableAccess
accessFor access t =
fromMaybe fullAccess (HM.lookup (QualifiedIdentifier (tableSchema t) (tableName t)) access)
where
fullAccess = TableAccess allCols allCols allCols True
allCols = colName <$> tableColumnsList t
makeMimeList :: [MediaType] -> MimeList
makeMimeList cs = MimeList $ fmap (fromString . BS.unpack . toMime) cs
@@ -97,14 +107,41 @@ parseDefault colType colDefault =
where
wrapInQuotations text = "\"" <> text <> "\""
makeTableDef :: RelationshipsMap -> Table -> (Text, Schema)
makeTableDef rels t =
let tn = tableName t in
makeTableDef :: RelationshipsMap -> (Table, TableAccess) -> (Text, Schema)
makeTableDef rels (t, access) =
(tn, (mempty :: Schema)
& description .~ tableDescription t
& description .~ tblDescription
& type_ ?~ SwaggerObject
& properties .~ fromList (makeProperty t rels <$> tableColumnsList t)
& required .~ fmap colName (filter (not . colNullable) $ tableColumnsList t))
& properties .~ fromList (makeProperty t rels <$> cols)
& required .~ fmap colName (filter (not . colNullable) cols))
where
tn = tableName t
cols = accessibleCols t (taSelectCols access)
tblDescription = case m2mMarkers t rels of
[] -> tableDescription t
ms -> Just $ maybe "" (`T.append` "\n\n") (tableDescription t) <> T.intercalate "\n" ms
-- | Emits markers for the many-to-many relationships of a table, so that clients
-- can render these relations. The marker includes the target table(embedding key),
-- the junction table and the junction columns referencing source and target.
m2mMarkers :: Table -> RelationshipsMap -> [Text]
m2mMarkers tbl rels = mapMaybe m2mMarker searchedRels
where
searchedRels = fromMaybe mempty $ HM.lookup (QualifiedIdentifier (tableSchema tbl) (tableName tbl), tableSchema tbl) rels
m2mMarker Relationship{relForeignTable, relCardinality=M2M junction} =
Just $ T.intercalate ""
[ "<m2m table='", qiName relForeignTable
, "' junction='", qiName (junTable junction)
, "' source='", junctionSourceCol junction
, "' target='", junctionTargetCol junction
, "'/>"
]
m2mMarker _ = Nothing
junctionSourceCol junction = maybe mempty snd (headMay $ junColsSource junction)
junctionTargetCol junction = maybe mempty snd (headMay $ junColsTarget junction)
accessibleCols :: Table -> [FieldName] -> [Column]
accessibleCols t cols = filter ((`elem` cols) . colName) (tableColumnsList t)
makeProperty :: Table -> RelationshipsMap -> Column -> (Text, Referenced Schema)
makeProperty tbl rels col = (colName col, Inline s)
@@ -129,11 +166,18 @@ makeProperty tbl rels col = (colName col, Inline s)
(\(a, b) -> T.intercalate "" ["This is a Foreign Key to `", a, ".", b, "`.<fk table='", a, "' column='", b, "'/>"]) <$> fTblCol
pk :: Bool
pk = colName col `elem` tablePKCols tbl
uniqueNotes :: [Text]
uniqueNotes = mapMaybe uniqueNote (filter (colName col `elem`) (tableUniqueCols tbl))
where
uniqueNote cols
| length cols == 1 = Just "This is a Unique column.<unique/>"
| otherwise = Just $ "This is part of a composite unique constraint.<unique cols='" <> T.intercalate "," cols <> "'/>"
n = catMaybes
[ Just "Note:"
, if pk then Just "This is a Primary Key.<pk/>" else Nothing
, fk
]
<> uniqueNotes
<> catMaybes [fk]
d =
if length n > 1 then
Just $ T.append (maybe "" (`T.append` "\n\n") $ colDescription col) (T.intercalate "\n" n)
@@ -222,8 +266,8 @@ makeProcPostParams pd =
, Ref $ Reference "preferParams"
]
makeParamDefs :: [Table] -> [(Text, Param)]
makeParamDefs ti =
makeParamDefs :: RelationshipsMap -> [(Table, TableAccess)] -> [(Text, Param)]
makeParamDefs rels tis =
-- TODO: create Prefer for each method (GET, PATCH, etc.)
[ ("preferParams", makePreferParam ["params"])
, ("preferReturn", makePreferParam ["return"])
@@ -280,17 +324,27 @@ makeParamDefs ti =
& in_ .~ ParamQuery
& type_ ?~ SwaggerString))
]
<> concat [ makeObjectBody (tableName t) : makeRowFilters (tableName t) (tableColumnsList t)
| t <- ti
<> concat [ makeObjectBody rels t access <> makeRowFilters (tableName t) (accessibleCols t (taSelectCols access))
| (t, access) <- tis
]
makeObjectBody :: Text -> (Text, Param)
makeObjectBody tn =
("body." <> tn, (mempty :: Param)
makeObjectBody :: RelationshipsMap -> Table -> TableAccess -> [(Text, Param)]
makeObjectBody rels t access =
[ ("body." <> tn, makeBodyParam (taInsertCols access))
, ("body." <> tn <> ".patch", makeBodyParam (taUpdateCols access))
]
where
tn = tableName t
makeBodyParam cols = (mempty :: Param)
& name .~ tn
& description ?~ tn
& required ?~ False
& schema .~ ParamBody (Ref (Reference tn)))
& schema .~ ParamBody (Inline bodySchema)
where
bodySchema = (mempty :: Schema)
& type_ ?~ SwaggerObject
& properties .~ fromList (makeProperty t rels <$> accessibleCols t cols)
& required .~ fmap colName (filter (not . colNullable) (accessibleCols t cols))
makeRowFilter :: Text -> Column -> (Text, Param)
makeRowFilter tn c =
@@ -305,8 +359,8 @@ makeRowFilter tn c =
makeRowFilters :: Text -> [Column] -> [(Text, Param)]
makeRowFilters tn = fmap (makeRowFilter tn)
makePathItem :: Table -> (FilePath, PathItem)
makePathItem t = ("/" ++ T.unpack tn, p $ tableInsertable t || tableUpdatable t || tableDeletable t)
makePathItem :: (Table, TableAccess) -> (FilePath, PathItem)
makePathItem (t, access) = ("/" ++ T.unpack tn, p)
where
-- Use first line of table description as summary; rest as description (if present)
-- We strip leading newlines from description so that users can include a blank line between summary and description
@@ -327,20 +381,26 @@ makePathItem t = ("/" ++ T.unpack tn, p $ tableInsertable t || tableUpdatable t
)
)
postOp = tOp
& parameters .~ fmap ref ["body." <> tn, "select", "preferPost"]
& parameters .~ fmap ref [bodyParam, "select", "preferPost"]
& at 201 ?~ "Created"
patchOp = tOp
& parameters .~ fmap ref (rs <> ["body." <> tn, "preferReturn"])
& parameters .~ fmap ref (rs <> [patchBodyParam, "preferReturn"])
& at 204 ?~ "No Content"
deletOp = tOp
& parameters .~ fmap ref (rs <> ["preferReturn"])
& at 204 ?~ "No Content"
pr = (mempty :: PathItem) & get ?~ getOp
pw = pr & post ?~ postOp & patch ?~ patchOp & delete ?~ deletOp
p False = pr
p True = pw
p = (mempty :: PathItem)
& get .~ (if not (null selCols) then Just getOp else Nothing)
& post .~ (if tableInsertable t && not (null insCols) then Just postOp else Nothing)
& patch .~ (if tableUpdatable t && not (null updCols) then Just patchOp else Nothing)
& delete .~ (if tableDeletable t && taDelete access then Just deletOp else Nothing)
tn = tableName t
rs = [ T.intercalate "." ["rowFilter", tn, colName c ] | c <- tableColumnsList t ]
selCols = accessibleCols t (taSelectCols access)
insCols = accessibleCols t (taInsertCols access)
updCols = accessibleCols t (taUpdateCols access)
rs = [ T.intercalate "." ["rowFilter", tn, colName c ] | c <- selCols ]
bodyParam = "body." <> tn
patchBodyParam = "body." <> tn <> ".patch"
ref = Ref . Reference
makeProcPathItem :: Routine -> (FilePath, PathItem)
@@ -375,9 +435,9 @@ makeRootPathItem = ("/", p)
pr = (mempty :: PathItem) & get ?~ getOp
p = pr
makePathItems :: [Routine] -> [Table] -> InsOrdHashMap FilePath PathItem
makePathItems pds ti = fromList $ makeRootPathItem :
fmap makePathItem ti ++ fmap makeProcPathItem pds
makePathItems :: [Routine] -> [(Table, TableAccess)] -> InsOrdHashMap FilePath PathItem
makePathItems pds tis = fromList $ makeRootPathItem :
fmap makePathItem tis ++ fmap makeProcPathItem pds
makeSecurityDefinitions :: Text -> Bool -> SecurityDefinitions
makeSecurityDefinitions secName allow
@@ -387,8 +447,8 @@ makeSecurityDefinitions secName allow
secSchType = SecuritySchemeApiKey (ApiKeyParams "Authorization" ApiKeyHeader)
secSchDescription = Just "Add the token prepending \"Bearer \" (without quotes) to it"
postgrestSpec :: (Text, Text) -> RelationshipsMap -> [Routine] -> [Table] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
postgrestSpec (prettyVersion, docsVersion) rels pds ti (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
postgrestSpec :: (Text, Text) -> RelationshipsMap -> [Routine] -> [(Table, TableAccess)] -> (Text, Text, Integer, Text) -> Maybe Text -> Bool -> Swagger
postgrestSpec (prettyVersion, docsVersion) rels pds tis (s, h, p, b) sd allowSecurityDef = (mempty :: Swagger)
& basePath ?~ T.unpack b
& schemes ?~ [s']
& info .~ ((mempty :: Info)
@@ -399,9 +459,9 @@ postgrestSpec (prettyVersion, docsVersion) rels pds ti (s, h, p, b) sd allowSecu
& description ?~ "PostgREST Documentation"
& url .~ URL ("https://postgrest.org/en/" <> docsVersion <> "/references/api.html"))
& host .~ h'
& definitions .~ fromList (makeTableDef rels <$> ti)
& parameters .~ fromList (makeParamDefs ti)
& paths .~ makePathItems pds ti
& definitions .~ fromList (makeTableDef rels <$> tis)
& parameters .~ fromList (makeParamDefs rels tis)
& paths .~ makePathItems pds tis
& produces .~ makeMimeList [MTApplicationJSON, MTVndSingularJSON True, MTVndSingularJSON False, MTTextCSV]
& consumes .~ makeMimeList [MTApplicationJSON, MTVndSingularJSON True, MTVndSingularJSON False, MTTextCSV]
& securityDefinitions .~ makeSecurityDefinitions securityDefName allowSecurityDef
+122 -5
View File
@@ -150,6 +150,7 @@ querySchemaCache pgVer conf@AppConfig{..} = do
m2oRels <- sqlTimedStmt gucRels mempty allM2OandO2ORels
funcs <- sqlTimedStmt gucFuncs conf (allFunctions pgVer configDbPreparedStatements)
cRels <- sqlTimedStmt gucCRels mempty allComputedRels
rlsPols <- sqlTimedStmt gucRLS conf allRlsPolicies
reps <- sqlTimedStmt gucDReps conf dataRepresentations
mHdlers <- sqlTimedStmt gucMHdrs conf mediaHandlers
@@ -161,10 +162,11 @@ querySchemaCache pgVer conf@AppConfig{..} = do
else pure Nothing
let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
tabsWithRls = addRlsQuals tabsWViewsPks (combineRlsPolicies rlsPols)
rels = addInverseRels $ addM2MRels tabsWViewsPks $ addViewM2OAndO2ORels keyDeps m2oRels
return (removeInternal schemas $ SchemaCache {
dbTables = tabsWViewsPks
dbTables = tabsWithRls
, dbRelationships = getOverrideRelationshipsMap rels cRels
, dbRoutines = funcs
, dbRepresentations = reps
@@ -232,6 +234,7 @@ decodeTables =
<*> column HD.bool
<*> column HD.bool
<*> arrayColumn HD.text
<*> column (HD.refine parseUniqueCols HD.jsonb)
<*> parseCols (compositeArrayColumn
(Column
<$> compositeField HD.text
@@ -242,11 +245,19 @@ decodeTables =
<*> nullableCompositeField HD.int4
<*> nullableCompositeField HD.text
<*> compositeFieldArray HD.text))
<*> pure (Nothing :: Maybe Text)
<*> pure (Nothing :: Maybe Text)
parseCols :: HD.Row [Column] -> HD.Row ColumnMap
parseCols = fmap (HMI.fromList . map (\col@Column{colName} -> (colName, col)))
parseUniqueCols :: JSON.Value -> Either Text [[FieldName]]
parseUniqueCols val =
case JSON.fromJSON val of
JSON.Success cols -> Right cols
JSON.Error err -> Left ("Invalid unique columns: " <> T.pack err)
decodeRels :: HD.Result [Relationship]
decodeRels =
HD.rowList relRow
@@ -676,6 +687,25 @@ tablesSqlQuery pgVer =
AND NOT pg_is_other_temp_schema(r.relnamespace)
AND NOT a.attisdropped
GROUP BY r.oid
),
tbl_unique_cols AS (
SELECT
r.oid AS relid,
jsonb_agg(cols ORDER BY c.oid) AS unique_cols
FROM pg_class r
JOIN pg_constraint c
ON r.oid = c.conrelid
JOIN LATERAL (
SELECT jsonb_agg(a.attname::text ORDER BY k.ord) AS cols
FROM unnest(c.conkey) WITH ORDINALITY AS k(attnum, ord)
JOIN pg_attribute a ON a.attrelid = r.oid AND a.attnum = k.attnum
) col_info ON TRUE
WHERE
c.contype = 'u'
AND r.relkind IN ('r', 'p')
AND r.relnamespace NOT IN ('pg_catalog'::regnamespace, 'information_schema'::regnamespace)
AND NOT pg_is_other_temp_schema(r.relnamespace)
GROUP BY r.oid
)
SELECT
n.nspname AS table_schema,
@@ -709,11 +739,13 @@ tablesSqlQuery pgVer =
)
) AS deletable,
coalesce(tpks.pk_cols, '{}') as pk_cols,
coalesce(tunq.unique_cols, '[]'::jsonb) as unique_cols,
coalesce(cols_agg.columns, '{}') as columns
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_description d on d.objoid = c.oid and d.objsubid = 0 and d.classoid = 'pg_class'::regclass
LEFT JOIN tbl_pk_cols tpks ON c.oid = tpks.relid
LEFT JOIN tbl_unique_cols tunq ON c.oid = tunq.relid
LEFT JOIN columns_agg cols_agg ON c.oid = cols_agg.relid
WHERE c.relkind IN ('v','r','m','f','p')
AND c.relnamespace NOT IN ('pg_catalog'::regnamespace, 'information_schema'::regnamespace)
@@ -810,6 +842,88 @@ allComputedRels =
column HD.bool <*>
column HD.bool
-- | A row-level security policy of an exposed table, gathered from pg_policies.
data RlsPolicyRow = RlsPolicyRow
{ rlsTable :: QualifiedIdentifier
, rlsRowSec :: Bool
, rlsCmd :: Text -- ^ "w" (UPDATE), "d" (DELETE), "*" (ALL)
, rlsPermiss :: Bool
, rlsQual :: Maybe Text -- ^ USING qualifier; Nothing means the policy has no USING restriction
}
-- | Returns the UPDATE/DELETE RLS policies of the exposed tables, so that the
-- planner can surface per-row can_edit/can_delete fields on SELECT *.
allRlsPolicies :: SQL.Statement AppConfig [RlsPolicyRow]
allRlsPolicies =
SQL.Statement sql params decodeRlsPolicies True
where
params = map escapeIdent . toList . configDbSchemas >$< arrayParam HE.text
sql = encodeUtf8 [trimming|
SELECT
n.nspname::text AS table_schema,
c.relname::text AS table_name,
c.relrowsecurity AS row_security,
p.polcmd::text AS cmd,
p.polpermissive AS permissive,
pg_get_expr(p.polqual, p.polrelid) AS qual
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
JOIN pg_policy p ON p.polrelid = c.oid
WHERE c.relkind IN ('r','p')
AND c.relnamespace = ANY($$1::regnamespace[])
AND p.polcmd IN ('w','d','*')
ORDER BY n.nspname, c.relname|]
decodeRlsPolicies :: HD.Result [RlsPolicyRow]
decodeRlsPolicies =
HD.rowList rlsRow
where
rlsRow = RlsPolicyRow
<$> (QualifiedIdentifier <$> column HD.text <*> column HD.text)
<*> column HD.bool
<*> column HD.text
<*> column HD.bool
<*> nullableColumn HD.text
-- | Combines the per-command RLS policies of each table into the SQL expression
-- used to compute can_edit/can_delete. Following PostgreSQL semantics, multiple
-- permissive policies combine with OR and restrictive ones with AND.
combineRlsPolicies :: [RlsPolicyRow] -> HM.HashMap QualifiedIdentifier (Maybe Text, Maybe Text)
combineRlsPolicies rows = HM.fromList $ mapMaybe toEntry $ HM.toList byTable
where
byTable = HM.fromListWith (<>) [ (rlsTable r, [r]) | r <- rows ]
toEntry (qi, rs)
| not (any rlsRowSec rs) = Nothing
| isNothing editQ && isNothing delQ = Nothing
| otherwise = Just (qi, (editQ, delQ))
where
editQ = combineCmd ["w", "*"] rs
delQ = combineCmd ["d", "*"] rs
combineCmd cmds rs
| null policies = Nothing
| null permissiveQs = Nothing
| otherwise = Just $ wrapQual combined
where
policies = [ r | r <- rs, rlsCmd r `elem` cmds ]
permissiveQs = [ fromMaybe "true" (rlsQual r) | r <- policies, rlsPermiss r ]
restrictiveQs = [ fromMaybe "true" (rlsQual r) | r <- policies, not (rlsPermiss r) ]
permissiveExpr = T.intercalate " OR " permissiveQs
combined = case restrictiveQs of
[] -> permissiveExpr
_ -> "(" <> permissiveExpr <> ") AND (" <> T.intercalate " AND " restrictiveQs <> ")"
wrapQual q = "COALESCE(" <> q <> ", false)"
-- | Attaches the combined RLS qualifiers to the corresponding tables.
addRlsQuals :: TablesMap -> HM.HashMap QualifiedIdentifier (Maybe Text, Maybe Text) -> TablesMap
addRlsQuals tabs rlsMap = HM.mapWithKey setRls tabs
where
setRls qi tbl = case HM.lookup qi rlsMap of
Nothing -> tbl
Just (editQ, delQ) -> tbl { tableRlsEditQual = editQ, tableRlsDeleteQual = delQ }
-- | Returns all the views' primary keys and foreign keys dependencies
allViewsKeyDependencies :: SQL.Statement AppConfig [ViewKeyDependency]
allViewsKeyDependencies =
@@ -1150,15 +1264,15 @@ extractTimings = SQL.Statement sql HE.noParams decodeThem True
qFrag setting = "extract('milliseconds' from current_setting('pgrst." <> setting <> "', false)::interval)::text"
sql = "SELECT " <> BS.intercalate ","
[ qFrag gucTbls, qFrag gucKDeps, qFrag gucRels
, qFrag gucFuncs, qFrag gucCRels, qFrag gucDReps
, qFrag gucMHdrs
, qFrag gucFuncs, qFrag gucCRels, qFrag gucRLS
, qFrag gucDReps, qFrag gucMHdrs
]
decodeThem :: HD.Result QueryTimings
decodeThem = HD.singleRow $
QueryTimings
<$> column HD.text <*> column HD.text <*> column HD.text
<*> column HD.text <*> column HD.text <*> column HD.text
<*> column HD.text
<*> column HD.text <*> column HD.text
data QueryTimings = QueryTimings
{ qtTables :: Text
@@ -1166,6 +1280,7 @@ data QueryTimings = QueryTimings
, qtRels :: Text
, qtFuncs :: Text
, qtCRels :: Text
, qtRls :: Text
, qtDReps :: Text
, qtMHdrs :: Text
} deriving (Show)
@@ -1177,15 +1292,17 @@ queryTimingsWLabels qt =
, (gucRels, qtRels qt)
, (gucFuncs, qtFuncs qt)
, (gucCRels, qtCRels qt)
, (gucRLS, qtRls qt)
, (gucDReps, qtDReps qt)
, (gucMHdrs, qtMHdrs qt)
]
gucTbls, gucKDeps, gucRels, gucFuncs, gucCRels, gucDReps, gucMHdrs :: ByteString
gucTbls, gucKDeps, gucRels, gucFuncs, gucCRels, gucRLS, gucDReps, gucMHdrs :: ByteString
gucTbls = "tables"
gucKDeps = "keydeps"
gucRels = "rels"
gucFuncs = "funcs"
gucCRels = "comprels"
gucRLS = "rls"
gucDReps = "dreps"
gucMHdrs = "mhandlers"
@@ -29,7 +29,18 @@ data Table = Table
, tableUpdatable :: Bool
, tableDeletable :: Bool
, tablePKCols :: [FieldName]
-- ^ Each element is the position-ordered column list of a unique
-- constraint. A single-column unique constraint is represented by a
-- single-element list.
, tableUniqueCols :: [[FieldName]]
, tableColumns :: ColumnMap
, tableRlsEditQual :: Maybe Text
-- ^ Combined RLS UPDATE USING qualifier, COALESCE-wrapped. Nothing means
-- the table has no row-level edit restriction to surface (RLS disabled or
-- no applicable UPDATE policy).
, tableRlsDeleteQual :: Maybe Text
-- ^ Combined RLS DELETE USING qualifier, COALESCE-wrapped. Nothing means
-- the table has no row-level delete restriction to surface.
}
deriving (Show, Generic, JSON.ToJSON)
@@ -7,7 +7,44 @@
tableIsView: false
tableName: authors_only
tablePKCols: []
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: no_rls_items
qiSchema: public
- tableColumns:
id:
colDefault: null
colDescription: null
colEnum: []
colMaxLen: null
colName: id
colNominalType: integer
colNullable: false
colType: integer
name:
colDefault: null
colDescription: null
colEnum: []
colMaxLen: null
colName: name
colNominalType: text
colNullable: true
colType: text
tableDeletable: true
tableDescription: null
tableInsertable: true
tableIsView: false
tableName: no_rls_items
tablePKCols:
- id
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: cats
@@ -38,7 +75,55 @@
tableName: cats
tablePKCols:
- id
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: rls_items
qiSchema: public
- tableColumns:
account_id:
colDefault: null
colDescription: null
colEnum: []
colMaxLen: null
colName: account_id
colNominalType: bigint
colNullable: true
colType: bigint
id:
colDefault: null
colDescription: null
colEnum: []
colMaxLen: null
colName: id
colNominalType: integer
colNullable: false
colType: integer
name:
colDefault: null
colDescription: null
colEnum: []
colMaxLen: null
colName: name
colNominalType: text
colNullable: true
colType: text
tableDeletable: true
tableDescription: null
tableInsertable: true
tableIsView: false
tableName: rls_items
tablePKCols:
- id
tableRlsDeleteQual: COALESCE(((((current_setting('request.jwt.claims'::text, true))::json
->> 'account_id'::text))::bigint = account_id), false)
tableRlsEditQual: COALESCE(((((current_setting('request.jwt.claims'::text, true))::json
->> 'account_id'::text))::bigint = account_id), false)
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: items_w_isolation_level
@@ -68,7 +153,10 @@
tableIsView: true
tableName: items_w_isolation_level
tablePKCols: []
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: directors
@@ -99,7 +187,10 @@
tableName: directors
tablePKCols:
- id
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: projects
@@ -111,7 +202,10 @@
tableIsView: false
tableName: projects
tablePKCols: []
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: infinite_recursion
@@ -123,7 +217,10 @@
tableIsView: true
tableName: infinite_recursion
tablePKCols: []
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: false
- - qiName: awards
@@ -181,7 +278,10 @@
tableName: awards
tablePKCols:
- id
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: films
@@ -221,7 +321,10 @@
tableName: films
tablePKCols:
- id
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
- - qiName: items
@@ -242,5 +345,8 @@
tableIsView: false
tableName: items
tablePKCols: []
tableRlsDeleteQual: null
tableRlsEditQual: null
tableSchema: public
tableUniqueCols: []
tableUpdatable: true
+5
View File
@@ -7,3 +7,8 @@ GRANT SELECT ON directors, films, awards TO postgrest_test_anonymous, postgrest_
GRANT ALL ON cats TO postgrest_test_anonymous;
GRANT ALL ON items_w_isolation_level TO postgrest_test_anonymous, postgrest_test_repeatable_read, postgrest_test_serializable;
GRANT SELECT ON rls_items TO postgrest_test_author;
GRANT UPDATE(name) ON rls_items TO postgrest_test_author;
GRANT DELETE ON rls_items TO postgrest_test_author;
GRANT SELECT ON no_rls_items TO postgrest_test_author;
+31
View File
@@ -268,3 +268,34 @@ $$ language sql;
create or replace function get_work_mem() returns text as $$
select current_setting('work_mem', true);
$$ language sql;
-- RLS fixtures for testing can_edit/can_delete computed fields
create table rls_items(
id int primary key,
account_id bigint,
name text
);
alter table rls_items enable row level security;
create policy rls_items_select on rls_items for select
using (
account_id is null
or (current_setting('request.jwt.claims', true)::json ->> 'account_id')::bigint = account_id
);
create policy rls_items_update on rls_items for update
using ((current_setting('request.jwt.claims', true)::json ->> 'account_id')::bigint = account_id);
create policy rls_items_delete on rls_items for delete
using ((current_setting('request.jwt.claims', true)::json ->> 'account_id')::bigint = account_id);
insert into rls_items(id, account_id, name) values (1, 1, 'own'), (2, null, 'public'), (3, 2, 'other');
-- no RLS at all: can_edit/can_delete must be omitted
create table no_rls_items(
id int primary key,
name text
);
insert into no_rls_items(id, name) values (1, 'a'), (2, 'b');
-221
View File
@@ -1,221 +0,0 @@
"Auth related IO tests for PostgREST"
from operator import attrgetter
import signal
import pytest
from config import BASEDIR, CONFIGSDIR, FIXTURES, SECRET
from util import authheader, jwtauthheader
from postgrest import (
run,
sleep_until_postgrest_config_reload,
sleep_until_postgrest_scache_reload,
wait_until_exit,
)
@pytest.mark.parametrize(
"secretpath",
[path for path in (BASEDIR / "secrets").iterdir() if path.suffix != ".jwt"],
ids=attrgetter("name"),
)
def test_read_secret_from_file(secretpath, defaultenv):
"Authorization should succeed when the secret is read from a file."
env = {**defaultenv, "PGRST_JWT_SECRET": f"@{secretpath}"}
if secretpath.suffix == ".b64":
env["PGRST_JWT_SECRET_IS_BASE64"] = "true"
secret = secretpath.read_bytes()
headers = authheader(secretpath.with_suffix(".jwt").read_text())
with run(stdin=secret, env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
print(response.text)
assert response.status_code == 200
def test_read_secret_from_stdin(defaultenv):
"Authorization should succeed when the secret is read from stdin."
env = {**defaultenv, "PGRST_DB_CONFIG": "false", "PGRST_JWT_SECRET": "@/dev/stdin"}
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
with run(stdin=SECRET.encode(), env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
print(response.text)
assert response.status_code == 200
# TODO: This test would fail right now, because of
# https://github.com/PostgREST/postgrest/issues/2126
@pytest.mark.skip
def test_read_secret_from_stdin_dbconfig(defaultenv):
"Authorization should succeed when the secret is read from stdin with db-config=true."
env = {**defaultenv, "PGRST_DB_CONFIG": "true", "PGRST_JWT_SECRET": "@/dev/stdin"}
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
with run(stdin=SECRET.encode(), env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
print(response.text)
assert response.status_code == 200
def test_fail_with_invalid_password(defaultenv):
"Connecting with an invalid password should fail without retries."
uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
env = {**defaultenv, "PGRST_DB_URI": uri}
with run(env=env, wait_for=None) as postgrest:
exitCode = wait_until_exit(postgrest)
assert exitCode == 1
@pytest.mark.parametrize(
"roleclaim", FIXTURES["roleclaims"], ids=lambda claim: claim["key"]
)
def test_role_claim_key(roleclaim, defaultenv):
"Authorization should depend on a correct role-claim-key and JWT claim."
env = {
**defaultenv,
"PGRST_JWT_ROLE_CLAIM_KEY": roleclaim["key"],
"PGRST_JWT_SECRET": SECRET,
}
headers = jwtauthheader(roleclaim["data"], SECRET)
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == roleclaim["expected_status"]
@pytest.mark.parametrize(
"jwtaudroleclaim",
FIXTURES["jwtaudroleclaims"],
ids=lambda claim: claim["key"] + "_" + str(claim["expected_status"]),
)
def test_jwt_aud_in_role_claim_key(jwtaudroleclaim, defaultenv):
"Allows authorization with JWT aud claim in role-claim-key"
env = {
**defaultenv,
"PGRST_JWT_AUD": "postgrest_test_author",
"PGRST_JWT_ROLE_CLAIM_KEY": jwtaudroleclaim["key"],
"PGRST_JWT_SECRET": SECRET,
}
headers = jwtauthheader(jwtaudroleclaim["data"], SECRET)
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == jwtaudroleclaim["expected_status"]
def test_jwt_secret_reload(tmp_path, defaultenv):
"JWT secret should be reloaded from file when PostgREST is sent SIGUSR2."
config = (CONFIGSDIR / "sigusr2-settings.config").read_text()
configfile = tmp_path / "test.config"
configfile.write_text(config)
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
with run(configfile, env=defaultenv) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
# change setting
configfile.write_text(config.replace("invalid" * 5, SECRET))
# reload config
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 200
def test_jwt_secret_external_file_reload(tmp_path, defaultenv):
"JWT secret external file should be reloaded when PostgREST is sent a SIGUSR2 or a NOTIFY."
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
external_secret_file = tmp_path / "jwt-secret-config"
external_secret_file.write_text("invalid" * 5)
env = {
**defaultenv,
"PGRST_JWT_SECRET": f"@{external_secret_file}",
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGRST_DB_CONFIG": "false",
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous", # required for NOTIFY
}
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
# change external file
external_secret_file.write_text(SECRET)
# SIGUSR1 doesn't reload external files, at least when db-config=false
postgrest.process.send_signal(signal.SIGUSR1)
sleep_until_postgrest_scache_reload()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
# reload config and external file with SIGUSR2
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 200
# change external file to wrong value again
external_secret_file.write_text("invalid" * 5)
# reload config and external file with NOTIFY
response = postgrest.session.post("/rpc/reload_pgrst_config")
assert response.text == ""
assert response.status_code == 204
sleep_until_postgrest_config_reload()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
def test_invalidate_jwt_cache_when_secret_changes(tmp_path, defaultenv):
"JWT cache should be emptied after jwt-secret is changed in a config reload"
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
external_secret_file = tmp_path / "jwt-secret-config"
external_secret_file.write_text(SECRET)
env = {
**defaultenv,
"PGRST_JWT_SECRET": f"@{external_secret_file}",
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGRST_JWT_CACHE_MAX_ENTRIES": "86400", # enable cache
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous", # required for NOTIFY
}
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 200 # jwt gets cached
# change external file
external_secret_file.write_text("invalid" * 5)
# reload config and external file with NOTIFY
# jwt-cache should get empty
response = postgrest.session.post("/rpc/reload_pgrst_config")
assert response.text == ""
assert response.status_code == 204
sleep_until_postgrest_config_reload()
# now the request should fail because the cached token is removed
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
+251
View File
@@ -0,0 +1,251 @@
"Test PostgREST configuration related behavior"
import time
import pytest
from operator import attrgetter
from config import BASEDIR, FIXTURES, SECRET
from util import (
Thread,
authheader,
jwtauthheader,
)
from postgrest import (
PostgrestTimedOut,
freeport,
run,
)
def test_pool_size(defaultenv, metapostgrest):
"Verify that PGRST_DB_POOL setting allows the correct number of parallel requests"
env = {
**defaultenv,
"PGRST_DB_POOL": "2",
}
with run(env=env) as postgrest:
start = time.time()
threads = []
for i in range(4):
def sleep(i=i):
response = postgrest.session.get("/rpc/sleep?seconds=0.5")
assert response.text == ""
assert response.status_code == 204, "thread {}".format(i)
t = Thread(target=sleep)
t.start()
threads.append(t)
for t in threads:
t.join()
end = time.time()
delta = end - start
# sleep 4 times for 0.5s each, with 2 requests in parallel
# => total time roughly 1s
assert delta > 1 and delta < 1.5
@pytest.mark.parametrize(
"secretpath",
[path for path in (BASEDIR / "secrets").iterdir() if path.suffix != ".jwt"],
ids=attrgetter("name"),
)
def test_read_secret_from_file(secretpath, defaultenv):
"Authorization should succeed when the secret is read from a file."
env = {**defaultenv, "PGRST_JWT_SECRET": f"@{secretpath}"}
if secretpath.suffix == ".b64":
env["PGRST_JWT_SECRET_IS_BASE64"] = "true"
secret = secretpath.read_bytes()
headers = authheader(secretpath.with_suffix(".jwt").read_text())
with run(stdin=secret, env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
print(response.text)
assert response.status_code == 200
def test_read_secret_from_stdin(defaultenv):
"Authorization should succeed when the secret is read from stdin."
env = {**defaultenv, "PGRST_DB_CONFIG": "false", "PGRST_JWT_SECRET": "@/dev/stdin"}
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
with run(stdin=SECRET.encode(), env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
print(response.text)
assert response.status_code == 200
# TODO: This test would fail right now, because of
# https://github.com/PostgREST/postgrest/issues/2126
@pytest.mark.skip
def test_read_secret_from_stdin_dbconfig(defaultenv):
"Authorization should succeed when the secret is read from stdin with db-config=true."
env = {**defaultenv, "PGRST_DB_CONFIG": "true", "PGRST_JWT_SECRET": "@/dev/stdin"}
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
with run(stdin=SECRET.encode(), env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
print(response.text)
assert response.status_code == 200
@pytest.mark.parametrize(
"roleclaim", FIXTURES["roleclaims"], ids=lambda claim: claim["key"]
)
def test_role_claim_key(roleclaim, defaultenv):
"Authorization should depend on a correct role-claim-key and JWT claim."
env = {
**defaultenv,
"PGRST_JWT_ROLE_CLAIM_KEY": roleclaim["key"],
"PGRST_JWT_SECRET": SECRET,
}
headers = jwtauthheader(roleclaim["data"], SECRET)
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == roleclaim["expected_status"]
@pytest.mark.parametrize(
"jwtaudroleclaim",
FIXTURES["jwtaudroleclaims"],
ids=lambda claim: claim["key"] + "_" + str(claim["expected_status"]),
)
def test_jwt_aud_in_role_claim_key(jwtaudroleclaim, defaultenv):
"Allows authorization with JWT aud claim in role-claim-key"
env = {
**defaultenv,
"PGRST_JWT_AUD": "postgrest_test_author",
"PGRST_JWT_ROLE_CLAIM_KEY": jwtaudroleclaim["key"],
"PGRST_JWT_SECRET": SECRET,
}
headers = jwtauthheader(jwtaudroleclaim["data"], SECRET)
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == jwtaudroleclaim["expected_status"]
def test_random_port_bound(defaultenv):
"PostgREST should bind to a random port when PGRST_SERVER_PORT is 0."
with run(env=defaultenv, port="0"):
assert True # liveness check is done by run(), so we just need to check that it doesn't fail
def test_so_reuseport_defaults_to_false(defaultenv):
"A second PostgREST instance should not bind to the same port by default."
host = "0.0.0.0"
port = freeport()
admin_port = freeport(used_ports=[port])
with run(
env={**defaultenv},
port=port,
host=host,
admin_port=admin_port,
):
with pytest.raises(PostgrestTimedOut):
with run(
env={**defaultenv},
port=port,
host=host,
admin_port=freeport(used_ports=[port, admin_port]),
wait_max_seconds=1,
):
pass
def test_schema_cache_startup_load_with_in_db_config(defaultenv, metapostgrest):
"verify that the Schema Cache loads correctly at startup, using the in-db `pgrst.db_schemas` config"
response = metapostgrest.session.post("/rpc/change_db_schemas_config")
assert response.text == ""
assert response.status_code == 204
with run(env=defaultenv) as postgrest:
response = postgrest.session.get("/rpc/get_current_schema")
assert response.text == '"test"'
assert response.status_code == 200
response = metapostgrest.session.post("/rpc/reset_db_schemas_config")
assert response.text == ""
assert response.status_code == 204
def test_allow_configs_to_be_set_to_empty(defaultenv):
'configs that are explicitly set to empty (= "<empty>") should not throw parse error'
env = {
**defaultenv,
"PGRST_DB_EXTRA_SEARCH_PATH": "",
}
with run(env=env) as postgrest:
response = postgrest.session.get("/projects")
assert response.status_code == 200
def test_connection_error_message_does_not_claim_retry(defaultenv):
"The connection error message should not claim retrying, since PostgREST stops on fatal errors."
uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
env = {**defaultenv, "PGRST_DB_URI": uri}
with run(env=env, no_startup_stdout=False, wait_for=None) as postgrest:
output = postgrest.read_stdout(nlines=8)
assert any('"message":"Database connection error."' in line for line in output)
def test_db_pre_config_with_non_existent_function(defaultenv):
"Log error when db-pre-config is set to non-existent function"
env = {
**defaultenv,
"PGRST_DB_PRE_CONFIG": "select", # no "select" function in our fixtures, fail gracefully at startup
}
with run(env=env, no_startup_stdout=False, wait_for=None) as postgrest:
output = postgrest.read_stdout(nlines=8)
assert any("function select() does not exist" in line for line in output)
@pytest.mark.parametrize("enabled", ["true", "false"])
def test_use_legacy_target_names(enabled, defaultenv):
"Show a warning when a target name is used instead of an alias, only when config is enabled"
env = {
**defaultenv,
"PGRST_URL_USE_LEGACY_TARGET_NAMES": enabled,
}
with run(env=env) as postgrest:
response = postgrest.session.get(
"/directors?select=name,all_films:films(title),awards_2026:awards(name)&films.order=title&awards.year=eq.2026"
)
output = postgrest.read_stdout(nlines=10)
log_err_warning = "WARNING: Embedded resource was referenced by relation name even though it has an alias. This is deprecated and will stop working in a future release."
log_err_hint = "Update filters, orders or limits that use `films` to `all_films`, `awards` to `awards_2026` in `GET /directors?select=name,all_films:films(title),awards_2026:awards(name)&films.order=title&awards.year=eq.2026`"
has_warning_log = any(log_err_warning in line for line in output)
has_hint_log = any(log_err_hint in line for line in output)
if enabled == "true":
assert response.status_code == 200
assert has_warning_log and has_hint_log
else:
assert response.status_code == 400
assert not has_warning_log and not has_hint_log
+317
View File
@@ -0,0 +1,317 @@
"Tests related to PostgREST connection and connections pools"
import os
import re
import signal
import time
import pytest
from config import SECRET
from util import (
Thread,
jwtauthheader,
drain_stdout,
)
from postgrest import (
Admin,
run,
run_pgproxy,
wait_until_exit,
)
def test_fail_with_invalid_password(defaultenv):
"Connecting with an invalid password should fail without retries."
uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
env = {**defaultenv, "PGRST_DB_URI": uri}
with run(env=env, wait_for=None) as postgrest:
exitCode = wait_until_exit(postgrest)
assert exitCode == 1
def test_connect_with_dburi(dburi, defaultenv):
"Connecting with db-uri instead of LIPQ* environment variables should work."
defaultenv_without_libpq = {
key: value
for key, value in defaultenv.items()
if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
}
env = {**defaultenv_without_libpq, "PGRST_DB_URI": dburi.decode()}
with run(env=env):
pass
@pytest.mark.parametrize("dburi_type", ["no_params", "no_params_qmark", "with_params"])
def test_get_pgrst_version_with_uri_connection_string(dburi_type, dburi, defaultenv):
"The fallback_application_name should be added to the db-uri if it has a URI format"
defaultenv_without_libpq = {
key: value
for key, value in defaultenv.items()
if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
}
env = {
"no_params": {**defaultenv, "PGRST_DB_URI": "postgresql://"},
"no_params_qmark": {**defaultenv, "PGRST_DB_URI": "postgresql://?"},
"with_params": {**defaultenv_without_libpq, "PGRST_DB_URI": dburi.decode()},
}
with run(env=env[dburi_type]) as postgrest:
response = postgrest.session.post("/rpc/get_pgrst_version")
version = '"%s"' % response.headers["Server"].replace(
"postgrest/", "PostgREST "
)
assert response.text == version
def test_get_pgrst_version_with_keyval_connection_string(defaultenv):
"The fallback_application_name should be added to the db-uri if it has a keyword/value format"
uri = f'dbname={defaultenv["PGDATABASE"]} host={defaultenv["PGHOST"]} user={defaultenv["PGUSER"]}'
defaultenv_without_libpq = {
key: value
for key, value in defaultenv.items()
if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
}
env = {**defaultenv_without_libpq, "PGRST_DB_URI": uri}
with run(env=env) as postgrest:
response = postgrest.session.post("/rpc/get_pgrst_version")
version = '"%s"' % response.headers["Server"].replace(
"postgrest/", "PostgREST "
)
assert response.text == version
def test_fail_with_invalid_dbname_and_automatic_recovery_disabled(defaultenv):
"Should fail without retries when automatic recovery is disabled and dbname is invalid"
dbname = "INVALID"
uri = f'postgresql://?dbname={dbname}&host={defaultenv["PGHOST"]}&user={defaultenv["PGUSER"]}'
env = {
**defaultenv,
"PGRST_DB_URI": uri,
"PGRST_DB_POOL_AUTOMATIC_RECOVERY": "false",
}
with run(env=env, wait_for=None) as postgrest:
exitCode = wait_until_exit(postgrest)
assert exitCode == 1
def test_fail_with_automatic_recovery_disabled_and_terminated_using_query(defaultenv):
"Should fail without retries when automatic recovery is disabled and pg_terminate_backend(pid) is called"
env = {
**defaultenv,
"PGRST_DB_POOL_AUTOMATIC_RECOVERY": "false",
"PGAPPNAME": "target",
}
app_name = "'{}'".format(env["PGAPPNAME"])
with run(env=env) as postgrest:
os.system(
f'psql -d {env["PGDATABASE"]} -U {env["PGUSER"]} -h {env["PGHOST"]} --set ON_ERROR_STOP=1 -a -c "SELECT terminate_pgrst({app_name})"'
)
exitCode = wait_until_exit(postgrest)
assert exitCode == 1
def test_read_dburi_from_stdin_without_eol(dburi, defaultenv):
"Reading the dburi from stdin with a single line should work."
defaultenv_without_libpq = {
key: value
for key, value in defaultenv.items()
if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
}
env = {**defaultenv_without_libpq, "PGRST_DB_URI": "@/dev/stdin"}
with run(env=env, stdin=dburi):
pass
def test_read_dburi_from_stdin_with_eol(dburi, defaultenv):
"Reading the dburi from stdin containing a newline should work."
defaultenv_without_libpq = {
key: value
for key, value in defaultenv.items()
if key not in ["PGDATABASE", "PGHOST", "PGUSER"]
}
env = {**defaultenv_without_libpq, "PGRST_DB_URI": "@/dev/stdin"}
with run(env=env, stdin=dburi + b"\n"):
pass
def test_flush_pool_no_interrupt(defaultenv):
"Flushing the pool via SIGUSR1 doesn't interrupt ongoing requests"
with run(env=defaultenv) as postgrest:
def sleep():
response = postgrest.session.get("/rpc/sleep?seconds=0.5")
assert response.text == ""
assert response.status_code == 204
t = Thread(target=sleep)
t.start()
# make sure the request has started
time.sleep(0.1)
# SIGUSR1 causes the postgres connection pool to be flushed
postgrest.process.send_signal(signal.SIGUSR1)
t.join()
def test_no_pool_connection_required_on_bad_http_logic(defaultenv):
"no pool connection should be consumed for failing on invalid http logic"
with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
# not found nested route shouldn't require opening a connection
response = postgrest.session.head("/path/notfound")
assert response.status_code == 404
# an invalid http method on a resource shouldn't require opening a connection
response = postgrest.session.request("TRACE", "/projects")
assert response.status_code == 405
response = postgrest.session.patch("/rpc/hello")
assert response.status_code == 405
def test_no_pool_connection_required_on_options(defaultenv):
"no pool connection should be consumed for OPTIONS requests"
with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
# OPTIONS on a table shouldn't require opening a connection
response = postgrest.session.options("/projects")
assert response.status_code == 200
# OPTIONS on RPC shouldn't require opening a connection
response = postgrest.session.options("/rpc/hello")
assert response.status_code == 200
# OPTIONS on root shouldn't require opening a connection
response = postgrest.session.options("/")
assert response.status_code == 200
def test_no_pool_connection_required_on_bad_jwt_claim(defaultenv):
"no pool connection should be consumed for failing on invalid jwt"
env = {**defaultenv, "PGRST_JWT_SECRET": SECRET}
with run(env=env, no_pool_connection_available=True) as postgrest:
# A JWT with an invalid signature shouldn't open a connection
wrong_secret = "This is the most wrong secret of all secrets"
headers = jwtauthheader({"role": "postgrest_test_author"}, wrong_secret)
response = postgrest.session.get("/projects", headers=headers)
assert response.status_code == 401
def test_no_pool_connection_required_on_bad_embedding(defaultenv):
"no pool connection should be consumed for failing to embed"
with run(env=defaultenv, no_pool_connection_available=True) as postgrest:
# OPTIONS on a table shouldn't require opening a connection
response = postgrest.session.get("/projects?select=*,unexistent(*)")
assert response.status_code == 400
@pytest.mark.parametrize("level", ["crit", "error", "warn", "info", "debug"])
def test_pool_acquisition_timeout(level, defaultenv, metapostgrest):
"Verify that PGRST_DB_POOL_ACQUISITION_TIMEOUT times out when the pool is empty"
env = {
**defaultenv,
"PGRST_DB_POOL": "1",
"PGRST_DB_POOL_ACQUISITION_TIMEOUT": "1", # 1 second
"PGRST_LOG_LEVEL": level,
}
with run(
env=env, no_pool_connection_available=True, wait_max_seconds=3
) as postgrest:
response = postgrest.session.get("/projects")
assert response.status_code == 504
data = response.json()
assert data["message"] == "Timed out acquiring connection from connection pool."
# ensure the message appears on the logs as well
output = sorted(drain_stdout(postgrest))
if level == "crit":
assert len(output) == 0
else:
assert any(" 504 " in line for line in output)
assert any(
"Timed out acquiring connection from connection pool." in line
for line in output
)
def test_pool_acquisition_timeout_logs_are_debounced(defaultenv):
"Pool acquisition timeout diagnostic logs should be debounced over a burst of failures"
env = {
**defaultenv,
"PGRST_DB_POOL": "1",
"PGRST_DB_POOL_ACQUISITION_TIMEOUT": "1",
"PGRST_LOG_LEVEL": "error",
}
total_requests = 6
with run(
env=env, no_pool_connection_available=True, wait_max_seconds=3
) as postgrest:
def request_timeout():
response = postgrest.session.get("/projects")
assert response.status_code == 504
assert (
response.json()["message"]
== "Timed out acquiring connection from connection pool."
)
return response
request_timeout()
threads = [Thread(target=request_timeout) for _ in range(total_requests - 1)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# Logger debouncing logs the first timeout immediately and, if more
# timeouts happen during the cooldown, logs one more time afterwards.
time.sleep(6)
output = drain_stdout(postgrest)
access_logs = [line for line in output if ' "GET /projects HTTP/1.1" 504 ' in line]
timeout_logs = [
line
for line in output
if "Timed out acquiring connection from connection pool." in line
]
assert len(access_logs) == total_requests
assert len(timeout_logs) == 2
def test_positive_pool_metric(defaultenv):
"When a network failure is caused on the pg connection, pgrst_db_pool_available stays positive"
with run_pgproxy(defaultenv, proxy_timeout="1ms") as pgproxyhost:
env = {**defaultenv, "PGHOST": pgproxyhost}
with run(env=env, wait_for=Admin.live) as postgrest:
response = postgrest.admin.get("/metrics", timeout=1)
assert response.status_code == 200
metrics = float(
re.search(
r"pgrst_db_pool_available (-?\d+(?:\.\d+)?)", response.text
).group(1)
)
assert metrics >= 0
+25
View File
@@ -0,0 +1,25 @@
import time
from util import Thread
from postgrest import run
def test_graceful_shutdown_waits_for_in_flight_request(defaultenv):
"SIGTERM should allow in-flight requests to finish before exiting"
with run(env=defaultenv, wait_max_seconds=5) as postgrest:
def sleep():
response = postgrest.session.get("/rpc/sleep?seconds=3", timeout=10)
assert response.text == ""
assert response.status_code == 204
t = Thread(target=sleep)
t.start()
# Wait for the request to be in-flight before shutting down.
time.sleep(1)
postgrest.process.terminate()
t.join()
-1555
View File
File diff suppressed because it is too large Load Diff
+621
View File
@@ -0,0 +1,621 @@
"Test PostgREST logs and observations"
import re
import signal
import time
import pytest
import requests
from config import SECRET
from util import (
jwtauthheader,
relativeSeconds,
drain_stdout,
match_log,
)
from postgrest import (
Admin,
freeport,
is_ipv6,
reset_statement_timeout,
run,
set_statement_timeout,
wait_until_exit,
)
@pytest.mark.parametrize("level", ["crit", "error", "warn", "info", "debug"])
def test_log_level(level, defaultenv):
"log_level should filter request logging"
env = {**defaultenv, "PGRST_LOG_LEVEL": level}
# any token to test 500 response for "Server lacks JWT secret"
claim = {"role": "postgrest_test_author"}
headers = jwtauthheader(claim, SECRET)
with run(env=env) as postgrest:
response = postgrest.session.get("/", headers=headers)
assert response.status_code == 500
response = postgrest.session.get("/unknown")
assert response.status_code == 404
response = postgrest.session.get("/")
assert response.status_code == 200
output = drain_stdout(postgrest)
if level == "crit":
assert len(output) == 0
elif level == "error":
match_log(
output,
[r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"'],
)
assert len(output) == 1
elif level == "warn":
match_log(
output,
[
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
],
)
assert len(output) == 2
elif level == "info":
match_log(
output,
[
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
],
)
assert len(output) == 3
elif level == "debug":
match_log(
output,
[
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
],
)
assert len(output) > 3
assert any("Connection" and "is available" in line for line in output)
assert any("Connection" and "is used" in line for line in output)
@pytest.mark.parametrize("level", ["crit", "error", "warn", "info", "debug"])
def test_log_query(level, defaultenv):
"log_query=true should log the SQL query according to the log_level"
env = {
**defaultenv,
"PGRST_LOG_LEVEL": level,
"PGRST_LOG_QUERY": "true",
}
with run(env=env) as postgrest:
response = postgrest.session.get("/")
assert response.status_code == 200
response = postgrest.session.get("/projects")
assert response.status_code == 200
response = postgrest.session.get(
"/projects", headers={"Prefer": "count=estimated"}
)
assert response.status_code == 200
response = postgrest.session.get(
"/projects", headers={"Prefer": "count=planned"}
)
assert response.status_code == 200
response = postgrest.session.get("/infinite_recursion")
assert response.status_code == 500
get_2xx_regx = r'.+: WITH pgrst_source AS.+SELECT "public"\."projects"\.\* FROM "public"\."projects".+_postgrest_t'
get_2xx_count_regx = (
r'.+: EXPLAIN \(FORMAT JSON\) SELECT 1 FROM "public"."projects"'
)
infinite_recursion_5xx_regx = r'.+: WITH pgrst_source AS.+SELECT "public"\."infinite_recursion"\.\* FROM "public"\."infinite_recursion".+_postgrest_t'
root_tables_regx = r".+: SELECT n.nspname AS table_schema, .+ FROM pg_class c .+ ORDER BY table_schema, table_name"
root_procs_regx = r".+: WITH.+base_types AS.+pn\.nspname AS proc_schema.+FROM pg_proc p.+p\.pronamespace = \$1::regnamespace"
root_descr_regx = r".+: SELECT pg_catalog\.obj_description\(\$1::regnamespace, 'pg_namespace'\)"
set_config_regx = (
r".+: select set_config\('search_path', \$1, true\), set_config\("
)
output = drain_stdout(postgrest)
project_queries = [line for line in output if re.match(get_2xx_regx, line)]
project_counts = [line for line in output if re.match(get_2xx_count_regx, line)]
infinite_queries = [
line for line in output if re.match(infinite_recursion_5xx_regx, line)
]
root_tables = [line for line in output if re.match(root_tables_regx, line)]
root_procs = [line for line in output if re.match(root_procs_regx, line)]
root_descr = [line for line in output if re.match(root_descr_regx, line)]
set_configs = [line for line in output if re.match(set_config_regx, line)]
if level == "crit":
assert not set_configs
assert not project_queries
assert not project_counts
assert not infinite_queries
assert not root_tables
assert not root_procs
assert not root_descr
elif level in {"error", "warn"}:
assert len(set_configs) == 1
assert len(infinite_queries) == 1
assert not project_queries
assert not project_counts
assert not root_tables
assert not root_procs
assert not root_descr
elif level == "info":
assert len(set_configs) == 5
assert len(project_queries) == 3
assert len(project_counts) == 2
assert len(infinite_queries) == 1
assert len(root_tables) == 1
assert len(root_procs) == 1
assert len(root_descr) == 1
elif level == "debug":
assert len(set_configs) == 5
assert len(project_queries) == 3
assert len(project_counts) == 2
assert len(infinite_queries) == 1
assert len(root_tables) == 1
assert len(root_procs) == 1
assert len(root_descr) == 1
pre_req_env = {
**env,
"PGRST_DB_PRE_REQUEST": "do_nothing",
}
with run(env=pre_req_env) as postgrest:
response = postgrest.session.get("/projects")
assert response.status_code == 200
output = drain_stdout(postgrest)
pre_request_regx = r'.+: select "do_nothing"()'
pre_reqs = [line for line in output if re.match(pre_request_regx, line)]
if level == "crit":
assert not pre_reqs
elif level in {"error", "warn"}:
assert not pre_reqs
elif level == "info":
assert len(pre_reqs) == 1
elif level == "debug":
assert len(pre_reqs) == 1
def test_log_lacks_role_with_empty_anon_role(defaultenv):
"Requests are logged without a role when db-anon-role is empty."
env = {
**defaultenv,
"PGRST_DB_CONFIG": "false",
"PGRST_DB_ANON_ROLE": "",
}
with run(env=env) as postgrest:
response = postgrest.session.get("/projects")
assert response.status_code == 401
output = postgrest.read_stdout(nlines=1)
assert len(output) == 1
assert re.match(
r'- - - \[.+\] "GET /projects HTTP/1.1" 401 \d+ "" "python-requests/.+"',
output[0],
)
def test_log_postgrest_version(defaultenv):
"Should show the PostgREST version in the logs"
with run(env=defaultenv, no_startup_stdout=False) as postgrest:
version = postgrest.session.head("/").headers["Server"].split("/")[1]
output = postgrest.read_stdout(nlines=1)
assert "Starting PostgREST %s..." % version in output[0]
@pytest.mark.parametrize(
"host", ["127.0.0.1", "::1", None], ids=["IPv4", "IPv6", "Unix"]
)
def test_log_postgrest_host_and_port(host, defaultenv):
"PostgREST should output the host and port it is bound to."
# We run postgrest on unix socket when host and port are set to None
is_unix = host is None
port = None if is_unix else freeport()
with run(
env=defaultenv, host=host, port=port, no_startup_stdout=False
) as postgrest:
output = postgrest.read_stdout(nlines=11)
# Cannot assume a particular log entry order
# Listening on a socket happens after schema querying
# but is concurrent to the schema loading process
# and migh happen before or after writing of the
# "Schema cache loaded" log entry
if is_unix:
match_log(output, [r".*API server listening on .*/tmp/.*\.sock"])
elif is_ipv6(host):
match_log(output, [r".*API server listening on \[.+]:\d+"])
else: # IPv4
match_log(output, [r".*API server listening on .+:\d+"])
@pytest.mark.parametrize(
"host", ["127.0.0.1", "::1", None], ids=["IPv4", "IPv6", "Unix"]
)
def test_log_postgrest_admin_server_host_and_port(host, defaultenv):
"PostgREST should log the admin server host and port"
# We run admin server on unix socket when host and admin_port are set to None
is_unix = host is None
port = None if is_unix else freeport()
admin_port = None if is_unix else freeport(used_ports=[port])
with run(
env=defaultenv,
host=host,
port=port,
admin_port=admin_port,
no_startup_stdout=False,
wait_for=Admin.ready,
) as postgrest:
output = postgrest.read_stdout(nlines=11)
# Cannot assume a particular log entry order
# Listening on a socket happens after schema querying
# but is concurrent to the schema loading process
# and migh happen before or after writing of the
# "Schema cache loaded" log entry
if is_unix:
match_log(output, [r".*Admin server listening on .*/tmp/.*\.sock"])
elif is_ipv6(host):
match_log(output, [r".*Admin server listening on \[.+]:\d+"])
else: # IPv4
match_log(output, [r".*Admin server listening on .+:\d+"])
def test_log_error_when_schema_cache_load_error_on_startup_to_stderr(defaultenv):
"Should log the 503 error message when there is an error loading schema cache on startup"
env = {
**defaultenv,
"PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP_BEFORE_QUERIES": "1000",
"PGRST_DB_SCHEMAS": "non_existent_schema_aaaa",
}
with run(env=env, wait_for=None) as postgrest:
postgrest.wait_until_scache_starts_loading()
# First call should fail with connection refused
with pytest.raises(requests.ConnectionError):
postgrest.session.get("/projects")
# Next call should return 503
time.sleep(1)
response = postgrest.session.get("/projects")
assert response.status_code == 503
output_start = postgrest.read_stdout(nlines=10)
log_err_message = '{"code":"PGRST002","details":null,"hint":null,"message":"Could not query the database for the schema cache. Retrying."}'
assert any(log_err_message in line for line in output_start)
@pytest.mark.parametrize("level", ["crit", "error", "warn", "info", "debug"])
def test_log_pool_req_observation(level, defaultenv):
"PostgREST should log PoolRequest and PoolRequestFullfilled observation when log-level=debug"
env = {**defaultenv, "PGRST_LOG_LEVEL": level, "PGRST_JWT_SECRET": SECRET}
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
pool_req = r".*Trying to borrow a connection from pool.*"
pool_req_fullfill = r".*Borrowed a connection from the pool.*"
with run(env=env) as postgrest:
postgrest.session.get("/authors_only", headers=headers)
if level == "debug":
output = postgrest.read_stdout(nlines=7)
assert len(output) == 7
match_log(output, [pool_req, pool_req_fullfill])
elif level == "info":
output = postgrest.read_stdout(nlines=4)
assert len(output) == 1
else:
output = postgrest.read_stdout(nlines=4)
assert len(output) == 0
def test_log_listener_connection_errors(defaultenv):
"The logs should show the listener connection error message in a single line"
env = {
**defaultenv,
"PGHOST": "no_host",
"PGRST_DB_CHANNEL_ENABLED": "true",
}
with run(env=env, no_startup_stdout=False, wait_for=None) as postgrest:
output = postgrest.read_stdout(nlines=5)
assert any(
'Failed listening for database notifications on the "pgrst" channel. could not translate host name "no_host" to address:'
in line
for line in output
)
def test_log_listener_connection_start(defaultenv):
"The logs should show the listener connection start message in a single line"
env = {
**defaultenv,
"PGRST_DB_CHANNEL_ENABLED": "true",
}
with run(env=env, no_startup_stdout=False, wait_for=Admin.ready) as postgrest:
output = postgrest.read_stdout(nlines=10)
# Check for the listener start message containing host and port
# Do not check if pg version is displayed properly as it is tricky to test it
assert any(
f'"{defaultenv["PGHOST"]}:5432" and listening for database notifications on the "pgrst" channel'
in line
for line in output
)
@pytest.mark.parametrize("level", ["crit", "error", "warn", "info", "debug"])
def test_db_error_logging_to_stderr(level, defaultenv, metapostgrest):
"verify that DB errors are logged to stderr"
role = "timeout_authenticator"
set_statement_timeout(metapostgrest, role, 500)
env = {
**defaultenv,
"PGUSER": role,
"PGRST_DB_ANON_ROLE": role,
"PGRST_LOG_LEVEL": level,
}
with run(env=env) as postgrest:
response = postgrest.session.get("/rpc/sleep?seconds=1")
assert response.status_code == 500
# ensure the message appears on the logs
output = drain_stdout(postgrest)
if level == "crit":
assert len(output) == 0
elif level == "debug":
match_log(
output,
[
r".*canceling statement due to statement timeout.*",
r".*500.*",
],
)
else:
assert " 500 " in output[1]
assert "canceling statement due to statement timeout" in output[0]
reset_statement_timeout(metapostgrest, role)
def test_schema_cache_query_sleep_logs(defaultenv):
"""Schema cache sleep should be reflected in the logged query duration."""
env = {
**defaultenv,
"PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": "1000",
}
log_pattern = re.compile(r"Schema cache queried in ([\d.]+) milliseconds")
with run(env=env, wait_max_seconds=3, no_startup_stdout=False) as postgrest:
observed_ms = None
collected = []
lines = postgrest.read_stdout(nlines=10)
collected.extend(lines)
for line in lines:
match = log_pattern.search(line)
if match:
observed_ms = float(match.group(1))
break
assert observed_ms is not None
assert 1000 < observed_ms < 2000
@pytest.mark.parametrize("level", ["crit", "error", "warn", "info", "debug"])
def test_schema_cache_query_timings_log(level, defaultenv):
"Schema cache query timings should be logged on log-level=debug."
env = {
**defaultenv,
"PGRST_LOG_LEVEL": level,
}
log_pattern = re.compile(
r".+: tables: [\d.]+ ms, keydeps: [\d.]+ ms, rels: [\d.]+ ms, funcs: [\d.]+ ms, comprels: [\d.]+ ms, rls: [\d.]+ ms, dreps: [\d.]+ ms, mhandlers: [\d.]+ ms"
)
with run(env=env, no_startup_stdout=False) as postgrest:
output = drain_stdout(postgrest)
timing_matches = [
match for line in output if (match := log_pattern.match(line))
]
if level == "debug":
assert len(timing_matches) == 1
else:
assert not timing_matches
def test_empty_schema_cache_log_contains_jwt_role(defaultenv):
"Requests are logged with the role when the schema cache is empty on startup"
env = {
**defaultenv,
"PGRST_DB_SCHEMAS": "non_existent_schema_aaaa",
"PGRST_JWT_SECRET": SECRET,
}
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
with run(env=env, wait_for=None) as postgrest:
postgrest.wait_until_scache_starts_loading()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 503
output = drain_stdout(postgrest)
assert any(
re.match(
r'- - postgrest_test_author \[.+\] "GET /authors_only HTTP/1.1" 503 \d+ "" "python-requests/.+"',
line,
)
for line in output
)
def test_expired_jwt_log_lacks_role(defaultenv):
"Expired JWT requests are logged without a role."
env = {**defaultenv, "PGRST_JWT_SECRET": SECRET}
headers = jwtauthheader({"exp": relativeSeconds(-35)}, SECRET)
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
output = postgrest.read_stdout(nlines=1)
assert len(output) == 1
assert re.match(
r'- - - \[.+\] "GET /authors_only HTTP/1.1" 401 \d+ "" "python-requests/.+"',
output[0],
)
def test_schema_cache_error_observation(defaultenv):
"schema cache error observation should be logged with invalid db-schemas or db-extra-search-path"
env = {
**defaultenv,
"PGRST_DB_EXTRA_SEARCH_PATH": "x",
}
with run(env=env, no_startup_stdout=False, wait_for=None) as postgrest:
# TODO: postgrest should exit here, instead it keeps retrying
# exitCode = wait_until_exit(postgrest)
# assert exitCode == 1
output = postgrest.read_stdout(nlines=9)
assert (
"Failed to load the schema cache using db-schemas=public and db-extra-search-path=x"
in output[6]
)
def test_invalid_rpc_method_log_contains_role(defaultenv):
"Invalid RPC method requests are logged with the anonymous role."
with run(env=defaultenv) as postgrest:
response = postgrest.session.put("/rpc/sleep")
assert response.status_code == 405
output = postgrest.read_stdout(nlines=1)
assert len(output) == 1
assert re.match(
r'- - postgrest_test_anonymous \[.+\] "PUT /rpc/sleep HTTP/1.1" 405 \d+ "" "python-requests/.+"',
output[0],
)
def test_pgrst_log_503_client_error_to_stderr(defaultenv):
"PostgREST should log 503 errors to stderr"
env = {
**defaultenv,
"PGAPPNAME": "test-io",
}
with run(env=env) as postgrest:
postgrest.session.get("/rpc/terminate_pgrst?appname=test-io")
output = postgrest.read_stdout(nlines=6)
log_message = '{"code":"PGRST001","details":"no connection to the server\\n","hint":null,"message":"Database client error. Retrying the connection."}\n'
assert any(log_message in line for line in output)
def test_termination_unix_signal_logging(defaultenv):
"Server logs when handling termination unix signals."
with run(env=defaultenv) as postgrest:
postgrest.process.send_signal(signal.SIGTERM)
lines = postgrest.read_stdout(nlines=1)
wait_until_exit(postgrest)
assert any("SIGTERM" in line for line in lines)
with run(env=defaultenv) as postgrest:
postgrest.process.send_signal(signal.SIGINT)
lines = postgrest.read_stdout(nlines=1)
wait_until_exit(postgrest)
assert any("SIGINT" in line for line in lines)
def test_options_request_logs_but_cors_preflight_does_not(defaultenv):
"Plain OPTIONS requests should be logged, but CORS preflight requests should not."
env = {
**defaultenv,
"PGRST_LOG_LEVEL": "info",
"PGRST_SERVER_CORS_ALLOWED_ORIGINS": "http://example.com",
}
preflight_headers = {
"Origin": "http://example.com",
"Access-Control-Request-Method": "POST",
"Access-Control-Request-Headers": "Content-Type",
}
with run(env=env) as postgrest:
response = postgrest.session.options("/projects")
assert response.status_code == 200
response = postgrest.session.options("/projects", headers=preflight_headers)
assert response.status_code == 200
assert response.headers["Access-Control-Allow-Origin"] == "http://example.com"
output = drain_stdout(postgrest)
assert len(output) == 1
assert re.match(
r'- - postgrest_test_anonymous \[.+\] "OPTIONS /projects HTTP/1.1" 200 \d+ "" "python-requests/.+"',
output[0],
)
+26
View File
@@ -0,0 +1,26 @@
from util import psql_as_superuser
from postgrest import run
def test_listener_query_is_visible_in_pg_stat_activity(defaultenv):
"The listener connection should show the LISTEN pgrst statement in pg_stat_activity"
env = {
**defaultenv,
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGAPPNAME": "listener-query-test",
}
with run(env=env):
output = psql_as_superuser(
"""
select query
from pg_stat_activity
where application_name = 'listener-query-test'
and query = 'LISTEN "pgrst"'
limit 1;
""",
capture_output=True,
).strip()
assert output == 'LISTEN "pgrst"'
+508
View File
@@ -0,0 +1,508 @@
"Test reloading behaviors in PostgREST"
import signal
import time
import pytest
import requests
from config import CONFIGSDIR, SECRET
from util import (
jwtauthheader,
psql_as_superuser,
)
from postgrest import (
run,
sleep_until_postgrest_config_reload,
sleep_until_postgrest_full_reload,
sleep_until_postgrest_scache_reload,
)
def test_db_schema_notify_reload(defaultenv):
"DB schema and config should be reloaded when PostgREST is sent a NOTIFY"
env = {**defaultenv, "PGRST_DB_CONFIG": "true", "PGRST_DB_CHANNEL_ENABLED": "true"}
with run(env=env) as postgrest:
response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
assert response.text == '"\\"public\\", \\"public\\""'
# change db-schemas config on the db and reload config and cache with notify
postgrest.session.post(
"/rpc/change_db_schema_and_full_reload", data={"schemas": "v1"}
)
sleep_until_postgrest_full_reload()
response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
assert response.text == '"\\"v1\\", \\"public\\""'
# reset db-schemas config on the db
response = postgrest.session.post("/rpc/reset_db_schema_config")
assert response.text == ""
assert response.status_code == 204
def test_db_schema_reload(tmp_path, defaultenv):
"DB schema should be reloaded from file when PostgREST is sent SIGUSR2."
config = (CONFIGSDIR / "sigusr2-settings.config").read_text()
configfile = tmp_path / "test.config"
configfile.write_text(config)
with run(configfile, env=defaultenv) as postgrest:
response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
assert response.text == '"\\"public\\", \\"public\\""'
# change setting
configfile.write_text(
config.replace('db-schemas = "public"', 'db-schemas = "v1"')
)
# reload config
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
# reload schema cache to verify that the config reload actually happened
postgrest.process.send_signal(signal.SIGUSR1)
sleep_until_postgrest_scache_reload()
response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
assert response.text == '"\\"v1\\", \\"public\\""'
def test_invalid_role_claim_key_notify_reload(defaultenv):
"NOTIFY reload config should show an error if role-claim-key is invalid"
env = {
**defaultenv,
"PGRST_DB_CONFIG": "true",
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGRST_LOG_LEVEL": "crit",
}
with run(env=env) as postgrest:
postgrest.session.post("/rpc/invalid_role_claim_key_reload")
output = postgrest.read_stdout()
assert 'Received a config reload message on the "pgrst" channel' in output[0]
output = postgrest.read_stdout()
assert "failed to parse role-claim-key value" in output[0]
response = postgrest.session.post("/rpc/reset_invalid_role_claim_key")
assert response.text == ""
assert response.status_code == 204
def test_max_rows_reload(defaultenv):
"max-rows should be reloaded from role settings when PostgREST receives a SIGUSR2."
env = {
**defaultenv,
"PGRST_DB_CONFIG": "true",
}
with run(env=env) as postgrest:
response = postgrest.session.head("/projects")
assert response.status_code == 200
assert response.headers["Content-Range"] == "0-4/*"
# change max-rows config on the db
postgrest.session.post("/rpc/change_max_rows_config", data={"val": 1})
# reload config
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
response = postgrest.session.head("/projects")
assert response.status_code == 200
assert response.headers["Content-Range"] == "0-0/*"
# reset max-rows config on the db
response = postgrest.session.post("/rpc/reset_max_rows_config")
assert response.text == ""
assert response.status_code == 204
def test_max_rows_notify_reload(defaultenv):
"max-rows should be reloaded from role settings when PostgREST receives a NOTIFY"
env = {
**defaultenv,
"PGRST_DB_CONFIG": "true",
"PGRST_DB_CHANNEL_ENABLED": "true",
}
with run(env=env) as postgrest:
response = postgrest.session.head("/projects")
assert response.status_code == 200
assert response.headers["Content-Range"] == "0-4/*"
# change max-rows config on the db and reload with notify
postgrest.session.post(
"/rpc/change_max_rows_config", data={"val": 1, "notify": True}
)
sleep_until_postgrest_config_reload()
response = postgrest.session.head("/projects")
assert response.status_code == 200
assert response.headers["Content-Range"] == "0-0/*"
# reset max-rows config on the db
response = postgrest.session.post("/rpc/reset_max_rows_config")
assert response.text == ""
assert response.status_code == 204
def test_no_double_schema_cache_reload_on_empty_schema(defaultenv):
"Should only load the schema cache once when there's an empty schema cache on startup"
env = {
**defaultenv,
"PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": "300",
}
with run(env=env, wait_for=None) as postgrest:
postgrest.wait_until_scache_starts_loading()
with pytest.raises(requests.ConnectionError):
postgrest.session.get("/projects")
# Should wait enough time to load the schema cache twice to guarantee that the test is valid
time.sleep(1)
response = postgrest.session.get("/projects")
assert response.status_code == 200
response = postgrest.admin.get("/metrics")
assert response.status_code == 200
assert 'pgrst_schema_cache_loads_total{status="SUCCESS"} 1.0' in response.text
# https://github.com/PostgREST/postgrest/issues/2620
def test_notify_reloading_catalog_cache(defaultenv):
"notify should reload the connection catalog cache"
with run(env=defaultenv) as postgrest:
# first the id col is an uuid
response = postgrest.session.get(
"/cats?id=eq.dea27321-f988-4a57-93e4-8eeb38f3cf1e"
)
assert response.status_code == 200
# change it to a bigint
response = postgrest.session.post("/rpc/drop_change_cats")
assert response.text == ""
assert response.status_code == 204
sleep_until_postgrest_scache_reload()
# next request should succeed with a bigint value
response = postgrest.session.get("/cats?id=eq.1")
assert response.status_code == 200
def test_notify_do_nothing(defaultenv):
"NOTIFY with unknown message should do nothing"
env = {
**defaultenv,
"PGRST_DB_CONFIG": "true",
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGRST_LOG_LEVEL": "crit",
}
with run(env=env) as postgrest:
response = postgrest.session.post("/rpc/notify_do_nothing")
assert response.text == ""
assert response.status_code == 204
output = postgrest.read_stdout()
assert output == []
def test_schema_cache_concurrent_notifications(slow_schema_cache_env):
"schema cache should be up-to-date whenever a notification is sent while another reload is in progress, see https://github.com/PostgREST/postgrest/issues/2791"
internal_sleep = (
int(slow_schema_cache_env["PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP"]) / 1000
)
with run(env=slow_schema_cache_env, wait_for=None) as postgrest:
time.sleep(2 * internal_sleep + 0.1) # wait for readiness manually
# first request, create a function and set a schema cache reload in progress
response = postgrest.session.post("/rpc/create_function")
assert response.text == ""
assert response.status_code == 204
time.sleep(
internal_sleep / 2
) # wait to be inside the schema cache reload process
# second request, change the same function and do another schema cache reload
response = postgrest.session.post("/rpc/migrate_function")
assert response.text == ""
assert response.status_code == 204
time.sleep(
2 * internal_sleep
) # wait enough time to get the final schema cache state
# confirm the schema cache is up-to-date and the 2nd reload wasn't lost
response = postgrest.session.get("/rpc/mult_them?c=3&d=4")
assert response.text == "12"
assert response.status_code == 200
def test_jwt_secret_reload(tmp_path, defaultenv):
"JWT secret should be reloaded from file when PostgREST is sent SIGUSR2."
config = (CONFIGSDIR / "sigusr2-settings.config").read_text()
configfile = tmp_path / "test.config"
configfile.write_text(config)
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
with run(configfile, env=defaultenv) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
# change setting
configfile.write_text(config.replace("invalid" * 5, SECRET))
# reload config
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 200
def test_jwt_secret_external_file_reload(tmp_path, defaultenv):
"JWT secret external file should be reloaded when PostgREST is sent a SIGUSR2 or a NOTIFY."
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
external_secret_file = tmp_path / "jwt-secret-config"
external_secret_file.write_text("invalid" * 5)
env = {
**defaultenv,
"PGRST_JWT_SECRET": f"@{external_secret_file}",
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGRST_DB_CONFIG": "false",
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous", # required for NOTIFY
}
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
# change external file
external_secret_file.write_text(SECRET)
# SIGUSR1 doesn't reload external files, at least when db-config=false
postgrest.process.send_signal(signal.SIGUSR1)
sleep_until_postgrest_scache_reload()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
# reload config and external file with SIGUSR2
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 200
# change external file to wrong value again
external_secret_file.write_text("invalid" * 5)
# reload config and external file with NOTIFY
response = postgrest.session.post("/rpc/reload_pgrst_config")
assert response.text == ""
assert response.status_code == 204
sleep_until_postgrest_config_reload()
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
def test_invalidate_jwt_cache_when_secret_changes(tmp_path, defaultenv):
"JWT cache should be emptied after jwt-secret is changed in a config reload"
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
external_secret_file = tmp_path / "jwt-secret-config"
external_secret_file.write_text(SECRET)
env = {
**defaultenv,
"PGRST_JWT_SECRET": f"@{external_secret_file}",
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGRST_JWT_CACHE_MAX_ENTRIES": "86400", # enable cache
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous", # required for NOTIFY
}
with run(env=env) as postgrest:
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 200 # jwt gets cached
# change external file
external_secret_file.write_text("invalid" * 5)
# reload config and external file with NOTIFY
# jwt-cache should get empty
response = postgrest.session.post("/rpc/reload_pgrst_config")
assert response.text == ""
assert response.status_code == 204
sleep_until_postgrest_config_reload()
# now the request should fail because the cached token is removed
response = postgrest.session.get("/authors_only", headers=headers)
assert response.status_code == 401
def test_stale_schema_cache_dropped_table_returns_database_error(defaultenv):
"dropped table should return a database error while schema cache is stale"
internal_sleep = 2
env = {
**defaultenv,
"PGRST_DB_POOL": "2",
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": str(internal_sleep * 1000),
}
try:
psql_as_superuser("""
drop table if exists stale_schema_cache_items;
create table stale_schema_cache_items(id int primary key);
insert into stale_schema_cache_items values (1);
grant select on stale_schema_cache_items to postgrest_test_anonymous;
""")
with run(env=env, wait_max_seconds=10) as postgrest:
response = postgrest.session.get("/stale_schema_cache_items")
assert response.status_code == 200
psql_as_superuser("""
drop table stale_schema_cache_items;
notify pgrst, 'reload schema';
""")
response = postgrest.session.get("/stale_schema_cache_items")
payload = response.json()
assert response.status_code == 404
assert payload["code"] == "42P01"
assert (
payload["message"]
== 'relation "public.stale_schema_cache_items" does not exist'
)
time.sleep(internal_sleep + 0.3)
response = postgrest.session.get("/stale_schema_cache_items")
payload = response.json()
assert response.status_code == 404
assert payload["code"] == "PGRST205"
assert (
payload["message"]
== "Could not find the table 'public.stale_schema_cache_items' in the schema cache"
)
finally:
psql_as_superuser("drop table if exists stale_schema_cache_items;")
def test_config_log_level_is_reloadable(tmp_path, defaultenv):
"Config log-level should be reloadable on SIGUSR2"
config = (CONFIGSDIR / "sigusr2-settings.config").read_text()
configfile = tmp_path / "test.config"
configfile.write_text(config)
# Delete the env variable for "log-level" so the config file value isn't overridden
del defaultenv["PGRST_LOG_LEVEL"]
with run(configfile, env=defaultenv) as postgrest:
response = postgrest.session.get("/projects")
assert response.status_code == 200
output = postgrest.read_stdout(nlines=5)
# log-level = error, so this log line shouldn't be logged
assert not any(
"Trying to borrow a connection from pool" in line for line in output
)
# change setting
configfile.write_text(
config.replace('log-level = "error"', 'log-level = "debug"')
)
# reload
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
response = postgrest.session.get("/projects")
assert response.status_code == 200
output = postgrest.read_stdout(nlines=5)
# log-level = debug now, so this log line must be logged
assert any("Trying to borrow a connection from pool" in line for line in output)
def test_config_db_channel_enabled_is_reloadable(tmp_path, defaultenv):
"Config db-channel-enabled should be reloadable on SIGUSR2"
config = (CONFIGSDIR / "sigusr2-settings.config").read_text()
configfile = tmp_path / "test.config"
configfile.write_text(config)
with run(configfile, env=defaultenv, no_startup_stdout=False) as postgrest:
output = postgrest.read_stdout(nlines=7)
# db-channel-enabled = false, so this shouldn't be logged
assert not any(
f'"{defaultenv["PGHOST"]}:5432" and listening for database notifications on the "pgrst" channel'
in line
for line in output
)
# change setting
configfile.write_text(
config.replace(
'db-channel-enabled = "false"', 'db-channel-enabled = "true"'
)
)
# reload
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
output = postgrest.read_stdout(nlines=7)
# db-channel-enabled = true, so this logged
assert any(
f'"{defaultenv["PGHOST"]}:5432" and listening for database notifications on the "pgrst" channel'
in line
for line in output
)
# change setting back to false
configfile.write_text(
configfile.read_text().replace(
'db-channel-enabled = "true"', 'db-channel-enabled = "false"'
)
)
# reload
postgrest.process.send_signal(signal.SIGUSR2)
sleep_until_postgrest_config_reload()
output = postgrest.read_stdout(nlines=7)
# db-channel-enabled = false, so this shouldn't be logged
assert not any(
f'"{defaultenv["PGHOST"]}:5432" and listening for database notifications on the "pgrst" channel'
in line
for line in output
)
+63
View File
@@ -0,0 +1,63 @@
from config import SECRET
from postgrest import run
from util import jwtauthheader
def author_headers(account_id):
"Authorization header for postgrest_test_author with the given account id."
return jwtauthheader(
{"role": "postgrest_test_author", "account_id": account_id}, SECRET
)
def test_rls_can_edit_can_delete(defaultenv):
"select * on an RLS table exposes can_edit/can_delete computed from the policies"
env = {**defaultenv, "PGRST_JWT_SECRET": SECRET}
with run(env=env) as postgrest:
response = postgrest.session.get("/rls_items", headers=author_headers(1))
assert response.status_code == 200
rows = {r["id"]: r for r in response.json()}
# rows visible to account_id=1: own row and the public row
assert set(rows) == {1, 2}
# the own row can be edited and deleted
assert rows[1]["can_edit"] is True
assert rows[1]["can_delete"] is True
# the public row is visible but not editable or deletable
assert rows[2]["can_edit"] is False
assert rows[2]["can_delete"] is False
def test_no_rls_omits_can_edit_can_delete(defaultenv):
"select * on a table without RLS omits the computed columns"
env = {**defaultenv, "PGRST_JWT_SECRET": SECRET}
with run(env=env) as postgrest:
response = postgrest.session.get("/no_rls_items", headers=author_headers(1))
assert response.status_code == 200
rows = response.json()
assert len(rows) == 2
for row in rows:
assert "can_edit" not in row
assert "can_delete" not in row
def test_rls_columns_not_in_openapi(defaultenv):
"The OpenAPI spec must not advertise the computed columns"
env = {**defaultenv, "PGRST_JWT_SECRET": SECRET}
with run(env=env) as postgrest:
response = postgrest.session.get("/", headers=author_headers(1))
assert response.status_code == 200
spec = response.json()
properties = spec["definitions"]["rls_items"]["properties"]
assert "can_edit" not in properties
assert "can_delete" not in properties
+75
View File
@@ -0,0 +1,75 @@
import time
from util import Thread
from postgrest import (
freeport,
run,
wait_until_exit,
)
def test_so_reuseport_zero_downtime_handover(defaultenv):
"A second PostgREST instance should take over on the same main/admin ports without request failures."
# set host to _all_ addresses to force port conflict without SO_REUSEPORT
# setting to localhost (which is the default)
# might allow running multiple instances on the same port
# as the name might be resolved to many IP addresses
host = "0.0.0.0"
port = freeport()
admin_port = freeport(used_ports=[port])
failures = []
# mutable location shared between threads
keep_running = {"value": True}
# 1. Start first PostgREST instance
# 2. Start a "client" thread issuing requests in a loop
# remembering all received errors
# 3. Start second PostgREST instance on the same port as the first one
# 4. Wait a little and terminate the first instance
#
# We expect the client does not get any errors after stopping the first instance
# and seamlessly migrate to the second instance.
#
# 5. Stop client thread
# 6. Stop second PostgREST instance
# 7. Verify client did not get any errors
with run(
env={**defaultenv, "PGRST_SERVER_REUSEPORT": "true"},
port=port,
host=host,
admin_port=admin_port,
) as first:
def continuously_request():
while keep_running["value"]:
try:
response = first.session.get("/projects", timeout=1)
assert response.status_code == 200
except Exception as exc:
failures.append(exc)
break
time.sleep(0.2)
requester = Thread(target=continuously_request)
requester.start()
try:
time.sleep(1)
with run(
env={**defaultenv, "PGRST_SERVER_REUSEPORT": "true"},
port=port,
host=host,
# we do not set SO_REUSEPORT on admin socket
admin_port=freeport(used_ports=[port, admin_port]),
):
time.sleep(1)
first.process.terminate()
wait_until_exit(first, 2)
time.sleep(1)
finally:
keep_running["value"] = False
requester.join()
assert failures == []
+134 -1
View File
@@ -222,6 +222,58 @@ spec withConfig = withConfig baseCfg $ describe "OpenAPI" $ do
. nth 0
liftIO $ tableTag `shouldBe` Just [aesonQQ|"authors_only"|]
it "reflects table privileges in the HTTP methods" $ do
r <- simpleBody <$> get "/"
let selectonlyGet = r ^? key "paths" . key "/selectonly" . key "get"
selectonlyPost = r ^? key "paths" . key "/selectonly" . key "post"
insertonlyGet = r ^? key "paths" . key "/insertonly" . key "get"
insertonlyPost = r ^? key "paths" . key "/insertonly" . key "post"
insertonlyDelete = r ^? key "paths" . key "/insertonly" . key "delete"
limitedStarsGet = r ^? key "paths" . key "/limited_article_stars" . key "get"
limitedStarsPost = r ^? key "paths" . key "/limited_article_stars" . key "post"
limitedStarsPatch = r ^? key "paths" . key "/limited_article_stars" . key "patch"
limitedStarsDelete = r ^? key "paths" . key "/limited_article_stars" . key "delete"
liftIO $ do
selectonlyGet `shouldNotBe` Nothing
selectonlyPost `shouldBe` Nothing
insertonlyGet `shouldBe` Nothing
insertonlyPost `shouldNotBe` Nothing
insertonlyDelete `shouldBe` Nothing
limitedStarsGet `shouldNotBe` Nothing
limitedStarsPost `shouldNotBe` Nothing
limitedStarsPatch `shouldNotBe` Nothing
limitedStarsDelete `shouldBe` Nothing
it "reflects column privileges in the table definition" $ do
r <- simpleBody <$> get "/"
let appUsersId = r ^? key "definitions" . key "app_users" . key "properties" . key "id"
appUsersEmail = r ^? key "definitions" . key "app_users" . key "properties" . key "email"
appUsersPassword = r ^? key "definitions" . key "app_users" . key "properties" . key "password"
appUsersRequired = r ^? key "definitions" . key "app_users" . key "required"
liftIO $ do
appUsersId `shouldNotBe` Nothing
appUsersEmail `shouldNotBe` Nothing
appUsersPassword `shouldBe` Nothing
appUsersRequired `shouldBe` Just [aesonQQ|["id", "email"]|]
it "reflects column privileges in the rowFilter parameters" $ do
r <- simpleBody <$> get "/"
let filterId = r ^? key "parameters" . key "rowFilter.app_users.id"
filterEmail = r ^? key "parameters" . key "rowFilter.app_users.email"
filterPassword = r ^? key "parameters" . key "rowFilter.app_users.password"
liftIO $ do
filterId `shouldNotBe` Nothing
filterEmail `shouldNotBe` Nothing
filterPassword `shouldBe` Nothing
it "includes a fk description for a O2O relationship" $ do
r <- simpleBody <$> get "/"
@@ -233,10 +285,91 @@ spec withConfig = withConfig baseCfg $ describe "OpenAPI" $ do
{
"format": "int32",
"type": "integer",
"description": "Note:\nThis is a Foreign Key to `second.id`.<fk table='second' column='id'/>"
"description": "Note:\nThis is a Unique column.<unique/>\nThis is a Foreign Key to `second.id`.<fk table='second' column='id'/>"
}
|]
it "includes a unique description for a column with a unique constraint" $ do
r <- simpleBody <$> get "/"
let uniqueKey = r ^? key "definitions" . key "single_unique" . key "properties" . key "unique_key"
liftIO $
uniqueKey `shouldBe` Just
[aesonQQ|
{
"format": "int32",
"type": "integer",
"description": "Note:\nThis is a Unique column.<unique/>"
}
|]
it "includes the column list of a composite unique constraint" $ do
r <- simpleBody <$> get "/"
let compoundKey1 = r ^? key "definitions" . key "compound_unique" . key "properties" . key "key1"
compoundKey2 = r ^? key "definitions" . key "compound_unique" . key "properties" . key "key2"
liftIO $ do
compoundKey1 `shouldBe` Just
[aesonQQ|
{
"format": "int32",
"type": "integer",
"description": "Note:\nThis is part of a composite unique constraint.<unique cols='key1,key2'/>"
}
|]
compoundKey2 `shouldBe` Just
[aesonQQ|
{
"format": "int32",
"type": "integer",
"description": "Note:\nThis is part of a composite unique constraint.<unique cols='key1,key2'/>"
}
|]
it "includes the column list for mixed single and composite unique constraints" $ do
r <- simpleBody <$> get "/"
let uniqueCol = r ^? key "definitions" . key "mixed_unique" . key "properties" . key "id"
compoundKey1 = r ^? key "definitions" . key "mixed_unique" . key "properties" . key "key1"
compoundKey2 = r ^? key "definitions" . key "mixed_unique" . key "properties" . key "key2"
liftIO $ do
uniqueCol `shouldBe` Just
[aesonQQ|
{
"format": "int32",
"type": "integer",
"description": "Note:\nThis is a Unique column.<unique/>"
}
|]
compoundKey1 `shouldBe` Just
[aesonQQ|
{
"format": "int32",
"type": "integer",
"description": "Note:\nThis is part of a composite unique constraint.<unique cols='key1,key2'/>"
}
|]
compoundKey2 `shouldBe` Just
[aesonQQ|
{
"format": "int32",
"type": "integer",
"description": "Note:\nThis is part of a composite unique constraint.<unique cols='key1,key2'/>"
}
|]
it "includes m2m relationship markers in the table description" $ do
r <- simpleBody <$> get "/"
let beingDescription = r ^? key "definitions" . key "being" . key "description"
liftIO $
beingDescription `shouldBe` Just
[aesonQQ|"<m2m table='part' junction='being_part' source='being' target='part'/>"|]
describe "Foreign table" $
it "includes foreign table properties" $ do
+5 -2
View File
@@ -120,9 +120,12 @@ spec withConfig = withConfig baseCfg $
}
context "table with limited privileges" $ do
it "fails deleting the row when return=representation and selecting all the columns" $
it "succeeds deleting the row when return=representation and selecting all columns, returning only the privileged columns" $
request methodDelete "/app_users?id=eq.1" [("Prefer", "return=representation")] mempty
`shouldRespondWith` 401
`shouldRespondWith` [json|[ { "id": 1, "email": "test@123.com" } ]|]
{ matchStatus = 200
, matchHeaders = ["Content-Range" <:> "*/*"]
}
it "succeeds deleting the row when return=representation and selecting only the privileged columns" $
request methodDelete "/app_users?id=eq.1&select=id,email" [("Prefer", "return=representation")]
+3 -4
View File
@@ -720,11 +720,10 @@ spec withConfig = withConfig baseCfg $ do
, matchHeaders = []
}
it "fails inserting if select is not specified" $
it "succeeds inserting if select is not specified, returning only the accessible columns" $
request methodPost "/limited_article_stars" [("Prefer", "return=representation")]
[json| {"article_id": 3, "user_id": 1} |] `shouldRespondWith`
[json|{"hint":null,"details":null,"code":"42501","message":"permission denied for view limited_article_stars"}|]
{ matchStatus = 401
[json| {"article_id": 3, "user_id": 1} |] `shouldRespondWith` [json|[{"article_id":3,"user_id":1}]|]
{ matchStatus = 201
, matchHeaders = []
}
+13
View File
@@ -36,6 +36,19 @@ spec actualPgVersion withConfig = withConfig baseCfg $ do
, matchHeaders = ["Content-Length" <:> "120"]
}
describe "Column-level privileges" $ do
it "selects only the accessible columns when no select is specified" $
get "/app_users?id=eq.1"
`shouldRespondWith`
[json| [{"id":1,"email":"test@123.com"}] |]
{ matchStatus = 200 }
it "can still select the accessible columns explicitly" $
get "/app_users?id=eq.1&select=id,email"
`shouldRespondWith`
[json| [{"id":1,"email":"test@123.com"}] |]
{ matchStatus = 200 }
describe "Filtering response" $ do
it "matches with equality" $
get "/items?id=eq.5"
+3
View File
@@ -28,10 +28,13 @@ REVOKE ALL PRIVILEGES ON TABLE
, authors_only
, insertonly
, limited_article_stars
, selectonly
FROM postgrest_test_anonymous;
GRANT INSERT ON TABLE insertonly TO postgrest_test_anonymous;
GRANT SELECT ON TABLE selectonly TO postgrest_test_anonymous;
GRANT USAGE ON SEQUENCE
auto_incrementing_pk_id_seq
, items_id_seq
+14
View File
@@ -1451,6 +1451,15 @@ create table test.compound_unique(
unique(key1, key2)
);
create table test.mixed_unique(
id integer not null,
key1 integer not null,
key2 integer not null,
value text,
unique(id),
unique(key1, key2)
);
create table test.family_tree (
id text not null primary key,
name text not null,
@@ -1926,6 +1935,11 @@ create table app_users (
password text not null
);
create table selectonly (
id integer primary key,
name text
);
create table private.pages (
link int not null unique
, url text