Compare commits

..
18 Commits
Author SHA1 Message Date
steve-chavez 1f13e43abe bump version to 11.1.0 2023-06-07 19:03:31 -05:00
steve-chavez fac797c766 changelog: clarify 2781 2023-06-07 18:28:01 -05:00
steve-chavez 07cb0b582e feat: recover on pg_terminate_backend 2023-06-06 14:41:37 -05:00
steve-chavez d54a2f48de refactor: move checkIsFatal to AppState 2023-06-06 14:41:37 -05:00
steve-chavez bcce7b1c53 fix: stop retrying on "no password supplied" 2023-06-06 14:41:37 -05:00
steve-chavez 8d1961ce07 refactor: cache the isolation level 2023-06-06 14:41:37 -05:00
steve-chavez 9a19dff83e refactor: move debounceLogAcquisitionTimeout
Move it to AppState
2023-06-06 14:41:37 -05:00
steve-chavez 54b9a0b8b3 test: fix hello function for pg 9.6 2023-06-02 17:45:25 -05:00
steve-chavez 9a3d453bf4 test: remove big_schema.sql from io tests
It was unused(test skipped) and made the scache reload slower, causing
test failure on CI.
2023-06-02 17:45:25 -05:00
steve-chavez 4f6c466031 nix: postgrest-run uses similar config to tests
Otherwise manual tests can be deceiving
2023-06-02 17:45:25 -05:00
Steve ChavezandGitHub a852b766eb Fix dropping schema cache reload notifications
* test: bad schema reload
* refactor: DRY using the "extra" lib
* refactor: move worker funtions inside AppState
* Also rename Workers.hs to Admin.hs
2023-06-02 12:22:17 -05:00
steve-chavez 14be3fb671 feat: add pre-config function
Allows using the in-database configuration without SUPERUSER
2023-05-28 10:21:13 -05:00
steve-chavez 8a3686d86b refactor: whitelist in-db config settings 2023-05-28 10:21:13 -05:00
steve-chavez 009250006e feat: add db-pool-timeout alias
For db-pool-max-idletime
2023-05-22 18:34:18 -03:00
steve-chavez 38ad8c04e1 nix: withTmpDb set search_path to test
It gets repetitive to set the search_path every time you go in the test
db with psql.
2023-05-22 13:46:24 -03:00
dependabot[bot]andGitHub 54cbf147e3 build(deps): bump codecov/codecov-action from 3.1.3 to 3.1.4 (#2794) 2023-05-22 11:34:18 -05:00
RobertandGitHub f9f0f79fa9 feat: use hasql-pool-0.10, add db-pool-max-idletime (#2786)
- new option db-pool-max-idletime limits the maximal idle time of a
  postgresql connection, defaults to 30 seconds
- this essentially restores the db-pool-timeout option from versions
  <=10, with more specific name
2023-05-16 23:00:40 +02:00
steve-chavez a867d79c42 nix: withTmpDb psql notice can't mutate the db
Use the postgres role instead of the authenticator on the notice
2023-05-05 20:03:09 -03:00
36 changed files with 764 additions and 537 deletions
+1 -1
View File
@@ -46,7 +46,7 @@ jobs:
- name: Run coverage (IO tests and Spec tests against PostgreSQL 15)
run: postgrest-coverage
- name: Upload coverage to codecov
uses: codecov/codecov-action@v3.1.3
uses: codecov/codecov-action@v3.1.4
with:
files: ./coverage/codecov.json
+4 -6
View File
@@ -55,12 +55,6 @@ It builds the OpenAPI response using the schema cache.
This module provides functions to deal with JWT authorization.
### Workers.hs
This spawns threads which are used to execute concurrent jobs.
Jobs include connection recovery, a listener for the PostgreSQL LISTEN command, and an admin server.
### SchemaCache.hs
This queries the PostgreSQL system catalogs and caches the metadata into a SchemaCache type,
@@ -68,3 +62,7 @@ This queries the PostgreSQL system catalogs and caches the metadata into a Schem
### AppState.hs
The state of the App which is kept across requests.
This spawns threads which are used to execute concurrent jobs.
Jobs include connection recover and a listener for the PostgreSQL LISTEN command.
+19
View File
@@ -5,6 +5,25 @@ This project adheres to [Semantic Versioning](http://semver.org/).
## Unreleased
## [11.1.0] - 2023-06-07
### Added
- #2786, Limit idle postgresql connection lifetime - @robx
+ New option `db-pool-max-idletime` (default 30s).
+ This is equivalent to the old option `db-pool-timeout` of PostgREST 10.0.0.
+ A config alias for `db-pool-timeout` is included.
- #2703, Add pre-config function - @steve-chavez
+ New config option `db-pre-config`(empty by default)
+ Allows using the in-database configuration without SUPERUSER
- #2781, When `db-channel-enabled` is false, start automatic connection recovery on a new request when pool connections are closed with `pg_terminate_backend` - @steve-chavez
+ Mitigates the lack of LISTEN/NOTIFY for schema cache reloading on read replicas.
### Fixed
- #2791, Fix dropping schema cache reload notifications - @steve-chavez
- #2801, Stop retrying connection when "no password supplied" - @steve-chavez
## [11.0.1] - 2023-04-27
### Fixed
+4 -4
View File
@@ -43,8 +43,8 @@ let
(prev.callHackageDirect
{
pkg = "hasql-notifications";
ver = "0.2.0.4";
sha256 = "sha256-fm1xiDyvDkb5WLOJ73/s8wrWEW23XFS7luAv2brfr8I=";
ver = "0.2.0.5";
sha256 = "sha256-KV36zs/RTgJh/oBBPZaNiQshusmakWnjYTcdoNa4JLA=";
}
{ });
@@ -52,8 +52,8 @@ let
(prev.callHackageDirect
{
pkg = "hasql-pool";
ver = "0.9";
sha256 = "sha256-5UshbbaBVY8eJ/9VagNVVxonRwMcd7UmGqDc35pJNFY=";
ver = "0.10";
sha256 = "sha256-kHzoqtNV9BFWnn1h560JRqMooQRwxokVKgDRBexamNI=";
}
{ });
} // extraOverrides final prev;
+4
View File
@@ -41,6 +41,8 @@ let
args =
[
"ARG_USE_ENV([PGRST_DB_ANON_ROLE], [postgrest_test_anonymous], [PostgREST anonymous role])"
"ARG_USE_ENV([PGRST_DB_POOL], [1], [PostgREST pool size])"
"ARG_USE_ENV([PGRST_DB_POOL_ACQUISITION_TIMEOUT], [1], [PostgREST pool size])"
"ARG_LEFTOVERS([PostgREST arguments])"
];
inRootDir = true;
@@ -48,6 +50,8 @@ let
}
''
export PGRST_DB_ANON_ROLE
export PGRST_DB_POOL
export PGRST_DB_POOL_ACQUISITION_TIMEOUT
exec ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
postgrest "''${_arg_leftovers[@]}"
+3 -1
View File
@@ -32,6 +32,7 @@ let
"ARG_USE_ENV([PGDATABASE], [postgres], [PG database name])"
"ARG_USE_ENV([PGRST_DB_SCHEMAS], [test], [Schema to expose])"
"ARG_USE_ENV([PGTZ], [utc], [Timezone to use])"
"ARG_USE_ENV([PGOPTIONS], [-c search_path=public,test], [PG options to use])"
];
positionalCompletion = "_command";
inRootDir = true;
@@ -61,6 +62,7 @@ let
export PGDATABASE
export PGRST_DB_SCHEMAS
export PGTZ
export PGOPTIONS
HBA_FILE="$tmpdir/pg_hba.conf"
echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
@@ -94,7 +96,7 @@ let
log "Done. Running command..."
echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U $PGUSER"
echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U ${superuserRole}"
echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log"
("$_arg_command" "''${_arg_leftovers[@]}")
+6 -5
View File
@@ -1,5 +1,5 @@
name: postgrest
version: 11.0.1
version: 11.1.0
synopsis: REST API for any Postgres database
description: Reads the schema of a PostgreSQL database and creates RESTful routes
for tables, views, and functions, supporting all HTTP methods that security
@@ -34,7 +34,8 @@ library
default-extensions: OverloadedStrings
NoImplicitPrelude
hs-source-dirs: src
exposed-modules: PostgREST.App
exposed-modules: PostgREST.Admin
PostgREST.App
PostgREST.AppState
PostgREST.Auth
PostgREST.CLI
@@ -70,7 +71,6 @@ library
PostgREST.Response.OpenAPI
PostgREST.Response.GucHeader
PostgREST.Version
PostgREST.Workers
other-modules: Paths_postgrest
build-depends: base >= 4.9 && < 4.17
, HTTP >= 4000.3.7 && < 4000.5
@@ -86,12 +86,13 @@ library
, contravariant-extras >= 0.3.3 && < 0.4
, cookie >= 0.4.2 && < 0.5
, either >= 4.4.1 && < 5.1
, extra >= 1.7.0 && < 2.0
, fuzzyset >= 0.2.3
, gitrev >= 1.2 && < 1.4
, hasql >= 1.6.1.1 && < 1.7
, hasql-dynamic-statements >= 0.3.1 && < 0.4
, hasql-notifications >= 0.1 && < 0.3
, hasql-pool >= 0.9 && < 0.10
, hasql-pool >= 0.10 && < 0.11
, hasql-transaction >= 1.0.1 && < 1.1
, heredoc >= 0.2 && < 0.3
, http-types >= 0.12.2 && < 0.13
@@ -232,7 +233,7 @@ test-suite spec
, bytestring >= 0.10.8 && < 0.12
, case-insensitive >= 1.2 && < 1.3
, containers >= 0.5.7 && < 0.7
, hasql-pool >= 0.9 && < 0.10
, hasql-pool >= 0.10 && < 0.11
, hasql-transaction >= 1.0.1 && < 1.1
, heredoc >= 0.2 && < 0.3
, hspec >= 2.3 && < 2.10
+87
View File
@@ -0,0 +1,87 @@
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE RecordWildCards #-}
module PostgREST.Admin
( runAdmin
) where
import qualified Data.Text as T
import qualified Hasql.Session as SQL
import qualified Network.HTTP.Types.Status as HTTP
import qualified Network.Wai as Wai
import qualified Network.Wai.Handler.Warp as Warp
import Control.Monad.Extra (whenJust)
import Network.Socket
import Network.Socket.ByteString
import PostgREST.AppState (AppState)
import PostgREST.Config (AppConfig (..))
import qualified PostgREST.AppState as AppState
import Protolude
runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
runAdmin conf@AppConfig{configAdminServerPort} appState settings =
whenJust configAdminServerPort $ \adminPort -> do
AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
where
adminApp = admin appState conf
-- | PostgREST admin application
admin :: AppState.AppState -> AppConfig -> Wai.Application
admin appState appConfig req respond = do
isMainAppReachable <- any isRight <$> reachMainApp appConfig
isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
isConnectionUp <-
if configDbChannelEnabled appConfig
then AppState.getIsListenerOn appState
else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
case Wai.pathInfo req of
["ready"] ->
respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
["live"] ->
respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
_ ->
respond $ Wai.responseLBS HTTP.status404 [] mempty
-- Try to connect to the main app socket
-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
-- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
-- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
reachMainApp :: AppConfig -> IO [Either IOException ()]
reachMainApp AppConfig{..} =
case configServerUnixSocket of
Just path -> do
sock <- socket AF_UNIX Stream 0
(:[]) <$> try (do
connect sock $ SockAddrUnix path
withSocketsDo $ bracket (pure sock) close sendEmpty)
Nothing -> do
let
host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
| otherwise = Just configServerHost
filterAddrs xs =
case configServerHost of
"*4" -> ipv4Addrs xs ++ ipv6Addrs xs
"!4" -> ipv4Addrs xs
"*6" -> ipv6Addrs xs ++ ipv4Addrs xs
"!6" -> ipv6Addrs xs
_ -> xs
ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
tryAddr `traverse` filterAddrs addrs
where
sendEmpty sock = void $ send sock mempty
tryAddr :: AddrInfo -> IO (Either IOException ())
tryAddr addr = do
sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
try $ do
connect sock $ addrAddress addr
withSocketsDo $ bracket (pure sock) close sendEmpty
+10 -22
View File
@@ -9,7 +9,6 @@ Some of its functionality includes:
- Producing HTTP Headers according to RFCs.
- Content Negotiation
-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE RecordWildCards #-}
module PostgREST.App
( SignalHandlerInstaller
@@ -20,7 +19,7 @@ module PostgREST.App
import Control.Monad.Except (liftEither)
import Data.Either.Combinators (mapLeft, whenLeft)
import Data.Either.Combinators (mapLeft)
import Data.Maybe (fromJust)
import Data.String (IsString (..))
import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
@@ -28,11 +27,11 @@ import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
import System.Posix.Types (FileMode)
import qualified Data.HashMap.Strict as HM
import qualified Hasql.Pool as SQL
import qualified Hasql.Transaction.Sessions as SQL
import qualified Network.Wai as Wai
import qualified Network.Wai.Handler.Warp as Warp
import qualified PostgREST.Admin as Admin
import qualified PostgREST.ApiRequest as ApiRequest
import qualified PostgREST.ApiRequest.Types as ApiRequestTypes
import qualified PostgREST.AppState as AppState
@@ -43,7 +42,6 @@ import qualified PostgREST.Logger as Logger
import qualified PostgREST.Plan as Plan
import qualified PostgREST.Query as Query
import qualified PostgREST.Response as Response
import qualified PostgREST.Workers as Workers
import PostgREST.ApiRequest (Action (..), ApiRequest (..),
Mutation (..), Target (..))
@@ -68,14 +66,14 @@ type SocketRunner = Warp.Settings -> Wai.Application -> FileMode -> FilePath ->
run :: SignalHandlerInstaller -> Maybe SocketRunner -> AppState -> IO ()
run installHandlers maybeRunWithSocket appState = do
conf@AppConfig{..} <- AppState.getConfig appState
Workers.connectionWorker appState -- Loads the initial SchemaCache
AppState.connectionWorker appState -- Loads the initial SchemaCache
installHandlers appState
-- reload schema cache + config on NOTIFY
Workers.runListener conf appState
AppState.runListener conf appState
Workers.runAdmin conf appState $ serverSettings conf
Admin.runAdmin conf appState $ serverSettings conf
let app = postgrest conf appState (Workers.connectionWorker appState)
let app = postgrest conf appState (AppState.connectionWorker appState)
case configServerUnixSocket of
Just socket ->
@@ -154,27 +152,17 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@
Response.optionalRollback conf apiRequest $
handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache
runDbHandler :: AppState.AppState -> Maybe Text -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
runDbHandler :: AppState.AppState -> SQL.IsolationLevel -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
runDbHandler appState isoLvl mode authenticated prepared handler = do
dbResp <- lift $ do
let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction
res <- AppState.usePool appState . transaction (toIsolationLevel isoLvl) mode $ runExceptT handler
whenLeft res (\case
SQL.AcquisitionTimeoutUsageError -> AppState.debounceLogAcquisitionTimeout appState -- this can happen rapidly for many requests, so we debounce
_ -> pure ())
return res
AppState.usePool appState . transaction isoLvl mode $ runExceptT handler
resp <-
liftEither . mapLeft Error.PgErr $
mapLeft (Error.PgError authenticated) dbResp
liftEither resp
where
toIsolationLevel = \case
Nothing -> SQL.ReadCommitted
Just "repeatable read" -> SQL.RepeatableRead
Just "serializable" -> SQL.Serializable
_ -> SQL.ReadCommitted
handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> Bool -> Bool -> PgVersion -> ApiRequest -> SchemaCache -> Handler IO Wai.Response
handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@ApiRequest{..} sCache =
@@ -206,7 +194,7 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A
(ActionInvoke invMethod, TargetProc identifier _) -> do
cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq invMethod
resultSet <- runQuery (roleIsoLvl <|> pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
resultSet <- runQuery (fromMaybe roleIsoLvl $ pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
return $ Response.invokeResponse invMethod (Plan.crProc cPlan) apiReq resultSet
(ActionInspect headersOnly, TargetDefaultSpec tSchema) -> do
@@ -229,7 +217,7 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A
throwError $ Error.ApiRequestError ApiRequestTypes.NotFound
where
roleSettings = fromMaybe mempty (HM.lookup authRole $ configRoleSettings conf)
roleIsoLvl = decodeUtf8 <$> HM.lookup "default_transaction_isolation" roleSettings
roleIsoLvl = HM.findWithDefault SQL.ReadCommitted authRole $ configRoleIsoLvl conf
runQuery isoLvl mode query =
runDbHandler appState isoLvl mode authenticated prepared $ do
Query.setPgLocals conf authClaims authRole (HM.toList roleSettings) apiReq pgVer
+310 -25
View File
@@ -1,9 +1,10 @@
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE RecordWildCards #-}
module PostgREST.AppState
( AppState
, destroy
, flushPool
, getConfig
, getSchemaCache
, getIsListenerOn
@@ -11,40 +12,50 @@ module PostgREST.AppState
, getPgVersion
, getRetryNextIn
, getTime
, getWorkerSem
, init
, initWithPool
, logWithZTime
, logPgrstError
, putConfig
, putSchemaCache
, putIsListenerOn
, putPgVersion
, putRetryNextIn
, signalListener
, usePool
, waitListener
, debounceLogAcquisitionTimeout
, loadSchemaCache
, reReadConfig
, connectionWorker
, runListener
) where
import qualified Data.ByteString.Lazy as LBS
import qualified Data.Text.Encoding as T
import qualified Hasql.Pool as SQL
import qualified Hasql.Session as SQL
import qualified PostgREST.Error as Error
import qualified Data.ByteString.Char8 as BS
import qualified Data.ByteString.Lazy as LBS
import Data.Either.Combinators (whenLeft)
import qualified Data.Text.Encoding as T
import Hasql.Connection (acquire)
import qualified Hasql.Notifications as SQL
import qualified Hasql.Pool as SQL
import qualified Hasql.Session as SQL
import qualified Hasql.Transaction.Sessions as SQL
import qualified PostgREST.Error as Error
import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
updateAction)
import Control.Debounce
import Control.Retry (RetryStatus, capDelay, exponentialBackoff,
retrying, rsPreviousDelay)
import Data.IORef (IORef, atomicWriteIORef, newIORef,
readIORef)
import Data.Time (ZonedTime, defaultTimeLocale, formatTime,
getZonedTime)
import Data.Time.Clock (UTCTime, getCurrentTime)
import PostgREST.Config (AppConfig (..))
import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
import PostgREST.SchemaCache (SchemaCache)
import PostgREST.Config (AppConfig (..),
readAppConfig)
import PostgREST.Config.Database (queryDbSettings,
queryPgVersion,
queryRoleSettings)
import PostgREST.Config.PgVersion (PgVersion (..),
minimumPgVersion)
import PostgREST.SchemaCache (SchemaCache,
querySchemaCache)
import PostgREST.SchemaCache.Identifiers (dumpQi)
import Protolude
@@ -56,8 +67,8 @@ data AppState = AppState
, statePgVersion :: IORef PgVersion
-- | No schema cache at the start. Will be filled in by the connectionWorker
, stateSchemaCache :: IORef (Maybe SchemaCache)
-- | Binary semaphore to make sure just one connectionWorker can run at a time
, stateWorkerSem :: MVar ()
-- | starts the connection worker with a debounce
, debouncedConnectionWorker :: IO ()
-- | Binary semaphore used to sync the listener(NOTIFY reload) with the connectionWorker.
, stateListener :: MVar ()
-- | State of the LISTEN channel, used for the admin server checks
@@ -86,7 +97,7 @@ initWithPool pool conf = do
appState <- AppState pool
<$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step
<*> newIORef Nothing
<*> newEmptyMVar
<*> pure (pure ())
<*> newEmptyMVar
<*> newIORef False
<*> newIORef conf
@@ -96,7 +107,8 @@ initWithPool pool conf = do
<*> newIORef 0
<*> pure (pure ())
deb <-
debLogTimeout <-
let oneSecond = 1000000 in
mkDebounce defaultDebounceSettings
{ debounceAction = logPgrstError appState SQL.AcquisitionTimeoutUsageError
@@ -104,7 +116,15 @@ initWithPool pool conf = do
, debounceEdge = leadingEdge -- logs at the start and the end
}
return appState { debounceLogAcquisitionTimeout = deb }
debWorker <-
let decisecond = 100000 in
mkDebounce defaultDebounceSettings
{ debounceAction = internalConnectionWorker appState
, debounceFreq = decisecond
, debounceEdge = leadingEdge -- runs the worker at the start and the end
}
return appState { debounceLogAcquisitionTimeout = debLogTimeout, debouncedConnectionWorker = debWorker }
destroy :: AppState -> IO ()
destroy = destroyPool
@@ -115,11 +135,17 @@ initPool AppConfig{..} =
configDbPoolSize
(fromIntegral configDbPoolAcquisitionTimeout)
(fromIntegral configDbPoolMaxLifetime)
(fromIntegral configDbPoolMaxIdletime)
(toUtf8 configDbUri)
-- | Run an action with a database connection.
usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a)
usePool AppState{..} = SQL.use statePool
usePool AppState{..} x = do
res <- SQL.use statePool x
whenLeft res (\case
SQL.AcquisitionTimeoutUsageError -> debounceLogAcquisitionTimeout -- this can happen rapidly for many requests, so we debounce
_ -> pure ())
return res
-- | Flush the connection pool so that any future use of the pool will
-- use connections freshly established after this call.
@@ -142,8 +168,8 @@ getSchemaCache = readIORef . stateSchemaCache
putSchemaCache :: AppState -> Maybe SchemaCache -> IO ()
putSchemaCache appState = atomicWriteIORef (stateSchemaCache appState)
getWorkerSem :: AppState -> MVar ()
getWorkerSem = stateWorkerSem
connectionWorker :: AppState -> IO ()
connectionWorker = debouncedConnectionWorker
getRetryNextIn :: AppState -> IO Int
getRetryNextIn = readIORef . stateRetryNextIn
@@ -188,3 +214,262 @@ getIsListenerOn = readIORef . stateIsListenerOn
putIsListenerOn :: AppState -> Bool -> IO ()
putIsListenerOn = atomicWriteIORef . stateIsListenerOn
-- | Schema cache status
data SCacheStatus
= SCLoaded
| SCOnRetry
| SCFatalFail
-- | Load the SchemaCache by using a connection from the pool.
loadSchemaCache :: AppState -> IO SCacheStatus
loadSchemaCache appState = do
conf@AppConfig{..} <- getConfig appState
result <-
let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
usePool appState . transaction SQL.ReadCommitted SQL.Read $
querySchemaCache conf
case result of
Left e -> do
case checkIsFatal e of
Just hint -> do
logWithZTime appState "A fatal error ocurred when loading the schema cache"
logPgrstError appState e
logWithZTime appState hint
return SCFatalFail
Nothing -> do
putSchemaCache appState Nothing
logWithZTime appState "An error ocurred when loading the schema cache"
logPgrstError appState e
return SCOnRetry
Right sCache -> do
putSchemaCache appState (Just sCache)
logWithZTime appState "Schema cache loaded"
return SCLoaded
-- | Current database connection status data ConnectionStatus
data ConnectionStatus
= NotConnected
| Connected PgVersion
| FatalConnectionError Text
deriving (Eq)
-- | The purpose of this worker is to obtain a healthy connection to pg and an
-- up-to-date schema cache(SchemaCache). This method is meant to be called
-- multiple times by the same thread, but does nothing if the previous
-- invocation has not terminated. In all cases this method does not halt the
-- calling thread, the work is performed in a separate thread.
--
-- Background thread that does the following :
-- 1. Tries to connect to pg server and will keep trying until success.
-- 2. Checks if the pg version is supported and if it's not it kills the main
-- program.
-- 3. Obtains the sCache. If this fails, it goes back to 1.
internalConnectionWorker :: AppState -> IO ()
internalConnectionWorker appState = work
where
work = do
AppConfig{..} <- getConfig appState
logWithZTime appState "Attempting to connect to the database..."
connected <- establishConnection appState
case connected of
FatalConnectionError reason ->
-- Fatal error when connecting
logWithZTime appState reason >> killThread (getMainThreadId appState)
NotConnected ->
-- Unreachable because establishConnection will keep trying to connect
return ()
Connected actualPgVersion -> do
-- Procede with initialization
putPgVersion appState actualPgVersion
when configDbChannelEnabled $
signalListener appState
logWithZTime appState "Connection successful"
-- this could be fail because the connection drops, but the loadSchemaCache will pick the error and retry again
-- We cannot retry after it fails immediately, because db-pre-config could have user errors. We just log the error and continue.
when configDbConfig $ reReadConfig False appState
scStatus <- loadSchemaCache appState
case scStatus of
SCLoaded ->
-- do nothing and proceed if the load was successful
return ()
SCOnRetry ->
-- retry reloading the schema cache
work
SCFatalFail ->
-- die if our schema cache query has an error
killThread $ getMainThreadId appState
-- | Repeatedly flush the pool, and check if a connection from the
-- pool allows access to the PostgreSQL database.
--
-- Releasing the pool is key for rapid recovery. Otherwise, the pool
-- timeout would have to be reached for new healthy connections to be acquired.
-- Which might not happen if the server is busy with requests. No idle
-- connection, no pool timeout.
--
-- The connection tries are capped, but if the connection times out no error is
-- thrown, just 'False' is returned.
establishConnection :: AppState -> IO ConnectionStatus
establishConnection appState =
retrying retrySettings shouldRetry $
const $ flushPool appState >> getConnectionStatus
where
retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
delayMicroseconds = 32000000 -- 32 seconds
backoffMicroseconds = 1000000 -- 1 second
getConnectionStatus :: IO ConnectionStatus
getConnectionStatus = do
pgVersion <- usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
case pgVersion of
Left e -> do
logPgrstError appState e
case checkIsFatal e of
Just reason ->
return $ FatalConnectionError reason
Nothing ->
return NotConnected
Right version ->
if version < minimumPgVersion then
return . FatalConnectionError $
"Cannot run in this PostgreSQL version, PostgREST needs at least "
<> pgvName minimumPgVersion
else
return . Connected $ version
shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
shouldRetry rs isConnSucc = do
let
delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
itShould = NotConnected == isConnSucc
when itShould . logWithZTime appState $
"Attempting to reconnect to the database in "
<> (show delay::Text)
<> " seconds..."
when itShould $ putRetryNextIn appState delay
return itShould
-- | Re-reads the config plus config options from the db
reReadConfig :: Bool -> AppState -> IO ()
reReadConfig startingUp appState = do
AppConfig{..} <- getConfig appState
dbSettings <-
if configDbConfig then do
qDbSettings <- usePool appState $ queryDbSettings (dumpQi <$> configDbPreConfig) configDbPreparedStatements
case qDbSettings of
Left e -> do
logWithZTime appState
"An error ocurred when trying to query database settings for the config parameters"
case checkIsFatal e of
Just hint -> do
logPgrstError appState e
logWithZTime appState hint
killThread (getMainThreadId appState)
Nothing -> do
logPgrstError appState e
pure mempty
Right x -> pure x
else
pure mempty
(roleSettings, roleIsolationLvl) <-
if configDbConfig then do
rSettings <- usePool appState $ queryRoleSettings configDbPreparedStatements
case rSettings of
Left e -> do
logWithZTime appState "An error ocurred when trying to query the role settings"
logPgrstError appState e
pure (mempty, mempty)
Right x -> pure x
else
pure mempty
readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings roleIsolationLvl >>= \case
Left err ->
if startingUp then
panic err -- die on invalid config if the program is starting up
else
logWithZTime appState $ "Failed reloading config: " <> err
Right newConf -> do
putConfig appState newConf
if startingUp then
pass
else
logWithZTime appState "Config reloaded"
runListener :: AppConfig -> AppState -> IO ()
runListener AppConfig{configDbChannelEnabled} appState =
when configDbChannelEnabled $ listener appState
-- | Starts a dedicated pg connection to LISTEN for notifications. When a
-- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
-- cache. It uses the connectionWorker in case the LISTEN connection dies.
listener :: AppState -> IO ()
listener appState = do
AppConfig{..} <- getConfig appState
let dbChannel = toS configDbChannel
-- The listener has to wait for a signal from the connectionWorker.
-- This is because when the connection to the db is lost, the listener also
-- tries to recover the connection, but not with the same pace as the connectionWorker.
-- Not waiting makes stderr quickly fill with connection retries messages from the listener.
waitListener appState
-- forkFinally allows to detect if the thread dies
void . flip forkFinally (handleFinally dbChannel) $ do
dbOrError <- acquire $ toUtf8 configDbUri
case dbOrError of
Right db -> do
logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
putIsListenerOn appState True
SQL.listen db $ SQL.toPgIdentifier dbChannel
SQL.waitForNotifications handleNotification db
_ ->
die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
where
handleFinally dbChannel _ = do
-- if the thread dies, we try to recover
logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
putIsListenerOn appState False
-- assume the pool connection was also lost, call the connection worker
connectionWorker appState
-- retry the listener
listener appState
handleNotification _ msg
| BS.null msg = cacheReloader
| msg == "reload schema" = cacheReloader
| msg == "reload config" = reReadConfig False appState
| otherwise = pure () -- Do nothing if anything else than an empty message is sent
cacheReloader =
-- reloads the schema cache + restarts pool connections
-- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
connectionWorker appState
checkIsFatal :: SQL.UsageError -> Maybe Text
checkIsFatal (SQL.ConnectionUsageError e)
| isAuthFailureMessage = Just $ toS failureMessage
| otherwise = Nothing
where isAuthFailureMessage =
("FATAL: password authentication failed" `isInfixOf` failureMessage) ||
("no password supplied" `isInfixOf` failureMessage)
failureMessage = BS.unpack $ fromMaybe mempty e
checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
= case serverError of
-- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
SQL.ServerError "42601" _ _ _ _
-> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
-- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
-- This would mean that a connection pooler in transaction mode is being used
-- while prepared statements are enabled in the PostgREST configuration,
-- both of which are incompatible with each other.
SQL.ServerError "42P05" _ _ _ _
-> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
-- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
-- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
-> Just "Hint: Connection poolers in statement mode are not supported."
_ -> Nothing
checkIsFatal _ = Nothing
+10 -8
View File
@@ -21,7 +21,6 @@ import PostgREST.AppState (AppState)
import PostgREST.Config (AppConfig (..))
import PostgREST.SchemaCache (querySchemaCache)
import PostgREST.Version (prettyVersion)
import PostgREST.Workers (reReadConfig)
import qualified PostgREST.App as App
import qualified PostgREST.AppState as AppState
@@ -33,7 +32,7 @@ import Protolude hiding (hPutStrLn)
main :: App.SignalHandlerInstaller -> Maybe App.SocketRunner -> CLI -> IO ()
main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
conf@AppConfig{..} <-
either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty
either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty mempty
-- Per https://github.com/PostgREST/postgrest/issues/268, we want to
-- explicitly close the connections to PostgreSQL on shutdown.
@@ -43,7 +42,7 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
AppState.destroy
(\appState -> case cliCommand of
CmdDumpConfig -> do
when configDbConfig $ reReadConfig True appState
when configDbConfig $ AppState.reReadConfig True appState
putStr . Config.toText =<< AppState.getConfig appState
CmdDumpSchema -> putStrLn =<< dumpSchema appState
CmdRun -> App.run installSignalHandlers runAppWithSocket appState)
@@ -51,15 +50,12 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
-- | Dump SchemaCache schema to JSON
dumpSchema :: AppState -> IO LBS.ByteString
dumpSchema appState = do
AppConfig{..} <- AppState.getConfig appState
conf@AppConfig{..} <- AppState.getConfig appState
result <-
let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
AppState.usePool appState $
transaction SQL.ReadCommitted SQL.Read $
querySchemaCache
(toList configDbSchemas)
configDbExtraSearchPath
configDbPreparedStatements
querySchemaCache conf
case result of
Left e -> do
hPutStrLn stderr $ "An error ocurred when loading the schema cache:\n" <> show e
@@ -136,6 +132,9 @@ exampleConfigFile =
|## Enable in-database configuration
|db-config = true
|
|## Function for in-database configuration
|## db-pre-config = "postgrest.pre_config"
|
|## Extra schemas to add to the search_path of every request
|db-extra-search-path = "public"
|
@@ -154,6 +153,9 @@ exampleConfigFile =
|## Time in seconds after which to recycle pool connections
|# db-pool-max-lifetime = 1800
|
|## Time in seconds after which to recycle unused pool connections
|# db-pool-max-idletime = 30
|
|## Stored proc to exec immediately after auth
|# db-pre-request = "stored_proc_name"
|
+21 -16
View File
@@ -51,7 +51,8 @@ import Numeric (readOct, showOct)
import System.Environment (getEnvironment)
import System.Posix.Types (FileMode)
import PostgREST.Config.Database (RoleSettings)
import PostgREST.Config.Database (RoleIsolationLvl,
RoleSettings)
import PostgREST.Config.JSPath (JSPath, JSPathExp (..),
dumpJSPath, pRoleClaimKey)
import PostgREST.Config.Proxy (Proxy (..),
@@ -74,11 +75,13 @@ data AppConfig = AppConfig
, configDbPoolSize :: Int
, configDbPoolAcquisitionTimeout :: Int
, configDbPoolMaxLifetime :: Int
, configDbPoolMaxIdletime :: Int
, configDbPreRequest :: Maybe QualifiedIdentifier
, configDbPreparedStatements :: Bool
, configDbRootSpec :: Maybe QualifiedIdentifier
, configDbSchemas :: NonEmpty Text
, configDbConfig :: Bool
, configDbPreConfig :: Maybe QualifiedIdentifier
, configDbTxAllowOverride :: Bool
, configDbTxRollbackAll :: Bool
, configDbUri :: Text
@@ -101,6 +104,8 @@ data AppConfig = AppConfig
, configServerUnixSocketMode :: FileMode
, configAdminServerPort :: Maybe Int
, configRoleSettings :: RoleSettings
, configRoleIsoLvl :: RoleIsolationLvl
, configInternalSCSleep :: Maybe Int32
}
data LogLevel = LogCrit | LogError | LogWarn | LogInfo
@@ -137,11 +142,13 @@ toText conf =
,("db-pool", show . configDbPoolSize)
,("db-pool-acquisition-timeout", show . configDbPoolAcquisitionTimeout)
,("db-pool-max-lifetime", show . configDbPoolMaxLifetime)
,("db-pool-max-idletime", show . configDbPoolMaxIdletime)
,("db-pre-request", q . maybe mempty dumpQi . configDbPreRequest)
,("db-prepared-statements", T.toLower . show . configDbPreparedStatements)
,("db-root-spec", q . maybe mempty dumpQi . configDbRootSpec)
,("db-schemas", q . T.intercalate "," . toList . configDbSchemas)
,("db-config", T.toLower . show . configDbConfig)
,("db-pre-config", q . maybe mempty dumpQi . configDbPreConfig)
,("db-tx-end", q . showTxEnd)
,("db-uri", q . configDbUri)
,("db-use-legacy-gucs", T.toLower . show . configDbUseLegacyGucs)
@@ -193,13 +200,13 @@ instance JustIfMaybe a (Maybe a) where
-- | Reads and parses the config and overrides its parameters from env vars,
-- files or db settings.
readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> IO (Either Text AppConfig)
readAppConfig dbSettings optPath prevDbUri roleSettings = do
readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> RoleIsolationLvl -> IO (Either Text AppConfig)
readAppConfig dbSettings optPath prevDbUri roleSettings roleIsolationLvl = do
env <- readPGRSTEnvironment
-- if no filename provided, start with an empty map to read config from environment
conf <- maybe (return $ Right M.empty) loadConfig optPath
case C.runParser (parser optPath env dbSettings roleSettings) =<< mapLeft show conf of
case C.runParser (parser optPath env dbSettings roleSettings roleIsolationLvl) =<< mapLeft show conf of
Left err ->
return . Left $ "Error in config " <> err
Right parsedConfig ->
@@ -214,8 +221,8 @@ readAppConfig dbSettings optPath prevDbUri roleSettings = do
decodeJWKS <$>
(decodeSecret =<< readSecretFile =<< readDbUriFile prevDbUri parsedConfig)
parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> C.Parser C.Config AppConfig
parser optPath env dbSettings roleSettings =
parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> RoleIsolationLvl -> C.Parser C.Config AppConfig
parser optPath env dbSettings roleSettings roleIsolationLvl =
AppConfig
<$> parseAppSettings "app.settings"
<*> (fmap encodeUtf8 <$> optString "db-anon-role")
@@ -228,6 +235,8 @@ parser optPath env dbSettings roleSettings =
<*> (fromMaybe 10 <$> optInt "db-pool")
<*> (fromMaybe 10 <$> optInt "db-pool-acquisition-timeout")
<*> (fromMaybe 1800 <$> optInt "db-pool-max-lifetime")
<*> (fromMaybe 30 <$> optWithAlias (optInt "db-pool-timeout")
(optInt "db-pool-max-idletime"))
<*> (fmap toQi <$> optWithAlias (optString "db-pre-request")
(optString "pre-request"))
<*> (fromMaybe True <$> optBool "db-prepared-statements")
@@ -236,6 +245,7 @@ parser optPath env dbSettings roleSettings =
<*> (fromList . maybe ["public"] splitOnCommas <$> optWithAlias (optValue "db-schemas")
(optValue "db-schema"))
<*> (fromMaybe True <$> optBool "db-config")
<*> (fmap toQi <$> optString "db-pre-config")
<*> parseTxEnd "db-tx-end" snd
<*> parseTxEnd "db-tx-end" fst
<*> (fromMaybe "postgresql://" <$> optString "db-uri")
@@ -260,6 +270,8 @@ parser optPath env dbSettings roleSettings =
<*> parseSocketFileMode "server-unix-socket-mode"
<*> optInt "admin-server-port"
<*> pure roleSettings
<*> pure roleIsolationLvl
<*> optInt "internal-schema-cache-sleep"
where
parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
@@ -354,21 +366,14 @@ parser optPath env dbSettings roleSettings =
(C.Key -> C.Parser C.Value a -> C.Parser C.Config b) ->
C.Key -> (C.Value -> a) -> C.Parser C.Config b
overrideFromDbOrEnvironment necessity key coercion =
case reloadableDbSetting <|> M.lookup envVarName env of
case dbConf <|> M.lookup envVarName env of
Just dbOrEnvVal -> pure $ justIfMaybe $ coercion $ C.String dbOrEnvVal
Nothing -> necessity key (coercion <$> C.value)
Nothing -> necessity key (coercion <$> C.value)
where
dashToUnderscore '-' = '_'
dashToUnderscore c = c
envVarName = "PGRST_" <> (toUpper . dashToUnderscore <$> toS key)
reloadableDbSetting =
let dbSettingName = T.pack $ dashToUnderscore <$> toS key in
if dbSettingName `notElem` [
"server_host", "server_port", "server_unix_socket", "server_unix_socket_mode", "admin_server_port", "log_level",
"db_uri", "db_channel_enabled", "db_channel", "db_pool", "db_pool_acquisition_timeout",
"db_pool_max_lifetime", "db_config"]
then lookup dbSettingName dbSettings
else Nothing
dbConf = lookup (T.pack $ dashToUnderscore <$> toS key) dbSettings
coerceText :: C.Value -> Text
coerceText (C.String s) = s
+117 -35
View File
@@ -6,6 +6,8 @@ module PostgREST.Config.Database
, queryRoleSettings
, queryPgVersion
, RoleSettings
, RoleIsolationLvl
, toIsolationLevel
) where
import Control.Arrow ((***))
@@ -21,11 +23,47 @@ import qualified Hasql.Statement as SQL
import qualified Hasql.Transaction as SQL
import qualified Hasql.Transaction.Sessions as SQL
import Text.InterpolatedString.Perl6 (q)
import Text.InterpolatedString.Perl6 (q, qc)
import Protolude
type RoleSettings = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
type RoleSettings = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
type RoleIsolationLvl = HM.HashMap ByteString SQL.IsolationLevel
toIsolationLevel :: (Eq a, IsString a) => a -> SQL.IsolationLevel
toIsolationLevel a = case a of
"repeatable read" -> SQL.RepeatableRead
"serializable" -> SQL.Serializable
_ -> SQL.ReadCommitted
prefix :: Text
prefix = "pgrst."
-- | In-db settings names
dbSettingsNames :: [Text]
dbSettingsNames =
(prefix <>) <$>
["db_anon_role"
,"db_pre_config"
,"db_extra_search_path"
,"db_max_rows"
,"db_plan_enabled"
,"db_pre_request"
,"db_prepared_statements"
,"db_root_spec"
,"db_schemas"
,"db_tx_end"
,"db_use_legacy_gucs"
,"jwt_aud"
,"jwt_role_claim_key"
,"jwt_secret"
,"jwt_secret_is_base64"
,"openapi_mode"
,"openapi_security_active"
,"openapi_server_proxy_uri"
,"raw_media_types"
,"server_trace_header"
]
queryPgVersion :: Bool -> Session PgVersion
queryPgVersion prepared = statement mempty $ pgVersionStatement prepared
@@ -36,46 +74,62 @@ pgVersionStatement = SQL.Statement sql HE.noParams versionRow
sql = "SELECT current_setting('server_version_num')::integer, current_setting('server_version')"
versionRow = HD.singleRow $ PgVersion <$> column HD.int4 <*> column HD.text
queryDbSettings :: Bool -> Session [(Text, Text)]
queryDbSettings prepared =
-- | Query the in-database configuration. The settings have the following priorities:
--
-- 1. Role + with database-specific settings:
-- ALTER ROLE authenticator IN DATABASE postgres SET <prefix>jwt_aud = 'val';
-- 2. Role + with settings:
-- ALTER ROLE authenticator SET <prefix>jwt_aud = 'overridden';
-- 3. pre-config function:
-- CREATE FUNCTION pre_config() .. PERFORM set_config(<prefix>jwt_aud, 'pre_config_aud'..)
--
-- The example above will result in <prefix>jwt_aud = 'val'
-- A setting on the database only will have no effect: ALTER DATABASE postgres SET <prefix>jwt_aud = 'xx'
queryDbSettings :: Maybe Text -> Bool -> Session [(Text, Text)]
queryDbSettings preConfFunc prepared =
let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ dbSettingsStatement prepared
-- | Get db settings from the connection role. Global settings will be overridden by database specific settings.
dbSettingsStatement :: Bool -> SQL.Statement () [(Text, Text)]
dbSettingsStatement = SQL.Statement sql HE.noParams decodeSettings
transaction SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared
where
sql = [q|
sql = [qc|
WITH
role_setting (database, setting) AS (
SELECT setdatabase,
unnest(setconfig)
FROM pg_catalog.pg_db_role_setting
WHERE setrole = CURRENT_USER::regrole::oid
AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
role_setting AS (
SELECT setdatabase as database,
unnest(setconfig) as setting
FROM pg_catalog.pg_db_role_setting
WHERE setrole = CURRENT_USER::regrole::oid
AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
),
kv_settings (database, k, v) AS (
kv_settings AS (
SELECT database,
substr(setting, 1, strpos(setting, '=') - 1),
substr(setting, strpos(setting, '=') + 1)
FROM role_setting
WHERE setting LIKE 'pgrst.%'
substr(setting, 1, strpos(setting, '=') - 1) as k,
substr(setting, strpos(setting, '=') + 1) as v
FROM role_setting
{preConfigF}
)
SELECT DISTINCT ON (key)
replace(k, 'pgrst.', '') AS key,
replace(k, '{prefix}', '') AS key,
v AS value
FROM kv_settings
ORDER BY key, database DESC;
FROM kv_settings
WHERE k = ANY($1) AND v IS NOT NULL
ORDER BY key, database DESC NULLS LAST;
|]
preConfigF = case preConfFunc of
Nothing -> mempty
Just func -> [qc|
UNION
SELECT
null as database,
x as k,
current_setting(x, true) as v
FROM unnest($1) x
JOIN {func}() _ ON TRUE
|]::Text
decodeSettings = HD.rowList $ (,) <$> column HD.text <*> column HD.text
queryRoleSettings :: Bool -> Session RoleSettings
queryRoleSettings :: Bool -> Session (RoleSettings, RoleIsolationLvl)
queryRoleSettings prepared =
let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ roleSettingsStatement prepared
roleSettingsStatement :: Bool -> SQL.Statement () RoleSettings
roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared
where
sql = [q|
with
@@ -91,18 +145,40 @@ roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
substr(setting, 1, strpos(setting, '=') - 1) as key,
lower(substr(setting, strpos(setting, '=') + 1)) as value
FROM role_setting
),
iso_setting AS (
SELECT rolname, value
FROM kv_settings
WHERE key = 'default_transaction_isolation'
)
select rolname, array_agg(row(key, value))
from kv_settings
group by rolname;
select
kv.rolname,
i.value as iso_lvl,
array_agg(row(kv.key, kv.value)) filter (where key <> 'default_transation_isolation') as role_settings
from kv_settings kv
left join iso_setting i on i.rolname = kv.rolname
group by kv.rolname, i.value;
|]
decodeRoleSettings = HM.fromList . map (bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>))) <$> HD.rowList aRow
aRow :: HD.Row (Text, [(Text, Text)])
aRow = (,) <$> column HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
processRows :: [(Text, Maybe Text, [(Text, Text)])] -> (RoleSettings, RoleIsolationLvl)
processRows rs =
let
rowsWRoleSettings = [ (x, z) | (x, _, z) <- rs ]
rowsWIsolation = [ (x, y) | (x, Just y, _) <- rs ]
in
( HM.fromList $ bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>)) <$> rowsWRoleSettings
, HM.fromList $ (encodeUtf8 *** toIsolationLevel) <$> rowsWIsolation
)
rows :: HD.Result [(Text, Maybe Text, [(Text, Text)])]
rows = HD.rowList $ (,,) <$> column HD.text <*> nullableColumn HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
column :: HD.Value a -> HD.Row a
column = HD.column . HD.nonNullable
nullableColumn :: HD.Value a -> HD.Row (Maybe a)
nullableColumn = HD.column . HD.nullable
compositeField :: HD.Value a -> HD.Composite a
compositeField = HD.field . HD.nonNullable
@@ -111,3 +187,9 @@ compositeArrayColumn = arrayColumn . HD.composite
arrayColumn :: HD.Value a -> HD.Row [a]
arrayColumn = column . HD.listArray . HD.nonNullable
param :: HE.Value a -> HE.Params a
param = HE.param . HE.nonNullable
arrayParam :: HE.Value a -> HE.Params [a]
arrayParam = param . HE.foldableArray . HE.nonNullable
+1 -24
View File
@@ -11,7 +11,6 @@ module PostgREST.Error
, PgError(..)
, Error(..)
, errorPayload
, checkIsFatal
, singularityError
) where
@@ -429,6 +428,7 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
'5':'3':_ -> HTTP.status503 -- insufficient resources
'5':'4':_ -> HTTP.status413 -- too complex
'5':'5':_ -> HTTP.status500 -- obj not on prereq state
'5':'7':'P':'0':'1':_ -> HTTP.status503 -- terminating connection due to administrator command
'5':'7':_ -> HTTP.status500 -- operator intervention
'5':'8':_ -> HTTP.status500 -- system error
'F':'0':_ -> HTTP.status500 -- conf file error
@@ -446,29 +446,6 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
_ -> HTTP.status500
checkIsFatal :: SQL.UsageError -> Maybe Text
checkIsFatal (SQL.ConnectionUsageError e)
| isAuthFailureMessage = Just $ toS failureMessage
| otherwise = Nothing
where isAuthFailureMessage = "FATAL: password authentication failed" `isInfixOf` failureMessage
failureMessage = BS.unpack $ fromMaybe mempty e
checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
= case serverError of
-- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
SQL.ServerError "42601" _ _ _ _
-> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
-- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
-- This would mean that a connection pooler in transaction mode is being used
-- while prepared statements are enabled in the PostgREST configuration,
-- both of which are incompatible with each other.
SQL.ServerError "42P05" _ _ _ _
-> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
-- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
-- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
-> Just "Hint: Connection poolers in statement mode are not supported."
_ -> Nothing
checkIsFatal _ = Nothing
data Error
+15 -5
View File
@@ -26,6 +26,8 @@ module PostgREST.SchemaCache
, schemaDescription
) where
import Control.Monad.Extra (whenJust)
import qualified Data.Aeson as JSON
import qualified Data.HashMap.Strict as HM
import qualified Data.HashMap.Strict.InsOrd as HMI
@@ -38,7 +40,9 @@ import qualified Hasql.Transaction as SQL
import Contravariant.Extras (contrazip2)
import Text.InterpolatedString.Perl6 (q)
import PostgREST.Config.Database (pgVersionStatement)
import PostgREST.Config (AppConfig (..))
import PostgREST.Config.Database (pgVersionStatement,
toIsolationLevel)
import PostgREST.Config.PgVersion (PgVersion, pgVersion100,
pgVersion110, pgVersion120)
import PostgREST.SchemaCache.Identifiers (AccessSet, FieldName,
@@ -103,15 +107,18 @@ data KeyDep
-- | A SQL query that can be executed independently
type SqlQuery = ByteString
querySchemaCache :: [Schema] -> [Schema] -> Bool -> SQL.Transaction SchemaCache
querySchemaCache schemas extraSearchPath prepared = do
querySchemaCache :: AppConfig -> SQL.Transaction SchemaCache
querySchemaCache AppConfig{..} = do
SQL.sql "set local schema ''" -- This voids the search path. The following queries need this for getting the fully qualified name(schema.name) of every db object
pgVer <- SQL.statement mempty $ pgVersionStatement prepared
tabs <- SQL.statement schemas $ allTables pgVer prepared
keyDeps <- SQL.statement (schemas, extraSearchPath) $ allViewsKeyDependencies prepared
keyDeps <- SQL.statement (schemas, configDbExtraSearchPath) $ allViewsKeyDependencies prepared
m2oRels <- SQL.statement mempty $ allM2OandO2ORels pgVer prepared
funcs <- SQL.statement schemas $ allFunctions pgVer prepared
cRels <- SQL.statement mempty $ allComputedRels prepared
_ <-
let sleepCall = SQL.Statement "select pg_sleep($1)" (param HE.int4) HD.noResult prepared in
whenJust configInternalSCSleep (`SQL.statement` sleepCall) -- only used for testing
let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
rels = addInverseRels $ addM2MRels tabsWViewsPks $ addViewM2OAndO2ORels keyDeps m2oRels
@@ -121,6 +128,9 @@ querySchemaCache schemas extraSearchPath prepared = do
, dbRelationships = getOverrideRelationshipsMap rels cRels
, dbRoutines = funcs
}
where
schemas = toList configDbSchemas
prepared = configDbPreparedStatements
-- | overrides detected relationships with the computed relationships and gets the RelationshipsMap
getOverrideRelationshipsMap :: [Relationship] -> [Relationship] -> RelationshipsMap
@@ -250,7 +260,7 @@ decodeFuncs =
<*> column HD.bool)
<*> (parseVolatility <$> column HD.char)
<*> column HD.bool
<*> nullableColumn HD.text
<*> nullableColumn (toIsolationLevel <$> HD.text)
addKey :: Routine -> (QualifiedIdentifier, Routine)
addKey pd = (QualifiedIdentifier (pdSchema pd) (pdName pd), pd)
+18 -4
View File
@@ -16,8 +16,10 @@ module PostgREST.SchemaCache.Routine
, funcReturnsCompositeAlias
) where
import qualified Data.Aeson as JSON
import qualified Data.HashMap.Strict as HM
import Data.Aeson ((.=))
import qualified Data.Aeson as JSON
import qualified Data.HashMap.Strict as HM
import qualified Hasql.Transaction.Sessions as SQL
import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
Schema, TableName)
@@ -48,9 +50,21 @@ data Routine = Function
, pdReturnType :: RetType
, pdVolatility :: FuncVolatility
, pdHasVariadic :: Bool
, pdIsoLvl :: Maybe Text
, pdIsoLvl :: Maybe SQL.IsolationLevel
}
deriving (Eq, Generic, JSON.ToJSON)
deriving (Eq, Generic)
-- need to define JSON manually bc SQL.IsolationLevel doesn't have a JSON instance(and we can't define one for that type without getting a compiler error)
instance JSON.ToJSON Routine where
toJSON (Function sch nam desc params ret vol hasVar _) = JSON.object
[
"pdSchema" .= sch
, "pdName" .= nam
, "pdDescription" .= desc
, "pdParams" .= JSON.toJSON params
, "pdReturnType" .= JSON.toJSON ret
, "pdVolatility" .= JSON.toJSON vol
, "pdHasVariadic" .= JSON.toJSON hasVar
]
data RoutineParam = RoutineParam
{ ppName :: Text
+2 -3
View File
@@ -14,7 +14,6 @@ import System.Posix.Files (setFileMode)
import System.Posix.Types (FileMode)
import qualified PostgREST.AppState as AppState
import qualified PostgREST.Workers as Workers
import Protolude
@@ -49,10 +48,10 @@ installSignalHandlers appState = do
-- The SIGUSR1 signal updates the internal 'SchemaCache' by running
-- 'connectionWorker' exactly as before.
install Signals.sigUSR1 $ Workers.connectionWorker appState
install Signals.sigUSR1 $ AppState.connectionWorker appState
-- Re-read the config on SIGUSR2
install Signals.sigUSR2 $ Workers.reReadConfig False appState
install Signals.sigUSR2 $ AppState.reReadConfig False appState
where
install signal handler =
void $ Signals.installHandler signal (Signals.Catch handler) Nothing
-343
View File
@@ -1,343 +0,0 @@
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE RecordWildCards #-}
module PostgREST.Workers
( connectionWorker
, reReadConfig
, runListener
, runAdmin
) where
import qualified Data.ByteString as BS
import qualified Data.Text as T
import qualified Hasql.Notifications as SQL
import qualified Hasql.Session as SQL
import qualified Hasql.Transaction.Sessions as SQL
import qualified Network.HTTP.Types.Status as HTTP
import qualified Network.Wai as Wai
import qualified Network.Wai.Handler.Warp as Warp
import Control.Retry (RetryStatus, capDelay, exponentialBackoff,
retrying, rsPreviousDelay)
import Hasql.Connection (acquire)
import Network.Socket
import Network.Socket.ByteString
import PostgREST.AppState (AppState)
import PostgREST.Config (AppConfig (..), readAppConfig)
import PostgREST.Config.Database (queryDbSettings, queryPgVersion,
queryRoleSettings)
import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
import PostgREST.Error (checkIsFatal)
import PostgREST.SchemaCache (querySchemaCache)
import qualified PostgREST.AppState as AppState
import Protolude
-- | Current database connection status data ConnectionStatus
data ConnectionStatus
= NotConnected
| Connected PgVersion
| FatalConnectionError Text
deriving (Eq)
-- | Schema cache status
data SCacheStatus
= SCLoaded
| SCOnRetry
| SCFatalFail
-- | The purpose of this worker is to obtain a healthy connection to pg and an
-- up-to-date schema cache(SchemaCache). This method is meant to be called
-- multiple times by the same thread, but does nothing if the previous
-- invocation has not terminated. In all cases this method does not halt the
-- calling thread, the work is performed in a separate thread.
--
-- Background thread that does the following :
-- 1. Tries to connect to pg server and will keep trying until success.
-- 2. Checks if the pg version is supported and if it's not it kills the main
-- program.
-- 3. Obtains the sCache. If this fails, it goes back to 1.
connectionWorker :: AppState -> IO ()
connectionWorker appState = do
runExclusively (AppState.getWorkerSem appState) work
-- Prevents multiple workers to be running at the same time. Could happen on
-- too many SIGUSR1s.
where
runExclusively mvar action = mask_ $ do
success <- tryPutMVar mvar ()
when success $ do
void $ forkIO $ action `finally` takeMVar mvar
work = do
AppConfig{..} <- AppState.getConfig appState
AppState.logWithZTime appState "Attempting to connect to the database..."
connected <- establishConnection appState
case connected of
FatalConnectionError reason ->
-- Fatal error when connecting
AppState.logWithZTime appState reason >> killThread (AppState.getMainThreadId appState)
NotConnected ->
-- Unreachable because establishConnection will keep trying to connect
return ()
Connected actualPgVersion -> do
-- Procede with initialization
AppState.putPgVersion appState actualPgVersion
when configDbChannelEnabled $
AppState.signalListener appState
AppState.logWithZTime appState "Connection successful"
-- this could be fail because the connection drops, but the
-- loadSchemaCache will pick the error and retry again
when configDbConfig $ reReadConfig False appState
scStatus <- loadSchemaCache appState
case scStatus of
SCLoaded ->
-- do nothing and proceed if the load was successful
return ()
SCOnRetry ->
-- retry reloading the schema cache
work
SCFatalFail ->
-- die if our schema cache query has an error
killThread $ AppState.getMainThreadId appState
-- | Repeatedly flush the pool, and check if a connection from the
-- pool allows access to the PostgreSQL database.
--
-- Releasing the pool is key for rapid recovery. Otherwise, the pool
-- timeout would have to be reached for new healthy connections to be acquired.
-- Which might not happen if the server is busy with requests. No idle
-- connection, no pool timeout.
--
-- The connection tries are capped, but if the connection times out no error is
-- thrown, just 'False' is returned.
establishConnection :: AppState -> IO ConnectionStatus
establishConnection appState =
retrying retrySettings shouldRetry $
const $ AppState.flushPool appState >> getConnectionStatus
where
retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
delayMicroseconds = 32000000 -- 32 seconds
backoffMicroseconds = 1000000 -- 1 second
getConnectionStatus :: IO ConnectionStatus
getConnectionStatus = do
pgVersion <- AppState.usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
case pgVersion of
Left e -> do
AppState.logPgrstError appState e
case checkIsFatal e of
Just reason ->
return $ FatalConnectionError reason
Nothing ->
return NotConnected
Right version ->
if version < minimumPgVersion then
return . FatalConnectionError $
"Cannot run in this PostgreSQL version, PostgREST needs at least "
<> pgvName minimumPgVersion
else
return . Connected $ version
shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
shouldRetry rs isConnSucc = do
let
delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
itShould = NotConnected == isConnSucc
when itShould . AppState.logWithZTime appState $
"Attempting to reconnect to the database in "
<> (show delay::Text)
<> " seconds..."
when itShould $ AppState.putRetryNextIn appState delay
return itShould
-- | Load the SchemaCache by using a connection from the pool.
loadSchemaCache :: AppState -> IO SCacheStatus
loadSchemaCache appState = do
AppConfig{..} <- AppState.getConfig appState
result <-
let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
AppState.usePool appState . transaction SQL.ReadCommitted SQL.Read $
querySchemaCache (toList configDbSchemas) configDbExtraSearchPath configDbPreparedStatements
case result of
Left e -> do
case checkIsFatal e of
Just hint -> do
AppState.logWithZTime appState "A fatal error ocurred when loading the schema cache"
AppState.logPgrstError appState e
AppState.logWithZTime appState hint
return SCFatalFail
Nothing -> do
AppState.putSchemaCache appState Nothing
AppState.logWithZTime appState "An error ocurred when loading the schema cache"
AppState.logPgrstError appState e
return SCOnRetry
Right sCache -> do
AppState.putSchemaCache appState (Just sCache)
AppState.logWithZTime appState "Schema cache loaded"
return SCLoaded
runListener :: AppConfig -> AppState -> IO ()
runListener AppConfig{configDbChannelEnabled} appState =
when configDbChannelEnabled $ listener appState
-- | Starts a dedicated pg connection to LISTEN for notifications. When a
-- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
-- cache. It uses the connectionWorker in case the LISTEN connection dies.
listener :: AppState -> IO ()
listener appState = do
AppConfig{..} <- AppState.getConfig appState
let dbChannel = toS configDbChannel
-- The listener has to wait for a signal from the connectionWorker.
-- This is because when the connection to the db is lost, the listener also
-- tries to recover the connection, but not with the same pace as the connectionWorker.
-- Not waiting makes stderr quickly fill with connection retries messages from the listener.
AppState.waitListener appState
-- forkFinally allows to detect if the thread dies
void . flip forkFinally (handleFinally dbChannel) $ do
dbOrError <- acquire $ toUtf8 configDbUri
case dbOrError of
Right db -> do
AppState.logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
AppState.putIsListenerOn appState True
SQL.listen db $ SQL.toPgIdentifier dbChannel
SQL.waitForNotifications handleNotification db
_ ->
die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
where
handleFinally dbChannel _ = do
-- if the thread dies, we try to recover
AppState.logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
AppState.putIsListenerOn appState False
-- assume the pool connection was also lost, call the connection worker
connectionWorker appState
-- retry the listener
listener appState
handleNotification _ msg
| BS.null msg = cacheReloader
| msg == "reload schema" = cacheReloader
| msg == "reload config" = reReadConfig False appState
| otherwise = pure () -- Do nothing if anything else than an empty message is sent
cacheReloader =
-- reloads the schema cache + restarts pool connections
-- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
connectionWorker appState
-- | Re-reads the config plus config options from the db
reReadConfig :: Bool -> AppState -> IO ()
reReadConfig startingUp appState = do
AppConfig{..} <- AppState.getConfig appState
dbSettings <-
if configDbConfig then do
qDbSettings <- AppState.usePool appState $ queryDbSettings configDbPreparedStatements
case qDbSettings of
Left e -> do
AppState.logWithZTime appState
"An error ocurred when trying to query database settings for the config parameters"
case checkIsFatal e of
Just hint -> do
AppState.logPgrstError appState e
AppState.logWithZTime appState hint
killThread (AppState.getMainThreadId appState)
Nothing -> do
AppState.logPgrstError appState e
pure mempty
Right x -> pure x
else
pure mempty
roleSettings <-
if configDbConfig then do
rSettings <- AppState.usePool appState $ queryRoleSettings configDbPreparedStatements
case rSettings of
Left e -> do
AppState.logWithZTime appState "An error ocurred when trying to query the role settings"
AppState.logPgrstError appState e
pure mempty
Right x -> pure x
else
pure mempty
readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings >>= \case
Left err ->
if startingUp then
panic err -- die on invalid config if the program is starting up
else
AppState.logWithZTime appState $ "Failed reloading config: " <> err
Right newConf -> do
AppState.putConfig appState newConf
if startingUp then
pass
else
AppState.logWithZTime appState "Config reloaded"
runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
runAdmin conf@AppConfig{configAdminServerPort} appState settings =
whenJust configAdminServerPort $ \adminPort -> do
AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
where
whenJust :: Applicative m => Maybe a -> (a -> m ()) -> m ()
whenJust mg f = maybe (pure ()) f mg
adminApp = admin appState conf
-- | PostgREST admin application
admin :: AppState.AppState -> AppConfig -> Wai.Application
admin appState appConfig req respond = do
isMainAppReachable <- any isRight <$> reachMainApp appConfig
isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
isConnectionUp <-
if configDbChannelEnabled appConfig
then AppState.getIsListenerOn appState
else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
case Wai.pathInfo req of
["ready"] ->
respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
["live"] ->
respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
_ ->
respond $ Wai.responseLBS HTTP.status404 [] mempty
-- Try to connect to the main app socket
-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
-- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
-- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
reachMainApp :: AppConfig -> IO [Either IOException ()]
reachMainApp AppConfig{..} =
case configServerUnixSocket of
Just path -> do
sock <- socket AF_UNIX Stream 0
(:[]) <$> try (do
connect sock $ SockAddrUnix path
withSocketsDo $ bracket (pure sock) close sendEmpty)
Nothing -> do
let
host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
| otherwise = Just configServerHost
filterAddrs xs =
case configServerHost of
"*4" -> ipv4Addrs xs ++ ipv6Addrs xs
"!4" -> ipv4Addrs xs
"*6" -> ipv6Addrs xs ++ ipv4Addrs xs
"!6" -> ipv6Addrs xs
_ -> xs
ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
tryAddr `traverse` filterAddrs addrs
where
sendEmpty sock = void $ send sock mempty
tryAddr :: AddrInfo -> IO (Either IOException ())
tryAddr addr = do
sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
try $ do
connect sock $ addrAddress addr
withSocketsDo $ bracket (pure sock) close sendEmpty
+2 -2
View File
@@ -12,5 +12,5 @@ nix:
extra-deps:
- git: https://github.com/PostgREST/postgresql-libpq.git
commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
- hasql-notifications-0.2.0.4
- hasql-pool-0.9
- hasql-notifications-0.2.0.5
- hasql-pool-0.10
+6 -6
View File
@@ -16,19 +16,19 @@ packages:
commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
git: https://github.com/PostgREST/postgresql-libpq.git
- completed:
hackage: hasql-notifications-0.2.0.4@sha256:9a09fa9b97feadd9492c8bd8bc6b9cffe0513510102f08374b0c45ecd479ed67,2028
hackage: hasql-notifications-0.2.0.5@sha256:6c67d2ee42e948162e89176a2f52a2c6bb562678b33688e96e048be9b3d74122,2028
pantry-tree:
sha256: 56f9e240728e7a65711dde45fa2e2075b914e32cd370424aaa4572392378a60e
sha256: 8493755b0817a36b910a37d568769ca49ab715ef5dd935a0d09f0716aa39da51
size: 452
original:
hackage: hasql-notifications-0.2.0.4
hackage: hasql-notifications-0.2.0.5
- completed:
hackage: hasql-pool-0.9@sha256:db7a37f6b3a922c37adc3c7ced47a7c10786d1f171e47a735a6e812a587ba44c,2111
hackage: hasql-pool-0.10@sha256:912197a328acb85505f98bb9700d61f366b87659ca45126c5c2d636687b801c3,2112
pantry-tree:
sha256: 49b1181d28c6f5317e794671c2dae155754b834bdcfa30f7e5dbad28e4cf0249
sha256: b655c540a49764a8d16b62941137e295b936b96edc0785eb9250972f0f92dc47
size: 346
original:
hackage: hasql-pool-0.9
hackage: hasql-pool-0.10
snapshots:
- completed:
sha256: 4905c93319aa94aa53da8f41d614d7bacdbfe6c63a8c6132d32e6e62f24a9af4
+1
View File
@@ -1,4 +1,5 @@
db-schema = "provided_through_alias"
db-pool-timeout = 5
max-rows = 1000
pre-request = "check_alias"
role-claim-key = ".aliased"
+2
View File
@@ -7,11 +7,13 @@ db-plan-enabled = false
db-pool = 10
db-pool-acquisition-timeout = 10
db-pool-max-lifetime = 1800
db-pool-max-idletime = 5
db-pre-request = "check_alias"
db-prepared-statements = true
db-root-spec = "open_alias"
db-schemas = "provided_through_alias"
db-config = true
db-pre-config = ""
db-tx-end = "commit"
db-uri = "postgresql://"
db-use-legacy-gucs = true
@@ -7,11 +7,13 @@ db-plan-enabled = false
db-pool = 10
db-pool-acquisition-timeout = 10
db-pool-max-lifetime = 1800
db-pool-max-idletime = 30
db-pre-request = ""
db-prepared-statements = false
db-root-spec = ""
db-schemas = "public"
db-config = true
db-pre-config = ""
db-tx-end = "commit"
db-uri = "postgresql://"
db-use-legacy-gucs = true
@@ -7,11 +7,13 @@ db-plan-enabled = false
db-pool = 10
db-pool-acquisition-timeout = 10
db-pool-max-lifetime = 1800
db-pool-max-idletime = 30
db-pre-request = ""
db-prepared-statements = false
db-root-spec = ""
db-schemas = "public"
db-config = true
db-pre-config = ""
db-tx-end = "commit"
db-uri = "postgresql://"
db-use-legacy-gucs = true
+2
View File
@@ -7,11 +7,13 @@ db-plan-enabled = false
db-pool = 10
db-pool-acquisition-timeout = 10
db-pool-max-lifetime = 1800
db-pool-max-idletime = 30
db-pre-request = ""
db-prepared-statements = true
db-root-spec = ""
db-schemas = "public"
db-config = false
db-pre-config = ""
db-tx-end = "commit"
db-uri = "postgresql://"
db-use-legacy-gucs = true
@@ -1,4 +1,4 @@
db-anon-role = "other"
db-anon-role = "pre_config_role"
db-channel = "postgrest"
db-channel-enabled = false
db-extra-search-path = "public,extensions,other"
@@ -7,16 +7,18 @@ db-plan-enabled = true
db-pool = 1
db-pool-acquisition-timeout = 30
db-pool-max-lifetime = 3600
db-pool-max-idletime = 60
db-pre-request = "test.other_custom_headers"
db-prepared-statements = false
db-root-spec = "other_root"
db-schemas = "test,other_tenant1,other_tenant2"
db-config = true
db-pre-config = "postgrest.pre_config"
db-tx-end = "rollback-allow-override"
db-uri = "postgresql://"
db-use-legacy-gucs = false
jwt-aud = "https://otherexample.org"
jwt-role-claim-key = ".\"other\".\"role\""
jwt-role-claim-key = ".\"other\".\"pre_config_role\""
jwt-secret = "ODERREALLYREALLYREALLYREALLYVERYSAFE"
jwt-secret-is-base64 = true
log-level = "info"
@@ -7,11 +7,13 @@ db-plan-enabled = true
db-pool = 1
db-pool-acquisition-timeout = 30
db-pool-max-lifetime = 3600
db-pool-max-idletime = 60
db-pre-request = "test.custom_headers"
db-prepared-statements = false
db-root-spec = "root"
db-schemas = "test,tenant1,tenant2"
db-config = true
db-pre-config = "postgrest.preconf"
db-tx-end = "commit-allow-override"
db-uri = "postgresql://"
db-use-legacy-gucs = false
@@ -7,11 +7,13 @@ db-plan-enabled = true
db-pool = 1
db-pool-acquisition-timeout = 30
db-pool-max-lifetime = 3600
db-pool-max-idletime = 60
db-pre-request = "please_run_fast"
db-prepared-statements = false
db-root-spec = "openapi_v3"
db-schemas = "multi,tenant,setup"
db-config = false
db-pre-config = "postgrest.pre_config"
db-tx-end = "rollback-allow-override"
db-uri = "tmp_db"
db-use-legacy-gucs = false
+2
View File
@@ -7,11 +7,13 @@ db-plan-enabled = false
db-pool = 10
db-pool-acquisition-timeout = 10
db-pool-max-lifetime = 1800
db-pool-max-idletime = 30
db-pre-request = ""
db-prepared-statements = true
db-root-spec = ""
db-schemas = "public"
db-config = true
db-pre-config = ""
db-tx-end = "commit"
db-uri = "postgresql://"
db-use-legacy-gucs = true
+2 -1
View File
@@ -9,14 +9,15 @@ PGRST_DB_PLAN_ENABLED: true
PGRST_DB_POOL: 1
PGRST_DB_POOL_ACQUISITION_TIMEOUT: 30
PGRST_DB_POOL_MAX_LIFETIME: 3600
PGRST_DB_POOL_MAX_IDLETIME: 60
PGRST_DB_PREPARED_STATEMENTS: false
PGRST_DB_PRE_REQUEST: please_run_fast
PGRST_DB_ROOT_SPEC: openapi_v3
PGRST_DB_SCHEMAS: multi, tenant,setup
PGRST_DB_CONFIG: false
PGRST_DB_PRE_CONFIG: "postgrest.pre_config"
PGRST_DB_TX_END: rollback-allow-override
PGRST_DB_URI: tmp_db
PGRST_DB_EMBED_DEFAULT_JOIN: inner
PGRST_DB_USE_LEGACY_GUCS: false
PGRST_JWT_AUD: 'https://postgrest.org'
PGRST_JWT_ROLE_CLAIM_KEY: '.user[0]."real-role"'
+2
View File
@@ -7,11 +7,13 @@ db-plan-enabled = true
db-pool = 1
db-pool-acquisition-timeout = 30
db-pool-max-lifetime = 3600
db-pool-max-idletime = 60
db-pre-request = "please_run_fast"
db-prepared-statements = false
db-root-spec = "openapi_v3"
db-schemas = "multi, tenant,setup"
db-config = false
db-pre-config = "postgrest.pre_config"
db-tx-end = "rollback-allow-override"
db-uri = "tmp_db"
db-use-legacy-gucs = false
+31 -4
View File
@@ -9,6 +9,7 @@ ALTER ROLE db_config_authenticator SET pgrst.jwt_secret_is_base64 = 'false';
ALTER ROLE db_config_authenticator SET pgrst.jwt_role_claim_key = '."a"."role"';
ALTER ROLE db_config_authenticator SET pgrst.db_anon_role = 'anonymous';
ALTER ROLE db_config_authenticator SET pgrst.db_tx_end = 'commit-allow-override';
ALTER ROLE db_config_authenticator SET pgrst.db_pre_config = 'postgrest.preconf';
ALTER ROLE db_config_authenticator SET pgrst.db_schemas = 'test, tenant1, tenant2';
ALTER ROLE db_config_authenticator SET pgrst.db_root_spec = 'root';
ALTER ROLE db_config_authenticator SET pgrst.db_plan_enabled = 'true';
@@ -40,7 +41,10 @@ ALTER ROLE db_config_authenticator SET pgrst.db_channel_enabled = 'ignored';
ALTER ROLE db_config_authenticator SET pgrst.db_channel = 'ignored';
ALTER ROLE db_config_authenticator SET pgrst.db_pool = 'ignored';
ALTER ROLE db_config_authenticator SET pgrst.db_pool_timeout = 'ignored';
ALTER ROLE db_config_authenticator SET pgrst.db_config = 'ignored';
ALTER ROLE db_config_authenticator SET pgrst.db_pool_acquisition_timeout = 'ignored';
ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_lifetime = 'ignored';
ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_idletime = 'ignored';
ALTER ROLE db_config_authenticator SET pgrst.db_config = 'true';
-- other authenticator reloadable config options
CREATE ROLE other_authenticator LOGIN NOINHERIT;
@@ -49,9 +53,6 @@ ALTER ROLE other_authenticator SET pgrst.openapi_server_proxy_uri = 'https://oth
ALTER ROLE other_authenticator SET pgrst.raw_media_types = 'application/vnd.pgrst.other-db-config';
ALTER ROLE other_authenticator SET pgrst.jwt_secret = 'ODERREALLYREALLYREALLYREALLYVERYSAFE';
ALTER ROLE other_authenticator SET pgrst.jwt_secret_is_base64 = 'true';
ALTER ROLE other_authenticator SET pgrst.jwt_role_claim_key = '."other"."role"';
ALTER ROLE other_authenticator SET pgrst.db_anon_role = 'other';
ALTER ROLE other_authenticator SET pgrst.db_tx_end = 'rollback-allow-override';
ALTER ROLE other_authenticator SET pgrst.db_schemas = 'test, other_tenant1, other_tenant2';
ALTER ROLE other_authenticator SET pgrst.db_root_spec = 'other_root';
ALTER ROLE other_authenticator SET pgrst.db_plan_enabled = 'true';
@@ -62,6 +63,32 @@ ALTER ROLE other_authenticator SET pgrst.db_extra_search_path = 'public, extensi
ALTER ROLE other_authenticator SET pgrst.openapi_mode = 'disabled';
ALTER ROLE other_authenticator SET pgrst.openapi_security_active = 'false';
ALTER ROLE other_authenticator SET pgrst.server_trace_header = 'traceparent';
ALTER ROLE other_authenticator SET pgrst.db_pre_config = 'postgrest.pre_config';
create schema postgrest;
grant usage on schema postgrest to db_config_authenticator;
grant usage on schema postgrest to other_authenticator;
-- pre-config hook
create or replace function postgrest.pre_config()
returns void as $$
begin
if current_user = 'other_authenticator' then
perform
set_config('pgrst.jwt_role_claim_key', '."other"."pre_config_role"', true)
, set_config('pgrst.db_anon_role', 'pre_config_role', true)
, set_config('pgrst.db_schemas', 'will be overriden with the above ALTER ROLE.. db_schemas', true)
, set_config('pgrst.db_tx_end', 'rollback-allow-override', true);
else
null;
end if;
end $$ language plpgsql;
create or replace function postgrest.preconf()
returns void as $$
begin
null;
end $$ language plpgsql;
-- authenticator used for tests that manipulate statement timeout
CREATE ROLE timeout_authenticator LOGIN NOINHERIT;
+18 -2
View File
@@ -1,4 +1,4 @@
\ir big_schema.sql
-- \ir big_schema.sql big schema test currently skipped, see test_io.py
\ir db_config.sql
set search_path to public;
@@ -90,7 +90,7 @@ create or replace function sleep(seconds double precision) returns void as $$
$$ language sql;
create or replace function hello() returns text as $$
select 'hello';
select 'hello'::text;
$$ language sql;
create table cats(id uuid primary key, name text);
@@ -142,3 +142,19 @@ returns text as $$
select current_setting('transaction_isolation', true);
$$
language sql set default_transaction_isolation = 'REPEATABLE READ';
create or replace function create_function() returns void as $_$
drop function if exists mult_them(int, int);
create or replace function mult_them(a int, b int) returns int as $$
select a*b;
$$ language sql;
notify pgrst, 'reload schema';
$_$ language sql security definer;
create or replace function migrate_function() returns void as $_$
drop function if exists mult_them(int, int);
create or replace function mult_them(c int, d int) returns int as $$
select c*d;
$$ language sql;
notify pgrst, 'reload schema';
$_$ language sql security definer;
+41 -5
View File
@@ -308,13 +308,11 @@ def test_db_schema_reload(tmp_path, defaultenv):
# reload config
postgrest.process.send_signal(signal.SIGUSR2)
time.sleep(0.1)
# reload schema cache to verify that the config reload actually happened
postgrest.process.send_signal(signal.SIGUSR1)
# takes max 1 second to load the internal cache(big_schema.sql included now)
# TODO this could go back to time.sleep(0.1) if the big_schema is put in another test suite
time.sleep(1)
time.sleep(0.1)
response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
assert response.text == '"\\"v1\\", \\"public\\""'
@@ -551,7 +549,7 @@ def test_pool_size(defaultenv, metapostgrest):
def test_pool_acquisition_timeout(defaultenv, metapostgrest):
"Verify that PGRST_DB_POOL_ACQUISITON_TIMEOUT times out when the pool is empty"
"Verify that PGRST_DB_POOL_ACQUISITION_TIMEOUT times out when the pool is empty"
env = {
**defaultenv,
@@ -946,6 +944,44 @@ def test_isolation_level(defaultenv):
assert response.text == '"serializable"'
def test_schema_cache_reloading(defaultenv):
"schema cache should reload successfully"
# If DB_POOL=1, then the second request(/rpc/migrate_function) will just wait(PGRST_DB_POOL_ACQUISITION_TIMEOUT=10) for the schema cache reload to finish.
# This is bc the only pool connection will be busy with the PGRST_INTERNAL_SCHEMA_CACHE_SLEEP(does a pg_sleep)
# So this must be tested with a DB_POOL size of at least 2. That way the second request will pick the other pool connection and proceed.
env = {
**defaultenv,
"PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "1",
"PGRST_DB_CHANNEL_ENABLED": "true",
"PGRST_DB_POOL": "2",
}
internal_sleep = int(env["PGRST_INTERNAL_SCHEMA_CACHE_SLEEP"])
with run(env=env, wait_for_readiness=False) as postgrest:
time.sleep(2 * internal_sleep + 0.1) # wait for readiness manually
response = postgrest.session.post("/rpc/create_function")
assert response.status_code == 204
time.sleep(
internal_sleep / 2
) # wait to be inside the schema cache reload process
response = postgrest.session.post("/rpc/migrate_function")
assert response.status_code == 204
time.sleep(
2 * internal_sleep
) # wait enough time to ensure the schema cache state remains
response = postgrest.session.get("/rpc/mult_them?c=3&d=4")
assert response.text == "12"
assert response.status_code == 200
# TODO: This test fails now because of https://github.com/PostgREST/postgrest/pull/2122
# The stack size of 1K(-with-rtsopts=-K1K) is not enough and this fails with "stack overflow"
# A stack size of 200K seems to be enough for succeess
+7 -13
View File
@@ -3,8 +3,7 @@ module Main where
import qualified Hasql.Pool as P
import qualified Hasql.Transaction.Sessions as HT
import Data.Function (id)
import Data.List.NonEmpty (toList)
import Data.Function (id)
import Test.Hspec
@@ -66,14 +65,12 @@ import qualified Feature.RpcPreRequestGucsSpec
main :: IO ()
main = do
pool <- P.acquire 3 10 60 $ toUtf8 $ configDbUri testCfg
pool <- P.acquire 3 10 60 60 $ toUtf8 $ configDbUri testCfg
actualPgVersion <- either (panic . show) id <$> P.use pool (queryPgVersion False)
baseSchemaCache <-
loadSchemaCache pool
(configDbSchemas testCfg)
(configDbExtraSearchPath testCfg)
-- cached schema cache so most tests run fast
baseSchemaCache <- loadSchemaCache pool testCfg
let
-- For tests that run with the same refSchemaCache
@@ -85,10 +82,7 @@ main = do
-- For tests that run with a different SchemaCache(depends on configSchemas)
appDbs config = do
customSchemaCache <-
loadSchemaCache pool
(configDbSchemas config)
(configDbExtraSearchPath config)
customSchemaCache <- loadSchemaCache pool config
appState <- AppState.initWithPool pool config
AppState.putPgVersion appState actualPgVersion
AppState.putSchemaCache appState (Just customSchemaCache)
@@ -265,5 +259,5 @@ main = do
describe "Feature.RollbackForcedSpec" Feature.RollbackSpec.forced
where
loadSchemaCache pool schemas extraSearchPath =
either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache (toList schemas) extraSearchPath True)
loadSchemaCache pool conf =
either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache conf)
+4
View File
@@ -84,11 +84,13 @@ baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
, configDbPoolSize = 10
, configDbPoolAcquisitionTimeout = 10
, configDbPoolMaxLifetime = 1800
, configDbPoolMaxIdletime = 600
, configDbPreRequest = Just $ QualifiedIdentifier "test" "switch_role"
, configDbPreparedStatements = True
, configDbRootSpec = Nothing
, configDbSchemas = fromList ["test"]
, configDbConfig = False
, configDbPreConfig = Nothing
, configDbUri = "postgresql://"
, configDbUseLegacyGucs = True
, configFilePath = Nothing
@@ -111,6 +113,8 @@ baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
, configDbTxRollbackAll = True
, configAdminServerPort = Nothing
, configRoleSettings = mempty
, configRoleIsoLvl = mempty
, configInternalSCSleep = Nothing
}
testCfg :: AppConfig