fix: listener running with exception masked after first failure
This commit is contained in:
@@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file. From versio
|
|||||||
|
|
||||||
- Ensure Listener connections are released by @mkleczek in #4614
|
- Ensure Listener connections are released by @mkleczek in #4614
|
||||||
- Fix incorrectly filtering the returned representation for PATCH requests when using `or/and` filters by @laurenceisla in #3707
|
- Fix incorrectly filtering the returned representation for PATCH requests when using `or/and` filters by @laurenceisla in #3707
|
||||||
|
- Fix listener running with exception masked after first failure #4615
|
||||||
|
|
||||||
## [14.3] - 2026-01-03
|
## [14.3] - 2026-01-03
|
||||||
|
|
||||||
|
|||||||
@@ -31,15 +31,16 @@ runListener :: AppState -> IO ()
|
|||||||
runListener appState = do
|
runListener appState = do
|
||||||
AppConfig{..} <- getConfig appState
|
AppConfig{..} <- getConfig appState
|
||||||
when configDbChannelEnabled $
|
when configDbChannelEnabled $
|
||||||
void . forkIO $ retryingListen appState
|
void . forkIO . void $ retryingListen appState
|
||||||
|
|
||||||
-- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
|
-- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
|
||||||
retryingListen :: AppState -> IO ()
|
-- | This function never returns (but can throw) and return type enforces that.
|
||||||
|
retryingListen :: AppState -> IO Void
|
||||||
retryingListen appState = do
|
retryingListen appState = do
|
||||||
AppConfig{..} <- AppState.getConfig appState
|
AppConfig{..} <- AppState.getConfig appState
|
||||||
let
|
let
|
||||||
dbChannel = toS configDbChannel
|
dbChannel = toS configDbChannel
|
||||||
handleFinally err = do
|
onError err = do
|
||||||
AppState.putIsListenerOn appState False
|
AppState.putIsListenerOn appState False
|
||||||
observer $ DBListenFail dbChannel (Right err)
|
observer $ DBListenFail dbChannel (Right err)
|
||||||
when (isDbListenerBug err) $
|
when (isDbListenerBug err) $
|
||||||
@@ -53,10 +54,11 @@ retryingListen appState = do
|
|||||||
threadDelay (delay * oneSecondInMicro)
|
threadDelay (delay * oneSecondInMicro)
|
||||||
unless (delay == maxDelay) $
|
unless (delay == maxDelay) $
|
||||||
AppState.putNextListenerDelay appState (delay * 2)
|
AppState.putNextListenerDelay appState (delay * 2)
|
||||||
|
-- loop running the listener
|
||||||
retryingListen appState
|
retryingListen appState
|
||||||
|
|
||||||
-- forkFinally allows to detect if the thread dies
|
-- Execute the listener with with error handling
|
||||||
void . flip forkFinally handleFinally $ do
|
handle onError $ do
|
||||||
-- Make sure we don't leak connections on errors
|
-- Make sure we don't leak connections on errors
|
||||||
bracket
|
bracket
|
||||||
-- acquire connection
|
-- acquire connection
|
||||||
@@ -81,7 +83,9 @@ retryingListen appState = do
|
|||||||
|
|
||||||
observer $ DBListenStart pqHost pqPort pgFullName dbChannel
|
observer $ DBListenStart pqHost pqPort pgFullName dbChannel
|
||||||
|
|
||||||
SQL.waitForNotifications handleNotification db
|
-- wait for notifications
|
||||||
|
-- this will never return, in case of an error it will throw and be caught by onError
|
||||||
|
forever $ SQL.waitForNotifications handleNotification db
|
||||||
|
|
||||||
Left err -> do
|
Left err -> do
|
||||||
observer $ DBListenFail dbChannel (Left err)
|
observer $ DBListenFail dbChannel (Left err)
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ data Observation
|
|||||||
| SchemaCacheLoadedObs Double
|
| SchemaCacheLoadedObs Double
|
||||||
| ConnectionRetryObs Int
|
| ConnectionRetryObs Int
|
||||||
| DBListenStart (Maybe ByteString) (Maybe ByteString) Text Text -- host, port, version string, channel
|
| DBListenStart (Maybe ByteString) (Maybe ByteString) Text Text -- host, port, version string, channel
|
||||||
| DBListenFail Text (Either SQL.ConnectionError (Either SomeException ()))
|
| DBListenFail Text (Either SQL.ConnectionError SomeException)
|
||||||
| DBListenRetry Int
|
| DBListenRetry Int
|
||||||
| DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147
|
| DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147
|
||||||
| DBListenerGotSCacheMsg ByteString
|
| DBListenerGotSCacheMsg ByteString
|
||||||
@@ -171,14 +171,12 @@ observationMessage = \case
|
|||||||
showListenerConnError :: SQL.ConnectionError -> Text
|
showListenerConnError :: SQL.ConnectionError -> Text
|
||||||
showListenerConnError = maybe "Connection error" (showOnSingleLine '\t' . T.decodeUtf8)
|
showListenerConnError = maybe "Connection error" (showOnSingleLine '\t' . T.decodeUtf8)
|
||||||
|
|
||||||
showListenerException :: Either SomeException () -> Text
|
showListenerException :: SomeException -> Text
|
||||||
showListenerException (Right _) = "Failed getting notifications" -- should not happen as the listener will never finish (hasql-notifications uses `forever` internally) with a Right result
|
showListenerException = showOnSingleLine '\t' . show
|
||||||
showListenerException (Left e) = showOnSingleLine '\t' $ show e
|
|
||||||
|
|
||||||
|
|
||||||
showOnSingleLine :: Char -> Text -> Text
|
showOnSingleLine :: Char -> Text -> Text
|
||||||
showOnSingleLine split txt = T.intercalate " " $ T.filter (/= split) <$> T.lines txt -- the errors from hasql-notifications come intercalated with "\t\n"
|
showOnSingleLine split txt = T.intercalate " " $ T.filter (/= split) <$> T.lines txt -- the errors from hasql-notifications come intercalated with "\t\n"
|
||||||
|
|
||||||
isDbListenerBug :: Either SomeException () -> Bool
|
isDbListenerBug :: SomeException -> Bool
|
||||||
isDbListenerBug (Left e) = "could not access status of transaction" `T.isInfixOf` show e
|
isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e
|
||||||
isDbListenerBug _ = False
|
|
||||||
|
|||||||
Reference in New Issue
Block a user