diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d651d39..37136c6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ All notable changes to this project will be documented in this file. From versio - Remove automatic transaction retries on `40001 (serialization_failure)` errors to prevent replication lag by @laurenceisla in #3673 - Fix unexpected results when embedding and filtering the same table more than once by @laurenceisla in #4075 - Restore Listener query shape so it can be found in pg_stat_activity by @mkleczek in #4857 #4859 +- The LISTEN channel now automatically recovers when it stops working due to a PostgreSQL bug @laurenceisla in #3147 + ### Changed diff --git a/src/PostgREST/Listener.hs b/src/PostgREST/Listener.hs index f4628122a..4d4af1f2f 100644 --- a/src/PostgREST/Listener.hs +++ b/src/PostgREST/Listener.hs @@ -30,12 +30,12 @@ runListener :: AppState -> IO () runListener appState = do AppConfig{..} <- getConfig appState when configDbChannelEnabled $ - void . forkIO . void $ retryingListen appState + void . forkIO . void $ retryingListen appState False -- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost. -- | This function never returns (but can throw) and return type enforces that. -retryingListen :: AppState -> IO Void -retryingListen appState = do +retryingListen :: AppState -> Bool -> IO Void +retryingListen appState hasDbListenerBug = do cfg@AppConfig{..} <- AppState.getConfig appState let dbChannel = toS configDbChannel @@ -43,7 +43,7 @@ retryingListen appState = do AppState.putIsListenerOn appState False observer $ DBListenFail dbChannel (Right err) when (isDbListenerBug err) $ - observer DBListenBugHint + observer DBListenBugCallQueryFix unless configDbPoolAutomaticRecovery $ killThread mainThreadId @@ -54,7 +54,7 @@ retryingListen appState = do unless (delay == maxDelay) $ AppState.putNextListenerDelay appState (delay * 2) -- loop running the listener - retryingListen appState + retryingListen appState (isDbListenerBug err) -- Execute the listener with with error handling handle onError $ do @@ -70,6 +70,7 @@ retryingListen appState = do Right db -> do (pqHost, pqPort) <- SQL.withLibPQConnection db $ bisequence . (LibPQ.host &&& LibPQ.port) pgFullName <- SQL.run queryPgVersion db >>= either throwIO (pure . pgvFullName) + when hasDbListenerBug $ SQL.run callNotifQueryUsage db >>= either throwIO pure SQL.listen db $ SQL.toPgIdentifier dbChannel AppState.putIsListenerOn appState True @@ -108,3 +109,8 @@ retryingListen appState = do releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e + + -- Used to fix a Postgres bug in the listener, see: https://github.com/PostgREST/postgrest/issues/3147#issuecomment-3494591361 + -- This query advances the async notification query tail, which solves this issue. + callNotifQueryUsage :: SQL.Session () + callNotifQueryUsage = SQL.sql "SELECT pg_notification_queue_usage();" diff --git a/src/PostgREST/Logger.hs b/src/PostgREST/Logger.hs index b00bc775f..cb2a22ba8 100644 --- a/src/PostgREST/Logger.hs +++ b/src/PostgREST/Logger.hs @@ -188,8 +188,8 @@ observationMessages = \case either showListenerConnError showListenerException listenErr DBListenRetry delay -> pure $ "Retrying listening for database notifications in " <> (show delay::Text) <> " seconds..." - DBListenBugHint -> - pure "HINT: This is likely a bug in the notification queue, try executing the following to solve it: select pg_notification_queue_usage();" + DBListenBugCallQueryFix -> + pure "This is likely a PostgreSQL bug in the notification queue, executing the following to try to solve it: SELECT pg_notification_queue_usage();" DBListenerGotSCacheMsg channel -> pure $ "Received a schema cache reload message on the " <> show channel <> " channel" DBListenerGotConfigMsg channel -> diff --git a/src/PostgREST/Observation.hs b/src/PostgREST/Observation.hs index 6a37daab6..0fde28d01 100644 --- a/src/PostgREST/Observation.hs +++ b/src/PostgREST/Observation.hs @@ -38,7 +38,7 @@ data Observation | DBListenStart (Maybe ByteString) (Maybe ByteString) Text Text -- host, port, version string, channel | DBListenFail Text (Either SQL.ConnectionError SomeException) | DBListenRetry Int - | DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147 + | DBListenBugCallQueryFix | DBListenerGotSCacheMsg ByteString | DBListenerGotConfigMsg ByteString | DBListenerConnectionCleanupFail SomeException