From c09394517d41281c22126ca5d30290f500ce23b8 Mon Sep 17 00:00:00 2001 From: Laurence Isla Date: Thu, 30 Apr 2026 18:22:59 -0500 Subject: [PATCH] fix: call `pg_notification_queue_usage()` automatically when the LISTEN channel bug is detected (#4858) There's a PostgreSQL bug that doesn't let any listener to register in the DB: https://www.postgresql.org/message-id/flat/CAK98qZ3wZLE-RZJN_Y%2BTFjiTRPPFPBwNBpBi5K5CU8hUHkzDpw%40mail.gmail.com The only workaround is to advance the async notification queue tail, which can be done by executing: "SELECT pg_notification_queue_usage();". Before we just logged a HINT with this suggestion, but now we call that function directly and then let the listener to automatically recover. No automated tests were added here as it would be too complex and this is a PostgreSQL bug. But this was manually tested following the steps on https://github.com/PostgREST/postgrest/pull/4581#issuecomment-3690610592 --- CHANGELOG.md | 2 ++ src/PostgREST/Listener.hs | 16 +++++++++++----- src/PostgREST/Logger.hs | 4 ++-- src/PostgREST/Observation.hs | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d651d39..37136c6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ All notable changes to this project will be documented in this file. From versio - Remove automatic transaction retries on `40001 (serialization_failure)` errors to prevent replication lag by @laurenceisla in #3673 - Fix unexpected results when embedding and filtering the same table more than once by @laurenceisla in #4075 - Restore Listener query shape so it can be found in pg_stat_activity by @mkleczek in #4857 #4859 +- The LISTEN channel now automatically recovers when it stops working due to a PostgreSQL bug @laurenceisla in #3147 + ### Changed diff --git a/src/PostgREST/Listener.hs b/src/PostgREST/Listener.hs index f4628122a..4d4af1f2f 100644 --- a/src/PostgREST/Listener.hs +++ b/src/PostgREST/Listener.hs @@ -30,12 +30,12 @@ runListener :: AppState -> IO () runListener appState = do AppConfig{..} <- getConfig appState when configDbChannelEnabled $ - void . forkIO . void $ retryingListen appState + void . forkIO . void $ retryingListen appState False -- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost. -- | This function never returns (but can throw) and return type enforces that. -retryingListen :: AppState -> IO Void -retryingListen appState = do +retryingListen :: AppState -> Bool -> IO Void +retryingListen appState hasDbListenerBug = do cfg@AppConfig{..} <- AppState.getConfig appState let dbChannel = toS configDbChannel @@ -43,7 +43,7 @@ retryingListen appState = do AppState.putIsListenerOn appState False observer $ DBListenFail dbChannel (Right err) when (isDbListenerBug err) $ - observer DBListenBugHint + observer DBListenBugCallQueryFix unless configDbPoolAutomaticRecovery $ killThread mainThreadId @@ -54,7 +54,7 @@ retryingListen appState = do unless (delay == maxDelay) $ AppState.putNextListenerDelay appState (delay * 2) -- loop running the listener - retryingListen appState + retryingListen appState (isDbListenerBug err) -- Execute the listener with with error handling handle onError $ do @@ -70,6 +70,7 @@ retryingListen appState = do Right db -> do (pqHost, pqPort) <- SQL.withLibPQConnection db $ bisequence . (LibPQ.host &&& LibPQ.port) pgFullName <- SQL.run queryPgVersion db >>= either throwIO (pure . pgvFullName) + when hasDbListenerBug $ SQL.run callNotifQueryUsage db >>= either throwIO pure SQL.listen db $ SQL.toPgIdentifier dbChannel AppState.putIsListenerOn appState True @@ -108,3 +109,8 @@ retryingListen appState = do releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e + + -- Used to fix a Postgres bug in the listener, see: https://github.com/PostgREST/postgrest/issues/3147#issuecomment-3494591361 + -- This query advances the async notification query tail, which solves this issue. + callNotifQueryUsage :: SQL.Session () + callNotifQueryUsage = SQL.sql "SELECT pg_notification_queue_usage();" diff --git a/src/PostgREST/Logger.hs b/src/PostgREST/Logger.hs index b00bc775f..cb2a22ba8 100644 --- a/src/PostgREST/Logger.hs +++ b/src/PostgREST/Logger.hs @@ -188,8 +188,8 @@ observationMessages = \case either showListenerConnError showListenerException listenErr DBListenRetry delay -> pure $ "Retrying listening for database notifications in " <> (show delay::Text) <> " seconds..." - DBListenBugHint -> - pure "HINT: This is likely a bug in the notification queue, try executing the following to solve it: select pg_notification_queue_usage();" + DBListenBugCallQueryFix -> + pure "This is likely a PostgreSQL bug in the notification queue, executing the following to try to solve it: SELECT pg_notification_queue_usage();" DBListenerGotSCacheMsg channel -> pure $ "Received a schema cache reload message on the " <> show channel <> " channel" DBListenerGotConfigMsg channel -> diff --git a/src/PostgREST/Observation.hs b/src/PostgREST/Observation.hs index 6a37daab6..0fde28d01 100644 --- a/src/PostgREST/Observation.hs +++ b/src/PostgREST/Observation.hs @@ -38,7 +38,7 @@ data Observation | DBListenStart (Maybe ByteString) (Maybe ByteString) Text Text -- host, port, version string, channel | DBListenFail Text (Either SQL.ConnectionError SomeException) | DBListenRetry Int - | DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147 + | DBListenBugCallQueryFix | DBListenerGotSCacheMsg ByteString | DBListenerGotConfigMsg ByteString | DBListenerConnectionCleanupFail SomeException