From 0502488b705765898a4bac94ceada9a2a59f8798 Mon Sep 17 00:00:00 2001 From: Laurence Isla Date: Thu, 30 Apr 2026 18:22:59 -0500 Subject: [PATCH] fix: call `pg_notification_queue_usage()` automatically when the LISTEN channel bug is detected (#4858) There's a PostgreSQL bug that doesn't let any listener to register in the DB: https://www.postgresql.org/message-id/flat/CAK98qZ3wZLE-RZJN_Y%2BTFjiTRPPFPBwNBpBi5K5CU8hUHkzDpw%40mail.gmail.com The only workaround is to advance the async notification queue tail, which can be done by executing: "SELECT pg_notification_queue_usage();". Before we just logged a HINT with this suggestion, but now we call that function directly and then let the listener to automatically recover. No automated tests were added here as it would be too complex and this is a PostgreSQL bug. But this was manually tested following the steps on https://github.com/PostgREST/postgrest/pull/4581#issuecomment-3690610592 --- CHANGELOG.md | 1 + src/PostgREST/Listener.hs | 22 +++++++++++++++------- src/PostgREST/Observation.hs | 10 +++------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd817aa31..bb1b248bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. From versio - Fix login with uppercase and mixed case role names by @taimoorzaeem in #4678 - Restore Listener query shape so it can be found in `pg_stat_activity` by @mkleczek in #4857 #4859 +- The LISTEN channel now automatically recovers when it stops working due to a PostgreSQL bug @laurenceisla in #3147 ## [14.10] - 2026-04-16 diff --git a/src/PostgREST/Listener.hs b/src/PostgREST/Listener.hs index 9dc5d3edc..652dbe5af 100644 --- a/src/PostgREST/Listener.hs +++ b/src/PostgREST/Listener.hs @@ -10,8 +10,7 @@ import qualified Hasql.Connection as SQL import qualified Hasql.Notifications as SQL import PostgREST.AppState (AppState, getConfig) import PostgREST.Config (AppConfig (..)) -import PostgREST.Observation (Observation (..), - isDbListenerBug) +import PostgREST.Observation (Observation (..)) import PostgREST.Version (prettyVersion) import qualified PostgREST.AppState as AppState @@ -20,6 +19,7 @@ import qualified PostgREST.Config as Config import Control.Arrow ((&&&)) import Data.Bitraversable (bisequence) import Data.Either.Combinators (whenRight) +import qualified Data.Text as T import qualified Database.PostgreSQL.LibPQ as LibPQ import qualified Hasql.Session as SQL import PostgREST.Config.Database (queryPgVersion) @@ -31,12 +31,12 @@ runListener :: AppState -> IO () runListener appState = do AppConfig{..} <- getConfig appState when configDbChannelEnabled $ - void . forkIO . void $ retryingListen appState + void . forkIO . void $ retryingListen appState False -- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost. -- | This function never returns (but can throw) and return type enforces that. -retryingListen :: AppState -> IO Void -retryingListen appState = do +retryingListen :: AppState -> Bool -> IO Void +retryingListen appState hasDbListenerBug = do AppConfig{..} <- AppState.getConfig appState let dbChannel = toS configDbChannel @@ -44,7 +44,7 @@ retryingListen appState = do AppState.putIsListenerOn appState False observer $ DBListenFail dbChannel (Right err) when (isDbListenerBug err) $ - observer DBListenBugHint + observer DBListenBugCallQueryFix unless configDbPoolAutomaticRecovery $ killThread mainThreadId @@ -55,7 +55,7 @@ retryingListen appState = do unless (delay == maxDelay) $ AppState.putNextListenerDelay appState (delay * 2) -- loop running the listener - retryingListen appState + retryingListen appState (isDbListenerBug err) -- Execute the listener with with error handling handle onError $ do @@ -70,6 +70,7 @@ retryingListen appState = do Right db -> do (pqHost, pqPort) <- SQL.withLibPQConnection db $ bisequence . (LibPQ.host &&& LibPQ.port) pgFullName <- SQL.run (queryPgVersion False) db >>= either throwIO (pure . pgvFullName) + when hasDbListenerBug $ SQL.run callNotifQueryUsage db >>= either throwIO pure SQL.listen db $ SQL.toPgIdentifier dbChannel AppState.putIsListenerOn appState True @@ -106,3 +107,10 @@ retryingListen appState = do AppState.schemaCacheLoader appState releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release + + isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e + + -- Used to fix a Postgres bug in the listener, see: https://github.com/PostgREST/postgrest/issues/3147#issuecomment-3494591361 + -- This query advances the async notification query tail, which solves this issue. + callNotifQueryUsage :: SQL.Session () + callNotifQueryUsage = SQL.sql "SELECT pg_notification_queue_usage();" diff --git a/src/PostgREST/Observation.hs b/src/PostgREST/Observation.hs index 99d450601..e38bcb18a 100644 --- a/src/PostgREST/Observation.hs +++ b/src/PostgREST/Observation.hs @@ -13,7 +13,6 @@ module PostgREST.Observation , observationMessage , ObservationHandler , showOnSingleLine - , isDbListenerBug ) where import qualified Data.ByteString.Lazy as LBS @@ -48,7 +47,7 @@ data Observation | DBListenStart (Maybe ByteString) (Maybe ByteString) Text Text -- host, port, version string, channel | DBListenFail Text (Either SQL.ConnectionError SomeException) | DBListenRetry Int - | DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147 + | DBListenBugCallQueryFix | DBListenerGotSCacheMsg ByteString | DBListenerGotConfigMsg ByteString | DBListenerConnectionCleanupFail SomeException @@ -122,8 +121,8 @@ observationMessage = \case either showListenerConnError showListenerException listenErr DBListenRetry delay -> "Retrying listening for database notifications in " <> (show delay::Text) <> " seconds..." - DBListenBugHint -> - "HINT: This is likely a bug in the notification queue, try executing the following to solve it: select pg_notification_queue_usage();" + DBListenBugCallQueryFix -> + "This is likely a PostgreSQL bug in the notification queue, executing the following to try to solve it: SELECT pg_notification_queue_usage();" DBListenerGotSCacheMsg channel -> "Received a schema cache reload message on the " <> show channel <> " channel" DBListenerGotConfigMsg channel -> @@ -188,6 +187,3 @@ observationMessage = \case showOnSingleLine :: Char -> Text -> Text showOnSingleLine split txt = T.intercalate " " $ T.filter (/= split) <$> T.lines txt -- the errors from hasql-notifications come intercalated with "\t\n" - -isDbListenerBug :: SomeException -> Bool -isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e