fix: call pg_notification_queue_usage() automatically when the LISTEN channel bug is detected (#4858)

There's a PostgreSQL bug that doesn't let any listener to register in the DB:
https://www.postgresql.org/message-id/flat/CAK98qZ3wZLE-RZJN_Y%2BTFjiTRPPFPBwNBpBi5K5CU8hUHkzDpw%40mail.gmail.com
The only workaround is to advance the async notification queue tail,
which can be done by executing:  "SELECT pg_notification_queue_usage();".
Before we just logged a HINT with this suggestion, but now we call that function directly
and then let the listener to automatically recover.

No automated tests were added here as it would be too complex and this is a PostgreSQL bug. But this was manually tested following the steps on https://github.com/PostgREST/postgrest/pull/4581#issuecomment-3690610592

CHANGELOG.md

@@ -22,6 +22,8 @@ All notable changes to this project will be documented in this file. From versio
 - Remove automatic transaction retries on `40001 (serialization_failure)` errors to prevent replication lag by @laurenceisla in #3673
 - Fix unexpected results when embedding and filtering the same table more than once by @laurenceisla in #4075
 - Restore Listener query shape so it can be found in pg_stat_activity by @mkleczek in #4857 #4859
+- The LISTEN channel now automatically recovers when it stops working due to a PostgreSQL bug @laurenceisla in #3147
+
 
 ### Changed
 

src/PostgREST/Listener.hs

@@ -30,12 +30,12 @@ runListener :: AppState -> IO ()
 runListener appState = do
   AppConfig{..} <- getConfig appState
   when configDbChannelEnabled $
-    void . forkIO . void $ retryingListen appState
+    void . forkIO . void $ retryingListen appState False
 
 -- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
 -- | This function never returns (but can throw) and return type enforces that.
-retryingListen :: AppState -> IO Void
-retryingListen appState = do
+retryingListen :: AppState -> Bool -> IO Void
+retryingListen appState hasDbListenerBug = do
   cfg@AppConfig{..} <- AppState.getConfig appState
   let
     dbChannel = toS configDbChannel
@@ -43,7 +43,7 @@ retryingListen appState = do
       AppState.putIsListenerOn appState False
       observer $ DBListenFail dbChannel (Right err)
       when (isDbListenerBug err) $
-        observer DBListenBugHint
+        observer DBListenBugCallQueryFix
       unless configDbPoolAutomaticRecovery $
         killThread mainThreadId
 
@@ -54,7 +54,7 @@ retryingListen appState = do
       unless (delay == maxDelay) $
         AppState.putNextListenerDelay appState (delay * 2)
       -- loop running the listener
-      retryingListen appState
+      retryingListen appState (isDbListenerBug err)
 
   -- Execute the listener with with error handling
   handle onError $ do
@@ -70,6 +70,7 @@ retryingListen appState = do
         Right db -> do
           (pqHost, pqPort) <- SQL.withLibPQConnection db $ bisequence . (LibPQ.host &&& LibPQ.port)
           pgFullName <- SQL.run queryPgVersion db >>= either throwIO (pure . pgvFullName)
+          when hasDbListenerBug $ SQL.run callNotifQueryUsage db >>= either throwIO pure
           SQL.listen db $ SQL.toPgIdentifier dbChannel
 
           AppState.putIsListenerOn appState True
@@ -108,3 +109,8 @@ retryingListen appState = do
     releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release
 
     isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e
+
+    -- Used to fix a Postgres bug in the listener, see: https://github.com/PostgREST/postgrest/issues/3147#issuecomment-3494591361
+    -- This query advances the async notification query tail, which solves this issue.
+    callNotifQueryUsage :: SQL.Session ()
+    callNotifQueryUsage = SQL.sql "SELECT pg_notification_queue_usage();"

src/PostgREST/Logger.hs

@@ -188,8 +188,8 @@ observationMessages = \case
       either showListenerConnError showListenerException listenErr
   DBListenRetry delay ->
     pure $ "Retrying listening for database notifications in " <> (show delay::Text) <> " seconds..."
-  DBListenBugHint ->
-    pure "HINT:  This is likely a bug in the notification queue, try executing the following to solve it: select pg_notification_queue_usage();"
+  DBListenBugCallQueryFix ->
+    pure "This is likely a PostgreSQL bug in the notification queue, executing the following to try to solve it: SELECT pg_notification_queue_usage();"
   DBListenerGotSCacheMsg channel ->
     pure $ "Received a schema cache reload message on the " <> show channel <> " channel"
   DBListenerGotConfigMsg channel ->

src/PostgREST/Observation.hs

@@ -38,7 +38,7 @@ data Observation
   | DBListenStart (Maybe ByteString) (Maybe ByteString) Text Text -- host, port, version string, channel
   | DBListenFail Text (Either SQL.ConnectionError SomeException)
   | DBListenRetry Int
-  | DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147
+  | DBListenBugCallQueryFix
   | DBListenerGotSCacheMsg ByteString
   | DBListenerGotConfigMsg ByteString
   | DBListenerConnectionCleanupFail SomeException