prevent GSSAPI error between Listener and pool

Brings back the the signaling/waiting between the connection pool and
the Listener.

Prevents the GSSAPI error shown on https://github.com/PostgREST/postgrest/issues/3569

src/PostgREST/AppState.hs

@@ -29,6 +29,7 @@ module PostgREST.AppState
   , getObserver
   , isLoaded
   , isPending
+  , waitForListenerCanStart
   ) where
 
 import qualified Data.Aeson                 as JSON
@@ -98,6 +99,8 @@ data AppState = AppState
   , stateIsListenerOn         :: IORef Bool
   -- | starts the connection worker with a debounce
   , debouncedConnectionWorker :: IO ()
+  -- | Binary semaphore used to sync the listener with the connectionWorker.
+  , stateListenerCanStart     :: MVar ()
   -- | Config that can change at runtime
   , stateConf                 :: IORef AppConfig
   -- | Time used for verifying JWT expiration
@@ -156,6 +159,7 @@ initWithPool (sock, adminSock) pool conf loggerState metricsState observer = do
     <*> newIORef ConnPending
     <*> newIORef False
     <*> pure (pure ())
+    <*> newEmptyMVar
     <*> newIORef conf
     <*> mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime }
     <*> myThreadId
@@ -317,6 +321,26 @@ getNextListenerDelay = readIORef . stateNextListenerDelay
 putNextListenerDelay :: AppState -> Int -> IO ()
 putNextListenerDelay = atomicWriteIORef . stateNextListenerDelay
 
+--------------------------------------------------------------------------------------
+-------------------------------------------IMPORTANT----------------------------------
+--------------------------------------------------------------------------------------
+-- Both of these function ensure there's no parallel connection attempts between the listener and the connection pool.
+-- Doing that raised an error with GSSAPI as discussed on https://github.com/PostgREST/postgrest/issues/3569.
+-- Until the root cause is found and solved, we need to prevent parallel connection attempts.
+
+-- tryPutMVar doesn't lock the thread. It should always succeed since
+-- the connectionWorker is the only mvar producer.
+setListenerCanStart :: AppState -> IO ()
+setListenerCanStart appState = void $ tryPutMVar (stateListenerCanStart appState) ()
+
+-- | As this IO action uses `takeMVar` internally, it will only return once
+-- `stateListenerCanStart` has been set using `setListenerCanStart`.
+waitForListenerCanStart :: AppState -> IO ()
+waitForListenerCanStart = takeMVar . stateListenerCanStart
+
+--------------------------------------------------------------------------------------
+--------------------------------------------------------------------------------------
+
 getConfig :: AppState -> IO AppConfig
 getConfig = readIORef . stateConf
 
@@ -436,6 +460,9 @@ internalConnectionWorker appState@AppState{stateObserver=observer, stateMainThre
             observer $ ExitUnsupportedPgVersion actualPgVersion minimumPgVersion
             killThread mainThreadId
           observer (DBConnectedObs $ pgvFullName actualPgVersion)
+          -- Wake up the Listener
+          when configDbChannelEnabled $
+            setListenerCanStart appState
           -- this could be fail because the connection drops, but the loadSchemaCache will pick the error and retry again
           -- We cannot retry after it fails immediately, because db-pre-config could have user errors. We just log the error and continue.
           when configDbConfig $ reReadConfig False appState

src/PostgREST/Listener.hs

@@ -27,10 +27,16 @@ runListener appState = do
 -- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
 retryingListen :: AppState -> IO ()
 retryingListen appState = do
+  AppState.waitForListenerCanStart appState
   AppConfig{..} <- AppState.getConfig appState
   let
     dbChannel = toS configDbChannel
     handleFinally err = do
+      -- assume we lost notifications, call the connection worker which will also reload the schema cache
+      -- and will setListenerCanStart again
+      -- TODO: When the connection error is only on the Listener, it's wasteful to call the connectionWorker everytime.
+      AppState.connectionWorker appState
+
       AppState.putIsListenerOn appState False
       observer $ DBListenFail dbChannel (Right err)
       unless configDbPoolAutomaticRecovery $
@@ -54,8 +60,6 @@ retryingListen appState = do
 
         delay <- AppState.getNextListenerDelay appState
         when (delay > 1) $ do -- if we did a retry
-          -- assume we lost notifications, call the connection worker which will also reload the schema cache
-          AppState.connectionWorker appState
           -- reset the delay
           AppState.putNextListenerDelay appState 1