feat: /live and /ready respond with 500 on failure

503 is still used by /ready to indicate a transient state
that can be recovered from.

src/PostgREST/Admin.hs

@@ -42,12 +42,19 @@ admin :: AppState.AppState -> Wai.Application
 admin appState req respond  = do
   isMainAppReachable  <- isRight <$> reachMainApp (AppState.getSocketREST appState)
   isLoaded <- AppState.isLoaded appState
+  isPending <- AppState.isPending appState
 
   case Wai.pathInfo req of
-    ["ready"] ->
-      respond $ Wai.responseLBS (if isMainAppReachable && isLoaded then HTTP.status200 else HTTP.status503) [] mempty
     ["live"] ->
-      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
+      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status500) [] mempty
+    ["ready"] ->
+      let
+        status | not isMainAppReachable = HTTP.status500
+               | isPending              = HTTP.status503
+               | isLoaded               = HTTP.status200
+               | otherwise              = HTTP.status500
+      in
+      respond $ Wai.responseLBS status [] mempty
     ["config"] -> do
       config <- AppState.getConfig appState
       respond $ Wai.responseLBS HTTP.status200 [] (LBS.fromStrict $ encodeUtf8 $ Config.toText config)

src/PostgREST/AppState.hs

@@ -27,6 +27,7 @@ module PostgREST.AppState
   , runListener
   , getObserver
   , isLoaded
+  , isPending
   ) where
 
 import qualified Data.Aeson                 as JSON
@@ -315,6 +316,12 @@ isLoaded x = do
   connEstablished <- isConnEstablished x
   return $ scacheStatus == SCLoaded && connEstablished
 
+isPending :: AppState -> IO Bool
+isPending x = do
+  scacheStatus <- readIORef $ stateSCacheStatus x
+  connStatus <- readIORef $ stateConnStatus x
+  return $ scacheStatus == SCPending || connStatus == ConnPending
+
 putSCacheStatus :: AppState -> SchemaCacheStatus -> IO ()
 putSCacheStatus = atomicWriteIORef . stateSCacheStatus
 
@@ -338,12 +345,15 @@ loadSchemaCache appState@AppState{stateObserver=observer} = do
           observer $ SchemaCacheFatalErrorObs e hint
           return SCFatalFail
         Nothing -> do
+          putSCacheStatus appState SCPending
           putSchemaCache appState Nothing
           observer $ SchemaCacheNormalErrorObs e
-          putSCacheStatus appState SCPending
           return SCPending
 
     Right sCache -> do
+      -- IMPORTANT: While the pending schema cache state starts from running the above querySchemaCache, only at this stage we block API requests due to the usage of an
+      -- IORef on putSchemaCache. This is why SCacheStatus is put at SCPending here to signal the Admin server (using isPending) that we're on a recovery state.
+      putSCacheStatus appState SCPending
       putSchemaCache appState $ Just sCache
       observer $ SchemaCacheQueriedObs resultTime
       (t, _) <- timeItT $ observer $ SchemaCacheSummaryObs $ showSummary sCache