fix: shutdown should wait for in flight requests

Upgraded warp to 3.4.13 which fixed https://github.com/yesodweb/wai/issues/853
Changed interrupt handling so that instead of killing the main thread, listening sockets are closed which triggers warp graceful shutdown.

CHANGELOG.md

@@ -15,6 +15,10 @@ All notable changes to this project will be documented in this file. From versio
 - Add config `db-timezone-enabled` for optional querying of timezones by @taimoorzaeem in #4751
 - Log schema cache queries timings on `log-level=debug` by @steve-chavez in #4805
 
+### Fixed
+
+- Shutdown should wait for in flight requests by @mkleczek in #4702
+
 ### Changed
 
 - Drop support for PostgreSQL EOL version 13 by @wolfgangwalther in #4193

nix/overlays/haskell-packages.nix

@@ -59,6 +59,43 @@ let
       postgresql-binary = lib.dontCheck (lib.doJailbreak prev.postgresql-binary_0_13_1_3);
       text-builder = prev.text-builder_0_6_10;
       text-builder-dev = prev.text-builder-dev_0_3_10;
+
+      http2 =
+        prev.callHackageDirect
+          {
+            pkg = "http2";
+            ver = "5.4.0";
+            sha256 = "sha256-PeEWVd61bQ8G7LvfLeXklzXqNJFaAjE2ecRMWJZESPE=";
+          }
+          { };
+
+      http-semantics =
+        prev.callHackageDirect
+          {
+            pkg = "http-semantics";
+            ver = "0.4.0";
+            sha256 = "sha256-rh0z51EKvsu5rQd5n2z3fSRjjEObouNZSBPO9NFYOF0=";
+          }
+          { };
+
+      network-run =
+        prev.callHackageDirect
+          {
+            pkg = "network-run";
+            ver = "0.5.0";
+            sha256 = "sha256-vbXh+CzxDsGApjqHxCYf/ijpZtUCApFbkcF5gyN0THU=";
+          }
+          { };
+
+      warp =
+        lib.dontCheck (prev.callHackageDirect
+          {
+            pkg = "warp";
+            ver = "3.4.13";
+            sha256 = "sha256-jmr8kpeSPDkOhT0i9PhozZapX4nUs92cOX7POAGb7/M=";
+          }
+          { });
+
     };
 in
 {

postgrest.cabal

@@ -156,7 +156,7 @@ library
                     -- for unix sockets; this is tested in test/io/test_io.py. See
                     -- https://github.com/kazu-yamamoto/logger/commit/3a71ca70afdbb93d4ecf0083eeba1fbbbcab3fc3
                     , wai-logger                >= 2.4.0
-                    , warp                      >= 3.3.19 && < 3.5
+                    , warp                      >= 3.4.13 && < 3.5
                     , stm                       >= 2.5 && < 3
                     , stm-hamt                  >= 1.2 && < 2
                     , focus                     >= 1.0 && < 2

src/PostgREST/App.hs

@@ -22,6 +22,7 @@ import GHC.IO.Exception (IOErrorType (..))
 import System.IO.Error  (ioeGetErrorType)
 
 import Control.Monad.Except     (liftEither)
+import Control.Monad.Extra      (whenJust)
 import Data.Either.Combinators  (mapLeft, whenLeft)
 import Data.Maybe               (fromJust)
 import Data.String              (IsString (..))
@@ -79,8 +80,10 @@ run appState = do
 
   AppState.schemaCacheLoader appState -- Loads the initial SchemaCache
   (mainSocket, adminSocket) <- initSockets conf
-
+  let closeSockets = do
-  Unix.installSignalHandlers observer (AppState.getMainThreadId appState) (AppState.schemaCacheLoader appState) (AppState.readInDbConfig False appState)
+        whenJust adminSocket NS.close
+        NS.close mainSocket
+  Unix.installSignalHandlers observer closeSockets (AppState.schemaCacheLoader appState) (AppState.readInDbConfig False appState)
 
   Listener.runListener appState
 

src/PostgREST/Unix.hs

@@ -19,10 +19,9 @@ import           System.Directory      (removeFile)
 import           System.IO.Error       (isDoesNotExistError)
 
 -- | Set signal handlers, only for systems with signals
-installSignalHandlers :: Observation.ObservationHandler -> ThreadId -> IO () -> IO () -> IO ()
+installSignalHandlers :: Observation.ObservationHandler -> IO () -> IO () -> IO () -> IO ()
 #ifndef mingw32_HOST_OS
-installSignalHandlers observer tid usr1 usr2 = do
+installSignalHandlers observer interrupt usr1 usr2 = do
-  let interrupt = throwTo tid UserInterrupt
   install Signals.sigINT  $ observer (Observation.TerminationUnixSignalObs "SIGINT") >> interrupt
   install Signals.sigTERM $ observer (Observation.TerminationUnixSignalObs "SIGTERM") >> interrupt
   install Signals.sigUSR1 usr1

stack.yaml

@@ -16,9 +16,12 @@ extra-deps:
   - hasql-notifications-0.2.2.2
   - hasql-pool-1.0.1
   - hasql-transaction-1.1.0.1
+  - http-semantics-0.4.0
+  - http2-5.4.0
   - postgresql-binary-0.13.1.3
   - text-builder-0.6.10
   - text-builder-dev-0.3.10
+  - warp-3.4.13
 
 allow-newer: true
 allow-newer-deps:

stack.yaml.lock

@@ -60,6 +60,20 @@ packages:
       size: 1027
   original:
     hackage: hasql-transaction-1.1.0.1
+- completed:
+    hackage: http-semantics-0.4.0@sha256:da8a98d542b2032cc12590847179577b0208a52bb3b9aa9a07c08d27d2a1714c,1513
+    pantry-tree:
+      sha256: d0e08875907c0fbff71813747fd93ce7bfd4e3d4a6b979df5c137430a9130f1d
+      size: 1188
+  original:
+    hackage: http-semantics-0.4.0
+- completed:
+    hackage: http2-5.4.0@sha256:1e9f6f5f32bfb3176136f35e041aa279bc456e81d4674ddaaeaa7c0d091be0c7,10624
+    pantry-tree:
+      sha256: 5c89815392d85d854efe75cf2279f58ff9f5e4b8fc1f83c55de93191edd128da
+      size: 44864
+  original:
+    hackage: http2-5.4.0
 - completed:
     hackage: postgresql-binary-0.13.1.3@sha256:4de5ddc90d9d3e586c3edf2860280a0915a484e9b8de3f36316a4cab2b330852,4037
     pantry-tree:

test/io/test_io.py

@@ -128,7 +128,6 @@ def test_flush_pool_no_interrupt(defaultenv):
         t.join()
 
 
-@pytest.mark.xfail(reason="Graceful shutdown is currently failing", strict=True)
 def test_graceful_shutdown_waits_for_in_flight_request(defaultenv):
     "SIGTERM should allow in-flight requests to finish before exiting"
 

test/memory/memory-tests.sh

@@ -110,9 +110,9 @@ jsonKeyTest "10M" "POST" "/rpc/leak?columns=blob" "32M"
 jsonKeyTest "10M" "POST" "/leak?columns=blob" "32M"
 jsonKeyTest "10M" "PATCH" "/leak?id=eq.1&columns=blob" "50M"
 
-jsonKeyTest "50M" "POST" "/rpc/leak?columns=blob" "73M"
+jsonKeyTest "50M" "POST" "/rpc/leak?columns=blob" "78M"
-jsonKeyTest "50M" "POST" "/leak?columns=blob" "73M"
+jsonKeyTest "50M" "POST" "/leak?columns=blob" "78M"
-jsonKeyTest "50M" "PATCH" "/leak?id=eq.1&columns=blob" "73M"
+jsonKeyTest "50M" "PATCH" "/leak?id=eq.1&columns=blob" "78M"
 
 postJsonArrayTest "1000" "/perf_articles?columns=id,body" "21M"
 postJsonArrayTest "10000" "/perf_articles?columns=id,body" "22M"