nix(test): add test suite for observability tests

- Create separate test suite for observability tests

- Create wrapper script `postgrest-test-observability`

- Add to CI and `postgrest-check`

- Move JWT cache tests under observability tests

Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>

.github/workflows/test.yaml

@@ -83,7 +83,7 @@ jobs:
         uses: ./.github/actions/setup-nix
         with:
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: tests.testSpec.bin tests.testIO.bin tests.testBigSchema.bin withTools.pg-${{ matrix.pgVersion }}.bin cabalTools.update.bin
+          tools: tests.testSpec.bin tests.testObservability.bin tests.testIO.bin tests.testBigSchema.bin withTools.pg-${{ matrix.pgVersion }}.bin cabalTools.update.bin
 
       - run: postgrest-cabal-update
 
@@ -91,6 +91,10 @@ jobs:
         if: always()
         run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-spec
 
+      - name: Run observability tests
+        if: always()
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-observability
+
       - name: Run IO tests
         if: always()
         run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-io -vv

nix/tools/devTools.nix

@@ -82,6 +82,7 @@ let
       }
       ''
         ${tests}/bin/postgrest-test-spec
+        ${tests}/bin/postgrest-test-observability
         ${tests}/bin/postgrest-test-doctests
         ${tests}/bin/postgrest-test-io
         ${tests}/bin/postgrest-test-big-schema

nix/tools/tests.nix

@@ -32,6 +32,20 @@ let
           ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec -- "''${_arg_leftovers[@]}"
       '';
 
+  testObservability =
+    checkedShellScript
+      {
+        name = "postgrest-test-observability";
+        docs = "Run the Haskell observability test suite.";
+        args = [ "ARG_LEFTOVERS([hspec arguments])" ];
+        workingDir = "/";
+        withEnv = postgrest.env;
+      }
+      ''
+        ${withTools.withPg} -f test/observability/fixtures/load.sql \
+          ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:observability -- "''${_arg_leftovers[@]}"
+      '';
+
   testDoctests =
     checkedShellScript
       {
@@ -155,7 +169,7 @@ let
           rm -rf coverage/*
 
           # build once before running all the tests
-          ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest lib:postgrest test:spec
+          ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest lib:postgrest test:spec test:observability
 
           (
             trap 'echo Found dead code: Check file list above.' ERR ;
@@ -179,11 +193,16 @@ let
             ${withTools.withPg} -f test/spec/fixtures/load.sql \
             ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec
 
+          HPCTIXFILE="$tmpdir"/observability.tix \
+            ${withTools.withPg} -f test/observability/fixtures/load.sql \
+            ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:observability
+
           # Note: No coverage for doctests, as doctests leverage GHCi and GHCi does not support hpc
 
           # collect all the tix files
           ${ghc}/bin/hpc sum  --union --exclude=Paths_postgrest --output="$tmpdir"/tests.tix \
-            "$tmpdir"/io*.tix "$tmpdir"/big_schema*.tix "$tmpdir"/replica*.tix "$tmpdir"/spec.tix
+            "$tmpdir"/io*.tix "$tmpdir"/big_schema*.tix "$tmpdir"/replica*.tix "$tmpdir"/spec.tix \
+            "$tmpdir"/observability.tix
 
           # prepare the overlay
           ${ghc}/bin/hpc overlay --output="$tmpdir"/overlay.tix test/coverage.overlay
@@ -250,6 +269,7 @@ buildToolbox
   tools = {
     inherit
       testSpec
+      testObservability
       testDoctests
       testSpecIdempotence
       testIO

postgrest.cabal

@@ -213,7 +213,6 @@ test-suite spec
                       Feature.Auth.AudienceJwtSecretSpec
                       Feature.Auth.AuthSpec
                       Feature.Auth.BinaryJwtSecretSpec
-                      Feature.Auth.JwtCacheSpec
                       Feature.Auth.NoAnonSpec
                       Feature.Auth.NoJwtSecretSpec
                       Feature.ConcurrentSpec
@@ -296,6 +295,37 @@ test-suite spec
                       -- https://github.com/PostgREST/postgrest/issues/387
                       -with-rtsopts=-K33K
 
+test-suite observability
+  type:               exitcode-stdio-1.0
+  default-language:   Haskell2010
+  default-extensions: OverloadedStrings
+                      QuasiQuotes
+                      NoImplicitPrelude
+  hs-source-dirs:     test/observability
+  main-is:            Main.hs
+  other-modules:      ObsHelper
+                      Observation.JwtCache
+  build-depends:      base              >= 4.9 && < 4.20
+                    , base64-bytestring >= 1 && < 1.3
+                    , bytestring        >= 0.10.8 && < 0.13
+                    , hasql-pool        >= 1.0.1 && < 1.1
+                    , hasql-transaction >= 1.0.1 && < 1.2
+                    , hspec             >= 2.3 && < 2.12
+                    , hspec-expectations >= 0.8.4 && < 0.9
+                    , hspec-wai         >= 0.10 && < 0.12
+                    , hspec-wai-json    >= 0.10 && < 0.12
+                    , http-types        >= 0.12.3 && < 0.13
+                    , jose-jwt          >= 0.9.6 && < 0.11
+                    , postgrest
+                    , prometheus-client >= 1.1.1 && < 1.2.0
+                    , protolude         >= 0.3.1 && < 0.4
+                    , wai               >= 3.2.1 && < 3.3
+  ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
+                      -fno-spec-constr -optP-Wno-nonportable-include-path
+                      -fwrite-ide-info
+                      -- https://github.com/PostgREST/postgrest/issues/387
+                      -with-rtsopts=-K33K
+
 test-suite doctests
   type:               exitcode-stdio-1.0
   default-language:   Haskell2010

test/observability/Main.hs

@@ -0,0 +1,54 @@
+module Main where
+
+import qualified Hasql.Pool                 as P
+import qualified Hasql.Pool.Config          as P
+import qualified Hasql.Transaction.Sessions as HT
+
+import Data.Function (id)
+
+import           PostgREST.App             (postgrest)
+import qualified PostgREST.AppState        as AppState
+import           PostgREST.Config          (AppConfig (..))
+import           PostgREST.Config.Database (queryPgVersion)
+import qualified PostgREST.Logger          as Logger
+import qualified PostgREST.Metrics         as Metrics
+import           PostgREST.SchemaCache     (querySchemaCache)
+
+import qualified Observation.JwtCache
+
+import ObsHelper
+import Protolude  hiding (toList, toS)
+import Test.Hspec
+
+main :: IO ()
+main = do
+  pool <- P.acquire $ P.settings
+    [ P.size 3
+    , P.acquisitionTimeout 10
+    , P.agingTimeout 60
+    , P.idlenessTimeout 60
+    , P.staticConnectionSettings (toUtf8 $ configDbUri testCfg)
+    ]
+
+  actualPgVersion <- either (panic . show) id <$> P.use pool (queryPgVersion False)
+
+  -- cached schema cache so most tests run fast
+  baseSchemaCache <- loadSCache pool testCfg
+  loggerState <- Logger.init
+  metricsState <- Metrics.init (configDbPoolSize testCfg)
+
+  let
+    initApp sCache st config = do
+      appState <- AppState.initWithPool pool config loggerState metricsState (Metrics.observationMetrics metricsState)
+      AppState.putPgVersion appState actualPgVersion
+      AppState.putSchemaCache appState (Just sCache)
+      return (st, postgrest (configLogLevel config) appState (pure ()))
+
+  -- Run all test modules
+  hspec $ do
+    before (initApp baseSchemaCache metricsState testCfgJwtCache) $
+      describe "Observation.JwtCacheObs" Observation.JwtCache.spec
+
+  where
+    loadSCache pool conf =
+      either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache conf)

test/observability/ObsHelper.hs

@@ -0,0 +1,135 @@
+{-# LANGUAGE AllowAmbiguousTypes       #-}
+{-# LANGUAGE ExistentialQuantification #-}
+{-# LANGUAGE FlexibleContexts          #-}
+{-# LANGUAGE ScopedTypeVariables       #-}
+{-# LANGUAGE TupleSections             #-}
+{-# LANGUAGE TypeApplications          #-}
+module ObsHelper where
+
+import qualified Data.ByteString.Base64 as B64 (decodeLenient)
+import qualified Data.ByteString.Char8  as BS
+import qualified Data.ByteString.Lazy   as BL
+import qualified Jose.Jwa               as JWT
+import qualified Jose.Jws               as JWT
+import qualified Jose.Jwt               as JWT
+
+import PostgREST.Config (AppConfig (..), JSPathExp (..),
+                         LogLevel (..), OpenAPIMode (..),
+                         Verbosity (..), parseSecret)
+
+import Data.List.NonEmpty              (fromList)
+import Data.String                     (String)
+import Prometheus                      (Counter, getCounter)
+import Test.Hspec.Expectations.Contrib (annotate)
+
+import Network.HTTP.Types
+import Protolude
+import Test.Hspec
+import Test.Hspec.Wai
+
+
+baseCfg :: AppConfig
+baseCfg = let secret = encodeUtf8 "reallyreallyreallyreallyverysafe" in
+  AppConfig {
+    configAppSettings               = []
+  , configClientErrorVerbosity      = Verbose
+  , configDbAggregates              = False
+  , configDbAnonRole                = Just "postgrest_test_anonymous"
+  , configDbChannel                 = mempty
+  , configDbChannelEnabled          = True
+  , configDbExtraSearchPath         = []
+  , configDbHoistedTxSettings       = ["default_transaction_isolation","plan_filter.statement_cost_limit","statement_timeout"]
+  , configDbMaxRows                 = Nothing
+  , configDbPlanEnabled             = False
+  , configDbPoolSize                = 10
+  , configDbPoolAcquisitionTimeout  = 10
+  , configDbPoolMaxLifetime         = 1800
+  , configDbPoolMaxIdletime         = 600
+  , configDbPoolAutomaticRecovery   = True
+  , configDbPreRequest              = Nothing
+  , configDbPreparedStatements      = True
+  , configDbRootSpec                = Nothing
+  , configDbSchemas                 = fromList ["test"]
+  , configDbConfig                  = False
+  , configDbPreConfig               = Nothing
+  , configDbUri                     = "postgresql://"
+  , configFilePath                  = Nothing
+  , configJWKS                      = rightToMaybe $ parseSecret secret
+  , configJwtAudience               = Nothing
+  , configJwtRoleClaimKey           = [JSPKey "role"]
+  , configJwtSecret                 = Just secret
+  , configJwtSecretIsBase64         = False
+  , configJwtCacheMaxEntries        = 10
+  , configLogLevel                  = LogCrit
+  , configLogQuery                  = False
+  , configOpenApiMode               = OAFollowPriv
+  , configOpenApiSecurityActive     = False
+  , configOpenApiServerProxyUri     = Nothing
+  , configServerCorsAllowedOrigins  = Nothing
+  , configServerHost                = "localhost"
+  , configServerPort                = 3000
+  , configServerTraceHeader         = Nothing
+  , configServerUnixSocket          = Nothing
+  , configServerUnixSocketMode      = 432
+  , configDbTxAllowOverride         = True
+  , configDbTxRollbackAll           = True
+  , configAdminServerHost           = "localhost"
+  , configAdminServerPort           = Nothing
+  , configRoleSettings              = mempty
+  , configRoleIsoLvl                = mempty
+  , configInternalSCQuerySleep      = Nothing
+  , configInternalSCLoadSleep       = Nothing
+  , configInternalSCRelLoadSleep    = Nothing
+  , configServerTimingEnabled       = True
+  }
+
+testCfg :: AppConfig
+testCfg = baseCfg
+
+testCfgJwtCache :: AppConfig
+testCfgJwtCache =
+  baseCfg {
+    configJwtSecret = Just generateSecret
+  , configJWKS = rightToMaybe $ parseSecret generateSecret
+  , configJwtCacheMaxEntries = 2
+  }
+
+authHeader :: BS.ByteString -> BS.ByteString -> Header
+authHeader typ creds =
+  (hAuthorization, typ <> " " <> creds)
+
+authHeaderJWT :: BS.ByteString -> Header
+authHeaderJWT = authHeader "Bearer"
+
+generateSecret :: ByteString
+generateSecret = B64.decodeLenient "cmVhbGx5cmVhbGx5cmVhbGx5cmVhbGx5dmVyeXNhZmU="
+
+generateJWT :: BL.ByteString -> ByteString
+generateJWT claims =
+  either mempty JWT.unJwt $ JWT.hmacEncode JWT.HS256 generateSecret (BL.toStrict claims)
+
+-- state check helpers
+
+data StateCheck st m = forall a. StateCheck (st -> (String, m a)) (a -> a -> Expectation)
+
+stateCheck :: (Show a, Eq a) => (c -> m a) -> (st -> (String, c)) -> (a -> a) -> StateCheck st m
+stateCheck extractValue extractComponent expect = StateCheck (second extractValue . extractComponent) (flip shouldBe . expect)
+
+expectField :: forall s st a c m. (KnownSymbol s, Show a, Eq a, HasField s st c) => (c -> m a) -> (a -> a) -> StateCheck st m
+expectField extractValue = stateCheck extractValue ((symbolVal (Proxy @s),) . getField @s)
+
+checkState :: (Traversable t) => t (StateCheck st (WaiSession st)) -> WaiSession st b -> WaiSession st ()
+checkState checks act = getState >>= flip (`checkState'` checks) act
+
+checkState' :: (Traversable t, MonadIO m) => st -> t (StateCheck st m) -> m b -> m ()
+checkState' initialState checks act = do
+  expectations <- traverse (\(StateCheck g expect) -> let (msg, m) = g initialState in m >>= createExpectation msg m . expect) checks
+  void act
+  sequenceA_ expectations
+  where
+    createExpectation msg metrics expect = pure $ metrics >>= liftIO . annotate msg . expect
+
+expectCounter :: forall s st m. (KnownSymbol s, HasField s st Counter, MonadIO m) => (Int -> Int) -> StateCheck st m
+expectCounter = expectField @s intCounter
+  where
+    intCounter = ((round @Double @Int) <$>) . getCounter

test/observability/Observation/JwtCache.hs

@@ -1,14 +1,6 @@
-{-# LANGUAGE AllowAmbiguousTypes       #-}
+{-# LANGUAGE DataKinds        #-}
-{-# LANGUAGE DataKinds                 #-}
+{-# LANGUAGE TypeApplications #-}
-{-# LANGUAGE ExistentialQuantification #-}
+module Observation.JwtCache where
-{-# LANGUAGE FlexibleContexts          #-}
-{-# LANGUAGE ImpredicativeTypes        #-}
-{-# LANGUAGE OverloadedStrings         #-}
-{-# LANGUAGE ScopedTypeVariables       #-}
-{-# LANGUAGE TypeApplications          #-}
-module Feature.Auth.JwtCacheSpec
-
-where
 
 import Network.Wai (Application)
 
@@ -16,9 +8,9 @@ import Network.HTTP.Types
 import Test.Hspec         (SpecWith, describe, it)
 import Test.Hspec.Wai
 
+import ObsHelper
 import PostgREST.Metrics   (MetricsState (..))
 import Protolude
-import SpecHelper
 import Test.Hspec.Wai.JSON (json)
 
 spec :: SpecWith (MetricsState, Application)
@@ -32,7 +24,7 @@ spec = describe "Server started with JWT and metrics enabled" $ do
       , hits     (+ 0)
       ] $
 
-         request methodGet "/authors_only" [auth] ""
+         request methodGet "/authors_only" [auth] "" `shouldRespondWith` 200
 
   it "Should have JWT in cache" $ do
     let auth = genToken [json|{"exp": 9999999999, "role": "postgrest_test_author", "id": "jdoe2"}|]

test/observability/fixtures/database.sql

@@ -0,0 +1,2 @@
+-- Suppress NOTICE: ... messages
+SET client_min_messages TO warning;

test/observability/fixtures/load.sql

@@ -0,0 +1,8 @@
+-- Loads all fixtures for the PostgREST observability tests
+
+\set ON_ERROR_STOP on
+
+\ir database.sql
+\ir roles.sql
+\ir schema.sql
+\ir privileges.sql

test/observability/fixtures/privileges.sql

@@ -0,0 +1,9 @@
+-- Schema test objects
+SET search_path = test, pg_catalog;
+
+GRANT USAGE ON SCHEMA test TO postgrest_test_anonymous;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA test TO postgrest_test_anonymous;
+REVOKE ALL PRIVILEGES ON TABLE authors_only FROM postgrest_test_anonymous;
+
+GRANT USAGE ON SCHEMA test TO postgrest_test_author;
+GRANT ALL ON TABLE authors_only TO postgrest_test_author;

test/observability/fixtures/roles.sql

@@ -0,0 +1,5 @@
+DROP ROLE IF EXISTS postgrest_test_anonymous, postgrest_test_author;
+CREATE ROLE postgrest_test_anonymous;
+CREATE ROLE postgrest_test_author;
+
+GRANT postgrest_test_anonymous, postgrest_test_author TO :PGUSER;

test/observability/fixtures/schema.sql

@@ -0,0 +1,21 @@
+DROP SCHEMA IF EXISTS test;
+
+CREATE SCHEMA test;
+
+SET search_path = test, pg_catalog;
+
+--
+-- Name: authors_only; Type: TABLE; Schema: test; Owner: -
+--
+
+CREATE TABLE authors_only (
+    owner character varying NOT NULL,
+    secret character varying NOT NULL
+);
+
+--
+-- Name: authors_only_pkey; Type: CONSTRAINT; Schema: test; Owner: -
+--
+
+ALTER TABLE ONLY authors_only
+    ADD CONSTRAINT authors_only_pkey PRIMARY KEY (secret);

test/spec/Main.hs

@@ -23,7 +23,6 @@ import qualified Feature.Auth.AsymmetricJwtSpec
 import qualified Feature.Auth.AudienceJwtSecretSpec
 import qualified Feature.Auth.AuthSpec
 import qualified Feature.Auth.BinaryJwtSecretSpec
-import qualified Feature.Auth.JwtCacheSpec
 import qualified Feature.Auth.NoAnonSpec
 import qualified Feature.Auth.NoJwtSecretSpec
 import qualified Feature.ConcurrentSpec
@@ -274,9 +273,6 @@ main = do
     before pgSafeUpdateApp $
       describe "Feature.Query.PgSafeUpdateSpec.spec" Feature.Query.PgSafeUpdateSpec.spec
 
-    before (initApp baseSchemaCache metricsState testCfgJwtCache) $
-      describe "Feature.Auth.JwtCacheSpec" Feature.Auth.JwtCacheSpec.spec
-
   where
     loadSCache pool conf =
       either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache conf)

test/spec/SpecHelper.hs

@@ -1,10 +1,3 @@
-{-# LANGUAGE AllowAmbiguousTypes       #-}
-{-# LANGUAGE ExistentialQuantification #-}
-{-# LANGUAGE FlexibleContexts          #-}
-{-# LANGUAGE RankNTypes                #-}
-{-# LANGUAGE ScopedTypeVariables       #-}
-{-# LANGUAGE TupleSections             #-}
-{-# LANGUAGE TypeApplications          #-}
 module SpecHelper where
 
 import           Control.Lens           ((^?))
@@ -42,10 +35,8 @@ import PostgREST.Config                  (AppConfig (..),
                                           OpenAPIMode (..),
                                           Verbosity (..), parseSecret)
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..))
-import Prometheus                        (Counter, getCounter)
 import Protolude                         hiding (get, toS)
 import Protolude.Conv                    (toS)
-import Test.Hspec.Expectations.Contrib   (annotate)
 
 filterAndMatchCT :: BS.ByteString -> MatchHeader
 filterAndMatchCT val = MatchHeader $ \headers _ ->
@@ -215,14 +206,6 @@ testCfgBinaryJWT =
   , configJWKS = rightToMaybe $ parseSecret generateSecret
   }
 
-testCfgJwtCache :: AppConfig
-testCfgJwtCache =
-  baseCfg {
-    configJwtSecret = Just generateSecret
-  , configJWKS = rightToMaybe $ parseSecret generateSecret
-  , configJwtCacheMaxEntries = 2
-  }
-
 testCfgAudienceJWT :: AppConfig
 testCfgAudienceJWT =
   baseCfg {
@@ -355,29 +338,3 @@ getInsertDataForTiobePlsTable rows =
 
 readFixtureFile :: FilePath -> BL.ByteString
 readFixtureFile file = unsafePerformIO $ BL.readFile $ "test/spec/fixtures/" <> file
-
--- state check helpers
-
-data StateCheck st m = forall a. StateCheck (st -> (String, m a)) (a -> a -> Expectation)
-
-stateCheck :: (Show a, Eq a) => (c -> m a) -> (st -> (String, c)) -> (a -> a) -> StateCheck st m
-stateCheck extractValue extractComponent expect = StateCheck (second extractValue . extractComponent) (flip shouldBe . expect)
-
-expectField :: forall s st a c m. (KnownSymbol s, Show a, Eq a, HasField s st c) => (c -> m a) -> (a -> a) -> StateCheck st m
-expectField extractValue = stateCheck extractValue ((symbolVal (Proxy @s),) . getField @s)
-
-checkState :: (Traversable t) => t (StateCheck st (WaiSession st)) -> WaiSession st b -> WaiSession st ()
-checkState checks act = getState >>= flip (`checkState'` checks) act
-
-checkState' :: (Traversable t, MonadIO m) => st -> t (StateCheck st m) -> m b -> m ()
-checkState' initialState checks act = do
-  expectations <- traverse (\(StateCheck g expect) -> let (msg, m) = g initialState in m >>= createExpectation msg m . expect) checks
-  void act
-  sequenceA_ expectations
-  where
-    createExpectation msg metrics expect = pure $ metrics >>= liftIO . annotate msg . expect
-
-expectCounter :: forall s st m. (KnownSymbol s, HasField s st Counter, MonadIO m) => (Int -> Int) -> StateCheck st m
-expectCounter = expectField @s intCounter
-  where
-    intCounter = ((round @Double @Int) <$>) . getCounter