Pass custom settings to the DB's SESSION (#1063)

- allows queries to refer to current_setting('app.settings.foo') to retrieve variables
- useful for 12-factor apps (app data can be in environment)
- provides workaround for AWS Relational Database Service (RDS) not
  allowing `ALTER DATABASE SET 'app.[KEY]' TO '[VALUE]'` on database.

CHANGELOG.md

@@ -17,6 +17,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Computed columns now only work if they belong to the db-schema - @steve-chavez
 - To use RPC now the `json_to_record/json_to_recordset` functions are needed, these are available starting from PostgreSQL 9.4 - @steve-chavez
 - Overloaded functions now depend on the `dbStructure`, restart/sighup may be needed for their correct functioning - @steve-chavez
+- The configuration (e.g. `postgrest.conf`) now accepts arbitrary settings that will be passed through as session-local database settings. This can be used to pass in secret keys directly as strings, or via OS environment variables. For instance: `app.settings.jwt_secret = "$(MYAPP_JWT_SECRET)"` will take `MYAPP_JWT_SECRET` from the environment and make it available to postgresql functions as `current_setting('app.settings.jwt_secret')`. Only `app.settings.*` values in the configuration file are treated in this way. - @canadaduane
 
 ## [0.4.4.0] - 2018-01-08
 

main/Main.hs

@@ -7,7 +7,8 @@ import           PostgREST.App            (postgrest)
 import           PostgREST.Config         (AppConfig (..),
                                            minimumPgVersion,
                                            prettyVersion, readOptions)
-import           PostgREST.DbStructure    (getDbStructure, getPgVersion)
+import           PostgREST.DbStructure    (getDbStructure, getPgVersion,
+                                           fillSessionWithSettings)
 import           PostgREST.Error          (encodeError)
 import           PostgREST.OpenAPI        (isMalformedProxyUri)
 import           PostgREST.Types          (DbStructure, Schema, PgVersion(..))
@@ -32,6 +33,7 @@ import           Network.Wai.Handler.Warp (defaultSettings,
                                            setTimeout)
 import           System.IO                (BufferMode (..),
                                            hSetBuffering)
+
 #ifndef mingw32_HOST_OS
 import           System.Posix.Signals
 #endif
@@ -58,10 +60,11 @@ connectionWorker
   :: ThreadId -- ^ This thread is killed if pg version is unsupported
   -> P.Pool   -- ^ The PostgreSQL connection pool
   -> Schema   -- ^ Schema PostgREST is serving up
+  -> [(Text, Text)] -- ^ Settings or Environment passed in through the config
   -> IORef (Maybe DbStructure) -- ^ mutable reference to 'DbStructure'
   -> IORef Bool                -- ^ Used as a binary Semaphore
   -> IO ()
-connectionWorker mainTid pool schema refDbStructure refIsWorkerOn = do
+connectionWorker mainTid pool schema settings refDbStructure refIsWorkerOn = do
   isWorkerOn <- readIORef refIsWorkerOn
   unless isWorkerOn $ do
     atomicWriteIORef refIsWorkerOn True
@@ -79,6 +82,7 @@ connectionWorker mainTid pool schema refDbStructure refIsWorkerOn = do
               ("Cannot run in this PostgreSQL version, PostgREST needs at least "
               <> pgvName minimumPgVersion)
             killThread mainTid
+          fillSessionWithSettings settings
           dbStructure <- getDbStructure schema actualPgVersion
           liftIO $ atomicWriteIORef refDbStructure $ Just dbStructure
         case result of
@@ -173,6 +177,7 @@ main = do
     mainTid
     pool
     (configSchema conf)
+    (configSettings conf)
     refDbStructure
     refIsWorkerOn
   --
@@ -195,6 +200,7 @@ main = do
               mainTid
               pool
               (configSchema conf)
+              (configSettings conf)
               refDbStructure
               refIsWorkerOn
     ) Nothing
@@ -211,6 +217,7 @@ main = do
          mainTid
          pool
          (configSchema conf)
+         (configSettings conf)
          refDbStructure
          refIsWorkerOn)
 

postgrest.cabal

@@ -57,6 +57,7 @@ library
                      , configurator-ng == 0.0.0.1
                      , containers
                      , contravariant
+                     , contravariant-extras
                      , either
                      , gitrev
                      , hasql

src/PostgREST/Config.hs

@@ -76,6 +76,7 @@ data AppConfig = AppConfig {
   , configMaxRows           :: Maybe Integer
   , configReqCheck          :: Maybe Text
   , configQuiet             :: Bool
+  , configSettings          :: [(Text, Text)]
   }
 
 defaultCorsPolicy :: CorsResourcePolicy
@@ -136,6 +137,7 @@ readOptions = do
           <*> (join . fmap coerceInt <$> C.key "max-rows")
           <*> (mfilter (/= "") <$> C.key "pre-request")
           <*> pure False
+          <*> (fmap parsedPairToTextPair <$> C.subassocs "app.settings")
 
   case mAppConf of
     Nothing -> do
@@ -145,6 +147,13 @@ readOptions = do
       return appConf
 
   where
+    parsedPairToTextPair :: (Name, Value) -> (Text, Text)
+    parsedPairToTextPair (k, v) = (k, newValue)
+      where
+        newValue = case v of
+          String textVal -> textVal
+          _ -> show v
+
     parseJwtAudience :: Name -> C.ConfigParserM (Maybe StringOrURI)
     parseJwtAudience k =
       C.key k >>= \case
@@ -159,9 +168,9 @@ readOptions = do
     coerceInt (String x) = readMaybe $ toS x
     coerceInt _          = Nothing
 
-    coerceBool ::  Value -> Maybe Bool
+    coerceBool :: Value -> Maybe Bool
     coerceBool (Bool b)   = Just b
-    coerceBool (String x) = readMaybe $ toS x
+    coerceBool (String b) = readMaybe $ toS b
     coerceBool _          = Nothing
 
     opts = info (helper <*> pathParser) $
@@ -218,7 +227,7 @@ pathParser =
 
 -- | Tells the minimum PostgreSQL version required by this version of PostgREST
 minimumPgVersion :: PgVersion
-minimumPgVersion = PgVersion 90300 "9.3"
+minimumPgVersion = PgVersion 90400 "9.4"
 
 pgVersion96 :: PgVersion
 pgVersion96 = PgVersion 90600 "9.6"

src/PostgREST/DbStructure.hs

@@ -9,6 +9,7 @@ module PostgREST.DbStructure (
 , accessibleProcs
 , schemaDescription
 , getPgVersion
+, fillSessionWithSettings
 ) where
 
 import qualified Hasql.Decoders                as HD
@@ -18,9 +19,11 @@ import qualified Hasql.Query                   as H
 import           Control.Applicative
 import qualified Data.HashMap.Strict           as M
 import           Data.List                     (elemIndex)
+import qualified Data.List                     as List
 import           Data.Maybe                    (fromJust)
 import           Data.Text                     (split, strip,
-                                                breakOn, dropAround, splitOn)
+                                                breakOn, dropAround,
+                                                splitOn)
 import qualified Data.Text                     as T
 import qualified Hasql.Session                 as H
 import           PostgREST.Types
@@ -30,6 +33,9 @@ import           GHC.Exts                      (groupWith)
 import           Protolude
 import           Unsafe (unsafeHead)
 
+import           Data.Functor.Contravariant    (contramap)
+import           Contravariant.Extras          (contrazip2)
+
 getDbStructure :: Schema -> PgVersion -> H.Session DbStructure
 getDbStructure schema pgVer = do
   tabs <- H.query () allTables
@@ -747,3 +753,17 @@ getPgVersion = H.query () $ H.statement sql HE.unit versionRow False
   where
     sql = "SELECT current_setting('server_version_num')::integer, current_setting('server_version')"
     versionRow = HD.singleRow $ PgVersion <$> HD.value HD.int4 <*> HD.value HD.text
+
+fillSessionWithSettings :: [(Text, Text)] -> H.Session ()
+fillSessionWithSettings settings =
+    -- Send all of the config settings to the set_config function, using pgsql's `unnest` to transform arrays of values
+    H.query settings $ H.statement "SELECT set_config(k, v, false) FROM unnest($1, $2) AS f1(k, v)" encoder HD.unit False
+  
+  where
+    -- Take a list of (key, value) pairs and encode each as an array to later bind to the query
+    -- see Insert Many section at https://hackage.haskell.org/package/hasql-1.1.1/docs/Hasql-Encoders.html
+    encoder = contramap List.unzip $ contrazip2 (vector HE.text) (vector HE.text)
+      where
+        vector value =
+          HE.value $ HE.array $ HE.arrayDimension foldl' $ HE.arrayValue value
+

test/Feature/QuerySpec.hs

@@ -699,7 +699,6 @@ spec = do
           { matchStatus = 200
           , matchHeaders = []
           }
-
     it "multiple cookies ends up as claims" $
       request methodPost "/rpc/get_guc_value" [("Cookie","acookie=cookievalue;secondcookie=anothervalue")]
         [json| {"name":"request.cookie.secondcookie"} |]
@@ -708,6 +707,14 @@ spec = do
           { matchStatus = 200
           , matchHeaders = []
           }
+    it "app settings available" $
+      request methodPost "/rpc/get_guc_value" []
+        [json| { "name": "app.settings.app_host" } |]
+          `shouldRespondWith`
+          [str|"localhost"|]
+          { matchStatus  = 200
+          , matchHeaders = [ matchContentTypeJson ]
+          }
     
   describe "values with quotes in IN and NOT IN" $ do
     it "succeeds when only quoted values are present" $ do

test/Main.hs

@@ -6,8 +6,8 @@ import SpecHelper
 import qualified Hasql.Pool as P
 
 import PostgREST.App (postgrest)
-import PostgREST.Config (pgVersion96)
+import PostgREST.Config (pgVersion96, configSettings)
-import PostgREST.DbStructure (getDbStructure, getPgVersion)
+import PostgREST.DbStructure (getDbStructure, getPgVersion, fillSessionWithSettings)
 import PostgREST.Types (DbStructure(..))
 import Data.Function (id)
 import Data.IORef
@@ -58,7 +58,9 @@ main = do
       asymJwkApp           = return $ postgrest (testCfgAsymJWK testDbConn)     refDbStructure pool $ pure ()
       nonexistentSchemaApp = return $ postgrest (testNonexistentSchemaCfg testDbConn)   refDbStructure pool $ pure ()
 
-  let reset = resetDb testDbConn
+  let reset :: IO ()
+      reset = P.use pool (fillSessionWithSettings (configSettings $ testCfg testDbConn)) >> resetDb testDbConn
+
       actualPgVersion = pgVersion dbStructure
       pg96spec | actualPgVersion >= pgVersion96 = [("Feature.PgVersion96Spec"  , Feature.PgVersion96Spec.spec)]
                | otherwise = []

test/SpecHelper.hs

@@ -73,6 +73,9 @@ _baseCfg =  -- Connection Settings
             10 Nothing (Just "test.switch_role")
             -- Debug Settings
             True
+            [ ("app.settings.app_host", "localhost")
+            , ("app.settings.external_api_secret", "0123456789abcdef")
+            ]
 
 testCfg :: Text -> AppConfig
 testCfg testDbConn = _baseCfg { configDatabase = testDbConn }