Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f13e43abe | ||
|
|
fac797c766 | ||
|
|
07cb0b582e | ||
|
|
d54a2f48de | ||
|
|
bcce7b1c53 | ||
|
|
8d1961ce07 | ||
|
|
9a19dff83e | ||
|
|
54b9a0b8b3 | ||
|
|
9a3d453bf4 | ||
|
|
4f6c466031 | ||
|
|
a852b766eb | ||
|
|
14be3fb671 | ||
|
|
8a3686d86b | ||
|
|
009250006e | ||
|
|
38ad8c04e1 | ||
|
|
54cbf147e3 | ||
|
|
f9f0f79fa9 | ||
|
|
a867d79c42 |
@@ -46,7 +46,7 @@ jobs:
|
||||
- name: Run coverage (IO tests and Spec tests against PostgreSQL 15)
|
||||
run: postgrest-coverage
|
||||
- name: Upload coverage to codecov
|
||||
uses: codecov/codecov-action@v3.1.3
|
||||
uses: codecov/codecov-action@v3.1.4
|
||||
with:
|
||||
files: ./coverage/codecov.json
|
||||
|
||||
|
||||
+4
-6
@@ -55,12 +55,6 @@ It builds the OpenAPI response using the schema cache.
|
||||
|
||||
This module provides functions to deal with JWT authorization.
|
||||
|
||||
### Workers.hs
|
||||
|
||||
This spawns threads which are used to execute concurrent jobs.
|
||||
|
||||
Jobs include connection recovery, a listener for the PostgreSQL LISTEN command, and an admin server.
|
||||
|
||||
### SchemaCache.hs
|
||||
|
||||
This queries the PostgreSQL system catalogs and caches the metadata into a SchemaCache type,
|
||||
@@ -68,3 +62,7 @@ This queries the PostgreSQL system catalogs and caches the metadata into a Schem
|
||||
### AppState.hs
|
||||
|
||||
The state of the App which is kept across requests.
|
||||
|
||||
This spawns threads which are used to execute concurrent jobs.
|
||||
|
||||
Jobs include connection recover and a listener for the PostgreSQL LISTEN command.
|
||||
|
||||
@@ -5,6 +5,25 @@ This project adheres to [Semantic Versioning](http://semver.org/).
|
||||
|
||||
## Unreleased
|
||||
|
||||
## [11.1.0] - 2023-06-07
|
||||
|
||||
### Added
|
||||
|
||||
- #2786, Limit idle postgresql connection lifetime - @robx
|
||||
+ New option `db-pool-max-idletime` (default 30s).
|
||||
+ This is equivalent to the old option `db-pool-timeout` of PostgREST 10.0.0.
|
||||
+ A config alias for `db-pool-timeout` is included.
|
||||
- #2703, Add pre-config function - @steve-chavez
|
||||
+ New config option `db-pre-config`(empty by default)
|
||||
+ Allows using the in-database configuration without SUPERUSER
|
||||
- #2781, When `db-channel-enabled` is false, start automatic connection recovery on a new request when pool connections are closed with `pg_terminate_backend` - @steve-chavez
|
||||
+ Mitigates the lack of LISTEN/NOTIFY for schema cache reloading on read replicas.
|
||||
|
||||
### Fixed
|
||||
|
||||
- #2791, Fix dropping schema cache reload notifications - @steve-chavez
|
||||
- #2801, Stop retrying connection when "no password supplied" - @steve-chavez
|
||||
|
||||
## [11.0.1] - 2023-04-27
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -43,8 +43,8 @@ let
|
||||
(prev.callHackageDirect
|
||||
{
|
||||
pkg = "hasql-notifications";
|
||||
ver = "0.2.0.4";
|
||||
sha256 = "sha256-fm1xiDyvDkb5WLOJ73/s8wrWEW23XFS7luAv2brfr8I=";
|
||||
ver = "0.2.0.5";
|
||||
sha256 = "sha256-KV36zs/RTgJh/oBBPZaNiQshusmakWnjYTcdoNa4JLA=";
|
||||
}
|
||||
{ });
|
||||
|
||||
@@ -52,8 +52,8 @@ let
|
||||
(prev.callHackageDirect
|
||||
{
|
||||
pkg = "hasql-pool";
|
||||
ver = "0.9";
|
||||
sha256 = "sha256-5UshbbaBVY8eJ/9VagNVVxonRwMcd7UmGqDc35pJNFY=";
|
||||
ver = "0.10";
|
||||
sha256 = "sha256-kHzoqtNV9BFWnn1h560JRqMooQRwxokVKgDRBexamNI=";
|
||||
}
|
||||
{ });
|
||||
} // extraOverrides final prev;
|
||||
|
||||
@@ -41,6 +41,8 @@ let
|
||||
args =
|
||||
[
|
||||
"ARG_USE_ENV([PGRST_DB_ANON_ROLE], [postgrest_test_anonymous], [PostgREST anonymous role])"
|
||||
"ARG_USE_ENV([PGRST_DB_POOL], [1], [PostgREST pool size])"
|
||||
"ARG_USE_ENV([PGRST_DB_POOL_ACQUISITION_TIMEOUT], [1], [PostgREST pool size])"
|
||||
"ARG_LEFTOVERS([PostgREST arguments])"
|
||||
];
|
||||
inRootDir = true;
|
||||
@@ -48,6 +50,8 @@ let
|
||||
}
|
||||
''
|
||||
export PGRST_DB_ANON_ROLE
|
||||
export PGRST_DB_POOL
|
||||
export PGRST_DB_POOL_ACQUISITION_TIMEOUT
|
||||
|
||||
exec ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
|
||||
postgrest "''${_arg_leftovers[@]}"
|
||||
|
||||
@@ -32,6 +32,7 @@ let
|
||||
"ARG_USE_ENV([PGDATABASE], [postgres], [PG database name])"
|
||||
"ARG_USE_ENV([PGRST_DB_SCHEMAS], [test], [Schema to expose])"
|
||||
"ARG_USE_ENV([PGTZ], [utc], [Timezone to use])"
|
||||
"ARG_USE_ENV([PGOPTIONS], [-c search_path=public,test], [PG options to use])"
|
||||
];
|
||||
positionalCompletion = "_command";
|
||||
inRootDir = true;
|
||||
@@ -61,6 +62,7 @@ let
|
||||
export PGDATABASE
|
||||
export PGRST_DB_SCHEMAS
|
||||
export PGTZ
|
||||
export PGOPTIONS
|
||||
|
||||
HBA_FILE="$tmpdir/pg_hba.conf"
|
||||
echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
|
||||
@@ -94,7 +96,7 @@ let
|
||||
|
||||
log "Done. Running command..."
|
||||
|
||||
echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U $PGUSER"
|
||||
echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U ${superuserRole}"
|
||||
echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log"
|
||||
|
||||
("$_arg_command" "''${_arg_leftovers[@]}")
|
||||
|
||||
+6
-5
@@ -1,5 +1,5 @@
|
||||
name: postgrest
|
||||
version: 11.0.1
|
||||
version: 11.1.0
|
||||
synopsis: REST API for any Postgres database
|
||||
description: Reads the schema of a PostgreSQL database and creates RESTful routes
|
||||
for tables, views, and functions, supporting all HTTP methods that security
|
||||
@@ -34,7 +34,8 @@ library
|
||||
default-extensions: OverloadedStrings
|
||||
NoImplicitPrelude
|
||||
hs-source-dirs: src
|
||||
exposed-modules: PostgREST.App
|
||||
exposed-modules: PostgREST.Admin
|
||||
PostgREST.App
|
||||
PostgREST.AppState
|
||||
PostgREST.Auth
|
||||
PostgREST.CLI
|
||||
@@ -70,7 +71,6 @@ library
|
||||
PostgREST.Response.OpenAPI
|
||||
PostgREST.Response.GucHeader
|
||||
PostgREST.Version
|
||||
PostgREST.Workers
|
||||
other-modules: Paths_postgrest
|
||||
build-depends: base >= 4.9 && < 4.17
|
||||
, HTTP >= 4000.3.7 && < 4000.5
|
||||
@@ -86,12 +86,13 @@ library
|
||||
, contravariant-extras >= 0.3.3 && < 0.4
|
||||
, cookie >= 0.4.2 && < 0.5
|
||||
, either >= 4.4.1 && < 5.1
|
||||
, extra >= 1.7.0 && < 2.0
|
||||
, fuzzyset >= 0.2.3
|
||||
, gitrev >= 1.2 && < 1.4
|
||||
, hasql >= 1.6.1.1 && < 1.7
|
||||
, hasql-dynamic-statements >= 0.3.1 && < 0.4
|
||||
, hasql-notifications >= 0.1 && < 0.3
|
||||
, hasql-pool >= 0.9 && < 0.10
|
||||
, hasql-pool >= 0.10 && < 0.11
|
||||
, hasql-transaction >= 1.0.1 && < 1.1
|
||||
, heredoc >= 0.2 && < 0.3
|
||||
, http-types >= 0.12.2 && < 0.13
|
||||
@@ -232,7 +233,7 @@ test-suite spec
|
||||
, bytestring >= 0.10.8 && < 0.12
|
||||
, case-insensitive >= 1.2 && < 1.3
|
||||
, containers >= 0.5.7 && < 0.7
|
||||
, hasql-pool >= 0.9 && < 0.10
|
||||
, hasql-pool >= 0.10 && < 0.11
|
||||
, hasql-transaction >= 1.0.1 && < 1.1
|
||||
, heredoc >= 0.2 && < 0.3
|
||||
, hspec >= 2.3 && < 2.10
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
{-# LANGUAGE NamedFieldPuns #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
|
||||
module PostgREST.Admin
|
||||
( runAdmin
|
||||
) where
|
||||
|
||||
import qualified Data.Text as T
|
||||
import qualified Hasql.Session as SQL
|
||||
import qualified Network.HTTP.Types.Status as HTTP
|
||||
import qualified Network.Wai as Wai
|
||||
import qualified Network.Wai.Handler.Warp as Warp
|
||||
|
||||
import Control.Monad.Extra (whenJust)
|
||||
|
||||
import Network.Socket
|
||||
import Network.Socket.ByteString
|
||||
|
||||
import PostgREST.AppState (AppState)
|
||||
import PostgREST.Config (AppConfig (..))
|
||||
|
||||
import qualified PostgREST.AppState as AppState
|
||||
|
||||
import Protolude
|
||||
|
||||
runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
|
||||
runAdmin conf@AppConfig{configAdminServerPort} appState settings =
|
||||
whenJust configAdminServerPort $ \adminPort -> do
|
||||
AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
|
||||
void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
|
||||
where
|
||||
adminApp = admin appState conf
|
||||
|
||||
-- | PostgREST admin application
|
||||
admin :: AppState.AppState -> AppConfig -> Wai.Application
|
||||
admin appState appConfig req respond = do
|
||||
isMainAppReachable <- any isRight <$> reachMainApp appConfig
|
||||
isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
|
||||
isConnectionUp <-
|
||||
if configDbChannelEnabled appConfig
|
||||
then AppState.getIsListenerOn appState
|
||||
else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
|
||||
|
||||
case Wai.pathInfo req of
|
||||
["ready"] ->
|
||||
respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
|
||||
["live"] ->
|
||||
respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
|
||||
_ ->
|
||||
respond $ Wai.responseLBS HTTP.status404 [] mempty
|
||||
|
||||
-- Try to connect to the main app socket
|
||||
-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
|
||||
-- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
|
||||
-- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
|
||||
reachMainApp :: AppConfig -> IO [Either IOException ()]
|
||||
reachMainApp AppConfig{..} =
|
||||
case configServerUnixSocket of
|
||||
Just path -> do
|
||||
sock <- socket AF_UNIX Stream 0
|
||||
(:[]) <$> try (do
|
||||
connect sock $ SockAddrUnix path
|
||||
withSocketsDo $ bracket (pure sock) close sendEmpty)
|
||||
Nothing -> do
|
||||
let
|
||||
host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
|
||||
| otherwise = Just configServerHost
|
||||
filterAddrs xs =
|
||||
case configServerHost of
|
||||
"*4" -> ipv4Addrs xs ++ ipv6Addrs xs
|
||||
"!4" -> ipv4Addrs xs
|
||||
"*6" -> ipv6Addrs xs ++ ipv4Addrs xs
|
||||
"!6" -> ipv6Addrs xs
|
||||
_ -> xs
|
||||
ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
|
||||
ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
|
||||
|
||||
addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
|
||||
tryAddr `traverse` filterAddrs addrs
|
||||
where
|
||||
sendEmpty sock = void $ send sock mempty
|
||||
tryAddr :: AddrInfo -> IO (Either IOException ())
|
||||
tryAddr addr = do
|
||||
sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
|
||||
try $ do
|
||||
connect sock $ addrAddress addr
|
||||
withSocketsDo $ bracket (pure sock) close sendEmpty
|
||||
+10
-22
@@ -9,7 +9,6 @@ Some of its functionality includes:
|
||||
- Producing HTTP Headers according to RFCs.
|
||||
- Content Negotiation
|
||||
-}
|
||||
{-# LANGUAGE LambdaCase #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
module PostgREST.App
|
||||
( SignalHandlerInstaller
|
||||
@@ -20,7 +19,7 @@ module PostgREST.App
|
||||
|
||||
|
||||
import Control.Monad.Except (liftEither)
|
||||
import Data.Either.Combinators (mapLeft, whenLeft)
|
||||
import Data.Either.Combinators (mapLeft)
|
||||
import Data.Maybe (fromJust)
|
||||
import Data.String (IsString (..))
|
||||
import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
|
||||
@@ -28,11 +27,11 @@ import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
|
||||
import System.Posix.Types (FileMode)
|
||||
|
||||
import qualified Data.HashMap.Strict as HM
|
||||
import qualified Hasql.Pool as SQL
|
||||
import qualified Hasql.Transaction.Sessions as SQL
|
||||
import qualified Network.Wai as Wai
|
||||
import qualified Network.Wai.Handler.Warp as Warp
|
||||
|
||||
import qualified PostgREST.Admin as Admin
|
||||
import qualified PostgREST.ApiRequest as ApiRequest
|
||||
import qualified PostgREST.ApiRequest.Types as ApiRequestTypes
|
||||
import qualified PostgREST.AppState as AppState
|
||||
@@ -43,7 +42,6 @@ import qualified PostgREST.Logger as Logger
|
||||
import qualified PostgREST.Plan as Plan
|
||||
import qualified PostgREST.Query as Query
|
||||
import qualified PostgREST.Response as Response
|
||||
import qualified PostgREST.Workers as Workers
|
||||
|
||||
import PostgREST.ApiRequest (Action (..), ApiRequest (..),
|
||||
Mutation (..), Target (..))
|
||||
@@ -68,14 +66,14 @@ type SocketRunner = Warp.Settings -> Wai.Application -> FileMode -> FilePath ->
|
||||
run :: SignalHandlerInstaller -> Maybe SocketRunner -> AppState -> IO ()
|
||||
run installHandlers maybeRunWithSocket appState = do
|
||||
conf@AppConfig{..} <- AppState.getConfig appState
|
||||
Workers.connectionWorker appState -- Loads the initial SchemaCache
|
||||
AppState.connectionWorker appState -- Loads the initial SchemaCache
|
||||
installHandlers appState
|
||||
-- reload schema cache + config on NOTIFY
|
||||
Workers.runListener conf appState
|
||||
AppState.runListener conf appState
|
||||
|
||||
Workers.runAdmin conf appState $ serverSettings conf
|
||||
Admin.runAdmin conf appState $ serverSettings conf
|
||||
|
||||
let app = postgrest conf appState (Workers.connectionWorker appState)
|
||||
let app = postgrest conf appState (AppState.connectionWorker appState)
|
||||
|
||||
case configServerUnixSocket of
|
||||
Just socket ->
|
||||
@@ -154,27 +152,17 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache pgVer authResult@
|
||||
Response.optionalRollback conf apiRequest $
|
||||
handleRequest authResult conf appState (Just authRole /= configDbAnonRole) configDbPreparedStatements pgVer apiRequest sCache
|
||||
|
||||
runDbHandler :: AppState.AppState -> Maybe Text -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
|
||||
runDbHandler :: AppState.AppState -> SQL.IsolationLevel -> SQL.Mode -> Bool -> Bool -> DbHandler b -> Handler IO b
|
||||
runDbHandler appState isoLvl mode authenticated prepared handler = do
|
||||
dbResp <- lift $ do
|
||||
let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction
|
||||
res <- AppState.usePool appState . transaction (toIsolationLevel isoLvl) mode $ runExceptT handler
|
||||
whenLeft res (\case
|
||||
SQL.AcquisitionTimeoutUsageError -> AppState.debounceLogAcquisitionTimeout appState -- this can happen rapidly for many requests, so we debounce
|
||||
_ -> pure ())
|
||||
return res
|
||||
AppState.usePool appState . transaction isoLvl mode $ runExceptT handler
|
||||
|
||||
resp <-
|
||||
liftEither . mapLeft Error.PgErr $
|
||||
mapLeft (Error.PgError authenticated) dbResp
|
||||
|
||||
liftEither resp
|
||||
where
|
||||
toIsolationLevel = \case
|
||||
Nothing -> SQL.ReadCommitted
|
||||
Just "repeatable read" -> SQL.RepeatableRead
|
||||
Just "serializable" -> SQL.Serializable
|
||||
_ -> SQL.ReadCommitted
|
||||
|
||||
handleRequest :: AuthResult -> AppConfig -> AppState.AppState -> Bool -> Bool -> PgVersion -> ApiRequest -> SchemaCache -> Handler IO Wai.Response
|
||||
handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@ApiRequest{..} sCache =
|
||||
@@ -206,7 +194,7 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A
|
||||
|
||||
(ActionInvoke invMethod, TargetProc identifier _) -> do
|
||||
cPlan <- liftEither $ Plan.callReadPlan identifier conf sCache apiReq invMethod
|
||||
resultSet <- runQuery (roleIsoLvl <|> pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
|
||||
resultSet <- runQuery (fromMaybe roleIsoLvl $ pdIsoLvl (Plan.crProc cPlan))(Plan.crTxMode cPlan) $ Query.invokeQuery (Plan.crProc cPlan) cPlan apiReq conf pgVer
|
||||
return $ Response.invokeResponse invMethod (Plan.crProc cPlan) apiReq resultSet
|
||||
|
||||
(ActionInspect headersOnly, TargetDefaultSpec tSchema) -> do
|
||||
@@ -229,7 +217,7 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A
|
||||
throwError $ Error.ApiRequestError ApiRequestTypes.NotFound
|
||||
where
|
||||
roleSettings = fromMaybe mempty (HM.lookup authRole $ configRoleSettings conf)
|
||||
roleIsoLvl = decodeUtf8 <$> HM.lookup "default_transaction_isolation" roleSettings
|
||||
roleIsoLvl = HM.findWithDefault SQL.ReadCommitted authRole $ configRoleIsoLvl conf
|
||||
runQuery isoLvl mode query =
|
||||
runDbHandler appState isoLvl mode authenticated prepared $ do
|
||||
Query.setPgLocals conf authClaims authRole (HM.toList roleSettings) apiReq pgVer
|
||||
|
||||
+310
-25
@@ -1,9 +1,10 @@
|
||||
{-# LANGUAGE LambdaCase #-}
|
||||
{-# LANGUAGE NamedFieldPuns #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
|
||||
module PostgREST.AppState
|
||||
( AppState
|
||||
, destroy
|
||||
, flushPool
|
||||
, getConfig
|
||||
, getSchemaCache
|
||||
, getIsListenerOn
|
||||
@@ -11,40 +12,50 @@ module PostgREST.AppState
|
||||
, getPgVersion
|
||||
, getRetryNextIn
|
||||
, getTime
|
||||
, getWorkerSem
|
||||
, init
|
||||
, initWithPool
|
||||
, logWithZTime
|
||||
, logPgrstError
|
||||
, putConfig
|
||||
, putSchemaCache
|
||||
, putIsListenerOn
|
||||
, putPgVersion
|
||||
, putRetryNextIn
|
||||
, signalListener
|
||||
, usePool
|
||||
, waitListener
|
||||
, debounceLogAcquisitionTimeout
|
||||
, loadSchemaCache
|
||||
, reReadConfig
|
||||
, connectionWorker
|
||||
, runListener
|
||||
) where
|
||||
|
||||
import qualified Data.ByteString.Lazy as LBS
|
||||
import qualified Data.Text.Encoding as T
|
||||
import qualified Hasql.Pool as SQL
|
||||
import qualified Hasql.Session as SQL
|
||||
import qualified PostgREST.Error as Error
|
||||
import qualified Data.ByteString.Char8 as BS
|
||||
import qualified Data.ByteString.Lazy as LBS
|
||||
import Data.Either.Combinators (whenLeft)
|
||||
import qualified Data.Text.Encoding as T
|
||||
import Hasql.Connection (acquire)
|
||||
import qualified Hasql.Notifications as SQL
|
||||
import qualified Hasql.Pool as SQL
|
||||
import qualified Hasql.Session as SQL
|
||||
import qualified Hasql.Transaction.Sessions as SQL
|
||||
import qualified PostgREST.Error as Error
|
||||
|
||||
import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
|
||||
updateAction)
|
||||
import Control.Debounce
|
||||
import Control.Retry (RetryStatus, capDelay, exponentialBackoff,
|
||||
retrying, rsPreviousDelay)
|
||||
import Data.IORef (IORef, atomicWriteIORef, newIORef,
|
||||
readIORef)
|
||||
import Data.Time (ZonedTime, defaultTimeLocale, formatTime,
|
||||
getZonedTime)
|
||||
import Data.Time.Clock (UTCTime, getCurrentTime)
|
||||
|
||||
import PostgREST.Config (AppConfig (..))
|
||||
import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
|
||||
import PostgREST.SchemaCache (SchemaCache)
|
||||
import PostgREST.Config (AppConfig (..),
|
||||
readAppConfig)
|
||||
import PostgREST.Config.Database (queryDbSettings,
|
||||
queryPgVersion,
|
||||
queryRoleSettings)
|
||||
import PostgREST.Config.PgVersion (PgVersion (..),
|
||||
minimumPgVersion)
|
||||
import PostgREST.SchemaCache (SchemaCache,
|
||||
querySchemaCache)
|
||||
import PostgREST.SchemaCache.Identifiers (dumpQi)
|
||||
|
||||
import Protolude
|
||||
|
||||
@@ -56,8 +67,8 @@ data AppState = AppState
|
||||
, statePgVersion :: IORef PgVersion
|
||||
-- | No schema cache at the start. Will be filled in by the connectionWorker
|
||||
, stateSchemaCache :: IORef (Maybe SchemaCache)
|
||||
-- | Binary semaphore to make sure just one connectionWorker can run at a time
|
||||
, stateWorkerSem :: MVar ()
|
||||
-- | starts the connection worker with a debounce
|
||||
, debouncedConnectionWorker :: IO ()
|
||||
-- | Binary semaphore used to sync the listener(NOTIFY reload) with the connectionWorker.
|
||||
, stateListener :: MVar ()
|
||||
-- | State of the LISTEN channel, used for the admin server checks
|
||||
@@ -86,7 +97,7 @@ initWithPool pool conf = do
|
||||
appState <- AppState pool
|
||||
<$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step
|
||||
<*> newIORef Nothing
|
||||
<*> newEmptyMVar
|
||||
<*> pure (pure ())
|
||||
<*> newEmptyMVar
|
||||
<*> newIORef False
|
||||
<*> newIORef conf
|
||||
@@ -96,7 +107,8 @@ initWithPool pool conf = do
|
||||
<*> newIORef 0
|
||||
<*> pure (pure ())
|
||||
|
||||
deb <-
|
||||
|
||||
debLogTimeout <-
|
||||
let oneSecond = 1000000 in
|
||||
mkDebounce defaultDebounceSettings
|
||||
{ debounceAction = logPgrstError appState SQL.AcquisitionTimeoutUsageError
|
||||
@@ -104,7 +116,15 @@ initWithPool pool conf = do
|
||||
, debounceEdge = leadingEdge -- logs at the start and the end
|
||||
}
|
||||
|
||||
return appState { debounceLogAcquisitionTimeout = deb }
|
||||
debWorker <-
|
||||
let decisecond = 100000 in
|
||||
mkDebounce defaultDebounceSettings
|
||||
{ debounceAction = internalConnectionWorker appState
|
||||
, debounceFreq = decisecond
|
||||
, debounceEdge = leadingEdge -- runs the worker at the start and the end
|
||||
}
|
||||
|
||||
return appState { debounceLogAcquisitionTimeout = debLogTimeout, debouncedConnectionWorker = debWorker }
|
||||
|
||||
destroy :: AppState -> IO ()
|
||||
destroy = destroyPool
|
||||
@@ -115,11 +135,17 @@ initPool AppConfig{..} =
|
||||
configDbPoolSize
|
||||
(fromIntegral configDbPoolAcquisitionTimeout)
|
||||
(fromIntegral configDbPoolMaxLifetime)
|
||||
(fromIntegral configDbPoolMaxIdletime)
|
||||
(toUtf8 configDbUri)
|
||||
|
||||
-- | Run an action with a database connection.
|
||||
usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a)
|
||||
usePool AppState{..} = SQL.use statePool
|
||||
usePool AppState{..} x = do
|
||||
res <- SQL.use statePool x
|
||||
whenLeft res (\case
|
||||
SQL.AcquisitionTimeoutUsageError -> debounceLogAcquisitionTimeout -- this can happen rapidly for many requests, so we debounce
|
||||
_ -> pure ())
|
||||
return res
|
||||
|
||||
-- | Flush the connection pool so that any future use of the pool will
|
||||
-- use connections freshly established after this call.
|
||||
@@ -142,8 +168,8 @@ getSchemaCache = readIORef . stateSchemaCache
|
||||
putSchemaCache :: AppState -> Maybe SchemaCache -> IO ()
|
||||
putSchemaCache appState = atomicWriteIORef (stateSchemaCache appState)
|
||||
|
||||
getWorkerSem :: AppState -> MVar ()
|
||||
getWorkerSem = stateWorkerSem
|
||||
connectionWorker :: AppState -> IO ()
|
||||
connectionWorker = debouncedConnectionWorker
|
||||
|
||||
getRetryNextIn :: AppState -> IO Int
|
||||
getRetryNextIn = readIORef . stateRetryNextIn
|
||||
@@ -188,3 +214,262 @@ getIsListenerOn = readIORef . stateIsListenerOn
|
||||
|
||||
putIsListenerOn :: AppState -> Bool -> IO ()
|
||||
putIsListenerOn = atomicWriteIORef . stateIsListenerOn
|
||||
|
||||
-- | Schema cache status
|
||||
data SCacheStatus
|
||||
= SCLoaded
|
||||
| SCOnRetry
|
||||
| SCFatalFail
|
||||
|
||||
-- | Load the SchemaCache by using a connection from the pool.
|
||||
loadSchemaCache :: AppState -> IO SCacheStatus
|
||||
loadSchemaCache appState = do
|
||||
conf@AppConfig{..} <- getConfig appState
|
||||
result <-
|
||||
let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
|
||||
usePool appState . transaction SQL.ReadCommitted SQL.Read $
|
||||
querySchemaCache conf
|
||||
case result of
|
||||
Left e -> do
|
||||
case checkIsFatal e of
|
||||
Just hint -> do
|
||||
logWithZTime appState "A fatal error ocurred when loading the schema cache"
|
||||
logPgrstError appState e
|
||||
logWithZTime appState hint
|
||||
return SCFatalFail
|
||||
Nothing -> do
|
||||
putSchemaCache appState Nothing
|
||||
logWithZTime appState "An error ocurred when loading the schema cache"
|
||||
logPgrstError appState e
|
||||
return SCOnRetry
|
||||
|
||||
Right sCache -> do
|
||||
putSchemaCache appState (Just sCache)
|
||||
logWithZTime appState "Schema cache loaded"
|
||||
return SCLoaded
|
||||
|
||||
-- | Current database connection status data ConnectionStatus
|
||||
data ConnectionStatus
|
||||
= NotConnected
|
||||
| Connected PgVersion
|
||||
| FatalConnectionError Text
|
||||
deriving (Eq)
|
||||
|
||||
-- | The purpose of this worker is to obtain a healthy connection to pg and an
|
||||
-- up-to-date schema cache(SchemaCache). This method is meant to be called
|
||||
-- multiple times by the same thread, but does nothing if the previous
|
||||
-- invocation has not terminated. In all cases this method does not halt the
|
||||
-- calling thread, the work is performed in a separate thread.
|
||||
--
|
||||
-- Background thread that does the following :
|
||||
-- 1. Tries to connect to pg server and will keep trying until success.
|
||||
-- 2. Checks if the pg version is supported and if it's not it kills the main
|
||||
-- program.
|
||||
-- 3. Obtains the sCache. If this fails, it goes back to 1.
|
||||
internalConnectionWorker :: AppState -> IO ()
|
||||
internalConnectionWorker appState = work
|
||||
where
|
||||
work = do
|
||||
AppConfig{..} <- getConfig appState
|
||||
logWithZTime appState "Attempting to connect to the database..."
|
||||
connected <- establishConnection appState
|
||||
case connected of
|
||||
FatalConnectionError reason ->
|
||||
-- Fatal error when connecting
|
||||
logWithZTime appState reason >> killThread (getMainThreadId appState)
|
||||
NotConnected ->
|
||||
-- Unreachable because establishConnection will keep trying to connect
|
||||
return ()
|
||||
Connected actualPgVersion -> do
|
||||
-- Procede with initialization
|
||||
putPgVersion appState actualPgVersion
|
||||
when configDbChannelEnabled $
|
||||
signalListener appState
|
||||
logWithZTime appState "Connection successful"
|
||||
-- this could be fail because the connection drops, but the loadSchemaCache will pick the error and retry again
|
||||
-- We cannot retry after it fails immediately, because db-pre-config could have user errors. We just log the error and continue.
|
||||
when configDbConfig $ reReadConfig False appState
|
||||
scStatus <- loadSchemaCache appState
|
||||
case scStatus of
|
||||
SCLoaded ->
|
||||
-- do nothing and proceed if the load was successful
|
||||
return ()
|
||||
SCOnRetry ->
|
||||
-- retry reloading the schema cache
|
||||
work
|
||||
SCFatalFail ->
|
||||
-- die if our schema cache query has an error
|
||||
killThread $ getMainThreadId appState
|
||||
|
||||
-- | Repeatedly flush the pool, and check if a connection from the
|
||||
-- pool allows access to the PostgreSQL database.
|
||||
--
|
||||
-- Releasing the pool is key for rapid recovery. Otherwise, the pool
|
||||
-- timeout would have to be reached for new healthy connections to be acquired.
|
||||
-- Which might not happen if the server is busy with requests. No idle
|
||||
-- connection, no pool timeout.
|
||||
--
|
||||
-- The connection tries are capped, but if the connection times out no error is
|
||||
-- thrown, just 'False' is returned.
|
||||
establishConnection :: AppState -> IO ConnectionStatus
|
||||
establishConnection appState =
|
||||
retrying retrySettings shouldRetry $
|
||||
const $ flushPool appState >> getConnectionStatus
|
||||
where
|
||||
retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
|
||||
delayMicroseconds = 32000000 -- 32 seconds
|
||||
backoffMicroseconds = 1000000 -- 1 second
|
||||
|
||||
getConnectionStatus :: IO ConnectionStatus
|
||||
getConnectionStatus = do
|
||||
pgVersion <- usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
|
||||
case pgVersion of
|
||||
Left e -> do
|
||||
logPgrstError appState e
|
||||
case checkIsFatal e of
|
||||
Just reason ->
|
||||
return $ FatalConnectionError reason
|
||||
Nothing ->
|
||||
return NotConnected
|
||||
Right version ->
|
||||
if version < minimumPgVersion then
|
||||
return . FatalConnectionError $
|
||||
"Cannot run in this PostgreSQL version, PostgREST needs at least "
|
||||
<> pgvName minimumPgVersion
|
||||
else
|
||||
return . Connected $ version
|
||||
|
||||
shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
|
||||
shouldRetry rs isConnSucc = do
|
||||
let
|
||||
delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
|
||||
itShould = NotConnected == isConnSucc
|
||||
when itShould . logWithZTime appState $
|
||||
"Attempting to reconnect to the database in "
|
||||
<> (show delay::Text)
|
||||
<> " seconds..."
|
||||
when itShould $ putRetryNextIn appState delay
|
||||
return itShould
|
||||
|
||||
-- | Re-reads the config plus config options from the db
|
||||
reReadConfig :: Bool -> AppState -> IO ()
|
||||
reReadConfig startingUp appState = do
|
||||
AppConfig{..} <- getConfig appState
|
||||
dbSettings <-
|
||||
if configDbConfig then do
|
||||
qDbSettings <- usePool appState $ queryDbSettings (dumpQi <$> configDbPreConfig) configDbPreparedStatements
|
||||
case qDbSettings of
|
||||
Left e -> do
|
||||
logWithZTime appState
|
||||
"An error ocurred when trying to query database settings for the config parameters"
|
||||
case checkIsFatal e of
|
||||
Just hint -> do
|
||||
logPgrstError appState e
|
||||
logWithZTime appState hint
|
||||
killThread (getMainThreadId appState)
|
||||
Nothing -> do
|
||||
logPgrstError appState e
|
||||
pure mempty
|
||||
Right x -> pure x
|
||||
else
|
||||
pure mempty
|
||||
(roleSettings, roleIsolationLvl) <-
|
||||
if configDbConfig then do
|
||||
rSettings <- usePool appState $ queryRoleSettings configDbPreparedStatements
|
||||
case rSettings of
|
||||
Left e -> do
|
||||
logWithZTime appState "An error ocurred when trying to query the role settings"
|
||||
logPgrstError appState e
|
||||
pure (mempty, mempty)
|
||||
Right x -> pure x
|
||||
else
|
||||
pure mempty
|
||||
readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings roleIsolationLvl >>= \case
|
||||
Left err ->
|
||||
if startingUp then
|
||||
panic err -- die on invalid config if the program is starting up
|
||||
else
|
||||
logWithZTime appState $ "Failed reloading config: " <> err
|
||||
Right newConf -> do
|
||||
putConfig appState newConf
|
||||
if startingUp then
|
||||
pass
|
||||
else
|
||||
logWithZTime appState "Config reloaded"
|
||||
|
||||
|
||||
runListener :: AppConfig -> AppState -> IO ()
|
||||
runListener AppConfig{configDbChannelEnabled} appState =
|
||||
when configDbChannelEnabled $ listener appState
|
||||
|
||||
-- | Starts a dedicated pg connection to LISTEN for notifications. When a
|
||||
-- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
|
||||
-- cache. It uses the connectionWorker in case the LISTEN connection dies.
|
||||
listener :: AppState -> IO ()
|
||||
listener appState = do
|
||||
AppConfig{..} <- getConfig appState
|
||||
let dbChannel = toS configDbChannel
|
||||
|
||||
-- The listener has to wait for a signal from the connectionWorker.
|
||||
-- This is because when the connection to the db is lost, the listener also
|
||||
-- tries to recover the connection, but not with the same pace as the connectionWorker.
|
||||
-- Not waiting makes stderr quickly fill with connection retries messages from the listener.
|
||||
waitListener appState
|
||||
|
||||
-- forkFinally allows to detect if the thread dies
|
||||
void . flip forkFinally (handleFinally dbChannel) $ do
|
||||
dbOrError <- acquire $ toUtf8 configDbUri
|
||||
case dbOrError of
|
||||
Right db -> do
|
||||
logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
|
||||
putIsListenerOn appState True
|
||||
SQL.listen db $ SQL.toPgIdentifier dbChannel
|
||||
SQL.waitForNotifications handleNotification db
|
||||
_ ->
|
||||
die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
|
||||
where
|
||||
handleFinally dbChannel _ = do
|
||||
-- if the thread dies, we try to recover
|
||||
logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
|
||||
putIsListenerOn appState False
|
||||
-- assume the pool connection was also lost, call the connection worker
|
||||
connectionWorker appState
|
||||
-- retry the listener
|
||||
listener appState
|
||||
|
||||
handleNotification _ msg
|
||||
| BS.null msg = cacheReloader
|
||||
| msg == "reload schema" = cacheReloader
|
||||
| msg == "reload config" = reReadConfig False appState
|
||||
| otherwise = pure () -- Do nothing if anything else than an empty message is sent
|
||||
|
||||
cacheReloader =
|
||||
-- reloads the schema cache + restarts pool connections
|
||||
-- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
|
||||
connectionWorker appState
|
||||
|
||||
checkIsFatal :: SQL.UsageError -> Maybe Text
|
||||
checkIsFatal (SQL.ConnectionUsageError e)
|
||||
| isAuthFailureMessage = Just $ toS failureMessage
|
||||
| otherwise = Nothing
|
||||
where isAuthFailureMessage =
|
||||
("FATAL: password authentication failed" `isInfixOf` failureMessage) ||
|
||||
("no password supplied" `isInfixOf` failureMessage)
|
||||
failureMessage = BS.unpack $ fromMaybe mempty e
|
||||
checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
|
||||
= case serverError of
|
||||
-- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
|
||||
SQL.ServerError "42601" _ _ _ _
|
||||
-> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
|
||||
-- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
|
||||
-- This would mean that a connection pooler in transaction mode is being used
|
||||
-- while prepared statements are enabled in the PostgREST configuration,
|
||||
-- both of which are incompatible with each other.
|
||||
SQL.ServerError "42P05" _ _ _ _
|
||||
-> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
|
||||
-- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
|
||||
-- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
|
||||
SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
|
||||
-> Just "Hint: Connection poolers in statement mode are not supported."
|
||||
_ -> Nothing
|
||||
checkIsFatal _ = Nothing
|
||||
|
||||
+10
-8
@@ -21,7 +21,6 @@ import PostgREST.AppState (AppState)
|
||||
import PostgREST.Config (AppConfig (..))
|
||||
import PostgREST.SchemaCache (querySchemaCache)
|
||||
import PostgREST.Version (prettyVersion)
|
||||
import PostgREST.Workers (reReadConfig)
|
||||
|
||||
import qualified PostgREST.App as App
|
||||
import qualified PostgREST.AppState as AppState
|
||||
@@ -33,7 +32,7 @@ import Protolude hiding (hPutStrLn)
|
||||
main :: App.SignalHandlerInstaller -> Maybe App.SocketRunner -> CLI -> IO ()
|
||||
main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
|
||||
conf@AppConfig{..} <-
|
||||
either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty
|
||||
either panic identity <$> Config.readAppConfig mempty cliPath Nothing mempty mempty
|
||||
|
||||
-- Per https://github.com/PostgREST/postgrest/issues/268, we want to
|
||||
-- explicitly close the connections to PostgreSQL on shutdown.
|
||||
@@ -43,7 +42,7 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
|
||||
AppState.destroy
|
||||
(\appState -> case cliCommand of
|
||||
CmdDumpConfig -> do
|
||||
when configDbConfig $ reReadConfig True appState
|
||||
when configDbConfig $ AppState.reReadConfig True appState
|
||||
putStr . Config.toText =<< AppState.getConfig appState
|
||||
CmdDumpSchema -> putStrLn =<< dumpSchema appState
|
||||
CmdRun -> App.run installSignalHandlers runAppWithSocket appState)
|
||||
@@ -51,15 +50,12 @@ main installSignalHandlers runAppWithSocket CLI{cliCommand, cliPath} = do
|
||||
-- | Dump SchemaCache schema to JSON
|
||||
dumpSchema :: AppState -> IO LBS.ByteString
|
||||
dumpSchema appState = do
|
||||
AppConfig{..} <- AppState.getConfig appState
|
||||
conf@AppConfig{..} <- AppState.getConfig appState
|
||||
result <-
|
||||
let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
|
||||
AppState.usePool appState $
|
||||
transaction SQL.ReadCommitted SQL.Read $
|
||||
querySchemaCache
|
||||
(toList configDbSchemas)
|
||||
configDbExtraSearchPath
|
||||
configDbPreparedStatements
|
||||
querySchemaCache conf
|
||||
case result of
|
||||
Left e -> do
|
||||
hPutStrLn stderr $ "An error ocurred when loading the schema cache:\n" <> show e
|
||||
@@ -136,6 +132,9 @@ exampleConfigFile =
|
||||
|## Enable in-database configuration
|
||||
|db-config = true
|
||||
|
|
||||
|## Function for in-database configuration
|
||||
|## db-pre-config = "postgrest.pre_config"
|
||||
|
|
||||
|## Extra schemas to add to the search_path of every request
|
||||
|db-extra-search-path = "public"
|
||||
|
|
||||
@@ -154,6 +153,9 @@ exampleConfigFile =
|
||||
|## Time in seconds after which to recycle pool connections
|
||||
|# db-pool-max-lifetime = 1800
|
||||
|
|
||||
|## Time in seconds after which to recycle unused pool connections
|
||||
|# db-pool-max-idletime = 30
|
||||
|
|
||||
|## Stored proc to exec immediately after auth
|
||||
|# db-pre-request = "stored_proc_name"
|
||||
|
|
||||
|
||||
+21
-16
@@ -51,7 +51,8 @@ import Numeric (readOct, showOct)
|
||||
import System.Environment (getEnvironment)
|
||||
import System.Posix.Types (FileMode)
|
||||
|
||||
import PostgREST.Config.Database (RoleSettings)
|
||||
import PostgREST.Config.Database (RoleIsolationLvl,
|
||||
RoleSettings)
|
||||
import PostgREST.Config.JSPath (JSPath, JSPathExp (..),
|
||||
dumpJSPath, pRoleClaimKey)
|
||||
import PostgREST.Config.Proxy (Proxy (..),
|
||||
@@ -74,11 +75,13 @@ data AppConfig = AppConfig
|
||||
, configDbPoolSize :: Int
|
||||
, configDbPoolAcquisitionTimeout :: Int
|
||||
, configDbPoolMaxLifetime :: Int
|
||||
, configDbPoolMaxIdletime :: Int
|
||||
, configDbPreRequest :: Maybe QualifiedIdentifier
|
||||
, configDbPreparedStatements :: Bool
|
||||
, configDbRootSpec :: Maybe QualifiedIdentifier
|
||||
, configDbSchemas :: NonEmpty Text
|
||||
, configDbConfig :: Bool
|
||||
, configDbPreConfig :: Maybe QualifiedIdentifier
|
||||
, configDbTxAllowOverride :: Bool
|
||||
, configDbTxRollbackAll :: Bool
|
||||
, configDbUri :: Text
|
||||
@@ -101,6 +104,8 @@ data AppConfig = AppConfig
|
||||
, configServerUnixSocketMode :: FileMode
|
||||
, configAdminServerPort :: Maybe Int
|
||||
, configRoleSettings :: RoleSettings
|
||||
, configRoleIsoLvl :: RoleIsolationLvl
|
||||
, configInternalSCSleep :: Maybe Int32
|
||||
}
|
||||
|
||||
data LogLevel = LogCrit | LogError | LogWarn | LogInfo
|
||||
@@ -137,11 +142,13 @@ toText conf =
|
||||
,("db-pool", show . configDbPoolSize)
|
||||
,("db-pool-acquisition-timeout", show . configDbPoolAcquisitionTimeout)
|
||||
,("db-pool-max-lifetime", show . configDbPoolMaxLifetime)
|
||||
,("db-pool-max-idletime", show . configDbPoolMaxIdletime)
|
||||
,("db-pre-request", q . maybe mempty dumpQi . configDbPreRequest)
|
||||
,("db-prepared-statements", T.toLower . show . configDbPreparedStatements)
|
||||
,("db-root-spec", q . maybe mempty dumpQi . configDbRootSpec)
|
||||
,("db-schemas", q . T.intercalate "," . toList . configDbSchemas)
|
||||
,("db-config", T.toLower . show . configDbConfig)
|
||||
,("db-pre-config", q . maybe mempty dumpQi . configDbPreConfig)
|
||||
,("db-tx-end", q . showTxEnd)
|
||||
,("db-uri", q . configDbUri)
|
||||
,("db-use-legacy-gucs", T.toLower . show . configDbUseLegacyGucs)
|
||||
@@ -193,13 +200,13 @@ instance JustIfMaybe a (Maybe a) where
|
||||
|
||||
-- | Reads and parses the config and overrides its parameters from env vars,
|
||||
-- files or db settings.
|
||||
readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> IO (Either Text AppConfig)
|
||||
readAppConfig dbSettings optPath prevDbUri roleSettings = do
|
||||
readAppConfig :: [(Text, Text)] -> Maybe FilePath -> Maybe Text -> RoleSettings -> RoleIsolationLvl -> IO (Either Text AppConfig)
|
||||
readAppConfig dbSettings optPath prevDbUri roleSettings roleIsolationLvl = do
|
||||
env <- readPGRSTEnvironment
|
||||
-- if no filename provided, start with an empty map to read config from environment
|
||||
conf <- maybe (return $ Right M.empty) loadConfig optPath
|
||||
|
||||
case C.runParser (parser optPath env dbSettings roleSettings) =<< mapLeft show conf of
|
||||
case C.runParser (parser optPath env dbSettings roleSettings roleIsolationLvl) =<< mapLeft show conf of
|
||||
Left err ->
|
||||
return . Left $ "Error in config " <> err
|
||||
Right parsedConfig ->
|
||||
@@ -214,8 +221,8 @@ readAppConfig dbSettings optPath prevDbUri roleSettings = do
|
||||
decodeJWKS <$>
|
||||
(decodeSecret =<< readSecretFile =<< readDbUriFile prevDbUri parsedConfig)
|
||||
|
||||
parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> C.Parser C.Config AppConfig
|
||||
parser optPath env dbSettings roleSettings =
|
||||
parser :: Maybe FilePath -> Environment -> [(Text, Text)] -> RoleSettings -> RoleIsolationLvl -> C.Parser C.Config AppConfig
|
||||
parser optPath env dbSettings roleSettings roleIsolationLvl =
|
||||
AppConfig
|
||||
<$> parseAppSettings "app.settings"
|
||||
<*> (fmap encodeUtf8 <$> optString "db-anon-role")
|
||||
@@ -228,6 +235,8 @@ parser optPath env dbSettings roleSettings =
|
||||
<*> (fromMaybe 10 <$> optInt "db-pool")
|
||||
<*> (fromMaybe 10 <$> optInt "db-pool-acquisition-timeout")
|
||||
<*> (fromMaybe 1800 <$> optInt "db-pool-max-lifetime")
|
||||
<*> (fromMaybe 30 <$> optWithAlias (optInt "db-pool-timeout")
|
||||
(optInt "db-pool-max-idletime"))
|
||||
<*> (fmap toQi <$> optWithAlias (optString "db-pre-request")
|
||||
(optString "pre-request"))
|
||||
<*> (fromMaybe True <$> optBool "db-prepared-statements")
|
||||
@@ -236,6 +245,7 @@ parser optPath env dbSettings roleSettings =
|
||||
<*> (fromList . maybe ["public"] splitOnCommas <$> optWithAlias (optValue "db-schemas")
|
||||
(optValue "db-schema"))
|
||||
<*> (fromMaybe True <$> optBool "db-config")
|
||||
<*> (fmap toQi <$> optString "db-pre-config")
|
||||
<*> parseTxEnd "db-tx-end" snd
|
||||
<*> parseTxEnd "db-tx-end" fst
|
||||
<*> (fromMaybe "postgresql://" <$> optString "db-uri")
|
||||
@@ -260,6 +270,8 @@ parser optPath env dbSettings roleSettings =
|
||||
<*> parseSocketFileMode "server-unix-socket-mode"
|
||||
<*> optInt "admin-server-port"
|
||||
<*> pure roleSettings
|
||||
<*> pure roleIsolationLvl
|
||||
<*> optInt "internal-schema-cache-sleep"
|
||||
where
|
||||
parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
|
||||
parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
|
||||
@@ -354,21 +366,14 @@ parser optPath env dbSettings roleSettings =
|
||||
(C.Key -> C.Parser C.Value a -> C.Parser C.Config b) ->
|
||||
C.Key -> (C.Value -> a) -> C.Parser C.Config b
|
||||
overrideFromDbOrEnvironment necessity key coercion =
|
||||
case reloadableDbSetting <|> M.lookup envVarName env of
|
||||
case dbConf <|> M.lookup envVarName env of
|
||||
Just dbOrEnvVal -> pure $ justIfMaybe $ coercion $ C.String dbOrEnvVal
|
||||
Nothing -> necessity key (coercion <$> C.value)
|
||||
Nothing -> necessity key (coercion <$> C.value)
|
||||
where
|
||||
dashToUnderscore '-' = '_'
|
||||
dashToUnderscore c = c
|
||||
envVarName = "PGRST_" <> (toUpper . dashToUnderscore <$> toS key)
|
||||
reloadableDbSetting =
|
||||
let dbSettingName = T.pack $ dashToUnderscore <$> toS key in
|
||||
if dbSettingName `notElem` [
|
||||
"server_host", "server_port", "server_unix_socket", "server_unix_socket_mode", "admin_server_port", "log_level",
|
||||
"db_uri", "db_channel_enabled", "db_channel", "db_pool", "db_pool_acquisition_timeout",
|
||||
"db_pool_max_lifetime", "db_config"]
|
||||
then lookup dbSettingName dbSettings
|
||||
else Nothing
|
||||
dbConf = lookup (T.pack $ dashToUnderscore <$> toS key) dbSettings
|
||||
|
||||
coerceText :: C.Value -> Text
|
||||
coerceText (C.String s) = s
|
||||
|
||||
@@ -6,6 +6,8 @@ module PostgREST.Config.Database
|
||||
, queryRoleSettings
|
||||
, queryPgVersion
|
||||
, RoleSettings
|
||||
, RoleIsolationLvl
|
||||
, toIsolationLevel
|
||||
) where
|
||||
|
||||
import Control.Arrow ((***))
|
||||
@@ -21,11 +23,47 @@ import qualified Hasql.Statement as SQL
|
||||
import qualified Hasql.Transaction as SQL
|
||||
import qualified Hasql.Transaction.Sessions as SQL
|
||||
|
||||
import Text.InterpolatedString.Perl6 (q)
|
||||
import Text.InterpolatedString.Perl6 (q, qc)
|
||||
|
||||
import Protolude
|
||||
|
||||
type RoleSettings = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
|
||||
type RoleSettings = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
|
||||
type RoleIsolationLvl = HM.HashMap ByteString SQL.IsolationLevel
|
||||
|
||||
toIsolationLevel :: (Eq a, IsString a) => a -> SQL.IsolationLevel
|
||||
toIsolationLevel a = case a of
|
||||
"repeatable read" -> SQL.RepeatableRead
|
||||
"serializable" -> SQL.Serializable
|
||||
_ -> SQL.ReadCommitted
|
||||
|
||||
prefix :: Text
|
||||
prefix = "pgrst."
|
||||
|
||||
-- | In-db settings names
|
||||
dbSettingsNames :: [Text]
|
||||
dbSettingsNames =
|
||||
(prefix <>) <$>
|
||||
["db_anon_role"
|
||||
,"db_pre_config"
|
||||
,"db_extra_search_path"
|
||||
,"db_max_rows"
|
||||
,"db_plan_enabled"
|
||||
,"db_pre_request"
|
||||
,"db_prepared_statements"
|
||||
,"db_root_spec"
|
||||
,"db_schemas"
|
||||
,"db_tx_end"
|
||||
,"db_use_legacy_gucs"
|
||||
,"jwt_aud"
|
||||
,"jwt_role_claim_key"
|
||||
,"jwt_secret"
|
||||
,"jwt_secret_is_base64"
|
||||
,"openapi_mode"
|
||||
,"openapi_security_active"
|
||||
,"openapi_server_proxy_uri"
|
||||
,"raw_media_types"
|
||||
,"server_trace_header"
|
||||
]
|
||||
|
||||
queryPgVersion :: Bool -> Session PgVersion
|
||||
queryPgVersion prepared = statement mempty $ pgVersionStatement prepared
|
||||
@@ -36,46 +74,62 @@ pgVersionStatement = SQL.Statement sql HE.noParams versionRow
|
||||
sql = "SELECT current_setting('server_version_num')::integer, current_setting('server_version')"
|
||||
versionRow = HD.singleRow $ PgVersion <$> column HD.int4 <*> column HD.text
|
||||
|
||||
queryDbSettings :: Bool -> Session [(Text, Text)]
|
||||
queryDbSettings prepared =
|
||||
-- | Query the in-database configuration. The settings have the following priorities:
|
||||
--
|
||||
-- 1. Role + with database-specific settings:
|
||||
-- ALTER ROLE authenticator IN DATABASE postgres SET <prefix>jwt_aud = 'val';
|
||||
-- 2. Role + with settings:
|
||||
-- ALTER ROLE authenticator SET <prefix>jwt_aud = 'overridden';
|
||||
-- 3. pre-config function:
|
||||
-- CREATE FUNCTION pre_config() .. PERFORM set_config(<prefix>jwt_aud, 'pre_config_aud'..)
|
||||
--
|
||||
-- The example above will result in <prefix>jwt_aud = 'val'
|
||||
-- A setting on the database only will have no effect: ALTER DATABASE postgres SET <prefix>jwt_aud = 'xx'
|
||||
queryDbSettings :: Maybe Text -> Bool -> Session [(Text, Text)]
|
||||
queryDbSettings preConfFunc prepared =
|
||||
let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
|
||||
transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ dbSettingsStatement prepared
|
||||
|
||||
-- | Get db settings from the connection role. Global settings will be overridden by database specific settings.
|
||||
dbSettingsStatement :: Bool -> SQL.Statement () [(Text, Text)]
|
||||
dbSettingsStatement = SQL.Statement sql HE.noParams decodeSettings
|
||||
transaction SQL.ReadCommitted SQL.Read $ SQL.statement dbSettingsNames $ SQL.Statement sql (arrayParam HE.text) decodeSettings prepared
|
||||
where
|
||||
sql = [q|
|
||||
sql = [qc|
|
||||
WITH
|
||||
role_setting (database, setting) AS (
|
||||
SELECT setdatabase,
|
||||
unnest(setconfig)
|
||||
FROM pg_catalog.pg_db_role_setting
|
||||
WHERE setrole = CURRENT_USER::regrole::oid
|
||||
AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
|
||||
role_setting AS (
|
||||
SELECT setdatabase as database,
|
||||
unnest(setconfig) as setting
|
||||
FROM pg_catalog.pg_db_role_setting
|
||||
WHERE setrole = CURRENT_USER::regrole::oid
|
||||
AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
|
||||
),
|
||||
kv_settings (database, k, v) AS (
|
||||
kv_settings AS (
|
||||
SELECT database,
|
||||
substr(setting, 1, strpos(setting, '=') - 1),
|
||||
substr(setting, strpos(setting, '=') + 1)
|
||||
FROM role_setting
|
||||
WHERE setting LIKE 'pgrst.%'
|
||||
substr(setting, 1, strpos(setting, '=') - 1) as k,
|
||||
substr(setting, strpos(setting, '=') + 1) as v
|
||||
FROM role_setting
|
||||
{preConfigF}
|
||||
)
|
||||
SELECT DISTINCT ON (key)
|
||||
replace(k, 'pgrst.', '') AS key,
|
||||
replace(k, '{prefix}', '') AS key,
|
||||
v AS value
|
||||
FROM kv_settings
|
||||
ORDER BY key, database DESC;
|
||||
FROM kv_settings
|
||||
WHERE k = ANY($1) AND v IS NOT NULL
|
||||
ORDER BY key, database DESC NULLS LAST;
|
||||
|]
|
||||
preConfigF = case preConfFunc of
|
||||
Nothing -> mempty
|
||||
Just func -> [qc|
|
||||
UNION
|
||||
SELECT
|
||||
null as database,
|
||||
x as k,
|
||||
current_setting(x, true) as v
|
||||
FROM unnest($1) x
|
||||
JOIN {func}() _ ON TRUE
|
||||
|]::Text
|
||||
decodeSettings = HD.rowList $ (,) <$> column HD.text <*> column HD.text
|
||||
|
||||
queryRoleSettings :: Bool -> Session RoleSettings
|
||||
queryRoleSettings :: Bool -> Session (RoleSettings, RoleIsolationLvl)
|
||||
queryRoleSettings prepared =
|
||||
let transaction = if prepared then SQL.transaction else SQL.unpreparedTransaction in
|
||||
transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ roleSettingsStatement prepared
|
||||
|
||||
roleSettingsStatement :: Bool -> SQL.Statement () RoleSettings
|
||||
roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
|
||||
transaction SQL.ReadCommitted SQL.Read $ SQL.statement mempty $ SQL.Statement sql HE.noParams (processRows <$> rows) prepared
|
||||
where
|
||||
sql = [q|
|
||||
with
|
||||
@@ -91,18 +145,40 @@ roleSettingsStatement = SQL.Statement sql HE.noParams decodeRoleSettings
|
||||
substr(setting, 1, strpos(setting, '=') - 1) as key,
|
||||
lower(substr(setting, strpos(setting, '=') + 1)) as value
|
||||
FROM role_setting
|
||||
),
|
||||
iso_setting AS (
|
||||
SELECT rolname, value
|
||||
FROM kv_settings
|
||||
WHERE key = 'default_transaction_isolation'
|
||||
)
|
||||
select rolname, array_agg(row(key, value))
|
||||
from kv_settings
|
||||
group by rolname;
|
||||
select
|
||||
kv.rolname,
|
||||
i.value as iso_lvl,
|
||||
array_agg(row(kv.key, kv.value)) filter (where key <> 'default_transation_isolation') as role_settings
|
||||
from kv_settings kv
|
||||
left join iso_setting i on i.rolname = kv.rolname
|
||||
group by kv.rolname, i.value;
|
||||
|]
|
||||
decodeRoleSettings = HM.fromList . map (bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>))) <$> HD.rowList aRow
|
||||
aRow :: HD.Row (Text, [(Text, Text)])
|
||||
aRow = (,) <$> column HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
|
||||
|
||||
processRows :: [(Text, Maybe Text, [(Text, Text)])] -> (RoleSettings, RoleIsolationLvl)
|
||||
processRows rs =
|
||||
let
|
||||
rowsWRoleSettings = [ (x, z) | (x, _, z) <- rs ]
|
||||
rowsWIsolation = [ (x, y) | (x, Just y, _) <- rs ]
|
||||
in
|
||||
( HM.fromList $ bimap encodeUtf8 (HM.fromList . ((encodeUtf8 *** encodeUtf8) <$>)) <$> rowsWRoleSettings
|
||||
, HM.fromList $ (encodeUtf8 *** toIsolationLevel) <$> rowsWIsolation
|
||||
)
|
||||
|
||||
rows :: HD.Result [(Text, Maybe Text, [(Text, Text)])]
|
||||
rows = HD.rowList $ (,,) <$> column HD.text <*> nullableColumn HD.text <*> compositeArrayColumn ((,) <$> compositeField HD.text <*> compositeField HD.text)
|
||||
|
||||
column :: HD.Value a -> HD.Row a
|
||||
column = HD.column . HD.nonNullable
|
||||
|
||||
nullableColumn :: HD.Value a -> HD.Row (Maybe a)
|
||||
nullableColumn = HD.column . HD.nullable
|
||||
|
||||
compositeField :: HD.Value a -> HD.Composite a
|
||||
compositeField = HD.field . HD.nonNullable
|
||||
|
||||
@@ -111,3 +187,9 @@ compositeArrayColumn = arrayColumn . HD.composite
|
||||
|
||||
arrayColumn :: HD.Value a -> HD.Row [a]
|
||||
arrayColumn = column . HD.listArray . HD.nonNullable
|
||||
|
||||
param :: HE.Value a -> HE.Params a
|
||||
param = HE.param . HE.nonNullable
|
||||
|
||||
arrayParam :: HE.Value a -> HE.Params [a]
|
||||
arrayParam = param . HE.foldableArray . HE.nonNullable
|
||||
|
||||
+1
-24
@@ -11,7 +11,6 @@ module PostgREST.Error
|
||||
, PgError(..)
|
||||
, Error(..)
|
||||
, errorPayload
|
||||
, checkIsFatal
|
||||
, singularityError
|
||||
) where
|
||||
|
||||
@@ -429,6 +428,7 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
|
||||
'5':'3':_ -> HTTP.status503 -- insufficient resources
|
||||
'5':'4':_ -> HTTP.status413 -- too complex
|
||||
'5':'5':_ -> HTTP.status500 -- obj not on prereq state
|
||||
'5':'7':'P':'0':'1':_ -> HTTP.status503 -- terminating connection due to administrator command
|
||||
'5':'7':_ -> HTTP.status500 -- operator intervention
|
||||
'5':'8':_ -> HTTP.status500 -- system error
|
||||
'F':'0':_ -> HTTP.status500 -- conf file error
|
||||
@@ -446,29 +446,6 @@ pgErrorStatus authed (SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError
|
||||
|
||||
_ -> HTTP.status500
|
||||
|
||||
checkIsFatal :: SQL.UsageError -> Maybe Text
|
||||
checkIsFatal (SQL.ConnectionUsageError e)
|
||||
| isAuthFailureMessage = Just $ toS failureMessage
|
||||
| otherwise = Nothing
|
||||
where isAuthFailureMessage = "FATAL: password authentication failed" `isInfixOf` failureMessage
|
||||
failureMessage = BS.unpack $ fromMaybe mempty e
|
||||
checkIsFatal(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ResultError serverError)))
|
||||
= case serverError of
|
||||
-- Check for a syntax error (42601 is the pg code). This would mean the error is on our part somehow, so we treat it as fatal.
|
||||
SQL.ServerError "42601" _ _ _ _
|
||||
-> Just "Hint: This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues"
|
||||
-- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
|
||||
-- This would mean that a connection pooler in transaction mode is being used
|
||||
-- while prepared statements are enabled in the PostgREST configuration,
|
||||
-- both of which are incompatible with each other.
|
||||
SQL.ServerError "42P05" _ _ _ _
|
||||
-> Just "Hint: If you are using connection poolers in transaction mode, try setting db-prepared-statements to false."
|
||||
-- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
|
||||
-- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
|
||||
SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _
|
||||
-> Just "Hint: Connection poolers in statement mode are not supported."
|
||||
_ -> Nothing
|
||||
checkIsFatal _ = Nothing
|
||||
|
||||
|
||||
data Error
|
||||
|
||||
@@ -26,6 +26,8 @@ module PostgREST.SchemaCache
|
||||
, schemaDescription
|
||||
) where
|
||||
|
||||
import Control.Monad.Extra (whenJust)
|
||||
|
||||
import qualified Data.Aeson as JSON
|
||||
import qualified Data.HashMap.Strict as HM
|
||||
import qualified Data.HashMap.Strict.InsOrd as HMI
|
||||
@@ -38,7 +40,9 @@ import qualified Hasql.Transaction as SQL
|
||||
import Contravariant.Extras (contrazip2)
|
||||
import Text.InterpolatedString.Perl6 (q)
|
||||
|
||||
import PostgREST.Config.Database (pgVersionStatement)
|
||||
import PostgREST.Config (AppConfig (..))
|
||||
import PostgREST.Config.Database (pgVersionStatement,
|
||||
toIsolationLevel)
|
||||
import PostgREST.Config.PgVersion (PgVersion, pgVersion100,
|
||||
pgVersion110, pgVersion120)
|
||||
import PostgREST.SchemaCache.Identifiers (AccessSet, FieldName,
|
||||
@@ -103,15 +107,18 @@ data KeyDep
|
||||
-- | A SQL query that can be executed independently
|
||||
type SqlQuery = ByteString
|
||||
|
||||
querySchemaCache :: [Schema] -> [Schema] -> Bool -> SQL.Transaction SchemaCache
|
||||
querySchemaCache schemas extraSearchPath prepared = do
|
||||
querySchemaCache :: AppConfig -> SQL.Transaction SchemaCache
|
||||
querySchemaCache AppConfig{..} = do
|
||||
SQL.sql "set local schema ''" -- This voids the search path. The following queries need this for getting the fully qualified name(schema.name) of every db object
|
||||
pgVer <- SQL.statement mempty $ pgVersionStatement prepared
|
||||
tabs <- SQL.statement schemas $ allTables pgVer prepared
|
||||
keyDeps <- SQL.statement (schemas, extraSearchPath) $ allViewsKeyDependencies prepared
|
||||
keyDeps <- SQL.statement (schemas, configDbExtraSearchPath) $ allViewsKeyDependencies prepared
|
||||
m2oRels <- SQL.statement mempty $ allM2OandO2ORels pgVer prepared
|
||||
funcs <- SQL.statement schemas $ allFunctions pgVer prepared
|
||||
cRels <- SQL.statement mempty $ allComputedRels prepared
|
||||
_ <-
|
||||
let sleepCall = SQL.Statement "select pg_sleep($1)" (param HE.int4) HD.noResult prepared in
|
||||
whenJust configInternalSCSleep (`SQL.statement` sleepCall) -- only used for testing
|
||||
|
||||
let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
|
||||
rels = addInverseRels $ addM2MRels tabsWViewsPks $ addViewM2OAndO2ORels keyDeps m2oRels
|
||||
@@ -121,6 +128,9 @@ querySchemaCache schemas extraSearchPath prepared = do
|
||||
, dbRelationships = getOverrideRelationshipsMap rels cRels
|
||||
, dbRoutines = funcs
|
||||
}
|
||||
where
|
||||
schemas = toList configDbSchemas
|
||||
prepared = configDbPreparedStatements
|
||||
|
||||
-- | overrides detected relationships with the computed relationships and gets the RelationshipsMap
|
||||
getOverrideRelationshipsMap :: [Relationship] -> [Relationship] -> RelationshipsMap
|
||||
@@ -250,7 +260,7 @@ decodeFuncs =
|
||||
<*> column HD.bool)
|
||||
<*> (parseVolatility <$> column HD.char)
|
||||
<*> column HD.bool
|
||||
<*> nullableColumn HD.text
|
||||
<*> nullableColumn (toIsolationLevel <$> HD.text)
|
||||
|
||||
addKey :: Routine -> (QualifiedIdentifier, Routine)
|
||||
addKey pd = (QualifiedIdentifier (pdSchema pd) (pdName pd), pd)
|
||||
|
||||
@@ -16,8 +16,10 @@ module PostgREST.SchemaCache.Routine
|
||||
, funcReturnsCompositeAlias
|
||||
) where
|
||||
|
||||
import qualified Data.Aeson as JSON
|
||||
import qualified Data.HashMap.Strict as HM
|
||||
import Data.Aeson ((.=))
|
||||
import qualified Data.Aeson as JSON
|
||||
import qualified Data.HashMap.Strict as HM
|
||||
import qualified Hasql.Transaction.Sessions as SQL
|
||||
|
||||
import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
|
||||
Schema, TableName)
|
||||
@@ -48,9 +50,21 @@ data Routine = Function
|
||||
, pdReturnType :: RetType
|
||||
, pdVolatility :: FuncVolatility
|
||||
, pdHasVariadic :: Bool
|
||||
, pdIsoLvl :: Maybe Text
|
||||
, pdIsoLvl :: Maybe SQL.IsolationLevel
|
||||
}
|
||||
deriving (Eq, Generic, JSON.ToJSON)
|
||||
deriving (Eq, Generic)
|
||||
-- need to define JSON manually bc SQL.IsolationLevel doesn't have a JSON instance(and we can't define one for that type without getting a compiler error)
|
||||
instance JSON.ToJSON Routine where
|
||||
toJSON (Function sch nam desc params ret vol hasVar _) = JSON.object
|
||||
[
|
||||
"pdSchema" .= sch
|
||||
, "pdName" .= nam
|
||||
, "pdDescription" .= desc
|
||||
, "pdParams" .= JSON.toJSON params
|
||||
, "pdReturnType" .= JSON.toJSON ret
|
||||
, "pdVolatility" .= JSON.toJSON vol
|
||||
, "pdHasVariadic" .= JSON.toJSON hasVar
|
||||
]
|
||||
|
||||
data RoutineParam = RoutineParam
|
||||
{ ppName :: Text
|
||||
|
||||
@@ -14,7 +14,6 @@ import System.Posix.Files (setFileMode)
|
||||
import System.Posix.Types (FileMode)
|
||||
|
||||
import qualified PostgREST.AppState as AppState
|
||||
import qualified PostgREST.Workers as Workers
|
||||
|
||||
import Protolude
|
||||
|
||||
@@ -49,10 +48,10 @@ installSignalHandlers appState = do
|
||||
|
||||
-- The SIGUSR1 signal updates the internal 'SchemaCache' by running
|
||||
-- 'connectionWorker' exactly as before.
|
||||
install Signals.sigUSR1 $ Workers.connectionWorker appState
|
||||
install Signals.sigUSR1 $ AppState.connectionWorker appState
|
||||
|
||||
-- Re-read the config on SIGUSR2
|
||||
install Signals.sigUSR2 $ Workers.reReadConfig False appState
|
||||
install Signals.sigUSR2 $ AppState.reReadConfig False appState
|
||||
where
|
||||
install signal handler =
|
||||
void $ Signals.installHandler signal (Signals.Catch handler) Nothing
|
||||
|
||||
@@ -1,343 +0,0 @@
|
||||
{-# LANGUAGE LambdaCase #-}
|
||||
{-# LANGUAGE NamedFieldPuns #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
|
||||
module PostgREST.Workers
|
||||
( connectionWorker
|
||||
, reReadConfig
|
||||
, runListener
|
||||
, runAdmin
|
||||
) where
|
||||
|
||||
import qualified Data.ByteString as BS
|
||||
import qualified Data.Text as T
|
||||
import qualified Hasql.Notifications as SQL
|
||||
import qualified Hasql.Session as SQL
|
||||
import qualified Hasql.Transaction.Sessions as SQL
|
||||
import qualified Network.HTTP.Types.Status as HTTP
|
||||
import qualified Network.Wai as Wai
|
||||
import qualified Network.Wai.Handler.Warp as Warp
|
||||
|
||||
import Control.Retry (RetryStatus, capDelay, exponentialBackoff,
|
||||
retrying, rsPreviousDelay)
|
||||
import Hasql.Connection (acquire)
|
||||
|
||||
import Network.Socket
|
||||
import Network.Socket.ByteString
|
||||
|
||||
import PostgREST.AppState (AppState)
|
||||
import PostgREST.Config (AppConfig (..), readAppConfig)
|
||||
import PostgREST.Config.Database (queryDbSettings, queryPgVersion,
|
||||
queryRoleSettings)
|
||||
import PostgREST.Config.PgVersion (PgVersion (..), minimumPgVersion)
|
||||
import PostgREST.Error (checkIsFatal)
|
||||
import PostgREST.SchemaCache (querySchemaCache)
|
||||
|
||||
import qualified PostgREST.AppState as AppState
|
||||
|
||||
import Protolude
|
||||
|
||||
|
||||
-- | Current database connection status data ConnectionStatus
|
||||
data ConnectionStatus
|
||||
= NotConnected
|
||||
| Connected PgVersion
|
||||
| FatalConnectionError Text
|
||||
deriving (Eq)
|
||||
|
||||
-- | Schema cache status
|
||||
data SCacheStatus
|
||||
= SCLoaded
|
||||
| SCOnRetry
|
||||
| SCFatalFail
|
||||
|
||||
-- | The purpose of this worker is to obtain a healthy connection to pg and an
|
||||
-- up-to-date schema cache(SchemaCache). This method is meant to be called
|
||||
-- multiple times by the same thread, but does nothing if the previous
|
||||
-- invocation has not terminated. In all cases this method does not halt the
|
||||
-- calling thread, the work is performed in a separate thread.
|
||||
--
|
||||
-- Background thread that does the following :
|
||||
-- 1. Tries to connect to pg server and will keep trying until success.
|
||||
-- 2. Checks if the pg version is supported and if it's not it kills the main
|
||||
-- program.
|
||||
-- 3. Obtains the sCache. If this fails, it goes back to 1.
|
||||
connectionWorker :: AppState -> IO ()
|
||||
connectionWorker appState = do
|
||||
runExclusively (AppState.getWorkerSem appState) work
|
||||
-- Prevents multiple workers to be running at the same time. Could happen on
|
||||
-- too many SIGUSR1s.
|
||||
where
|
||||
runExclusively mvar action = mask_ $ do
|
||||
success <- tryPutMVar mvar ()
|
||||
when success $ do
|
||||
void $ forkIO $ action `finally` takeMVar mvar
|
||||
work = do
|
||||
AppConfig{..} <- AppState.getConfig appState
|
||||
AppState.logWithZTime appState "Attempting to connect to the database..."
|
||||
connected <- establishConnection appState
|
||||
case connected of
|
||||
FatalConnectionError reason ->
|
||||
-- Fatal error when connecting
|
||||
AppState.logWithZTime appState reason >> killThread (AppState.getMainThreadId appState)
|
||||
NotConnected ->
|
||||
-- Unreachable because establishConnection will keep trying to connect
|
||||
return ()
|
||||
Connected actualPgVersion -> do
|
||||
-- Procede with initialization
|
||||
AppState.putPgVersion appState actualPgVersion
|
||||
when configDbChannelEnabled $
|
||||
AppState.signalListener appState
|
||||
AppState.logWithZTime appState "Connection successful"
|
||||
-- this could be fail because the connection drops, but the
|
||||
-- loadSchemaCache will pick the error and retry again
|
||||
when configDbConfig $ reReadConfig False appState
|
||||
scStatus <- loadSchemaCache appState
|
||||
case scStatus of
|
||||
SCLoaded ->
|
||||
-- do nothing and proceed if the load was successful
|
||||
return ()
|
||||
SCOnRetry ->
|
||||
-- retry reloading the schema cache
|
||||
work
|
||||
SCFatalFail ->
|
||||
-- die if our schema cache query has an error
|
||||
killThread $ AppState.getMainThreadId appState
|
||||
|
||||
-- | Repeatedly flush the pool, and check if a connection from the
|
||||
-- pool allows access to the PostgreSQL database.
|
||||
--
|
||||
-- Releasing the pool is key for rapid recovery. Otherwise, the pool
|
||||
-- timeout would have to be reached for new healthy connections to be acquired.
|
||||
-- Which might not happen if the server is busy with requests. No idle
|
||||
-- connection, no pool timeout.
|
||||
--
|
||||
-- The connection tries are capped, but if the connection times out no error is
|
||||
-- thrown, just 'False' is returned.
|
||||
establishConnection :: AppState -> IO ConnectionStatus
|
||||
establishConnection appState =
|
||||
retrying retrySettings shouldRetry $
|
||||
const $ AppState.flushPool appState >> getConnectionStatus
|
||||
where
|
||||
retrySettings = capDelay delayMicroseconds $ exponentialBackoff backoffMicroseconds
|
||||
delayMicroseconds = 32000000 -- 32 seconds
|
||||
backoffMicroseconds = 1000000 -- 1 second
|
||||
|
||||
getConnectionStatus :: IO ConnectionStatus
|
||||
getConnectionStatus = do
|
||||
pgVersion <- AppState.usePool appState $ queryPgVersion False -- No need to prepare the query here, as the connection might not be established
|
||||
case pgVersion of
|
||||
Left e -> do
|
||||
AppState.logPgrstError appState e
|
||||
case checkIsFatal e of
|
||||
Just reason ->
|
||||
return $ FatalConnectionError reason
|
||||
Nothing ->
|
||||
return NotConnected
|
||||
Right version ->
|
||||
if version < minimumPgVersion then
|
||||
return . FatalConnectionError $
|
||||
"Cannot run in this PostgreSQL version, PostgREST needs at least "
|
||||
<> pgvName minimumPgVersion
|
||||
else
|
||||
return . Connected $ version
|
||||
|
||||
shouldRetry :: RetryStatus -> ConnectionStatus -> IO Bool
|
||||
shouldRetry rs isConnSucc = do
|
||||
let
|
||||
delay = fromMaybe 0 (rsPreviousDelay rs) `div` backoffMicroseconds
|
||||
itShould = NotConnected == isConnSucc
|
||||
when itShould . AppState.logWithZTime appState $
|
||||
"Attempting to reconnect to the database in "
|
||||
<> (show delay::Text)
|
||||
<> " seconds..."
|
||||
when itShould $ AppState.putRetryNextIn appState delay
|
||||
return itShould
|
||||
|
||||
-- | Load the SchemaCache by using a connection from the pool.
|
||||
loadSchemaCache :: AppState -> IO SCacheStatus
|
||||
loadSchemaCache appState = do
|
||||
AppConfig{..} <- AppState.getConfig appState
|
||||
result <-
|
||||
let transaction = if configDbPreparedStatements then SQL.transaction else SQL.unpreparedTransaction in
|
||||
AppState.usePool appState . transaction SQL.ReadCommitted SQL.Read $
|
||||
querySchemaCache (toList configDbSchemas) configDbExtraSearchPath configDbPreparedStatements
|
||||
case result of
|
||||
Left e -> do
|
||||
case checkIsFatal e of
|
||||
Just hint -> do
|
||||
AppState.logWithZTime appState "A fatal error ocurred when loading the schema cache"
|
||||
AppState.logPgrstError appState e
|
||||
AppState.logWithZTime appState hint
|
||||
return SCFatalFail
|
||||
Nothing -> do
|
||||
AppState.putSchemaCache appState Nothing
|
||||
AppState.logWithZTime appState "An error ocurred when loading the schema cache"
|
||||
AppState.logPgrstError appState e
|
||||
return SCOnRetry
|
||||
|
||||
Right sCache -> do
|
||||
AppState.putSchemaCache appState (Just sCache)
|
||||
AppState.logWithZTime appState "Schema cache loaded"
|
||||
return SCLoaded
|
||||
|
||||
runListener :: AppConfig -> AppState -> IO ()
|
||||
runListener AppConfig{configDbChannelEnabled} appState =
|
||||
when configDbChannelEnabled $ listener appState
|
||||
|
||||
-- | Starts a dedicated pg connection to LISTEN for notifications. When a
|
||||
-- NOTIFY <db-channel> - with an empty payload - is done, it refills the schema
|
||||
-- cache. It uses the connectionWorker in case the LISTEN connection dies.
|
||||
listener :: AppState -> IO ()
|
||||
listener appState = do
|
||||
AppConfig{..} <- AppState.getConfig appState
|
||||
let dbChannel = toS configDbChannel
|
||||
|
||||
-- The listener has to wait for a signal from the connectionWorker.
|
||||
-- This is because when the connection to the db is lost, the listener also
|
||||
-- tries to recover the connection, but not with the same pace as the connectionWorker.
|
||||
-- Not waiting makes stderr quickly fill with connection retries messages from the listener.
|
||||
AppState.waitListener appState
|
||||
|
||||
-- forkFinally allows to detect if the thread dies
|
||||
void . flip forkFinally (handleFinally dbChannel) $ do
|
||||
dbOrError <- acquire $ toUtf8 configDbUri
|
||||
case dbOrError of
|
||||
Right db -> do
|
||||
AppState.logWithZTime appState $ "Listening for notifications on the " <> dbChannel <> " channel"
|
||||
AppState.putIsListenerOn appState True
|
||||
SQL.listen db $ SQL.toPgIdentifier dbChannel
|
||||
SQL.waitForNotifications handleNotification db
|
||||
_ ->
|
||||
die $ "Could not listen for notifications on the " <> dbChannel <> " channel"
|
||||
where
|
||||
handleFinally dbChannel _ = do
|
||||
-- if the thread dies, we try to recover
|
||||
AppState.logWithZTime appState $ "Retrying listening for notifications on the " <> dbChannel <> " channel.."
|
||||
AppState.putIsListenerOn appState False
|
||||
-- assume the pool connection was also lost, call the connection worker
|
||||
connectionWorker appState
|
||||
-- retry the listener
|
||||
listener appState
|
||||
|
||||
handleNotification _ msg
|
||||
| BS.null msg = cacheReloader
|
||||
| msg == "reload schema" = cacheReloader
|
||||
| msg == "reload config" = reReadConfig False appState
|
||||
| otherwise = pure () -- Do nothing if anything else than an empty message is sent
|
||||
|
||||
cacheReloader =
|
||||
-- reloads the schema cache + restarts pool connections
|
||||
-- it's necessary to restart the pg connections because they cache the pg catalog(see #2620)
|
||||
connectionWorker appState
|
||||
|
||||
-- | Re-reads the config plus config options from the db
|
||||
reReadConfig :: Bool -> AppState -> IO ()
|
||||
reReadConfig startingUp appState = do
|
||||
AppConfig{..} <- AppState.getConfig appState
|
||||
dbSettings <-
|
||||
if configDbConfig then do
|
||||
qDbSettings <- AppState.usePool appState $ queryDbSettings configDbPreparedStatements
|
||||
case qDbSettings of
|
||||
Left e -> do
|
||||
AppState.logWithZTime appState
|
||||
"An error ocurred when trying to query database settings for the config parameters"
|
||||
case checkIsFatal e of
|
||||
Just hint -> do
|
||||
AppState.logPgrstError appState e
|
||||
AppState.logWithZTime appState hint
|
||||
killThread (AppState.getMainThreadId appState)
|
||||
Nothing -> do
|
||||
AppState.logPgrstError appState e
|
||||
pure mempty
|
||||
Right x -> pure x
|
||||
else
|
||||
pure mempty
|
||||
roleSettings <-
|
||||
if configDbConfig then do
|
||||
rSettings <- AppState.usePool appState $ queryRoleSettings configDbPreparedStatements
|
||||
case rSettings of
|
||||
Left e -> do
|
||||
AppState.logWithZTime appState "An error ocurred when trying to query the role settings"
|
||||
AppState.logPgrstError appState e
|
||||
pure mempty
|
||||
Right x -> pure x
|
||||
else
|
||||
pure mempty
|
||||
readAppConfig dbSettings configFilePath (Just configDbUri) roleSettings >>= \case
|
||||
Left err ->
|
||||
if startingUp then
|
||||
panic err -- die on invalid config if the program is starting up
|
||||
else
|
||||
AppState.logWithZTime appState $ "Failed reloading config: " <> err
|
||||
Right newConf -> do
|
||||
AppState.putConfig appState newConf
|
||||
if startingUp then
|
||||
pass
|
||||
else
|
||||
AppState.logWithZTime appState "Config reloaded"
|
||||
|
||||
runAdmin :: AppConfig -> AppState -> Warp.Settings -> IO ()
|
||||
runAdmin conf@AppConfig{configAdminServerPort} appState settings =
|
||||
whenJust configAdminServerPort $ \adminPort -> do
|
||||
AppState.logWithZTime appState $ "Admin server listening on port " <> show adminPort
|
||||
void . forkIO $ Warp.runSettings (settings & Warp.setPort adminPort) adminApp
|
||||
where
|
||||
whenJust :: Applicative m => Maybe a -> (a -> m ()) -> m ()
|
||||
whenJust mg f = maybe (pure ()) f mg
|
||||
adminApp = admin appState conf
|
||||
|
||||
-- | PostgREST admin application
|
||||
admin :: AppState.AppState -> AppConfig -> Wai.Application
|
||||
admin appState appConfig req respond = do
|
||||
isMainAppReachable <- any isRight <$> reachMainApp appConfig
|
||||
isSchemaCacheLoaded <- isJust <$> AppState.getSchemaCache appState
|
||||
isConnectionUp <-
|
||||
if configDbChannelEnabled appConfig
|
||||
then AppState.getIsListenerOn appState
|
||||
else isRight <$> AppState.usePool appState (SQL.sql "SELECT 1")
|
||||
|
||||
case Wai.pathInfo req of
|
||||
["ready"] ->
|
||||
respond $ Wai.responseLBS (if isMainAppReachable && isConnectionUp && isSchemaCacheLoaded then HTTP.status200 else HTTP.status503) [] mempty
|
||||
["live"] ->
|
||||
respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status503) [] mempty
|
||||
_ ->
|
||||
respond $ Wai.responseLBS HTTP.status404 [] mempty
|
||||
|
||||
-- Try to connect to the main app socket
|
||||
-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
|
||||
-- The code for resolving the "*4", "!4", "*6", "!6", "*" special values is taken from
|
||||
-- https://hackage.haskell.org/package/streaming-commons-0.2.2.4/docs/src/Data.Streaming.Network.html#bindPortGenEx
|
||||
reachMainApp :: AppConfig -> IO [Either IOException ()]
|
||||
reachMainApp AppConfig{..} =
|
||||
case configServerUnixSocket of
|
||||
Just path -> do
|
||||
sock <- socket AF_UNIX Stream 0
|
||||
(:[]) <$> try (do
|
||||
connect sock $ SockAddrUnix path
|
||||
withSocketsDo $ bracket (pure sock) close sendEmpty)
|
||||
Nothing -> do
|
||||
let
|
||||
host | configServerHost `elem` ["*4", "!4", "*6", "!6", "*"] = Nothing
|
||||
| otherwise = Just configServerHost
|
||||
filterAddrs xs =
|
||||
case configServerHost of
|
||||
"*4" -> ipv4Addrs xs ++ ipv6Addrs xs
|
||||
"!4" -> ipv4Addrs xs
|
||||
"*6" -> ipv6Addrs xs ++ ipv4Addrs xs
|
||||
"!6" -> ipv6Addrs xs
|
||||
_ -> xs
|
||||
ipv4Addrs = filter ((/=) AF_INET6 . addrFamily)
|
||||
ipv6Addrs = filter ((==) AF_INET6 . addrFamily)
|
||||
|
||||
addrs <- getAddrInfo (Just $ defaultHints { addrSocketType = Stream }) (T.unpack <$> host) (Just . show $ configServerPort)
|
||||
tryAddr `traverse` filterAddrs addrs
|
||||
where
|
||||
sendEmpty sock = void $ send sock mempty
|
||||
tryAddr :: AddrInfo -> IO (Either IOException ())
|
||||
tryAddr addr = do
|
||||
sock <- socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
|
||||
try $ do
|
||||
connect sock $ addrAddress addr
|
||||
withSocketsDo $ bracket (pure sock) close sendEmpty
|
||||
+2
-2
@@ -12,5 +12,5 @@ nix:
|
||||
extra-deps:
|
||||
- git: https://github.com/PostgREST/postgresql-libpq.git
|
||||
commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
|
||||
- hasql-notifications-0.2.0.4
|
||||
- hasql-pool-0.9
|
||||
- hasql-notifications-0.2.0.5
|
||||
- hasql-pool-0.10
|
||||
|
||||
+6
-6
@@ -16,19 +16,19 @@ packages:
|
||||
commit: 890a0a16cf57dd401420fdc6c7d576fb696003bc
|
||||
git: https://github.com/PostgREST/postgresql-libpq.git
|
||||
- completed:
|
||||
hackage: hasql-notifications-0.2.0.4@sha256:9a09fa9b97feadd9492c8bd8bc6b9cffe0513510102f08374b0c45ecd479ed67,2028
|
||||
hackage: hasql-notifications-0.2.0.5@sha256:6c67d2ee42e948162e89176a2f52a2c6bb562678b33688e96e048be9b3d74122,2028
|
||||
pantry-tree:
|
||||
sha256: 56f9e240728e7a65711dde45fa2e2075b914e32cd370424aaa4572392378a60e
|
||||
sha256: 8493755b0817a36b910a37d568769ca49ab715ef5dd935a0d09f0716aa39da51
|
||||
size: 452
|
||||
original:
|
||||
hackage: hasql-notifications-0.2.0.4
|
||||
hackage: hasql-notifications-0.2.0.5
|
||||
- completed:
|
||||
hackage: hasql-pool-0.9@sha256:db7a37f6b3a922c37adc3c7ced47a7c10786d1f171e47a735a6e812a587ba44c,2111
|
||||
hackage: hasql-pool-0.10@sha256:912197a328acb85505f98bb9700d61f366b87659ca45126c5c2d636687b801c3,2112
|
||||
pantry-tree:
|
||||
sha256: 49b1181d28c6f5317e794671c2dae155754b834bdcfa30f7e5dbad28e4cf0249
|
||||
sha256: b655c540a49764a8d16b62941137e295b936b96edc0785eb9250972f0f92dc47
|
||||
size: 346
|
||||
original:
|
||||
hackage: hasql-pool-0.9
|
||||
hackage: hasql-pool-0.10
|
||||
snapshots:
|
||||
- completed:
|
||||
sha256: 4905c93319aa94aa53da8f41d614d7bacdbfe6c63a8c6132d32e6e62f24a9af4
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
db-schema = "provided_through_alias"
|
||||
db-pool-timeout = 5
|
||||
max-rows = 1000
|
||||
pre-request = "check_alias"
|
||||
role-claim-key = ".aliased"
|
||||
|
||||
@@ -7,11 +7,13 @@ db-plan-enabled = false
|
||||
db-pool = 10
|
||||
db-pool-acquisition-timeout = 10
|
||||
db-pool-max-lifetime = 1800
|
||||
db-pool-max-idletime = 5
|
||||
db-pre-request = "check_alias"
|
||||
db-prepared-statements = true
|
||||
db-root-spec = "open_alias"
|
||||
db-schemas = "provided_through_alias"
|
||||
db-config = true
|
||||
db-pre-config = ""
|
||||
db-tx-end = "commit"
|
||||
db-uri = "postgresql://"
|
||||
db-use-legacy-gucs = true
|
||||
|
||||
@@ -7,11 +7,13 @@ db-plan-enabled = false
|
||||
db-pool = 10
|
||||
db-pool-acquisition-timeout = 10
|
||||
db-pool-max-lifetime = 1800
|
||||
db-pool-max-idletime = 30
|
||||
db-pre-request = ""
|
||||
db-prepared-statements = false
|
||||
db-root-spec = ""
|
||||
db-schemas = "public"
|
||||
db-config = true
|
||||
db-pre-config = ""
|
||||
db-tx-end = "commit"
|
||||
db-uri = "postgresql://"
|
||||
db-use-legacy-gucs = true
|
||||
|
||||
@@ -7,11 +7,13 @@ db-plan-enabled = false
|
||||
db-pool = 10
|
||||
db-pool-acquisition-timeout = 10
|
||||
db-pool-max-lifetime = 1800
|
||||
db-pool-max-idletime = 30
|
||||
db-pre-request = ""
|
||||
db-prepared-statements = false
|
||||
db-root-spec = ""
|
||||
db-schemas = "public"
|
||||
db-config = true
|
||||
db-pre-config = ""
|
||||
db-tx-end = "commit"
|
||||
db-uri = "postgresql://"
|
||||
db-use-legacy-gucs = true
|
||||
|
||||
@@ -7,11 +7,13 @@ db-plan-enabled = false
|
||||
db-pool = 10
|
||||
db-pool-acquisition-timeout = 10
|
||||
db-pool-max-lifetime = 1800
|
||||
db-pool-max-idletime = 30
|
||||
db-pre-request = ""
|
||||
db-prepared-statements = true
|
||||
db-root-spec = ""
|
||||
db-schemas = "public"
|
||||
db-config = false
|
||||
db-pre-config = ""
|
||||
db-tx-end = "commit"
|
||||
db-uri = "postgresql://"
|
||||
db-use-legacy-gucs = true
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
db-anon-role = "other"
|
||||
db-anon-role = "pre_config_role"
|
||||
db-channel = "postgrest"
|
||||
db-channel-enabled = false
|
||||
db-extra-search-path = "public,extensions,other"
|
||||
@@ -7,16 +7,18 @@ db-plan-enabled = true
|
||||
db-pool = 1
|
||||
db-pool-acquisition-timeout = 30
|
||||
db-pool-max-lifetime = 3600
|
||||
db-pool-max-idletime = 60
|
||||
db-pre-request = "test.other_custom_headers"
|
||||
db-prepared-statements = false
|
||||
db-root-spec = "other_root"
|
||||
db-schemas = "test,other_tenant1,other_tenant2"
|
||||
db-config = true
|
||||
db-pre-config = "postgrest.pre_config"
|
||||
db-tx-end = "rollback-allow-override"
|
||||
db-uri = "postgresql://"
|
||||
db-use-legacy-gucs = false
|
||||
jwt-aud = "https://otherexample.org"
|
||||
jwt-role-claim-key = ".\"other\".\"role\""
|
||||
jwt-role-claim-key = ".\"other\".\"pre_config_role\""
|
||||
jwt-secret = "ODERREALLYREALLYREALLYREALLYVERYSAFE"
|
||||
jwt-secret-is-base64 = true
|
||||
log-level = "info"
|
||||
|
||||
@@ -7,11 +7,13 @@ db-plan-enabled = true
|
||||
db-pool = 1
|
||||
db-pool-acquisition-timeout = 30
|
||||
db-pool-max-lifetime = 3600
|
||||
db-pool-max-idletime = 60
|
||||
db-pre-request = "test.custom_headers"
|
||||
db-prepared-statements = false
|
||||
db-root-spec = "root"
|
||||
db-schemas = "test,tenant1,tenant2"
|
||||
db-config = true
|
||||
db-pre-config = "postgrest.preconf"
|
||||
db-tx-end = "commit-allow-override"
|
||||
db-uri = "postgresql://"
|
||||
db-use-legacy-gucs = false
|
||||
|
||||
@@ -7,11 +7,13 @@ db-plan-enabled = true
|
||||
db-pool = 1
|
||||
db-pool-acquisition-timeout = 30
|
||||
db-pool-max-lifetime = 3600
|
||||
db-pool-max-idletime = 60
|
||||
db-pre-request = "please_run_fast"
|
||||
db-prepared-statements = false
|
||||
db-root-spec = "openapi_v3"
|
||||
db-schemas = "multi,tenant,setup"
|
||||
db-config = false
|
||||
db-pre-config = "postgrest.pre_config"
|
||||
db-tx-end = "rollback-allow-override"
|
||||
db-uri = "tmp_db"
|
||||
db-use-legacy-gucs = false
|
||||
|
||||
@@ -7,11 +7,13 @@ db-plan-enabled = false
|
||||
db-pool = 10
|
||||
db-pool-acquisition-timeout = 10
|
||||
db-pool-max-lifetime = 1800
|
||||
db-pool-max-idletime = 30
|
||||
db-pre-request = ""
|
||||
db-prepared-statements = true
|
||||
db-root-spec = ""
|
||||
db-schemas = "public"
|
||||
db-config = true
|
||||
db-pre-config = ""
|
||||
db-tx-end = "commit"
|
||||
db-uri = "postgresql://"
|
||||
db-use-legacy-gucs = true
|
||||
|
||||
@@ -9,14 +9,15 @@ PGRST_DB_PLAN_ENABLED: true
|
||||
PGRST_DB_POOL: 1
|
||||
PGRST_DB_POOL_ACQUISITION_TIMEOUT: 30
|
||||
PGRST_DB_POOL_MAX_LIFETIME: 3600
|
||||
PGRST_DB_POOL_MAX_IDLETIME: 60
|
||||
PGRST_DB_PREPARED_STATEMENTS: false
|
||||
PGRST_DB_PRE_REQUEST: please_run_fast
|
||||
PGRST_DB_ROOT_SPEC: openapi_v3
|
||||
PGRST_DB_SCHEMAS: multi, tenant,setup
|
||||
PGRST_DB_CONFIG: false
|
||||
PGRST_DB_PRE_CONFIG: "postgrest.pre_config"
|
||||
PGRST_DB_TX_END: rollback-allow-override
|
||||
PGRST_DB_URI: tmp_db
|
||||
PGRST_DB_EMBED_DEFAULT_JOIN: inner
|
||||
PGRST_DB_USE_LEGACY_GUCS: false
|
||||
PGRST_JWT_AUD: 'https://postgrest.org'
|
||||
PGRST_JWT_ROLE_CLAIM_KEY: '.user[0]."real-role"'
|
||||
|
||||
@@ -7,11 +7,13 @@ db-plan-enabled = true
|
||||
db-pool = 1
|
||||
db-pool-acquisition-timeout = 30
|
||||
db-pool-max-lifetime = 3600
|
||||
db-pool-max-idletime = 60
|
||||
db-pre-request = "please_run_fast"
|
||||
db-prepared-statements = false
|
||||
db-root-spec = "openapi_v3"
|
||||
db-schemas = "multi, tenant,setup"
|
||||
db-config = false
|
||||
db-pre-config = "postgrest.pre_config"
|
||||
db-tx-end = "rollback-allow-override"
|
||||
db-uri = "tmp_db"
|
||||
db-use-legacy-gucs = false
|
||||
|
||||
+31
-4
@@ -9,6 +9,7 @@ ALTER ROLE db_config_authenticator SET pgrst.jwt_secret_is_base64 = 'false';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.jwt_role_claim_key = '."a"."role"';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_anon_role = 'anonymous';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_tx_end = 'commit-allow-override';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_pre_config = 'postgrest.preconf';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_schemas = 'test, tenant1, tenant2';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_root_spec = 'root';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_plan_enabled = 'true';
|
||||
@@ -40,7 +41,10 @@ ALTER ROLE db_config_authenticator SET pgrst.db_channel_enabled = 'ignored';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_channel = 'ignored';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_pool = 'ignored';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_pool_timeout = 'ignored';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_config = 'ignored';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_pool_acquisition_timeout = 'ignored';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_lifetime = 'ignored';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_pool_max_idletime = 'ignored';
|
||||
ALTER ROLE db_config_authenticator SET pgrst.db_config = 'true';
|
||||
|
||||
-- other authenticator reloadable config options
|
||||
CREATE ROLE other_authenticator LOGIN NOINHERIT;
|
||||
@@ -49,9 +53,6 @@ ALTER ROLE other_authenticator SET pgrst.openapi_server_proxy_uri = 'https://oth
|
||||
ALTER ROLE other_authenticator SET pgrst.raw_media_types = 'application/vnd.pgrst.other-db-config';
|
||||
ALTER ROLE other_authenticator SET pgrst.jwt_secret = 'ODERREALLYREALLYREALLYREALLYVERYSAFE';
|
||||
ALTER ROLE other_authenticator SET pgrst.jwt_secret_is_base64 = 'true';
|
||||
ALTER ROLE other_authenticator SET pgrst.jwt_role_claim_key = '."other"."role"';
|
||||
ALTER ROLE other_authenticator SET pgrst.db_anon_role = 'other';
|
||||
ALTER ROLE other_authenticator SET pgrst.db_tx_end = 'rollback-allow-override';
|
||||
ALTER ROLE other_authenticator SET pgrst.db_schemas = 'test, other_tenant1, other_tenant2';
|
||||
ALTER ROLE other_authenticator SET pgrst.db_root_spec = 'other_root';
|
||||
ALTER ROLE other_authenticator SET pgrst.db_plan_enabled = 'true';
|
||||
@@ -62,6 +63,32 @@ ALTER ROLE other_authenticator SET pgrst.db_extra_search_path = 'public, extensi
|
||||
ALTER ROLE other_authenticator SET pgrst.openapi_mode = 'disabled';
|
||||
ALTER ROLE other_authenticator SET pgrst.openapi_security_active = 'false';
|
||||
ALTER ROLE other_authenticator SET pgrst.server_trace_header = 'traceparent';
|
||||
ALTER ROLE other_authenticator SET pgrst.db_pre_config = 'postgrest.pre_config';
|
||||
|
||||
create schema postgrest;
|
||||
grant usage on schema postgrest to db_config_authenticator;
|
||||
grant usage on schema postgrest to other_authenticator;
|
||||
|
||||
-- pre-config hook
|
||||
create or replace function postgrest.pre_config()
|
||||
returns void as $$
|
||||
begin
|
||||
if current_user = 'other_authenticator' then
|
||||
perform
|
||||
set_config('pgrst.jwt_role_claim_key', '."other"."pre_config_role"', true)
|
||||
, set_config('pgrst.db_anon_role', 'pre_config_role', true)
|
||||
, set_config('pgrst.db_schemas', 'will be overriden with the above ALTER ROLE.. db_schemas', true)
|
||||
, set_config('pgrst.db_tx_end', 'rollback-allow-override', true);
|
||||
else
|
||||
null;
|
||||
end if;
|
||||
end $$ language plpgsql;
|
||||
|
||||
create or replace function postgrest.preconf()
|
||||
returns void as $$
|
||||
begin
|
||||
null;
|
||||
end $$ language plpgsql;
|
||||
|
||||
-- authenticator used for tests that manipulate statement timeout
|
||||
CREATE ROLE timeout_authenticator LOGIN NOINHERIT;
|
||||
|
||||
+18
-2
@@ -1,4 +1,4 @@
|
||||
\ir big_schema.sql
|
||||
-- \ir big_schema.sql big schema test currently skipped, see test_io.py
|
||||
\ir db_config.sql
|
||||
|
||||
set search_path to public;
|
||||
@@ -90,7 +90,7 @@ create or replace function sleep(seconds double precision) returns void as $$
|
||||
$$ language sql;
|
||||
|
||||
create or replace function hello() returns text as $$
|
||||
select 'hello';
|
||||
select 'hello'::text;
|
||||
$$ language sql;
|
||||
|
||||
create table cats(id uuid primary key, name text);
|
||||
@@ -142,3 +142,19 @@ returns text as $$
|
||||
select current_setting('transaction_isolation', true);
|
||||
$$
|
||||
language sql set default_transaction_isolation = 'REPEATABLE READ';
|
||||
|
||||
create or replace function create_function() returns void as $_$
|
||||
drop function if exists mult_them(int, int);
|
||||
create or replace function mult_them(a int, b int) returns int as $$
|
||||
select a*b;
|
||||
$$ language sql;
|
||||
notify pgrst, 'reload schema';
|
||||
$_$ language sql security definer;
|
||||
|
||||
create or replace function migrate_function() returns void as $_$
|
||||
drop function if exists mult_them(int, int);
|
||||
create or replace function mult_them(c int, d int) returns int as $$
|
||||
select c*d;
|
||||
$$ language sql;
|
||||
notify pgrst, 'reload schema';
|
||||
$_$ language sql security definer;
|
||||
|
||||
+41
-5
@@ -308,13 +308,11 @@ def test_db_schema_reload(tmp_path, defaultenv):
|
||||
|
||||
# reload config
|
||||
postgrest.process.send_signal(signal.SIGUSR2)
|
||||
time.sleep(0.1)
|
||||
|
||||
# reload schema cache to verify that the config reload actually happened
|
||||
postgrest.process.send_signal(signal.SIGUSR1)
|
||||
|
||||
# takes max 1 second to load the internal cache(big_schema.sql included now)
|
||||
# TODO this could go back to time.sleep(0.1) if the big_schema is put in another test suite
|
||||
time.sleep(1)
|
||||
time.sleep(0.1)
|
||||
|
||||
response = postgrest.session.get("/rpc/get_guc_value?name=search_path")
|
||||
assert response.text == '"\\"v1\\", \\"public\\""'
|
||||
@@ -551,7 +549,7 @@ def test_pool_size(defaultenv, metapostgrest):
|
||||
|
||||
|
||||
def test_pool_acquisition_timeout(defaultenv, metapostgrest):
|
||||
"Verify that PGRST_DB_POOL_ACQUISITON_TIMEOUT times out when the pool is empty"
|
||||
"Verify that PGRST_DB_POOL_ACQUISITION_TIMEOUT times out when the pool is empty"
|
||||
|
||||
env = {
|
||||
**defaultenv,
|
||||
@@ -946,6 +944,44 @@ def test_isolation_level(defaultenv):
|
||||
assert response.text == '"serializable"'
|
||||
|
||||
|
||||
def test_schema_cache_reloading(defaultenv):
|
||||
"schema cache should reload successfully"
|
||||
|
||||
# If DB_POOL=1, then the second request(/rpc/migrate_function) will just wait(PGRST_DB_POOL_ACQUISITION_TIMEOUT=10) for the schema cache reload to finish.
|
||||
# This is bc the only pool connection will be busy with the PGRST_INTERNAL_SCHEMA_CACHE_SLEEP(does a pg_sleep)
|
||||
# So this must be tested with a DB_POOL size of at least 2. That way the second request will pick the other pool connection and proceed.
|
||||
|
||||
env = {
|
||||
**defaultenv,
|
||||
"PGRST_INTERNAL_SCHEMA_CACHE_SLEEP": "1",
|
||||
"PGRST_DB_CHANNEL_ENABLED": "true",
|
||||
"PGRST_DB_POOL": "2",
|
||||
}
|
||||
|
||||
internal_sleep = int(env["PGRST_INTERNAL_SCHEMA_CACHE_SLEEP"])
|
||||
|
||||
with run(env=env, wait_for_readiness=False) as postgrest:
|
||||
time.sleep(2 * internal_sleep + 0.1) # wait for readiness manually
|
||||
|
||||
response = postgrest.session.post("/rpc/create_function")
|
||||
assert response.status_code == 204
|
||||
|
||||
time.sleep(
|
||||
internal_sleep / 2
|
||||
) # wait to be inside the schema cache reload process
|
||||
|
||||
response = postgrest.session.post("/rpc/migrate_function")
|
||||
assert response.status_code == 204
|
||||
|
||||
time.sleep(
|
||||
2 * internal_sleep
|
||||
) # wait enough time to ensure the schema cache state remains
|
||||
|
||||
response = postgrest.session.get("/rpc/mult_them?c=3&d=4")
|
||||
assert response.text == "12"
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
# TODO: This test fails now because of https://github.com/PostgREST/postgrest/pull/2122
|
||||
# The stack size of 1K(-with-rtsopts=-K1K) is not enough and this fails with "stack overflow"
|
||||
# A stack size of 200K seems to be enough for succeess
|
||||
|
||||
+7
-13
@@ -3,8 +3,7 @@ module Main where
|
||||
import qualified Hasql.Pool as P
|
||||
import qualified Hasql.Transaction.Sessions as HT
|
||||
|
||||
import Data.Function (id)
|
||||
import Data.List.NonEmpty (toList)
|
||||
import Data.Function (id)
|
||||
|
||||
import Test.Hspec
|
||||
|
||||
@@ -66,14 +65,12 @@ import qualified Feature.RpcPreRequestGucsSpec
|
||||
|
||||
main :: IO ()
|
||||
main = do
|
||||
pool <- P.acquire 3 10 60 $ toUtf8 $ configDbUri testCfg
|
||||
pool <- P.acquire 3 10 60 60 $ toUtf8 $ configDbUri testCfg
|
||||
|
||||
actualPgVersion <- either (panic . show) id <$> P.use pool (queryPgVersion False)
|
||||
|
||||
baseSchemaCache <-
|
||||
loadSchemaCache pool
|
||||
(configDbSchemas testCfg)
|
||||
(configDbExtraSearchPath testCfg)
|
||||
-- cached schema cache so most tests run fast
|
||||
baseSchemaCache <- loadSchemaCache pool testCfg
|
||||
|
||||
let
|
||||
-- For tests that run with the same refSchemaCache
|
||||
@@ -85,10 +82,7 @@ main = do
|
||||
|
||||
-- For tests that run with a different SchemaCache(depends on configSchemas)
|
||||
appDbs config = do
|
||||
customSchemaCache <-
|
||||
loadSchemaCache pool
|
||||
(configDbSchemas config)
|
||||
(configDbExtraSearchPath config)
|
||||
customSchemaCache <- loadSchemaCache pool config
|
||||
appState <- AppState.initWithPool pool config
|
||||
AppState.putPgVersion appState actualPgVersion
|
||||
AppState.putSchemaCache appState (Just customSchemaCache)
|
||||
@@ -265,5 +259,5 @@ main = do
|
||||
describe "Feature.RollbackForcedSpec" Feature.RollbackSpec.forced
|
||||
|
||||
where
|
||||
loadSchemaCache pool schemas extraSearchPath =
|
||||
either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache (toList schemas) extraSearchPath True)
|
||||
loadSchemaCache pool conf =
|
||||
either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache conf)
|
||||
|
||||
@@ -84,11 +84,13 @@ baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
|
||||
, configDbPoolSize = 10
|
||||
, configDbPoolAcquisitionTimeout = 10
|
||||
, configDbPoolMaxLifetime = 1800
|
||||
, configDbPoolMaxIdletime = 600
|
||||
, configDbPreRequest = Just $ QualifiedIdentifier "test" "switch_role"
|
||||
, configDbPreparedStatements = True
|
||||
, configDbRootSpec = Nothing
|
||||
, configDbSchemas = fromList ["test"]
|
||||
, configDbConfig = False
|
||||
, configDbPreConfig = Nothing
|
||||
, configDbUri = "postgresql://"
|
||||
, configDbUseLegacyGucs = True
|
||||
, configFilePath = Nothing
|
||||
@@ -111,6 +113,8 @@ baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
|
||||
, configDbTxRollbackAll = True
|
||||
, configAdminServerPort = Nothing
|
||||
, configRoleSettings = mempty
|
||||
, configRoleIsoLvl = mempty
|
||||
, configInternalSCSleep = Nothing
|
||||
}
|
||||
|
||||
testCfg :: AppConfig
|
||||
|
||||
Reference in New Issue
Block a user