Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d89826b982 | ||
|
|
94350fdde6 | ||
|
|
230eb3630c | ||
|
|
5485b8ca9a | ||
|
|
895e9c536c | ||
|
|
16c767134c | ||
|
|
0a8b836435 | ||
|
|
5796f86100 | ||
|
|
101eac1cce | ||
|
|
1ae14afdf2 | ||
|
|
69090bd224 | ||
|
|
5d5160fbd7 | ||
|
|
545f45d9de | ||
|
|
eb55e73645 | ||
|
|
e252a4900c | ||
|
|
01bdb05c89 | ||
|
|
122ed4d02e | ||
|
|
7ff6755af7 |
@@ -38,7 +38,7 @@ jobs:
|
||||
|
||||
# This is required for backport action to cherry-pick the PR
|
||||
- name: Fetch PR ref
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
|
||||
@@ -118,7 +118,7 @@ jobs:
|
||||
runs-on: ${{ matrix.runs-on }}
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
- uses: haskell-actions/setup@0512451d82f3ca8c147db62e30464e7c4ca63d30 # v2.9.1
|
||||
- uses: haskell-actions/setup@dc63c94789664bb2910876ec3dfeeaa24d23b96b # v2.10.2
|
||||
with:
|
||||
# This must match the version in stack.yaml's resolver
|
||||
ghc-version: 9.6.7
|
||||
@@ -177,7 +177,7 @@ jobs:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
- uses: haskell-actions/setup@0512451d82f3ca8c147db62e30464e7c4ca63d30 # v2.9.1
|
||||
- uses: haskell-actions/setup@dc63c94789664bb2910876ec3dfeeaa24d23b96b # v2.10.2
|
||||
with:
|
||||
ghc-version: ${{ matrix.ghc }}
|
||||
- name: Cache .cabal
|
||||
|
||||
@@ -144,7 +144,7 @@ jobs:
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
||||
with:
|
||||
name: postgrest-ubuntu-aarch64
|
||||
- uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
- uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ vars.DOCKER_USER }}
|
||||
|
||||
@@ -4,6 +4,40 @@ All notable changes to this project will be documented in this file. From versio
|
||||
|
||||
## Unreleased
|
||||
|
||||
## [14.5] - 2026-02-12
|
||||
|
||||
### Added
|
||||
|
||||
- Log error when `db-schemas` config contains schema `pg_catalog` or `information_schema` by @taimoorzaeem in #4359
|
||||
- Add a `HINT` when the LISTEN channel stops working due to a PostgreSQL bug by @laurenceisla in #4581
|
||||
- Add string slicing operator for `jwt-role-claim-key` by @taimoorzaeem in #4599
|
||||
- Log host, port and pg version of listener database connection by @mkleczek in #4617 #4618
|
||||
- Optimize requests with `Prefer: count=exact` that do not use ranges or `db-max-rows` by @laurenceisla in #3957
|
||||
+ Removed unnecessary double count when building the `Content-Range`.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Don't hide async exceptions in logs by @stevechavez in #4646
|
||||
|
||||
### Changed
|
||||
|
||||
- Log error when `db-schemas` config contains schema `pg_catalog` or `information_schema` by @taimoorzaeem in #4359
|
||||
+ Now fails at startup. Prior to this, it failed with `PGRST205` on requests related to these schemas.
|
||||
|
||||
## [14.4] - 2026-01-29
|
||||
|
||||
### Fixed
|
||||
|
||||
- Ensure Listener connections are released by @mkleczek in #4614
|
||||
- Fix incorrectly filtering the returned representation for PATCH requests when using `or/and` filters by @laurenceisla in #3707
|
||||
- Fix listener running with exception masked after first failure by @mkleczek #4615
|
||||
|
||||
## [14.3] - 2026-01-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix performance and high memory usage of relation hint calculation by @mkleczek in #4462, #4463
|
||||
|
||||
## [14.2] - 2025-12-18
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -302,6 +302,7 @@ linkcheck_ignore = [
|
||||
r"https://www.cybertec-postgresql.com/.*",
|
||||
# Odd SSL error
|
||||
r"https://www.dripdepot.com",
|
||||
r"https://www.euronodes.com",
|
||||
# New GitHub UI delays comment load, so anchor fails
|
||||
r"https://github.com/.*#issuecomment",
|
||||
# Random 500 Internal Server Error
|
||||
|
||||
@@ -16,7 +16,7 @@ Supported PostgreSQL versions
|
||||
=============================
|
||||
|
||||
=============== =================================
|
||||
**Supported** PostgreSQL >= 12
|
||||
**Supported** PostgreSQL >= 13
|
||||
=============== =================================
|
||||
|
||||
PostgREST works with all PostgreSQL versions still `officially supported <https://www.postgresql.org/support/versioning/>`_.
|
||||
|
||||
@@ -213,7 +213,6 @@ In Production
|
||||
Here are some companies that use PostgREST in production.
|
||||
|
||||
* `Catarse <https://www.catarse.me>`_
|
||||
* `Datrium <https://www.datrium.com>`_
|
||||
* `Drip Depot <https://www.dripdepot.com>`_
|
||||
* `Image-charts <https://www.image-charts.com>`_
|
||||
* `Netwo <https://www.netwo.io>`_
|
||||
|
||||
+16
-1
@@ -12,6 +12,7 @@
|
||||
, silver-searcher
|
||||
, statix
|
||||
, stylish-haskell
|
||||
, writeText
|
||||
}:
|
||||
let
|
||||
style =
|
||||
@@ -51,6 +52,20 @@ let
|
||||
${git}/bin/git diff-index --exit-code HEAD -- '*.hs' '*.lhs' '*.nix' '*.py'
|
||||
'';
|
||||
|
||||
hlintConfig = writeText "hlintConfig.yml" ''
|
||||
|
||||
# Arguments passed to hlint
|
||||
- arguments: [-j, -XQuasiQuotes, -XNoPatternSynonyms]
|
||||
|
||||
# Warnings
|
||||
- warn: { lhs: "a == a", rhs: "True", note: "This comparison always evaluates to True" }
|
||||
- warn: { lhs: "a /= a", rhs: "False", note: "This comparison always evaluates to False" }
|
||||
- warn: { lhs: "a < a", rhs: "False", note: "This comparison always evaluates to False" }
|
||||
- warn: { lhs: "a > a", rhs: "False", note: "This comparison always evaluates to False" }
|
||||
- warn: { lhs: "a <= a", rhs: "True", note: "This comparison always evaluates to True" }
|
||||
- warn: { lhs: "a >= a", rhs: "True", note: "This comparison always evaluates to True" }
|
||||
'';
|
||||
|
||||
lint =
|
||||
checkedShellScript
|
||||
{
|
||||
@@ -79,7 +94,7 @@ let
|
||||
echo "Linting Haskell files..."
|
||||
# --vimgrep fixes a bug in ag: https://github.com/ggreer/the_silver_searcher/issues/753
|
||||
${silver-searcher}/bin/ag -l --vimgrep -g '\.l?hs$' . \
|
||||
| xargs ${hlint}/bin/hlint -j -X QuasiQuotes -X NoPatternSynonyms
|
||||
| xargs ${hlint}/bin/hlint --hint=${hlintConfig}
|
||||
'';
|
||||
|
||||
in
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
name: postgrest
|
||||
version: 14.2
|
||||
version: 14.5
|
||||
synopsis: REST API for any Postgres database
|
||||
description: Reads the schema of a PostgreSQL database and creates RESTful routes
|
||||
for tables, views, and functions, supporting all HTTP methods that security
|
||||
|
||||
+26
-4
@@ -9,18 +9,24 @@ Some of its functionality includes:
|
||||
- Producing HTTP Headers according to RFCs.
|
||||
- Content Negotiation
|
||||
-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
{-# LANGUAGE ScopedTypeVariables #-}
|
||||
{-# LANGUAGE ViewPatterns #-}
|
||||
module PostgREST.App
|
||||
( postgrest
|
||||
, run
|
||||
) where
|
||||
|
||||
|
||||
import GHC.IO.Exception (IOErrorType (..))
|
||||
import System.IO.Error (ioeGetErrorType)
|
||||
|
||||
import Control.Monad.Except (liftEither)
|
||||
import Data.Either.Combinators (mapLeft, whenLeft)
|
||||
import Data.Maybe (fromJust)
|
||||
import Data.String (IsString (..))
|
||||
import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort,
|
||||
import Network.Wai.Handler.Warp (defaultSettings, setHost,
|
||||
setOnException, setPort,
|
||||
setServerName)
|
||||
|
||||
import qualified Data.Text.Encoding as T
|
||||
@@ -63,7 +69,6 @@ type Handler = ExceptT Error
|
||||
|
||||
run :: AppState -> IO ()
|
||||
run appState = do
|
||||
let observer = AppState.getObserver appState
|
||||
conf@AppConfig{..} <- AppState.getConfig appState
|
||||
|
||||
AppState.schemaCacheLoader appState -- Loads the initial SchemaCache
|
||||
@@ -79,7 +84,24 @@ run appState = do
|
||||
address <- resolveSocketToAddress (AppState.getSocketREST appState)
|
||||
observer $ AppServerAddressObs address
|
||||
|
||||
Warp.runSettingsSocket (serverSettings conf) (AppState.getSocketREST appState) app
|
||||
Warp.runSettingsSocket (serverSettings conf & setOnException onWarpException) (AppState.getSocketREST appState) app
|
||||
where
|
||||
observer = AppState.getObserver appState
|
||||
|
||||
onWarpException :: Maybe Wai.Request -> SomeException -> IO ()
|
||||
onWarpException _ ex =
|
||||
when (shouldDisplayException ex) $
|
||||
observer $ WarpErrorObs $ show ex
|
||||
|
||||
-- Similar to wai defaultShouldDisplayException in
|
||||
-- https://github.com/yesodweb/wai//blob/8c3882c60f6abe043889fc20c7efd3fa9747fa4a/warp/Network/Wai/Handler/Warp/Settings.hs#L251-L258
|
||||
-- but without omitting AsyncException since it's important to log for ThreadKilled, StackOverflow and other cases.
|
||||
-- We want to reuse this to avoid flooding the logs for some transient failure cases.
|
||||
shouldDisplayException :: SomeException -> Bool
|
||||
shouldDisplayException se
|
||||
| Just (_ :: Warp.InvalidRequest) <- fromException se = False
|
||||
| Just (ioeGetErrorType -> et) <- fromException se, et == ResourceVanished || et == InvalidArgument = False
|
||||
| otherwise = True
|
||||
|
||||
serverSettings :: AppConfig -> Warp.Settings
|
||||
serverSettings AppConfig{..} =
|
||||
|
||||
@@ -3,6 +3,7 @@ Module : PostgREST.Error
|
||||
Description : PostgREST error HTTP responses
|
||||
-}
|
||||
{-# OPTIONS_GHC -fno-warn-orphans #-}
|
||||
{-# LANGUAGE NamedFieldPuns #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
|
||||
module PostgREST.Error
|
||||
@@ -41,6 +42,7 @@ import Network.HTTP.Types.Header (Header)
|
||||
import PostgREST.MediaType (MediaType (..))
|
||||
import qualified PostgREST.MediaType as MediaType
|
||||
|
||||
import PostgREST.SchemaCache (SchemaCache (SchemaCache, dbTablesFuzzyIndex))
|
||||
import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
|
||||
Schema)
|
||||
import PostgREST.SchemaCache.Relationship (Cardinality (..),
|
||||
@@ -49,10 +51,8 @@ import PostgREST.SchemaCache.Relationship (Cardinality (..),
|
||||
RelationshipsMap)
|
||||
import PostgREST.SchemaCache.Routine (Routine (..),
|
||||
RoutineParam (..))
|
||||
import PostgREST.SchemaCache.Table (Table (..))
|
||||
import Protolude
|
||||
|
||||
|
||||
class (ErrorBody a, JSON.ToJSON a) => PgrstError a where
|
||||
status :: a -> HTTP.Status
|
||||
headers :: a -> [Header]
|
||||
@@ -250,7 +250,7 @@ data SchemaCacheError
|
||||
| NoRelBetween Text Text (Maybe Text) Text RelationshipsMap
|
||||
| NoRpc Text Text [Text] MediaType Bool [QualifiedIdentifier] [Routine]
|
||||
| ColumnNotFound Text Text
|
||||
| TableNotFound Text Text [Table]
|
||||
| TableNotFound Text Text SchemaCache
|
||||
deriving Show
|
||||
|
||||
instance PgrstError SchemaCacheError where
|
||||
@@ -313,7 +313,7 @@ instance ErrorBody SchemaCacheError where
|
||||
where
|
||||
onlySingleParams = isInvPost && contentType `elem` [MTTextPlain, MTTextXML, MTOctetStream]
|
||||
hint (AmbiguousRpc _) = Just "Try renaming the parameters or the function itself in the database so function overloading can be resolved"
|
||||
hint (TableNotFound schemaName relName tbls) = JSON.String <$> tableNotFoundHint schemaName relName tbls
|
||||
hint (TableNotFound schemaName relName schemaCache) = JSON.String <$> tableNotFoundHint schemaName relName schemaCache
|
||||
|
||||
hint _ = Nothing
|
||||
|
||||
@@ -428,13 +428,11 @@ noRpcHint schema procName params allProcs overloadedProcs =
|
||||
|
||||
-- |
|
||||
-- Do a fuzzy search in all tables in the same schema and return closest result
|
||||
tableNotFoundHint :: Text -> Text -> [Table] -> Maybe Text
|
||||
tableNotFoundHint schema tblName tblList
|
||||
tableNotFoundHint :: Text -> Text -> SchemaCache -> Maybe Text
|
||||
tableNotFoundHint schema tblName SchemaCache{dbTablesFuzzyIndex}
|
||||
= fmap (\tbl -> "Perhaps you meant the table '" <> schema <> "." <> tbl <> "'") perhapsTable
|
||||
where
|
||||
perhapsTable = Fuzzy.getOne fuzzyTableSet tblName
|
||||
fuzzyTableSet = Fuzzy.fromList [ tableName tbl | tbl <- tblList, tableSchema tbl == schema]
|
||||
|
||||
perhapsTable = (`Fuzzy.getOne` tblName) =<< HM.lookup schema dbTablesFuzzyIndex
|
||||
|
||||
compressedRel :: Relationship -> JSON.Value
|
||||
-- An ambiguousness error cannot happen for computed relationships TODO refactor so this mempty is not needed
|
||||
|
||||
+36
-21
@@ -1,3 +1,4 @@
|
||||
{-# LANGUAGE LambdaCase #-}
|
||||
{-# LANGUAGE MultiWayIf #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
|
||||
@@ -15,6 +16,7 @@ import PostgREST.Version (prettyVersion)
|
||||
import qualified PostgREST.AppState as AppState
|
||||
import qualified PostgREST.Config as Config
|
||||
|
||||
import Data.Either.Combinators (whenRight)
|
||||
import Protolude
|
||||
|
||||
-- | Starts the Listener in a thread
|
||||
@@ -22,15 +24,16 @@ runListener :: AppState -> IO ()
|
||||
runListener appState = do
|
||||
AppConfig{..} <- getConfig appState
|
||||
when configDbChannelEnabled $
|
||||
void . forkIO $ retryingListen appState
|
||||
void . forkIO . void $ retryingListen appState
|
||||
|
||||
-- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
|
||||
retryingListen :: AppState -> IO ()
|
||||
-- | This function never returns (but can throw) and return type enforces that.
|
||||
retryingListen :: AppState -> IO Void
|
||||
retryingListen appState = do
|
||||
AppConfig{..} <- AppState.getConfig appState
|
||||
let
|
||||
dbChannel = toS configDbChannel
|
||||
handleFinally err = do
|
||||
onError err = do
|
||||
AppState.putIsListenerOn appState False
|
||||
observer $ DBListenFail dbChannel (Right err)
|
||||
unless configDbPoolAutomaticRecovery $
|
||||
@@ -42,29 +45,39 @@ retryingListen appState = do
|
||||
threadDelay (delay * oneSecondInMicro)
|
||||
unless (delay == maxDelay) $
|
||||
AppState.putNextListenerDelay appState (delay * 2)
|
||||
-- loop running the listener
|
||||
retryingListen appState
|
||||
|
||||
-- forkFinally allows to detect if the thread dies
|
||||
void . flip forkFinally handleFinally $ do
|
||||
dbOrError <- SQL.acquire $ toUtf8 (Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri)
|
||||
case dbOrError of
|
||||
Right db -> do
|
||||
SQL.listen db $ SQL.toPgIdentifier dbChannel
|
||||
AppState.putIsListenerOn appState True
|
||||
-- Execute the listener with with error handling
|
||||
handle onError $ do
|
||||
-- Make sure we don't leak connections on errors
|
||||
bracket
|
||||
-- acquire connection
|
||||
(SQL.acquire $ toUtf8 (Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri))
|
||||
-- release connection
|
||||
(`whenRight` releaseConnection) $
|
||||
-- use connection
|
||||
\case
|
||||
Right db -> do
|
||||
SQL.listen db $ SQL.toPgIdentifier dbChannel
|
||||
AppState.putIsListenerOn appState True
|
||||
|
||||
delay <- AppState.getNextListenerDelay appState
|
||||
when (delay > 1) $ do -- if we did a retry
|
||||
-- assume we lost notifications, refresh the schema cache
|
||||
AppState.schemaCacheLoader appState
|
||||
-- reset the delay
|
||||
AppState.putNextListenerDelay appState 1
|
||||
delay <- AppState.getNextListenerDelay appState
|
||||
when (delay > 1) $ do -- if we did a retry
|
||||
-- assume we lost notifications, refresh the schema cache
|
||||
AppState.schemaCacheLoader appState
|
||||
-- reset the delay
|
||||
AppState.putNextListenerDelay appState 1
|
||||
|
||||
observer $ DBListenStart dbChannel
|
||||
SQL.waitForNotifications handleNotification db
|
||||
observer $ DBListenStart dbChannel
|
||||
|
||||
Left err -> do
|
||||
observer $ DBListenFail dbChannel (Left err)
|
||||
exitFailure
|
||||
-- wait for notifications
|
||||
-- this will never return, in case of an error it will throw and be caught by onError
|
||||
forever $ SQL.waitForNotifications handleNotification db
|
||||
|
||||
Left err -> do
|
||||
observer $ DBListenFail dbChannel (Left err)
|
||||
exitFailure
|
||||
where
|
||||
observer = AppState.getObserver appState
|
||||
mainThreadId = AppState.getMainThreadId appState
|
||||
@@ -79,3 +92,5 @@ retryingListen appState = do
|
||||
|
||||
cacheReloader =
|
||||
AppState.schemaCacheLoader appState
|
||||
|
||||
releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release
|
||||
|
||||
@@ -44,10 +44,11 @@ data Observation
|
||||
| SchemaCacheLoadedObs Double
|
||||
| ConnectionRetryObs Int
|
||||
| DBListenStart Text
|
||||
| DBListenFail Text (Either SQL.ConnectionError (Either SomeException ()))
|
||||
| DBListenFail Text (Either SQL.ConnectionError SomeException)
|
||||
| DBListenRetry Int
|
||||
| DBListenerGotSCacheMsg ByteString
|
||||
| DBListenerGotConfigMsg ByteString
|
||||
| DBListenerConnectionCleanupFail SomeException
|
||||
| QueryObs MainQuery Status
|
||||
| ConfigReadErrorObs SQL.UsageError
|
||||
| ConfigInvalidObs Text
|
||||
@@ -62,6 +63,7 @@ data Observation
|
||||
| PoolRequestFullfilled
|
||||
| JwtCacheLookup Bool
|
||||
| JwtCacheEviction
|
||||
| WarpErrorObs Text
|
||||
|
||||
data ObsFatalError = ServerAuthError | ServerPgrstBug | ServerError42P05 | ServerError08P01
|
||||
|
||||
@@ -118,6 +120,8 @@ observationMessage = \case
|
||||
"Received a schema cache reload message on the " <> show channel <> " channel"
|
||||
DBListenerGotConfigMsg channel ->
|
||||
"Received a config reload message on the " <> show channel <> " channel"
|
||||
DBListenerConnectionCleanupFail ex ->
|
||||
"Failed during listener connection cleanup: " <> showOnSingleLine '\t' (show ex)
|
||||
QueryObs{} ->
|
||||
mempty -- TODO pending refactor: The logic for printing the query cannot be done here. Join the observationMessage function into observationLogger to avoid this mempty.
|
||||
ConfigReadErrorObs usageErr ->
|
||||
@@ -154,6 +158,8 @@ observationMessage = \case
|
||||
"Looked up a JWT in JWT cache"
|
||||
JwtCacheEviction ->
|
||||
"Evicted entry from JWT cache"
|
||||
WarpErrorObs txt ->
|
||||
"Warp server error: " <> txt
|
||||
where
|
||||
showMillis :: Double -> Text
|
||||
showMillis x = toS $ showFFloat (Just 1) x ""
|
||||
@@ -164,9 +170,8 @@ observationMessage = \case
|
||||
showListenerConnError :: SQL.ConnectionError -> Text
|
||||
showListenerConnError = maybe "Connection error" (showOnSingleLine '\t' . T.decodeUtf8)
|
||||
|
||||
showListenerException :: Either SomeException () -> Text
|
||||
showListenerException (Right _) = "Failed getting notifications" -- should not happen as the listener will never finish (hasql-notifications uses `forever` internally) with a Right result
|
||||
showListenerException (Left e) = showOnSingleLine '\t' $ show e
|
||||
showListenerException :: SomeException -> Text
|
||||
showListenerException = showOnSingleLine '\t' . show
|
||||
|
||||
|
||||
showOnSingleLine :: Char -> Text -> Text
|
||||
|
||||
+14
-7
@@ -172,7 +172,7 @@ dbActionPlan dbAct conf apiReq sCache = case dbAct of
|
||||
|
||||
wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Bool -> Either Error CrudPlan
|
||||
wrappedReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} headersOnly = do
|
||||
qi <- findTable identifier (dbTables sCache)
|
||||
qi <- findTable identifier sCache
|
||||
rPlan <- readPlan qi conf sCache apiRequest
|
||||
(handler, mediaType) <- mapLeft ApiRequestError $ negotiateContent conf apiRequest qi iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan)
|
||||
if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
|
||||
@@ -180,7 +180,7 @@ wrappedReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Prefe
|
||||
|
||||
mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error CrudPlan
|
||||
mutateReadPlan mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf sCache = do
|
||||
qi <- findTable identifier (dbTables sCache)
|
||||
qi <- findTable identifier sCache
|
||||
rPlan <- readPlan qi conf sCache apiRequest
|
||||
mPlan <- mutatePlan mutation qi apiRequest sCache rPlan
|
||||
if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
|
||||
@@ -812,10 +812,10 @@ validateAggFunctions aggFunctionsAllowed (Node rp@ReadPlan {select} forest)
|
||||
| otherwise = Node rp <$> traverse (validateAggFunctions aggFunctionsAllowed) forest
|
||||
|
||||
-- | Lookup table in the schema cache before creating read plan
|
||||
findTable :: QualifiedIdentifier -> TablesMap -> Either Error QualifiedIdentifier
|
||||
findTable qi@QualifiedIdentifier{..} tableMap =
|
||||
case HM.lookup qi tableMap of
|
||||
Nothing -> Left $ SchemaCacheErr $ TableNotFound qiSchema qiName (HM.elems tableMap)
|
||||
findTable :: QualifiedIdentifier -> SchemaCache -> Either Error QualifiedIdentifier
|
||||
findTable qi@QualifiedIdentifier{..} sc@SchemaCache{dbTables} =
|
||||
case HM.lookup qi dbTables of
|
||||
Nothing -> Left $ SchemaCacheErr $ TableNotFound qiSchema qiName sc
|
||||
Just _ -> Right qi
|
||||
|
||||
addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
|
||||
@@ -965,10 +965,17 @@ addRanges ApiRequest{..} rReq =
|
||||
|
||||
addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
|
||||
addLogicTrees ctx ApiRequest{..} rReq =
|
||||
foldr addLogicTreeToNode (Right rReq) qsLogic
|
||||
foldr addLogicTreeToNode (Right rReq) logic
|
||||
where
|
||||
QueryParams.QueryParams{..} = iQueryParams
|
||||
|
||||
logic =
|
||||
case iAction of
|
||||
ActDb (ActRelationRead _ _) -> qsLogic
|
||||
ActDb (ActRoutine _ _) -> qsLogic
|
||||
-- For mutations, take the non-root logic filters. These will only affect the embeddings and not the top level of the returned representation.
|
||||
_ -> filter (not . null . fst) qsLogic
|
||||
|
||||
addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either Error ReadPlanTree -> Either Error ReadPlanTree
|
||||
addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)
|
||||
|
||||
|
||||
@@ -213,10 +213,10 @@ actionResponse (MaybeDbResult InspectPlan{ipHdrsOnly=headersOnly} body) _ versio
|
||||
in
|
||||
Right $ PgrstResponse HTTP.status200 (MediaType.toContentType MTOpenAPI : cLHeader ++ maybeToList (profileHeader schema negotiatedByProfile)) rsBody
|
||||
|
||||
actionResponse (NoDbResult (RelInfoPlan qi@QualifiedIdentifier{..})) _ _ _ SchemaCache{dbTables} _ _ =
|
||||
actionResponse (NoDbResult (RelInfoPlan qi@QualifiedIdentifier{..})) _ _ _ sc@SchemaCache{dbTables} _ _ =
|
||||
case HM.lookup qi dbTables of
|
||||
Just tbl -> respondInfo $ allowH tbl
|
||||
Nothing -> Left $ Error.SchemaCacheErr $ Error.TableNotFound qiSchema qiName (HM.elems dbTables)
|
||||
Nothing -> Left $ Error.SchemaCacheErr $ Error.TableNotFound qiSchema qiName sc
|
||||
where
|
||||
allowH table =
|
||||
let hasPK = not . null $ tablePKCols table in
|
||||
|
||||
@@ -20,6 +20,7 @@ These queries are executed once at startup or when PostgREST is reloaded.
|
||||
|
||||
module PostgREST.SchemaCache
|
||||
( SchemaCache(..)
|
||||
, TablesFuzzyIndex
|
||||
, querySchemaCache
|
||||
, showSummary
|
||||
, decodeFuncs
|
||||
@@ -66,21 +67,28 @@ import PostgREST.SchemaCache.Table (Column (..), ColumnMap,
|
||||
|
||||
import qualified PostgREST.MediaType as MediaType
|
||||
|
||||
import Control.Arrow ((&&&))
|
||||
import Protolude
|
||||
import System.IO.Unsafe (unsafePerformIO)
|
||||
import Control.Arrow ((&&&))
|
||||
import qualified Data.FuzzySet as Fuzzy
|
||||
import Protolude
|
||||
import System.IO.Unsafe (unsafePerformIO)
|
||||
|
||||
type TablesFuzzyIndex = HM.HashMap Schema Fuzzy.FuzzySet
|
||||
|
||||
data SchemaCache = SchemaCache
|
||||
{ dbTables :: TablesMap
|
||||
, dbRelationships :: RelationshipsMap
|
||||
, dbRoutines :: RoutineMap
|
||||
, dbRepresentations :: RepresentationsMap
|
||||
, dbMediaHandlers :: MediaHandlerMap
|
||||
, dbTimezones :: TimezoneNames
|
||||
}
|
||||
{ dbTables :: TablesMap
|
||||
, dbRelationships :: RelationshipsMap
|
||||
, dbRoutines :: RoutineMap
|
||||
, dbRepresentations :: RepresentationsMap
|
||||
, dbMediaHandlers :: MediaHandlerMap
|
||||
, dbTimezones :: TimezoneNames
|
||||
-- Memoized fuzzy index of table names per schema to support approximate matching
|
||||
-- Since index construction can be expensive, we build it once and store in the SchemaCache
|
||||
-- Haskell lazy evaluation ensures it's only built on first use and memoized afterwards
|
||||
, dbTablesFuzzyIndex :: TablesFuzzyIndex
|
||||
} deriving (Show)
|
||||
|
||||
instance JSON.ToJSON SchemaCache where
|
||||
toJSON (SchemaCache tabs rels routs reps hdlers tzs) = JSON.object [
|
||||
toJSON (SchemaCache tabs rels routs reps hdlers tzs _) = JSON.object [
|
||||
"dbTables" .= JSON.toJSON tabs
|
||||
, "dbRelationships" .= JSON.toJSON rels
|
||||
, "dbRoutines" .= JSON.toJSON routs
|
||||
@@ -90,7 +98,7 @@ instance JSON.ToJSON SchemaCache where
|
||||
]
|
||||
|
||||
showSummary :: SchemaCache -> Text
|
||||
showSummary (SchemaCache tbls rels routs reps mediaHdlrs tzs) =
|
||||
showSummary (SchemaCache tbls rels routs reps mediaHdlrs tzs _) =
|
||||
T.intercalate ", "
|
||||
[ show (HM.size tbls) <> " Relations"
|
||||
, show (HM.size rels) <> " Relationships"
|
||||
@@ -138,6 +146,8 @@ data KeyDep
|
||||
-- | A SQL query that can be executed independently
|
||||
type SqlQuery = ByteString
|
||||
|
||||
maxDbTablesForFuzzySearch :: Int
|
||||
maxDbTablesForFuzzySearch = 500
|
||||
|
||||
querySchemaCache :: AppConfig -> SQL.Transaction SchemaCache
|
||||
querySchemaCache conf@AppConfig{..} = do
|
||||
@@ -166,6 +176,11 @@ querySchemaCache conf@AppConfig{..} = do
|
||||
, dbRepresentations = reps
|
||||
, dbMediaHandlers = HM.union mHdlers initialMediaHandlers -- the custom handlers will override the initial ones
|
||||
, dbTimezones = tzones
|
||||
|
||||
, dbTablesFuzzyIndex =
|
||||
-- Only build fuzzy index for schemas with a reasonable number of tables
|
||||
-- Fuzzy.FuzzySet is memory heavy we just don't use it for large schemas
|
||||
Fuzzy.fromList <$> HM.filter ((< maxDbTablesForFuzzySearch) . length) (HM.fromListWith (<>) ((qiSchema &&& pure . qiName) <$> HM.keys tabsWViewsPks))
|
||||
}
|
||||
where
|
||||
schemas = toList configDbSchemas
|
||||
@@ -203,6 +218,7 @@ removeInternal schemas dbStruct =
|
||||
, dbRepresentations = dbRepresentations dbStruct -- no need to filter, not directly exposed through the API
|
||||
, dbMediaHandlers = dbMediaHandlers dbStruct
|
||||
, dbTimezones = dbTimezones dbStruct
|
||||
, dbTablesFuzzyIndex = dbTablesFuzzyIndex dbStruct
|
||||
}
|
||||
where
|
||||
hasInternalJunction ComputedRelationship{} = False
|
||||
|
||||
@@ -90,7 +90,7 @@ data RoutineParam = RoutineParam
|
||||
instance Ord Routine where
|
||||
Function schema1 name1 des1 prms1 rt1 vol1 hasVar1 iso1 sets1 `compare` Function schema2 name2 des2 prms2 rt2 vol2 hasVar2 iso2 sets2
|
||||
| schema1 == schema2 && name1 == name2 && length prms1 < length prms2 = LT
|
||||
| schema2 == schema2 && name1 == name2 && length prms1 > length prms2 = GT
|
||||
| schema1 == schema2 && name1 == name2 && length prms1 > length prms2 = GT
|
||||
| otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1, iso1, sets1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2, iso2, sets2)
|
||||
|
||||
-- | A map of all procs, all of which can be overloaded(one entry will have more than one Routine).
|
||||
|
||||
@@ -11375,12 +11375,34 @@ ALTER TABLE ONLY apflora.zielber
|
||||
|
||||
ALTER TABLE apflora."user" ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
|
||||
CREATE SCHEMA fuzzysearch;
|
||||
|
||||
-- Create many tables to test fuzzy string search
|
||||
-- computing hints for non existing tables
|
||||
DO
|
||||
$$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT
|
||||
format('CREATE TABLE fuzzysearch.unknown_table_%s ()', n) AS ct
|
||||
FROM
|
||||
generate_series(1, 499) n
|
||||
LOOP
|
||||
EXECUTE r.ct;
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
DROP ROLE IF EXISTS postgrest_test_anonymous;
|
||||
CREATE ROLE postgrest_test_anonymous;
|
||||
|
||||
GRANT postgrest_test_anonymous TO :PGUSER;
|
||||
|
||||
GRANT USAGE ON SCHEMA apflora TO postgrest_test_anonymous;
|
||||
GRANT USAGE ON SCHEMA fuzzysearch TO postgrest_test_anonymous;
|
||||
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA apflora
|
||||
TO postgrest_test_anonymous;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import re
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from postgrest import run
|
||||
|
||||
@@ -54,6 +55,25 @@ def test_openapi_in_big_schema(defaultenv):
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
def test_stackoverflow_is_logged(defaultenv):
|
||||
"Stack overflow errors should be logged with the Warp error message"
|
||||
|
||||
env = {
|
||||
**defaultenv,
|
||||
"PGRST_DB_SCHEMAS": "apflora",
|
||||
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous",
|
||||
}
|
||||
|
||||
with run(env=env, wait_max_seconds=30, no_startup_stdout=False) as postgrest:
|
||||
with pytest.raises(requests.exceptions.ConnectionError):
|
||||
postgrest.session.get("/")
|
||||
|
||||
output = postgrest.read_stdout(nlines=10)
|
||||
output.extend(postgrest.read_stdout(nlines=10))
|
||||
|
||||
assert any("Warp server error: stack overflow" in line for line in output)
|
||||
|
||||
|
||||
# See: https://github.com/PostgREST/postgrest/issues/3329
|
||||
def test_should_not_fail_with_stack_overflow(defaultenv):
|
||||
"requesting a non-existent relationship should not fail with stack overflow due to fuzzy search of candidates"
|
||||
@@ -70,3 +90,23 @@ def test_should_not_fail_with_stack_overflow(defaultenv):
|
||||
assert response.status_code == 404
|
||||
data = response.json()
|
||||
assert data["code"] == "PGRST205"
|
||||
|
||||
|
||||
def test_second_request_for_non_existent_table_should_be_quick(defaultenv):
|
||||
"requesting a non-existent relationship should be quick after the fuzzy search index is loaded (2nd request)"
|
||||
|
||||
env = {
|
||||
**defaultenv,
|
||||
"PGRST_DB_SCHEMAS": "fuzzysearch",
|
||||
"PGRST_DB_POOL": "2",
|
||||
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous",
|
||||
}
|
||||
|
||||
with run(env=env, wait_max_seconds=30) as postgrest:
|
||||
response = postgrest.session.get("/unknown-table")
|
||||
assert response.status_code == 404
|
||||
data = response.json()
|
||||
assert data["code"] == "PGRST205"
|
||||
first_duration = response.elapsed.total_seconds()
|
||||
response = postgrest.session.get("/unknown-table")
|
||||
assert response.elapsed.total_seconds() < first_duration / 10
|
||||
|
||||
@@ -252,21 +252,39 @@ spec =
|
||||
[json|[{"id": 7, "entities":null}, {"id": 8, "entities": {"id": 2}}, {"id": 9, "entities": {"id": 3}}]|]
|
||||
{ matchStatus = 201 }
|
||||
|
||||
context "used with PATCH" $
|
||||
context "used with PATCH" $ do
|
||||
it "succeeds when using and/or params" $
|
||||
request methodPatch "/grandchild_entities?or=(id.eq.1,id.eq.2)&select=id,name"
|
||||
[("Prefer", "return=representation")]
|
||||
[json|{ name : "updated grandchild entity"}|] `shouldRespondWith`
|
||||
[json|[{ "id": 1, "name" : "updated grandchild entity"},{ "id": 2, "name" : "updated grandchild entity"}]|]
|
||||
{ matchHeaders = [matchContentTypeJson] }
|
||||
it "succeeds when the filtered column is modified" $
|
||||
request methodPatch "/entities?select=id,name&or=(name.is.null,name.like.*test*)"
|
||||
[("Prefer", "return=representation")]
|
||||
[json|{ "name" : "updated entity" }|] `shouldRespondWith`
|
||||
[json|[{ "id": 4, "name": "updated entity" }]|]
|
||||
{ matchHeaders = [matchContentTypeJson] }
|
||||
it "succeeds when the filtered column is not selected in the returned representation" $
|
||||
request methodPatch "/entities?select=id&or=(name.is.null,name.like.*test*)"
|
||||
[("Prefer", "return=representation")]
|
||||
[json|{ "name" : "updated entity" }|] `shouldRespondWith`
|
||||
[json|[{ "id": 4 }]|]
|
||||
{ matchHeaders = [matchContentTypeJson] }
|
||||
|
||||
context "used with DELETE" $
|
||||
context "used with DELETE" $ do
|
||||
it "succeeds when using and/or params" $
|
||||
request methodDelete "/grandchild_entities?or=(id.eq.1,id.eq.2)&select=id,name"
|
||||
[("Prefer", "return=representation")]
|
||||
""
|
||||
`shouldRespondWith`
|
||||
[json|[{ "id": 1, "name" : "grandchild entity 1" },{ "id": 2, "name" : "grandchild entity 2" }]|]
|
||||
it "succeeds when the filtered column is not selected in the returned representation" $
|
||||
request methodDelete "/entities?select=id&or=(name.is.null,name.like.*test*)"
|
||||
[("Prefer", "return=representation")]
|
||||
""
|
||||
`shouldRespondWith`
|
||||
[json|[{ "id": 4 }]|]
|
||||
|
||||
it "can query columns that begin with and/or reserved words" $
|
||||
get "/grandchild_entities?or=(and_starting_col.eq.smth, or_starting_col.eq.smth)" `shouldRespondWith` 200
|
||||
|
||||
Reference in New Issue
Block a user