Compare commits

...
15 Commits
Author SHA1 Message Date
steve-chavez 5485b8ca9a bump version to 14.4 2026-01-29 14:13:49 -05:00
steve-chavez 895e9c536c chore: remove wrong entry in CHANGELOG 2026-01-29 13:24:20 -05:00
Michal Kleczekandsteve-chavez 16c767134c fix: listener running with exception masked after first failure 2026-01-29 13:22:46 -05:00
Laurence IslaandSteve Chavez 0a8b836435 fix: filtering the returned representation whenn using or/and filters on mutations
(cherry picked from commit 1682677297)
2026-01-29 09:16:08 -05:00
Michal KleczekandSteve Chavez 5796f86100 fix: ensure Listener connections are released
retryingListen function potentially leaks database connections. This patch ensures the connections are released in case of listen/notify errors.

(cherry picked from commit 00c7cb1a22)
2026-01-28 18:26:15 -05:00
Wolfgang Walther 101eac1cce docs: fix links
datrium.com doesn't exist anymore, while euronodes.com seems to only
fail SSL in CI.
2026-01-28 09:57:23 +01:00
renovate[bot]andWolfgang Walther 1ae14afdf2 chore(deps): update haskell-actions/setup action to v2.10.2 2026-01-11 17:36:28 +00:00
Wolfgang Walther 69090bd224 ci: pin backport action to version instead of default branch 2026-01-11 18:34:43 +01:00
renovate[bot]andWolfgang Walther 5d5160fbd7 chore(deps): update haskell-actions/setup action to v2.10.1 2026-01-05 19:01:32 +00:00
steve-chavez 545f45d9de bump version to 14.3 2026-01-03 16:45:56 +08:00
Taimoor Zaeemandsteve-chavez eb55e73645 chore: move changelog entry to unreleased section
Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>
2026-01-03 16:44:10 +08:00
Michał KłeczekandSteve Chavez e252a4900c fix: Performance and high memory usage of relation hint calculation
* Calculation of hint message when requested relation is not present in schema cache requires creation of a FuzzySet (to use fuzzy search to find candidate tables). For schemas with many tables it is costly.
This patch introduces dbTablesFuzzyIndex in SchemaCache to memoize the FuzzySet creation.

* Additionally, because of FuzzySet large memory requirements, this patch introduces a limit of 500 relations per schema, above which FuzzySet is not created and hint calculation disabled.

(cherry picked from commit e592d568c6)
2026-01-03 15:18:37 +08:00
Taimoor ZaeemandSteve Chavez 01bdb05c89 nix: add config file for hlint
Adds a config file for hlint containing arguments and
custom warnings.

Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>
2025-12-23 11:51:05 -05:00
122ed4d02e refactor: fix definition of Ord instance for Routine type (#4577)
The `Ord` instance definition for type `Routine` had a logical
error when comparing two routines. The error did not affect any
end users. However, for correctness and completeness reasons, this
commit fixes the error.

Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>
Co-authored-by: Joel Jacobson <joel@compiler.org>
2025-12-23 11:51:05 -05:00
renovate[bot]andWolfgang Walther 7ff6755af7 chore(deps): update docker/setup-buildx-action action to v3.12.0 2025-12-20 20:24:23 +00:00
18 changed files with 192 additions and 65 deletions
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
# This is required for backport action to cherry-pick the PR # This is required for backport action to cherry-pick the PR
- name: Fetch PR ref - name: Fetch PR ref
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with: with:
ref: ${{ github.event.pull_request.head.sha }} ref: ${{ github.event.pull_request.head.sha }}
token: ${{ steps.app-token.outputs.token }} token: ${{ steps.app-token.outputs.token }}
+2 -2
View File
@@ -118,7 +118,7 @@ jobs:
runs-on: ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }}
steps: steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: haskell-actions/setup@0512451d82f3ca8c147db62e30464e7c4ca63d30 # v2.9.1 - uses: haskell-actions/setup@dc63c94789664bb2910876ec3dfeeaa24d23b96b # v2.10.2
with: with:
# This must match the version in stack.yaml's resolver # This must match the version in stack.yaml's resolver
ghc-version: 9.6.7 ghc-version: 9.6.7
@@ -177,7 +177,7 @@ jobs:
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
steps: steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: haskell-actions/setup@0512451d82f3ca8c147db62e30464e7c4ca63d30 # v2.9.1 - uses: haskell-actions/setup@dc63c94789664bb2910876ec3dfeeaa24d23b96b # v2.10.2
with: with:
ghc-version: ${{ matrix.ghc }} ghc-version: ${{ matrix.ghc }}
- name: Cache .cabal - name: Cache .cabal
+1 -1
View File
@@ -144,7 +144,7 @@ jobs:
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with: with:
name: postgrest-ubuntu-aarch64 name: postgrest-ubuntu-aarch64
- uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
- uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with: with:
username: ${{ vars.DOCKER_USER }} username: ${{ vars.DOCKER_USER }}
+14
View File
@@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file. From versio
## Unreleased ## Unreleased
## [14.4] - 2026-01-29
### Fixed
- Ensure Listener connections are released by @mkleczek in #4614
- Fix incorrectly filtering the returned representation for PATCH requests when using `or/and` filters by @laurenceisla in #3707
- Fix listener running with exception masked after first failure by @mkleczek #4615
## [14.3] - 2026-01-03
### Fixed
- Fix performance and high memory usage of relation hint calculation by @mkleczek in #4462, #4463
## [14.2] - 2025-12-18 ## [14.2] - 2025-12-18
### Fixed ### Fixed
+1
View File
@@ -302,6 +302,7 @@ linkcheck_ignore = [
r"https://www.cybertec-postgresql.com/.*", r"https://www.cybertec-postgresql.com/.*",
# Odd SSL error # Odd SSL error
r"https://www.dripdepot.com", r"https://www.dripdepot.com",
r"https://www.euronodes.com",
# New GitHub UI delays comment load, so anchor fails # New GitHub UI delays comment load, so anchor fails
r"https://github.com/.*#issuecomment", r"https://github.com/.*#issuecomment",
# Random 500 Internal Server Error # Random 500 Internal Server Error
-1
View File
@@ -213,7 +213,6 @@ In Production
Here are some companies that use PostgREST in production. Here are some companies that use PostgREST in production.
* `Catarse <https://www.catarse.me>`_ * `Catarse <https://www.catarse.me>`_
* `Datrium <https://www.datrium.com>`_
* `Drip Depot <https://www.dripdepot.com>`_ * `Drip Depot <https://www.dripdepot.com>`_
* `Image-charts <https://www.image-charts.com>`_ * `Image-charts <https://www.image-charts.com>`_
* `Netwo <https://www.netwo.io>`_ * `Netwo <https://www.netwo.io>`_
+16 -1
View File
@@ -12,6 +12,7 @@
, silver-searcher , silver-searcher
, statix , statix
, stylish-haskell , stylish-haskell
, writeText
}: }:
let let
style = style =
@@ -51,6 +52,20 @@ let
${git}/bin/git diff-index --exit-code HEAD -- '*.hs' '*.lhs' '*.nix' '*.py' ${git}/bin/git diff-index --exit-code HEAD -- '*.hs' '*.lhs' '*.nix' '*.py'
''; '';
hlintConfig = writeText "hlintConfig.yml" ''
# Arguments passed to hlint
- arguments: [-j, -XQuasiQuotes, -XNoPatternSynonyms]
# Warnings
- warn: { lhs: "a == a", rhs: "True", note: "This comparison always evaluates to True" }
- warn: { lhs: "a /= a", rhs: "False", note: "This comparison always evaluates to False" }
- warn: { lhs: "a < a", rhs: "False", note: "This comparison always evaluates to False" }
- warn: { lhs: "a > a", rhs: "False", note: "This comparison always evaluates to False" }
- warn: { lhs: "a <= a", rhs: "True", note: "This comparison always evaluates to True" }
- warn: { lhs: "a >= a", rhs: "True", note: "This comparison always evaluates to True" }
'';
lint = lint =
checkedShellScript checkedShellScript
{ {
@@ -79,7 +94,7 @@ let
echo "Linting Haskell files..." echo "Linting Haskell files..."
# --vimgrep fixes a bug in ag: https://github.com/ggreer/the_silver_searcher/issues/753 # --vimgrep fixes a bug in ag: https://github.com/ggreer/the_silver_searcher/issues/753
${silver-searcher}/bin/ag -l --vimgrep -g '\.l?hs$' . \ ${silver-searcher}/bin/ag -l --vimgrep -g '\.l?hs$' . \
| xargs ${hlint}/bin/hlint -j -X QuasiQuotes -X NoPatternSynonyms | xargs ${hlint}/bin/hlint --hint=${hlintConfig}
''; '';
in in
+1 -1
View File
@@ -1,5 +1,5 @@
name: postgrest name: postgrest
version: 14.2 version: 14.4
synopsis: REST API for any Postgres database synopsis: REST API for any Postgres database
description: Reads the schema of a PostgreSQL database and creates RESTful routes description: Reads the schema of a PostgreSQL database and creates RESTful routes
for tables, views, and functions, supporting all HTTP methods that security for tables, views, and functions, supporting all HTTP methods that security
+7 -9
View File
@@ -3,6 +3,7 @@ Module : PostgREST.Error
Description : PostgREST error HTTP responses Description : PostgREST error HTTP responses
-} -}
{-# OPTIONS_GHC -fno-warn-orphans #-} {-# OPTIONS_GHC -fno-warn-orphans #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE RecordWildCards #-}
module PostgREST.Error module PostgREST.Error
@@ -41,6 +42,7 @@ import Network.HTTP.Types.Header (Header)
import PostgREST.MediaType (MediaType (..)) import PostgREST.MediaType (MediaType (..))
import qualified PostgREST.MediaType as MediaType import qualified PostgREST.MediaType as MediaType
import PostgREST.SchemaCache (SchemaCache (SchemaCache, dbTablesFuzzyIndex))
import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..), import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
Schema) Schema)
import PostgREST.SchemaCache.Relationship (Cardinality (..), import PostgREST.SchemaCache.Relationship (Cardinality (..),
@@ -49,10 +51,8 @@ import PostgREST.SchemaCache.Relationship (Cardinality (..),
RelationshipsMap) RelationshipsMap)
import PostgREST.SchemaCache.Routine (Routine (..), import PostgREST.SchemaCache.Routine (Routine (..),
RoutineParam (..)) RoutineParam (..))
import PostgREST.SchemaCache.Table (Table (..))
import Protolude import Protolude
class (ErrorBody a, JSON.ToJSON a) => PgrstError a where class (ErrorBody a, JSON.ToJSON a) => PgrstError a where
status :: a -> HTTP.Status status :: a -> HTTP.Status
headers :: a -> [Header] headers :: a -> [Header]
@@ -250,7 +250,7 @@ data SchemaCacheError
| NoRelBetween Text Text (Maybe Text) Text RelationshipsMap | NoRelBetween Text Text (Maybe Text) Text RelationshipsMap
| NoRpc Text Text [Text] MediaType Bool [QualifiedIdentifier] [Routine] | NoRpc Text Text [Text] MediaType Bool [QualifiedIdentifier] [Routine]
| ColumnNotFound Text Text | ColumnNotFound Text Text
| TableNotFound Text Text [Table] | TableNotFound Text Text SchemaCache
deriving Show deriving Show
instance PgrstError SchemaCacheError where instance PgrstError SchemaCacheError where
@@ -313,7 +313,7 @@ instance ErrorBody SchemaCacheError where
where where
onlySingleParams = isInvPost && contentType `elem` [MTTextPlain, MTTextXML, MTOctetStream] onlySingleParams = isInvPost && contentType `elem` [MTTextPlain, MTTextXML, MTOctetStream]
hint (AmbiguousRpc _) = Just "Try renaming the parameters or the function itself in the database so function overloading can be resolved" hint (AmbiguousRpc _) = Just "Try renaming the parameters or the function itself in the database so function overloading can be resolved"
hint (TableNotFound schemaName relName tbls) = JSON.String <$> tableNotFoundHint schemaName relName tbls hint (TableNotFound schemaName relName schemaCache) = JSON.String <$> tableNotFoundHint schemaName relName schemaCache
hint _ = Nothing hint _ = Nothing
@@ -428,13 +428,11 @@ noRpcHint schema procName params allProcs overloadedProcs =
-- | -- |
-- Do a fuzzy search in all tables in the same schema and return closest result -- Do a fuzzy search in all tables in the same schema and return closest result
tableNotFoundHint :: Text -> Text -> [Table] -> Maybe Text tableNotFoundHint :: Text -> Text -> SchemaCache -> Maybe Text
tableNotFoundHint schema tblName tblList tableNotFoundHint schema tblName SchemaCache{dbTablesFuzzyIndex}
= fmap (\tbl -> "Perhaps you meant the table '" <> schema <> "." <> tbl <> "'") perhapsTable = fmap (\tbl -> "Perhaps you meant the table '" <> schema <> "." <> tbl <> "'") perhapsTable
where where
perhapsTable = Fuzzy.getOne fuzzyTableSet tblName perhapsTable = (`Fuzzy.getOne` tblName) =<< HM.lookup schema dbTablesFuzzyIndex
fuzzyTableSet = Fuzzy.fromList [ tableName tbl | tbl <- tblList, tableSchema tbl == schema]
compressedRel :: Relationship -> JSON.Value compressedRel :: Relationship -> JSON.Value
-- An ambiguousness error cannot happen for computed relationships TODO refactor so this mempty is not needed -- An ambiguousness error cannot happen for computed relationships TODO refactor so this mempty is not needed
+36 -21
View File
@@ -1,3 +1,4 @@
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE MultiWayIf #-} {-# LANGUAGE MultiWayIf #-}
{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE RecordWildCards #-}
@@ -15,6 +16,7 @@ import PostgREST.Version (prettyVersion)
import qualified PostgREST.AppState as AppState import qualified PostgREST.AppState as AppState
import qualified PostgREST.Config as Config import qualified PostgREST.Config as Config
import Data.Either.Combinators (whenRight)
import Protolude import Protolude
-- | Starts the Listener in a thread -- | Starts the Listener in a thread
@@ -22,15 +24,16 @@ runListener :: AppState -> IO ()
runListener appState = do runListener appState = do
AppConfig{..} <- getConfig appState AppConfig{..} <- getConfig appState
when configDbChannelEnabled $ when configDbChannelEnabled $
void . forkIO $ retryingListen appState void . forkIO . void $ retryingListen appState
-- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost. -- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
retryingListen :: AppState -> IO () -- | This function never returns (but can throw) and return type enforces that.
retryingListen :: AppState -> IO Void
retryingListen appState = do retryingListen appState = do
AppConfig{..} <- AppState.getConfig appState AppConfig{..} <- AppState.getConfig appState
let let
dbChannel = toS configDbChannel dbChannel = toS configDbChannel
handleFinally err = do onError err = do
AppState.putIsListenerOn appState False AppState.putIsListenerOn appState False
observer $ DBListenFail dbChannel (Right err) observer $ DBListenFail dbChannel (Right err)
unless configDbPoolAutomaticRecovery $ unless configDbPoolAutomaticRecovery $
@@ -42,29 +45,39 @@ retryingListen appState = do
threadDelay (delay * oneSecondInMicro) threadDelay (delay * oneSecondInMicro)
unless (delay == maxDelay) $ unless (delay == maxDelay) $
AppState.putNextListenerDelay appState (delay * 2) AppState.putNextListenerDelay appState (delay * 2)
-- loop running the listener
retryingListen appState retryingListen appState
-- forkFinally allows to detect if the thread dies -- Execute the listener with with error handling
void . flip forkFinally handleFinally $ do handle onError $ do
dbOrError <- SQL.acquire $ toUtf8 (Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri) -- Make sure we don't leak connections on errors
case dbOrError of bracket
Right db -> do -- acquire connection
SQL.listen db $ SQL.toPgIdentifier dbChannel (SQL.acquire $ toUtf8 (Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri))
AppState.putIsListenerOn appState True -- release connection
(`whenRight` releaseConnection) $
-- use connection
\case
Right db -> do
SQL.listen db $ SQL.toPgIdentifier dbChannel
AppState.putIsListenerOn appState True
delay <- AppState.getNextListenerDelay appState delay <- AppState.getNextListenerDelay appState
when (delay > 1) $ do -- if we did a retry when (delay > 1) $ do -- if we did a retry
-- assume we lost notifications, refresh the schema cache -- assume we lost notifications, refresh the schema cache
AppState.schemaCacheLoader appState AppState.schemaCacheLoader appState
-- reset the delay -- reset the delay
AppState.putNextListenerDelay appState 1 AppState.putNextListenerDelay appState 1
observer $ DBListenStart dbChannel observer $ DBListenStart dbChannel
SQL.waitForNotifications handleNotification db
Left err -> do -- wait for notifications
observer $ DBListenFail dbChannel (Left err) -- this will never return, in case of an error it will throw and be caught by onError
exitFailure forever $ SQL.waitForNotifications handleNotification db
Left err -> do
observer $ DBListenFail dbChannel (Left err)
exitFailure
where where
observer = AppState.getObserver appState observer = AppState.getObserver appState
mainThreadId = AppState.getMainThreadId appState mainThreadId = AppState.getMainThreadId appState
@@ -79,3 +92,5 @@ retryingListen appState = do
cacheReloader = cacheReloader =
AppState.schemaCacheLoader appState AppState.schemaCacheLoader appState
releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release
+6 -4
View File
@@ -44,10 +44,11 @@ data Observation
| SchemaCacheLoadedObs Double | SchemaCacheLoadedObs Double
| ConnectionRetryObs Int | ConnectionRetryObs Int
| DBListenStart Text | DBListenStart Text
| DBListenFail Text (Either SQL.ConnectionError (Either SomeException ())) | DBListenFail Text (Either SQL.ConnectionError SomeException)
| DBListenRetry Int | DBListenRetry Int
| DBListenerGotSCacheMsg ByteString | DBListenerGotSCacheMsg ByteString
| DBListenerGotConfigMsg ByteString | DBListenerGotConfigMsg ByteString
| DBListenerConnectionCleanupFail SomeException
| QueryObs MainQuery Status | QueryObs MainQuery Status
| ConfigReadErrorObs SQL.UsageError | ConfigReadErrorObs SQL.UsageError
| ConfigInvalidObs Text | ConfigInvalidObs Text
@@ -118,6 +119,8 @@ observationMessage = \case
"Received a schema cache reload message on the " <> show channel <> " channel" "Received a schema cache reload message on the " <> show channel <> " channel"
DBListenerGotConfigMsg channel -> DBListenerGotConfigMsg channel ->
"Received a config reload message on the " <> show channel <> " channel" "Received a config reload message on the " <> show channel <> " channel"
DBListenerConnectionCleanupFail ex ->
"Failed during listener connection cleanup: " <> showOnSingleLine '\t' (show ex)
QueryObs{} -> QueryObs{} ->
mempty -- TODO pending refactor: The logic for printing the query cannot be done here. Join the observationMessage function into observationLogger to avoid this mempty. mempty -- TODO pending refactor: The logic for printing the query cannot be done here. Join the observationMessage function into observationLogger to avoid this mempty.
ConfigReadErrorObs usageErr -> ConfigReadErrorObs usageErr ->
@@ -164,9 +167,8 @@ observationMessage = \case
showListenerConnError :: SQL.ConnectionError -> Text showListenerConnError :: SQL.ConnectionError -> Text
showListenerConnError = maybe "Connection error" (showOnSingleLine '\t' . T.decodeUtf8) showListenerConnError = maybe "Connection error" (showOnSingleLine '\t' . T.decodeUtf8)
showListenerException :: Either SomeException () -> Text showListenerException :: SomeException -> Text
showListenerException (Right _) = "Failed getting notifications" -- should not happen as the listener will never finish (hasql-notifications uses `forever` internally) with a Right result showListenerException = showOnSingleLine '\t' . show
showListenerException (Left e) = showOnSingleLine '\t' $ show e
showOnSingleLine :: Char -> Text -> Text showOnSingleLine :: Char -> Text -> Text
+14 -7
View File
@@ -172,7 +172,7 @@ dbActionPlan dbAct conf apiReq sCache = case dbAct of
wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Bool -> Either Error CrudPlan wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Bool -> Either Error CrudPlan
wrappedReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} headersOnly = do wrappedReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} headersOnly = do
qi <- findTable identifier (dbTables sCache) qi <- findTable identifier sCache
rPlan <- readPlan qi conf sCache apiRequest rPlan <- readPlan qi conf sCache apiRequest
(handler, mediaType) <- mapLeft ApiRequestError $ negotiateContent conf apiRequest qi iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan) (handler, mediaType) <- mapLeft ApiRequestError $ negotiateContent conf apiRequest qi iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan)
if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right () if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
@@ -180,7 +180,7 @@ wrappedReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Prefe
mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error CrudPlan mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error CrudPlan
mutateReadPlan mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf sCache = do mutateReadPlan mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf sCache = do
qi <- findTable identifier (dbTables sCache) qi <- findTable identifier sCache
rPlan <- readPlan qi conf sCache apiRequest rPlan <- readPlan qi conf sCache apiRequest
mPlan <- mutatePlan mutation qi apiRequest sCache rPlan mPlan <- mutatePlan mutation qi apiRequest sCache rPlan
if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right () if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
@@ -812,10 +812,10 @@ validateAggFunctions aggFunctionsAllowed (Node rp@ReadPlan {select} forest)
| otherwise = Node rp <$> traverse (validateAggFunctions aggFunctionsAllowed) forest | otherwise = Node rp <$> traverse (validateAggFunctions aggFunctionsAllowed) forest
-- | Lookup table in the schema cache before creating read plan -- | Lookup table in the schema cache before creating read plan
findTable :: QualifiedIdentifier -> TablesMap -> Either Error QualifiedIdentifier findTable :: QualifiedIdentifier -> SchemaCache -> Either Error QualifiedIdentifier
findTable qi@QualifiedIdentifier{..} tableMap = findTable qi@QualifiedIdentifier{..} sc@SchemaCache{dbTables} =
case HM.lookup qi tableMap of case HM.lookup qi dbTables of
Nothing -> Left $ SchemaCacheErr $ TableNotFound qiSchema qiName (HM.elems tableMap) Nothing -> Left $ SchemaCacheErr $ TableNotFound qiSchema qiName sc
Just _ -> Right qi Just _ -> Right qi
addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
@@ -965,10 +965,17 @@ addRanges ApiRequest{..} rReq =
addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
addLogicTrees ctx ApiRequest{..} rReq = addLogicTrees ctx ApiRequest{..} rReq =
foldr addLogicTreeToNode (Right rReq) qsLogic foldr addLogicTreeToNode (Right rReq) logic
where where
QueryParams.QueryParams{..} = iQueryParams QueryParams.QueryParams{..} = iQueryParams
logic =
case iAction of
ActDb (ActRelationRead _ _) -> qsLogic
ActDb (ActRoutine _ _) -> qsLogic
-- For mutations, take the non-root logic filters. These will only affect the embeddings and not the top level of the returned representation.
_ -> filter (not . null . fst) qsLogic
addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either Error ReadPlanTree -> Either Error ReadPlanTree addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either Error ReadPlanTree -> Either Error ReadPlanTree
addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f) addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)
+2 -2
View File
@@ -213,10 +213,10 @@ actionResponse (MaybeDbResult InspectPlan{ipHdrsOnly=headersOnly} body) _ versio
in in
Right $ PgrstResponse HTTP.status200 (MediaType.toContentType MTOpenAPI : cLHeader ++ maybeToList (profileHeader schema negotiatedByProfile)) rsBody Right $ PgrstResponse HTTP.status200 (MediaType.toContentType MTOpenAPI : cLHeader ++ maybeToList (profileHeader schema negotiatedByProfile)) rsBody
actionResponse (NoDbResult (RelInfoPlan qi@QualifiedIdentifier{..})) _ _ _ SchemaCache{dbTables} _ _ = actionResponse (NoDbResult (RelInfoPlan qi@QualifiedIdentifier{..})) _ _ _ sc@SchemaCache{dbTables} _ _ =
case HM.lookup qi dbTables of case HM.lookup qi dbTables of
Just tbl -> respondInfo $ allowH tbl Just tbl -> respondInfo $ allowH tbl
Nothing -> Left $ Error.SchemaCacheErr $ Error.TableNotFound qiSchema qiName (HM.elems dbTables) Nothing -> Left $ Error.SchemaCacheErr $ Error.TableNotFound qiSchema qiName sc
where where
allowH table = allowH table =
let hasPK = not . null $ tablePKCols table in let hasPK = not . null $ tablePKCols table in
+28 -12
View File
@@ -20,6 +20,7 @@ These queries are executed once at startup or when PostgREST is reloaded.
module PostgREST.SchemaCache module PostgREST.SchemaCache
( SchemaCache(..) ( SchemaCache(..)
, TablesFuzzyIndex
, querySchemaCache , querySchemaCache
, showSummary , showSummary
, decodeFuncs , decodeFuncs
@@ -66,21 +67,28 @@ import PostgREST.SchemaCache.Table (Column (..), ColumnMap,
import qualified PostgREST.MediaType as MediaType import qualified PostgREST.MediaType as MediaType
import Control.Arrow ((&&&)) import Control.Arrow ((&&&))
import Protolude import qualified Data.FuzzySet as Fuzzy
import System.IO.Unsafe (unsafePerformIO) import Protolude
import System.IO.Unsafe (unsafePerformIO)
type TablesFuzzyIndex = HM.HashMap Schema Fuzzy.FuzzySet
data SchemaCache = SchemaCache data SchemaCache = SchemaCache
{ dbTables :: TablesMap { dbTables :: TablesMap
, dbRelationships :: RelationshipsMap , dbRelationships :: RelationshipsMap
, dbRoutines :: RoutineMap , dbRoutines :: RoutineMap
, dbRepresentations :: RepresentationsMap , dbRepresentations :: RepresentationsMap
, dbMediaHandlers :: MediaHandlerMap , dbMediaHandlers :: MediaHandlerMap
, dbTimezones :: TimezoneNames , dbTimezones :: TimezoneNames
} -- Memoized fuzzy index of table names per schema to support approximate matching
-- Since index construction can be expensive, we build it once and store in the SchemaCache
-- Haskell lazy evaluation ensures it's only built on first use and memoized afterwards
, dbTablesFuzzyIndex :: TablesFuzzyIndex
} deriving (Show)
instance JSON.ToJSON SchemaCache where instance JSON.ToJSON SchemaCache where
toJSON (SchemaCache tabs rels routs reps hdlers tzs) = JSON.object [ toJSON (SchemaCache tabs rels routs reps hdlers tzs _) = JSON.object [
"dbTables" .= JSON.toJSON tabs "dbTables" .= JSON.toJSON tabs
, "dbRelationships" .= JSON.toJSON rels , "dbRelationships" .= JSON.toJSON rels
, "dbRoutines" .= JSON.toJSON routs , "dbRoutines" .= JSON.toJSON routs
@@ -90,7 +98,7 @@ instance JSON.ToJSON SchemaCache where
] ]
showSummary :: SchemaCache -> Text showSummary :: SchemaCache -> Text
showSummary (SchemaCache tbls rels routs reps mediaHdlrs tzs) = showSummary (SchemaCache tbls rels routs reps mediaHdlrs tzs _) =
T.intercalate ", " T.intercalate ", "
[ show (HM.size tbls) <> " Relations" [ show (HM.size tbls) <> " Relations"
, show (HM.size rels) <> " Relationships" , show (HM.size rels) <> " Relationships"
@@ -138,6 +146,8 @@ data KeyDep
-- | A SQL query that can be executed independently -- | A SQL query that can be executed independently
type SqlQuery = ByteString type SqlQuery = ByteString
maxDbTablesForFuzzySearch :: Int
maxDbTablesForFuzzySearch = 500
querySchemaCache :: AppConfig -> SQL.Transaction SchemaCache querySchemaCache :: AppConfig -> SQL.Transaction SchemaCache
querySchemaCache conf@AppConfig{..} = do querySchemaCache conf@AppConfig{..} = do
@@ -166,6 +176,11 @@ querySchemaCache conf@AppConfig{..} = do
, dbRepresentations = reps , dbRepresentations = reps
, dbMediaHandlers = HM.union mHdlers initialMediaHandlers -- the custom handlers will override the initial ones , dbMediaHandlers = HM.union mHdlers initialMediaHandlers -- the custom handlers will override the initial ones
, dbTimezones = tzones , dbTimezones = tzones
, dbTablesFuzzyIndex =
-- Only build fuzzy index for schemas with a reasonable number of tables
-- Fuzzy.FuzzySet is memory heavy we just don't use it for large schemas
Fuzzy.fromList <$> HM.filter ((< maxDbTablesForFuzzySearch) . length) (HM.fromListWith (<>) ((qiSchema &&& pure . qiName) <$> HM.keys tabsWViewsPks))
} }
where where
schemas = toList configDbSchemas schemas = toList configDbSchemas
@@ -203,6 +218,7 @@ removeInternal schemas dbStruct =
, dbRepresentations = dbRepresentations dbStruct -- no need to filter, not directly exposed through the API , dbRepresentations = dbRepresentations dbStruct -- no need to filter, not directly exposed through the API
, dbMediaHandlers = dbMediaHandlers dbStruct , dbMediaHandlers = dbMediaHandlers dbStruct
, dbTimezones = dbTimezones dbStruct , dbTimezones = dbTimezones dbStruct
, dbTablesFuzzyIndex = dbTablesFuzzyIndex dbStruct
} }
where where
hasInternalJunction ComputedRelationship{} = False hasInternalJunction ComputedRelationship{} = False
+1 -1
View File
@@ -90,7 +90,7 @@ data RoutineParam = RoutineParam
instance Ord Routine where instance Ord Routine where
Function schema1 name1 des1 prms1 rt1 vol1 hasVar1 iso1 sets1 `compare` Function schema2 name2 des2 prms2 rt2 vol2 hasVar2 iso2 sets2 Function schema1 name1 des1 prms1 rt1 vol1 hasVar1 iso1 sets1 `compare` Function schema2 name2 des2 prms2 rt2 vol2 hasVar2 iso2 sets2
| schema1 == schema2 && name1 == name2 && length prms1 < length prms2 = LT | schema1 == schema2 && name1 == name2 && length prms1 < length prms2 = LT
| schema2 == schema2 && name1 == name2 && length prms1 > length prms2 = GT | schema1 == schema2 && name1 == name2 && length prms1 > length prms2 = GT
| otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1, iso1, sets1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2, iso2, sets2) | otherwise = (schema1, name1, des1, prms1, rt1, vol1, hasVar1, iso1, sets1) `compare` (schema2, name2, des2, prms2, rt2, vol2, hasVar2, iso2, sets2)
-- | A map of all procs, all of which can be overloaded(one entry will have more than one Routine). -- | A map of all procs, all of which can be overloaded(one entry will have more than one Routine).
+22
View File
@@ -11375,12 +11375,34 @@ ALTER TABLE ONLY apflora.zielber
ALTER TABLE apflora."user" ENABLE ROW LEVEL SECURITY; ALTER TABLE apflora."user" ENABLE ROW LEVEL SECURITY;
CREATE SCHEMA fuzzysearch;
-- Create many tables to test fuzzy string search
-- computing hints for non existing tables
DO
$$
DECLARE
r record;
BEGIN
FOR r IN
SELECT
format('CREATE TABLE fuzzysearch.unknown_table_%s ()', n) AS ct
FROM
generate_series(1, 499) n
LOOP
EXECUTE r.ct;
END LOOP;
END
$$;
DROP ROLE IF EXISTS postgrest_test_anonymous; DROP ROLE IF EXISTS postgrest_test_anonymous;
CREATE ROLE postgrest_test_anonymous; CREATE ROLE postgrest_test_anonymous;
GRANT postgrest_test_anonymous TO :PGUSER; GRANT postgrest_test_anonymous TO :PGUSER;
GRANT USAGE ON SCHEMA apflora TO postgrest_test_anonymous; GRANT USAGE ON SCHEMA apflora TO postgrest_test_anonymous;
GRANT USAGE ON SCHEMA fuzzysearch TO postgrest_test_anonymous;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA apflora GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA apflora
TO postgrest_test_anonymous; TO postgrest_test_anonymous;
+20
View File
@@ -70,3 +70,23 @@ def test_should_not_fail_with_stack_overflow(defaultenv):
assert response.status_code == 404 assert response.status_code == 404
data = response.json() data = response.json()
assert data["code"] == "PGRST205" assert data["code"] == "PGRST205"
def test_second_request_for_non_existent_table_should_be_quick(defaultenv):
"requesting a non-existent relationship should be quick after the fuzzy search index is loaded (2nd request)"
env = {
**defaultenv,
"PGRST_DB_SCHEMAS": "fuzzysearch",
"PGRST_DB_POOL": "2",
"PGRST_DB_ANON_ROLE": "postgrest_test_anonymous",
}
with run(env=env, wait_max_seconds=30) as postgrest:
response = postgrest.session.get("/unknown-table")
assert response.status_code == 404
data = response.json()
assert data["code"] == "PGRST205"
first_duration = response.elapsed.total_seconds()
response = postgrest.session.get("/unknown-table")
assert response.elapsed.total_seconds() < first_duration / 10
+20 -2
View File
@@ -252,21 +252,39 @@ spec =
[json|[{"id": 7, "entities":null}, {"id": 8, "entities": {"id": 2}}, {"id": 9, "entities": {"id": 3}}]|] [json|[{"id": 7, "entities":null}, {"id": 8, "entities": {"id": 2}}, {"id": 9, "entities": {"id": 3}}]|]
{ matchStatus = 201 } { matchStatus = 201 }
context "used with PATCH" $ context "used with PATCH" $ do
it "succeeds when using and/or params" $ it "succeeds when using and/or params" $
request methodPatch "/grandchild_entities?or=(id.eq.1,id.eq.2)&select=id,name" request methodPatch "/grandchild_entities?or=(id.eq.1,id.eq.2)&select=id,name"
[("Prefer", "return=representation")] [("Prefer", "return=representation")]
[json|{ name : "updated grandchild entity"}|] `shouldRespondWith` [json|{ name : "updated grandchild entity"}|] `shouldRespondWith`
[json|[{ "id": 1, "name" : "updated grandchild entity"},{ "id": 2, "name" : "updated grandchild entity"}]|] [json|[{ "id": 1, "name" : "updated grandchild entity"},{ "id": 2, "name" : "updated grandchild entity"}]|]
{ matchHeaders = [matchContentTypeJson] } { matchHeaders = [matchContentTypeJson] }
it "succeeds when the filtered column is modified" $
request methodPatch "/entities?select=id,name&or=(name.is.null,name.like.*test*)"
[("Prefer", "return=representation")]
[json|{ "name" : "updated entity" }|] `shouldRespondWith`
[json|[{ "id": 4, "name": "updated entity" }]|]
{ matchHeaders = [matchContentTypeJson] }
it "succeeds when the filtered column is not selected in the returned representation" $
request methodPatch "/entities?select=id&or=(name.is.null,name.like.*test*)"
[("Prefer", "return=representation")]
[json|{ "name" : "updated entity" }|] `shouldRespondWith`
[json|[{ "id": 4 }]|]
{ matchHeaders = [matchContentTypeJson] }
context "used with DELETE" $ context "used with DELETE" $ do
it "succeeds when using and/or params" $ it "succeeds when using and/or params" $
request methodDelete "/grandchild_entities?or=(id.eq.1,id.eq.2)&select=id,name" request methodDelete "/grandchild_entities?or=(id.eq.1,id.eq.2)&select=id,name"
[("Prefer", "return=representation")] [("Prefer", "return=representation")]
"" ""
`shouldRespondWith` `shouldRespondWith`
[json|[{ "id": 1, "name" : "grandchild entity 1" },{ "id": 2, "name" : "grandchild entity 2" }]|] [json|[{ "id": 1, "name" : "grandchild entity 1" },{ "id": 2, "name" : "grandchild entity 2" }]|]
it "succeeds when the filtered column is not selected in the returned representation" $
request methodDelete "/entities?select=id&or=(name.is.null,name.like.*test*)"
[("Prefer", "return=representation")]
""
`shouldRespondWith`
[json|[{ "id": 4 }]|]
it "can query columns that begin with and/or reserved words" $ it "can query columns that begin with and/or reserved words" $
get "/grandchild_entities?or=(and_starting_col.eq.smth, or_starting_col.eq.smth)" `shouldRespondWith` 200 get "/grandchild_entities?or=(and_starting_col.eq.smth, or_starting_col.eq.smth)" `shouldRespondWith` 200