Compare commits

...
10 Commits
Author SHA1 Message Date
steve-chavez 83e1bebcdc bump version to 14.10 2026-04-16 13:54:44 -05:00
renovate[bot]andWolfgang Walther 89132e886a chore(deps): update ubuntu:noble docker digest to c4a8d55 2026-04-16 10:44:03 +00:00
Michal KleczekandTaimoor Zaeem abd76ca8ac fix: Flush pool as late as possible during schema cache reloading (#4645)
retryingSchemaCacheLoad flushes the pool upon every retry before it starts reloading the schema. This is too early as schema reloading might take some time during which new connections might be acquired. The consequence is that:
* upon successful schema cache reload we might have some connections created with the old schema cache
* we close connections upon each retry and under load we will keep closing and re-opening connections until schema cache load succeeds

This change is to make sure we flush the pool only after successful schema cache querying but before loading (so that connections acquired during loading wait for it and do not interfere with timing the loading process).
2026-04-16 11:55:43 +05:00
Michał KłeczekandTaimoor Zaeem df87ce46ed add: log pool flushes
Emit a dedicated PoolFlushed observation when the DB pool is released during schema cache reload.
2026-04-16 11:18:44 +05:00
renovate[bot]andWolfgang Walther d369d2c41e chore(deps): update haskell-actions/setup action to v2.11.0 2026-04-15 18:05:21 +00:00
Wolfgang Walther 9ee366a903 docs: fix broken link 2026-04-15 09:30:05 +02:00
Michał KłeczekandTaimoor Zaeem 01f7f13389 test(refactor): make debug output testing more robust
This change introduces a match_log function, that implements lenient matching of log output lines. Tests of log output are retrofited to use the function.
2026-04-15 07:40:16 +05:00
steve-chavez 4922a9feaf docs: target_session_attrs=read-only in libpq>=14 2026-04-14 11:11:22 -05:00
renovate[bot]andWolfgang Walther 679b957355 chore(deps): update actions/cache action to v5.0.5 2026-04-13 21:04:21 +00:00
renovate[bot]andWolfgang Walther 05df2c5692 chore(deps): update all dependencies 2026-04-12 15:25:03 +00:00
16 changed files with 152 additions and 68 deletions
@@ -112,7 +112,7 @@ runs:
echo "artifacts=${artifacts}" >> "$GITHUB_OUTPUT"
- name: Save artifact to GitHub Actions
if: steps.find-task.outputs.task_found
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ inputs.upload }}
path: ${{ steps.download.outputs.artifacts }}
+2 -2
View File
@@ -19,14 +19,14 @@ inputs:
runs:
using: composite
steps:
- uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
- uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
if: ${{ startsWith(github.ref, 'refs/heads/') || (inputs.save-prs && startsWith(github.ref, 'refs/pull/')) }}
with:
path: ${{ inputs.path }}
key: ${{ runner.os }}-${{ inputs.prefix }}-${{ inputs.suffix }}
restore-keys: |
${{ runner.os }}-${{ inputs.prefix }}-
- uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
if: ${{ !startsWith(github.ref, 'refs/heads/') && !(inputs.save-prs && startsWith(github.ref, 'refs/pull/')) }}
with:
path: ${{ inputs.path }}
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# This actions creates the github token using the postgrest app secrets
- name: Create Github App Token
id: app-token
uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
with:
app-id: ${{ vars.POSTGREST_CI_APP_ID }}
private-key: ${{ secrets.POSTGREST_CI_PRIVATE_KEY }}
+5 -5
View File
@@ -42,7 +42,7 @@ jobs:
- name: Build static executable
run: nix-build -A postgrestStatic -A postgrestStatic.tests
- name: Save built executable as artifact
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: postgrest-linux-static-x86-64
path: result/bin/postgrest
@@ -51,7 +51,7 @@ jobs:
- name: Build Docker image
run: nix-build -A docker.image --out-link postgrest-docker.tar.gz
- name: Save built Docker image as artifact
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: postgrest-docker-x86-64
path: postgrest-docker.tar.gz
@@ -118,7 +118,7 @@ jobs:
runs-on: ${{ matrix.runs-on }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: haskell-actions/setup@de26526e12bc780fb9d384c1fb61c0bf02e3a40d # v2.10.4
- uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0
with:
# This must match the version in stack.yaml's resolver
ghc-version: 9.6.7
@@ -146,7 +146,7 @@ jobs:
- name: Strip Executable
run: strip result/postgrest*
- name: Save built executable as artifact
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ matrix.artifact }}
path: |
@@ -177,7 +177,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: haskell-actions/setup@de26526e12bc780fb9d384c1fb61c0bf02e3a40d # v2.10.4
- uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0
with:
ghc-version: ${{ matrix.ghc }}
- name: Cache .cabal
+2 -2
View File
@@ -49,7 +49,7 @@ jobs:
echo "Relevant extract from CHANGELOG.md:"
cat CHANGES.md
- name: Save CHANGES.md as artifact
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-changes
path: CHANGES.md
@@ -91,7 +91,7 @@ jobs:
artifacts/postgrest-windows-x86-64/postgrest.exe
- name: Save release bundle
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-bundle
path: release-bundle
+10
View File
@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. From versio
## Unreleased
## [14.10] - 2026-04-16
### Added
- Log when the pool is released during schema cache reload on `log-level=debug` by @mkleczek in #4668
### Fixed
- Fix unnecessary connection pool flushes during schema cache reloading by @mkleczek in #4645
## [14.9] - 2026-04-10
### Added
+1 -1
View File
@@ -2,7 +2,7 @@
# The x86-64 is a single-static-binary image built via Nix, see:
# nix/tools/docker/README.md
FROM ubuntu:noble@sha256:84e77dee7d1bc93fb029a45e3c6cb9d8aa4831ccfcc7103d36e876938d28895b AS postgrest
FROM ubuntu:noble@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69a85730fb49b5cfca2330194ebcc41c7b AS postgrest
RUN apt-get update -y \
&& apt install -y --no-install-recommends libpq-dev zlib1g-dev jq gcc libnuma-dev \
+1 -1
View File
@@ -6,7 +6,7 @@ Community Tutorials
* `Building a Contacts List with PostgREST and Vue.js <https://www.youtube.com/watch?v=iHtsALtD5-U>`_ -
In this video series, DigitalOcean shows how to build and deploy an Nginx + PostgREST(using a managed PostgreSQL database) + Vue.js webapp in an Ubuntu server droplet.
* `PostgREST + Auth0: Create REST API in mintutes, and add social login using Auth0 <https://samkhawase.com/blog/postgrest/>`_ - A step-by-step tutorial to show how to dockerize and integrate Auth0 to PostgREST service.
* `PostgREST + Auth0: Create REST API in minutes, and add social login using Auth0 <https://samkhawase.com/blog/postgrest-1-introduction/>`_ - A step-by-step tutorial to show how to dockerize and integrate Auth0 to PostgREST service.
* `"CodeLess" backend using postgres, postgrest and oauth2 authentication with keycloak <https://www.mathieupassenaud.fr/codeless_backend/>`_ -
A step-by-step tutorial for using PostgREST with KeyCloak(hosted on a managed service).
+3 -1
View File
@@ -46,7 +46,9 @@ This will cause the :ref:`connection_pool` to connect to the read replica host a
.. note::
Under the hood, PostgREST forces `target_session_attrs=read-write <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-TARGET-SESSION-ATTRS>`_ for the ``LISTEN`` session.
- Under the hood, PostgREST forces `target_session_attrs=read-write <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-TARGET-SESSION-ATTRS>`_ for the ``LISTEN`` session.
So if you specify ``target_session_attrs=read-only`` as mentioned above, PostgREST will override it for the ``LISTEN``.
- ``read-only`` is only available on libpq >= 14, if you use a lower version you will get an error like ``invalid target_session_attrs value: \"read-only\"``.
.. _listener_automatic_recovery:
+2 -1
View File
@@ -1,5 +1,5 @@
name: postgrest
version: 14.9
version: 14.10
synopsis: REST API for any Postgres database
description: Reads the schema of a PostgreSQL database and creates RESTful routes
for tables, views, and functions, supporting all HTTP methods that security
@@ -304,6 +304,7 @@ test-suite observability
other-modules: ObsHelper
Observation.JwtCache
Observation.MetricsSpec
Observation.SchemaCacheSpec
build-depends: base >= 4.9 && < 4.20
, base64-bytestring >= 1 && < 1.3
, bytestring >= 0.10.8 && < 0.13
+9 -3
View File
@@ -219,10 +219,14 @@ usePool AppState{stateObserver=observer, stateMainThreadId=mainThreadId, ..} ses
-- | Flush the connection pool so that any future use of the pool will
-- use connections freshly established after this call.
-- | Emits PoolFlushed observation
flushPool :: AppState -> IO ()
flushPool AppState{..} = SQL.release statePool
flushPool AppState{..} = do
SQL.release statePool
stateObserver PoolFlushed
-- | Destroy the pool on shutdown.
-- | Differs from flushPool in not emiting PoolFlushed observation.
destroyPool :: AppState -> IO ()
destroyPool AppState{..} = SQL.release statePool
@@ -309,8 +313,6 @@ retryingSchemaCacheLoad appState@AppState{stateObserver=observer, stateMainThrea
observer $ ConnectionRetryObs delay
putNextListenerDelay appState delay
flushPool appState
(,) <$> qPgVersion <*> (qInDbConfig *> qSchemaCache)
)
where
@@ -359,6 +361,10 @@ retryingSchemaCacheLoad appState@AppState{stateObserver=observer, stateMainThrea
-- IORef on putSchemaCache. This is why SCacheStatus is put at SCPending here to signal the Admin server (using isPending) that we're on a recovery state.
putSCacheStatus appState SCPending
putSchemaCache appState $ Just sCache
-- Flush the pool after loading the schema cache to reset any stale session cache entries
-- We do it after successfully querying the schema cache (because this can fail and during retries we would flush the pool repeatedly unnecessarily)
-- and after marking sCacheStatus as pending,
flushPool appState
observer $ SchemaCacheQueriedObs resultTime
(t, _) <- timeItT $ observer $ SchemaCacheSummaryObs $ showSummary sCache
observer $ SchemaCacheLoadedObs t
+3
View File
@@ -110,6 +110,9 @@ observationLogger loggerState logLevel obs = case obs of
o@PoolRequestFullfilled ->
when (logLevel >= LogDebug) $ do
logWithZTime loggerState $ observationMessage o
o@PoolFlushed ->
when (logLevel >= LogDebug) $ do
logWithZTime loggerState $ observationMessage o
o@JwtCacheEviction ->
when (logLevel >= LogDebug) $ do
logWithZTime loggerState $ observationMessage o
+3
View File
@@ -64,6 +64,7 @@ data Observation
| HasqlPoolObs SQL.Observation
| PoolRequest
| PoolRequestFullfilled
| PoolFlushed
| JwtCacheLookup Bool
| JwtCacheEviction
| TerminationUnixSignalObs Text
@@ -161,6 +162,8 @@ observationMessage = \case
"Trying to borrow a connection from pool"
PoolRequestFullfilled ->
"Borrowed a connection from the pool"
PoolFlushed ->
"Database connection pool flushed"
JwtCacheLookup _ ->
"Looked up a JWT in JWT cache"
JwtCacheEviction ->
+53 -46
View File
@@ -21,6 +21,19 @@ from postgrest import (
)
def match_log(output, matchers):
ito = iter(output)
itm = iter(matchers)
nextMatcher = next(itm, None)
while nextMatcher is not None and (line := next(ito, None)) is not None:
if re.match(nextMatcher, line) is not None:
nextMatcher = next(itm, None)
if nextMatcher is not None:
raise AssertionError(
f"Expected log line matching {nextMatcher} not found in output"
)
def test_connect_with_dburi(dburi, defaultenv):
"Connecting with db-uri instead of LIPQ* environment variables should work."
defaultenv_without_libpq = {
@@ -679,55 +692,45 @@ def test_log_level(level, defaultenv):
response = postgrest.session.get("/")
assert response.status_code == 200
output = sorted(postgrest.read_stdout(nlines=7))
output = postgrest.read_stdout(nlines=9)
if level == "crit":
assert len(output) == 0
elif level == "error":
assert re.match(
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
output[0],
match_log(
output,
[r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"'],
)
assert len(output) == 1
elif level == "warn":
assert re.match(
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
output[0],
)
assert re.match(
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
output[1],
match_log(
output,
[
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
],
)
assert len(output) == 2
elif level == "info":
assert re.match(
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
output[0],
)
assert re.match(
r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
output[1],
)
assert re.match(
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
output[2],
match_log(
output,
[
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
],
)
assert len(output) == 3
elif level == "debug":
assert re.match(
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
output[0],
match_log(
output,
[
r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
],
)
assert re.match(
r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
output[1],
)
assert re.match(
r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
output[2],
)
assert len(output) == 7
assert len(output) == 9
assert any("Connection" and "is available" in line for line in output)
assert any("Connection" and "is used" in line for line in output)
@@ -1364,16 +1367,21 @@ def test_db_error_logging_to_stderr(level, defaultenv, metapostgrest):
assert response.status_code == 500
# ensure the message appears on the logs
output = sorted(postgrest.read_stdout(nlines=6))
output = postgrest.read_stdout(nlines=8)
if level == "crit":
assert len(output) == 0
elif level == "debug":
assert " 500 " in output[0]
assert "canceling statement due to statement timeout" in output[5]
match_log(
output,
[
r".*canceling statement due to statement timeout.*",
r".*500.*",
],
)
else:
assert " 500 " in output[0]
assert "canceling statement due to statement timeout" in output[1]
assert " 500 " in output[1]
assert "canceling statement due to statement timeout" in output[0]
reset_statement_timeout(metapostgrest, role)
@@ -1567,18 +1575,17 @@ def test_log_pool_req_observation(level, defaultenv):
headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)
pool_req = "Trying to borrow a connection from pool"
pool_req_fullfill = "Borrowed a connection from the pool"
pool_req = r".*Trying to borrow a connection from pool.*"
pool_req_fullfill = r".*Borrowed a connection from the pool.*"
with run(env=env) as postgrest:
postgrest.session.get("/authors_only", headers=headers)
if level == "debug":
output = postgrest.read_stdout(nlines=5)
assert pool_req in output[1]
assert pool_req_fullfill in output[4]
assert len(output) == 5
output = postgrest.read_stdout(nlines=7)
assert len(output) == 7
match_log(output, [pool_req, pool_req_fullfill])
elif level == "info":
output = postgrest.read_stdout(nlines=4)
assert len(output) == 1
+7 -4
View File
@@ -17,10 +17,11 @@ import PostgREST.SchemaCache (querySchemaCache)
import qualified Observation.JwtCache
import qualified Observation.MetricsSpec
import ObsHelper
import PostgREST.Observation (Observation (HasqlPoolObs))
import Protolude hiding (toList, toS)
import Test.Hspec
import qualified Observation.SchemaCacheSpec
import ObsHelper
import PostgREST.Observation (Observation (HasqlPoolObs))
import Protolude hiding (toList, toS)
import Test.Hspec
main :: IO ()
main = do
@@ -64,6 +65,8 @@ main = do
describe "Observation.JwtCacheObs" Observation.JwtCache.spec
before (initApp baseSchemaCache testCfg) $
describe "Feature.MetricsSpec" Observation.MetricsSpec.spec
before (initApp baseSchemaCache testCfg) $
describe "Feature.SchemaCacheSpec" Observation.SchemaCacheSpec.spec
where
loadSCache pool conf =
@@ -0,0 +1,49 @@
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE MonadComprehensions #-}
{-# LANGUAGE NamedFieldPuns #-}
module Observation.SchemaCacheSpec where
import Network.Wai (Application)
import ObsHelper
import qualified PostgREST.AppState as AppState
import PostgREST.Config (configDbSchemas)
import PostgREST.Observation
import Protolude
import Test.Hspec (SpecWith, describe, it)
import Test.Hspec.Wai (getState)
spec :: SpecWith (SpecState, Application)
spec = describe "Server started with metrics enabled" $ do
it "Should emit PoolFlushed, SchemaCacheQueriedObs and SchemaCacheLoadedObs when schema cache is reloaded" $ do
SpecState{specAppState = appState, specObsChan} <- getState
let waitFor = waitForObs specObsChan
liftIO $ do
AppState.schemaCacheLoader appState
waitFor (1 * sec) "PoolFlushed" $ \x -> [ o | o@PoolFlushed <- pure x ]
waitFor (1 * sec) "SchemaCacheQueriedObs" $ \x -> [ o | o@SchemaCacheQueriedObs{} <- pure x ]
waitFor (1 * sec) "SchemaCacheLoadedObs" $ \x -> [ o | o@SchemaCacheLoadedObs{} <- pure x ]
it "Should flush pool once when schema reloading retries" $ do
SpecState{specAppState = appState, specObsChan} <- getState
let waitFor = waitForObs specObsChan
liftIO $ do
AppState.getConfig appState >>= \cfg -> do
AppState.putConfig appState $ cfg { configDbSchemas = pure "bad_schema" }
AppState.schemaCacheLoader appState
waitFor (1 * sec) "SchemaCacheErrorObs" $ \x -> [ o | o@SchemaCacheErrorObs{} <- pure x ]
-- Restore configuration
AppState.putConfig appState cfg
-- Wait for 2 seconds so that retry can happen
waitFor (2 * sec) "PoolFlushed" $ \x -> [ o | o@PoolFlushed <- pure x ]
waitFor (1 * sec) "SchemaCacheQueriedObs" $ \x -> [ o | o@SchemaCacheQueriedObs{} <- pure x ]
waitFor (1 * sec) "SchemaCacheLoadedObs" $ \x -> [ o | o@SchemaCacheLoadedObs{} <- pure x ]
where
sec = 1000000