chore: bump version to 14.12

test: adjust test_positive_pool_metric
Adjusted so it waits for liveness instead of requiring sleeping
2026-05-20 12:20:38 -05:00 · 2026-05-20 10:29:12 -05:00 · 2026-05-20 10:29:12 -05:00 · 2026-05-20 10:29:12 -05:00 · 2026-05-20 10:29:12 -05:00 · 2026-05-20 10:29:12 -05:00
43 changed files with 776 additions and 408 deletions
@@ -2,4 +2,6 @@
 # and made its way to us through nixpkgs.
 self-hosted-runner:
  labels:
+    - macos-15-intel
+    - macos-26
    - ubuntu-24.04-arm
@@ -112,7 +112,7 @@ runs:
        echo "artifacts=${artifacts}" >> "$GITHUB_OUTPUT"
    - name: Save artifact to GitHub Actions
      if: steps.find-task.outputs.task_found
-      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: ${{ inputs.upload }}
        path: ${{ steps.download.outputs.artifacts }}
@@ -19,14 +19,14 @@ inputs:
 runs:
  using: composite
  steps:
-    - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+    - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
      if: ${{ startsWith(github.ref, 'refs/heads/') || (inputs.save-prs && startsWith(github.ref, 'refs/pull/')) }}
      with:
        path: ${{ inputs.path }}
        key: ${{ runner.os }}-${{ inputs.prefix }}-${{ inputs.suffix }}
        restore-keys: |
          ${{ runner.os }}-${{ inputs.prefix }}-
-    - uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+    - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
      if: ${{ !startsWith(github.ref, 'refs/heads/') && !(inputs.save-prs && startsWith(github.ref, 'refs/pull/'))  }}
      with:
        path: ${{ inputs.path }}
@@ -16,7 +16,7 @@ runs:
        nix_conf: |-
          always-allow-substitutes = true
          max-jobs = auto
-    - uses: cachix/cachix-action@1eb2ef646ac0255473d23a5907ad7b04ce94065c # v17
+    - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
      with:
        name: postgrest
        authToken: ${{ inputs.authToken }}
@@ -28,7 +28,7 @@ jobs:
      # This actions creates the github token using the postgrest app secrets
      - name: Create Github App Token
        id: app-token
-        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
        with:
          app-id: ${{ vars.POSTGREST_CI_APP_ID }}
          private-key: ${{ secrets.POSTGREST_CI_PRIVATE_KEY }}
@@ -45,7 +45,7 @@ jobs:

      # Backport action that creates the PR with given settings
      - name: Create backport PR
-        uses: korthout/backport-action@3c06f323a58619da1e8522229ebc8d5de2633e46 # v4.3.0
+        uses: korthout/backport-action@66065406958f46e82238fd59546f5a99e69e22aa # v4.5
        with:
          github_token: ${{ steps.app-token.outputs.token }}
          pull_description: 'Backport for #${pull_number}.'
@@ -42,7 +42,7 @@ jobs:
      - name: Build static executable
        run: nix-build -A postgrestStatic -A postgrestStatic.tests
      - name: Save built executable as artifact
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: postgrest-linux-static-x86-64
          path: result/bin/postgrest
@@ -51,7 +51,7 @@ jobs:
      - name: Build Docker image
        run: nix-build -A docker.image --out-link postgrest-docker.tar.gz
      - name: Save built Docker image as artifact
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: postgrest-docker-x86-64
          path: postgrest-docker.tar.gz
@@ -60,7 +60,7 @@ jobs:

  macos:
    name: Nix - MacOS
-    runs-on: macos-15
+    runs-on: macos-26
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Setup Nix Environment
@@ -105,6 +105,15 @@ jobs:
            artifact: postgrest-macos-aarch64
            deps: brew link --force libpq

+          - name: MacOS x86-64
+            runs-on: macos-15-intel
+            cache: |
+              ~/.stack/pantry
+              ~/.stack/snapshots
+              ~/.stack/stack.sqlite3
+            artifact: postgrest-macos-x86-64
+            deps: brew link --force libpq
+
          - name: Windows
            runs-on: windows-2022
            cache: |
@@ -118,7 +127,7 @@ jobs:
    runs-on: ${{ matrix.runs-on }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - uses: haskell-actions/setup@de26526e12bc780fb9d384c1fb61c0bf02e3a40d # v2.10.4
+      - uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0
        with:
          # This must match the version in stack.yaml's resolver
          ghc-version: 9.6.7
@@ -146,7 +155,7 @@ jobs:
      - name: Strip Executable
        run: strip result/postgrest*
      - name: Save built executable as artifact
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ matrix.artifact }}
          path: |
@@ -177,7 +186,7 @@ jobs:
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - uses: haskell-actions/setup@de26526e12bc780fb9d384c1fb61c0bf02e3a40d # v2.10.4
+      - uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0
        with:
          ghc-version: ${{ matrix.ghc }}
      - name: Cache .cabal
@@ -41,8 +41,7 @@ jobs:
    concurrency:
      # Never tag outdated commits on the main branch by skipping superseded commits
      group: ci-tag-${{ (github.ref == 'refs/heads/main' && github.ref) || github.run_id }}
-      # TODO: Enable this once https://github.com/orgs/community/discussions/13015 is solved
-      cancel-in-progress: false
+      cancel-in-progress: true
    if: vars.RELEASE_ENABLED
    runs-on: ubuntu-24.04
    needs:
@@ -7,6 +7,7 @@ on:

 jobs:
  linkcheck:
+    name: Linkcheck
    runs-on: ubuntu-24.04
    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -15,4 +16,28 @@ jobs:
      with:
        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
        tools: docs.linkcheck.bin
-    - run: postgrest-docs-linkcheck
+
+    - name: Run Linkcheck
+      id: linkcheck
+      run: postgrest-docs-linkcheck
+
+    # This actions creates the github token using the postgrest app secrets
+    - name: Create Github App Token (Runs only on linkcheck failure)
+      id: app-token
+      if: ${{ failure() && steps.linkcheck.outcome == 'failure' }} # only create the token on linkcheck failure
+      uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+      with:
+        app-id: ${{ vars.POSTGREST_CI_APP_ID }}
+        private-key: ${{ secrets.POSTGREST_CI_PRIVATE_KEY }}
+        permission-issues: write # required for commenting on issues
+
+    - name: Notify on linkcheck failure by commenting on GH Issue 4106
+      if: ${{ failure() && steps.linkcheck.outcome == 'failure' }}
+      uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
+      with:
+        token: ${{ steps.app-token.outputs.token }}
+        issue-number: 4106
+        body: |
+          **Linkcheck Job Failed!**
+
+          A broken link was detected in the docs. Please check the [failed run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
@@ -9,8 +9,7 @@ on:
 concurrency:
  # Terminate all previous runs of the same workflow for the same tag.
  group: release-${{ github.ref }}
-  # TODO: Enable this once https://github.com/orgs/community/discussions/13015 is solved
-  cancel-in-progress: false
+  cancel-in-progress: true

 jobs:
  build:
@@ -49,7 +48,7 @@ jobs:
          echo "Relevant extract from CHANGELOG.md:"
          cat CHANGES.md
      - name: Save CHANGES.md as artifact
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: release-changes
          path: CHANGES.md
@@ -81,6 +80,9 @@ jobs:
          tar cJvf "release-bundle/postgrest-${GITHUB_REF_NAME}-macos-aarch64.tar.xz" \
            -C artifacts/postgrest-macos-aarch64 postgrest

+          tar cJvf "release-bundle/postgrest-${GITHUB_REF_NAME}-macos-x86-64.tar.xz" \
+            -C artifacts/postgrest-macos-x86-64 postgrest
+
          tar cJvf "release-bundle/postgrest-${GITHUB_REF_NAME}-freebsd-x86-64.tar.xz" \
            -C artifacts/postgrest-freebsd-x86-64 postgrest

@@ -91,7 +93,7 @@ jobs:
            artifacts/postgrest-windows-x86-64/postgrest.exe

      - name: Save release bundle
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: release-bundle
          path: release-bundle
@@ -51,7 +51,7 @@ jobs:
      - name: Run coverage (IO tests and Spec tests against PostgreSQL 15)
        run: postgrest-coverage
      - name: Upload coverage to codecov
-        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
+        uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
        with:
          files: ./coverage/codecov.json
          token: ${{ secrets.CODECOV_TOKEN }}
@@ -156,6 +156,7 @@ jobs:
      fail-fast: false
      matrix:
        runs-on:
+          - macos-15-intel # x86_64-darwin
          - macos-14 # aarch64-darwin
          - ubuntu-24.04 # x86_64-linux
          - ubuntu-24.04-arm # aarch64-linux
@@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file. From versio

 ## Unreleased

+## [14.12] - 2026-05-20
+
+### Fixed
+
+- Fix race condition in pool_available metric causing negative values during network instability by @mkleczek in #4622
+
+## [14.11] - 2026-05-04
+
+### Fixed
+
+- Fix login with uppercase and mixed case role names by @taimoorzaeem in #4678
+- Restore Listener query shape so it can be found in `pg_stat_activity` by @mkleczek in #4857 #4859
+- The LISTEN channel now automatically recovers when it stops working due to a PostgreSQL bug @laurenceisla in #3147
+- Fix misleading "Functions" name on schema cache summary in startup logs by @taimoorzaeem in #4821
+
+## [14.10] - 2026-04-16
+
+### Added
+
+- Log when the pool is released during schema cache reload on `log-level=debug` by @mkleczek in #4668
+
+### Fixed
+
+- Fix unnecessary connection pool flushes during schema cache reloading by @mkleczek in #4645
+
 ## [14.9] - 2026-04-10

 ### Added
@@ -2,7 +2,7 @@
 # The x86-64 is a single-static-binary image built via Nix, see:
 # nix/tools/docker/README.md

-FROM ubuntu:noble@sha256:84e77dee7d1bc93fb029a45e3c6cb9d8aa4831ccfcc7103d36e876938d28895b AS postgrest
+FROM ubuntu:resolute@sha256:f3d28607ddd78734bb7f71f117f3c6706c666b8b76cbff7c9ff6e5718d46ff64 AS postgrest

 RUN apt-get update -y \
    && apt install -y --no-install-recommends libpq-dev zlib1g-dev jq gcc libnuma-dev \
@@ -6,7 +6,7 @@ Community Tutorials
 * `Building a Contacts List with PostgREST and Vue.js <https://www.youtube.com/watch?v=iHtsALtD5-U>`_ -
  In this video series, DigitalOcean shows how to build and deploy an Nginx + PostgREST(using a managed PostgreSQL database) + Vue.js webapp in an Ubuntu server droplet.

-* `PostgREST + Auth0: Create REST API in mintutes, and add social login using Auth0 <https://samkhawase.com/blog/postgrest/>`_ - A step-by-step tutorial to show how to dockerize and integrate Auth0 to PostgREST service.
+* `PostgREST + Auth0: Create REST API in minutes, and add social login using Auth0 <https://samkhawase.com/blog/postgrest-1-introduction/>`_ - A step-by-step tutorial to show how to dockerize and integrate Auth0 to PostgREST service.

 * `"CodeLess" backend using postgres, postgrest and oauth2 authentication with keycloak <https://www.mathieupassenaud.fr/codeless_backend/>`_ -
  A step-by-step tutorial for using PostgREST with KeyCloak(hosted on a managed service).
@@ -318,144 +318,6 @@ You can insert a new product using a JSON object for the ``extra_info`` column:

 To query and filter the data see :ref:`json_columns` for a complete reference.

-.. _ww_postgis:
-
-PostGIS
-------
-
-You can use the string representation for `PostGIS <https://postgis.net/>`_ data types such as ``geometry`` or ``geography`` (you need to `install PostGIS <https://postgis.net/documentation/getting_started/>`_ first).
-
-.. code-block:: postgres
-
-  -- Activate the postgis module in the current database
-  create extension if not exists postgis;
-
-  create table coverage (
-    id int primary key,
-    name text unique,
-    area geometry
-  );
-
-To add areas in polygon format, you can use string representation:
-
-.. code-block:: bash
-
-  curl "http://localhost:3000/coverage" \
-    -X POST -H "Content-Type: application/json" \
-    -d @- << EOF
-    [
-      { "id": 1, "name": "small", "area": "SRID=4326;POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))" },
-      { "id": 2, "name": "big", "area": "SRID=4326;POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))" }
-    ]
-  EOF
-
-Now, when you request the information, PostgREST will automatically cast the ``area`` column into a ``Polygon`` geometry type. Although this is useful, you may need the whole output to be in `GeoJSON <https://geojson.org/>`_ format out of the box, which can be done by including the ``Accept: application/geo+json`` in the request. This will work for PostGIS versions 3.0.0 and up and will return the output as a `FeatureCollection Object <https://www.rfc-editor.org/rfc/rfc7946#section-3.3>`_:
-
-.. code-block:: bash
-
-  curl "http://localhost:3000/coverage" \
-    -H "Accept: application/geo+json"
-
-.. code-block:: json
-
-  {
-    "type": "FeatureCollection",
-    "features": [
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Polygon",
-          "coordinates": [
-            [[0,0],[1,0],[1,1],[0,1],[0,0]]
-          ]
-        },
-        "properties": {
-          "id": 1,
-          "name": "small"
-        }
-      },
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Polygon",
-          "coordinates": [
-            [[0,0],[10,0],[10,10],[0,10],[0,0]]
-          ]
-        },
-        "properties": {
-          "id": 2,
-          "name": "big"
-        }
-      }
-    ]
-  }
-
-If you need to add an extra property, like the area in square units by using ``st_area(area)``, you could add a generated column to the table and it will appear in the ``properties`` key of each ``Feature``.
-
-.. code-block:: postgres
-
-  alter table coverage
-    add square_units double precision generated always as ( st_area(area) ) stored;
-
-In the case that you are using older PostGIS versions, then creating a function is your best option:
-
-.. code-block:: postgres
-
-  create or replace function coverage_geo_collection() returns json as $$
-    select
-      json_build_object(
-        'type', 'FeatureCollection',
-        'features', json_agg(
-          json_build_object(
-            'type', 'Feature',
-            'geometry', st_AsGeoJSON(c.area)::json,
-            'properties', json_build_object('id', c.id, 'name', c.name)
-          )
-        )
-      )
-    from coverage c;
-  $$ language sql;
-
-Now this query will return the same results:
-
-.. code-block:: bash
-
-  curl "http://localhost:3000/rpc/coverage_geo_collection"
-
-.. code-block:: json
-
-  {
-    "type": "FeatureCollection",
-    "features": [
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Polygon",
-          "coordinates": [
-            [[0,0],[1,0],[1,1],[0,1],[0,0]]
-          ]
-        },
-        "properties": {
-          "id": 1,
-          "name": "small"
-        }
-      },
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Polygon",
-          "coordinates": [
-            [[0,0],[10,0],[10,10],[0,10],[0,0]]
-          ]
-        },
-        "properties": {
-          "id": 2,
-          "name": "big"
-        }
-      }
-    ]
-  }
-
 Ranges
 ------

@@ -609,3 +471,20 @@ You can use other comparative filters and also all the `PostgreSQL special date/
      "due_date": "2022-02-27T06:00:00-05:00"
    }
  ]
+
+.. raw:: html
+
+  <script type="text/javascript">
+    let hash = window.location.hash;
+
+    const redirects = {
+      // PostGIS
+      '#postgis': '../integrations/postgis.html#postgis',
+    };
+
+    let willRedirectTo = redirects[hash];
+
+    if (willRedirectTo) {
+      window.location.href = willRedirectTo;
+    }
+  </script>
@@ -0,0 +1,137 @@
+.. _ww_postgis:
+
+PostGIS
+=======
+
+You can use the string representation for `PostGIS <https://postgis.net/>`_ data types such as ``geometry`` or ``geography`` (you need to `install PostGIS <https://postgis.net/documentation/getting_started/>`_ first).
+
+.. code-block:: postgres
+
+  -- Activate the postgis module in the current database
+  create extension if not exists postgis;
+
+  create table coverage (
+    id int primary key,
+    name text unique,
+    area geometry
+  );
+
+To add areas in polygon format, you can use string representation:
+
+.. code-block:: bash
+
+  curl "http://localhost:3000/coverage" \
+    -X POST -H "Content-Type: application/json" \
+    -d @- << EOF
+    [
+      { "id": 1, "name": "small", "area": "SRID=4326;POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))" },
+      { "id": 2, "name": "big", "area": "SRID=4326;POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))" }
+    ]
+  EOF
+
+Now, when you request the information, PostgREST will automatically cast the ``area`` column into a ``Polygon`` geometry type. Although this is useful, you may need the whole output to be in `GeoJSON <https://geojson.org/>`_ format out of the box, which can be done by including the ``Accept: application/geo+json`` in the request. This will work for PostGIS versions 3.0.0 and up and will return the output as a `FeatureCollection Object <https://www.rfc-editor.org/rfc/rfc7946#section-3.3>`_:
+
+.. code-block:: bash
+
+  curl "http://localhost:3000/coverage" \
+    -H "Accept: application/geo+json"
+
+.. code-block:: json
+
+  {
+    "type": "FeatureCollection",
+    "features": [
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Polygon",
+          "coordinates": [
+            [[0,0],[1,0],[1,1],[0,1],[0,0]]
+          ]
+        },
+        "properties": {
+          "id": 1,
+          "name": "small"
+        }
+      },
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Polygon",
+          "coordinates": [
+            [[0,0],[10,0],[10,10],[0,10],[0,0]]
+          ]
+        },
+        "properties": {
+          "id": 2,
+          "name": "big"
+        }
+      }
+    ]
+  }
+
+If you need to add an extra property, like the area in square units by using ``st_area(area)``, you could add a generated column to the table and it will appear in the ``properties`` key of each ``Feature``.
+
+.. code-block:: postgres
+
+  alter table coverage
+    add square_units double precision generated always as ( st_area(area) ) stored;
+
+In the case that you are using older PostGIS versions, then creating a function is your best option:
+
+.. code-block:: postgres
+
+  create or replace function coverage_geo_collection() returns json as $$
+    select
+      json_build_object(
+        'type', 'FeatureCollection',
+        'features', json_agg(
+          json_build_object(
+            'type', 'Feature',
+            'geometry', st_AsGeoJSON(c.area)::json,
+            'properties', json_build_object('id', c.id, 'name', c.name)
+          )
+        )
+      )
+    from coverage c;
+  $$ language sql;
+
+Now this query will return the same results:
+
+.. code-block:: bash
+
+  curl "http://localhost:3000/rpc/coverage_geo_collection"
+
+.. code-block:: json
+
+  {
+    "type": "FeatureCollection",
+    "features": [
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Polygon",
+          "coordinates": [
+            [[0,0],[1,0],[1,1],[0,1],[0,0]]
+          ]
+        },
+        "properties": {
+          "id": 1,
+          "name": "small"
+        }
+      },
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Polygon",
+          "coordinates": [
+            [[0,0],[10,0],[10,10],[0,10],[0,0]]
+          ]
+        },
+        "properties": {
+          "id": 2,
+          "name": "big"
+        }
+      }
+    ]
+  }
@@ -46,7 +46,9 @@ This will cause the :ref:`connection_pool` to connect to the read replica host a

 .. note::

-  Under the hood, PostgREST forces `target_session_attrs=read-write <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-TARGET-SESSION-ATTRS>`_ for the ``LISTEN`` session.
+  - Under the hood, PostgREST forces `target_session_attrs=read-write <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-TARGET-SESSION-ATTRS>`_ for the ``LISTEN`` session.
+    So if you specify ``target_session_attrs=read-only`` as mentioned above, PostgREST will override it for the ``LISTEN``.
+  - ``read-only`` is only available on libpq >= 14, if you use a lower version you will get an error like ``invalid target_session_attrs value: \"read-only\"``.

 .. _listener_automatic_recovery:

@@ -39,7 +39,7 @@ For diagnostic information about the server itself, PostgREST logs to ``stderr``
   06/May/2024:08:16:11 -0500: Listening for database notifications on the "pgrst" channel
   06/May/2024:08:16:11 -0500: Config reloaded
   06/May/2024:08:16:11 -0500: Schema cache queried in 3.8 milliseconds
-   06/May/2024:08:16:11 -0500: Schema cache loaded 15 Relations, 8 Relationships, 8 Functions, 0 Domain Representations, 4 Media Type Handlers
+   06/May/2024:08:16:11 -0500: Schema cache loaded 15 Relations, 8 Relationships, 8 RPCs, 0 Domain Representations, 4 Media Type Handlers
   06/May/2024:14:11:27 -0500: Received a config reload message on the "pgrst" channel
   06/May/2024:14:11:27 -0500: Config reloaded

@@ -62,7 +62,7 @@ let
        git add CHANGELOG.md > /dev/null

        echo "Committing ..."
-        git commit -m "bump version to $new_version" > /dev/null
+        git commit -m "chore: bump version to $new_version" > /dev/null

        if [[ "$current_branch" == "main" ]]; then
          bump devel
@@ -74,7 +74,7 @@ let
          git branch "v$A"

          echo "Committing (devel bump)..."
-          git commit -m "bump version to $new_version" > /dev/null
+          git commit -m "chore: bump version to $new_version" > /dev/null
        fi

        trap "echo Remote not found. Please push manually ..." ERR
@@ -10,6 +10,7 @@
 , hostPlatform
 , jq
 , lib
+, nginx
 , postgrest
 , python3
 , runtimeShell
@@ -94,6 +95,7 @@ let
        args = [ "ARG_LEFTOVERS([pytest arguments])" ];
        workingDir = "/";
        withEnv = postgrest.env;
+        withPath = [ nginx ];
      }
      ''
        ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest
@@ -156,6 +158,7 @@ let
        redirectTixFiles = false;
        withEnv = postgrest.env;
        withTmpDir = true;
+        withPath = [ nginx ];
      }
      (
        # required for `hpc markup` in CI; glibcLocales is not available e.g. on Darwin
@@ -25,7 +25,7 @@ let
            "ARG_OPTIONAL_SINGLE([fixtures], [f], [SQL file to load fixtures from])"
            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
            "ARG_LEFTOVERS([command arguments])"
-            "ARG_USE_ENV([PGUSER], [postgrest_test_authenticator], [Authenticator PG role])"
+            "ARG_USE_ENV([PGUSER], [Postgrest_Test_Authenticator], [Authenticator PG role])" # user is written in mixed case to implicitly test that it is being properly quoted in schema cache queries
            "ARG_USE_ENV([PGDATABASE], [postgres], [PG database name])"
            "ARG_USE_ENV([PGRST_DB_SCHEMAS], [test], [Schema to expose])"
            "ARG_USE_ENV([PGTZ], [utc], [Timezone to use])"
@@ -1,5 +1,5 @@
 name:               postgrest
-version:            14.9
+version:            14.12
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                    for tables, views, and functions, supporting all HTTP methods that security
@@ -157,6 +157,7 @@ library
                    , stm-hamt                  >= 1.2 && < 2
                    , focus                     >= 1.0 && < 2
                    , some                      >= 1.0.4.1 && < 2
+                    , uuid                      >= 1.3 && < 2
                      -- -fno-spec-constr may help keep compile time memory use in check,
                      --   see https://gitlab.haskell.org/ghc/ghc/issues/16017#note_219304
                      -- -optP-Wno-nonportable-include-path
@@ -304,6 +305,7 @@ test-suite observability
  other-modules:      ObsHelper
                      Observation.JwtCache
                      Observation.MetricsSpec
+                      Observation.SchemaCacheSpec
  build-depends:      base              >= 4.9 && < 4.20
                    , base64-bytestring >= 1 && < 1.3
                    , bytestring        >= 0.10.8 && < 0.13
@@ -219,10 +219,14 @@ usePool AppState{stateObserver=observer, stateMainThreadId=mainThreadId, ..} ses

 -- | Flush the connection pool so that any future use of the pool will
 -- use connections freshly established after this call.
+-- | Emits PoolFlushed observation
 flushPool :: AppState -> IO ()
-flushPool AppState{..} = SQL.release statePool
+flushPool AppState{..} = do
+  SQL.release statePool
+  stateObserver PoolFlushed

 -- | Destroy the pool on shutdown.
+-- | Differs from flushPool in not emiting PoolFlushed observation.
 destroyPool :: AppState -> IO ()
 destroyPool AppState{..} = SQL.release statePool

@@ -309,8 +313,6 @@ retryingSchemaCacheLoad appState@AppState{stateObserver=observer, stateMainThrea
      observer $ ConnectionRetryObs delay
      putNextListenerDelay appState delay

-    flushPool appState
-
    (,) <$> qPgVersion <*> (qInDbConfig *> qSchemaCache)
  )
  where
@@ -359,6 +361,10 @@ retryingSchemaCacheLoad appState@AppState{stateObserver=observer, stateMainThrea
          -- IORef on putSchemaCache. This is why SCacheStatus is put at SCPending here to signal the Admin server (using isPending) that we're on a recovery state.
          putSCacheStatus appState SCPending
          putSchemaCache appState $ Just sCache
+          -- Flush the pool after loading the schema cache to reset any stale session cache entries
+          -- We do it after successfully querying the schema cache (because this can fail and during retries we would flush the pool repeatedly unnecessarily)
+          -- and after marking sCacheStatus as pending,
+          flushPool appState
          observer $ SchemaCacheQueriedObs resultTime
          (t, _) <- timeItT $ observer $ SchemaCacheSummaryObs $ showSummary sCache
          observer $ SchemaCacheLoadedObs t
@@ -101,7 +101,7 @@ queryDbSettings preConfFunc prepared =
        SELECT setdatabase as database,
               unnest(setconfig) as setting
        FROM pg_catalog.pg_db_role_setting
-        WHERE setrole = CURRENT_USER::regrole::oid
+        WHERE setrole = quote_ident(CURRENT_USER)::regrole::oid
          AND setdatabase IN (0, (SELECT oid FROM pg_catalog.pg_database WHERE datname = CURRENT_CATALOG))
      ),
      kv_settings AS (
@@ -142,7 +142,7 @@ queryRoleSettings pgVer prepared =
        select r.rolname, unnest(r.rolconfig) as setting
        from pg_auth_members m
        join pg_roles r on r.oid = m.roleid
-        where member = current_user::regrole::oid
+        where member = quote_ident(current_user)::regrole::oid
      ),
      kv_settings AS (
        SELECT
@@ -167,7 +167,7 @@ queryRoleSettings pgVer prepared =
    |]

    hasParameterPrivilege
-      | pgVer >= pgVersion150 = "or has_parameter_privilege(current_user::regrole::oid, ps.name, 'set')"
+      | pgVer >= pgVersion150 = "or has_parameter_privilege(quote_ident(current_user)::regrole::oid, ps.name, 'set')"
      | otherwise             = ""

    processRows :: [(Text, Maybe Text, [(Text, Text)])] -> (RoleSettings, RoleIsolationLvl)
@@ -10,8 +10,7 @@ import qualified Hasql.Connection      as SQL
 import qualified Hasql.Notifications   as SQL
 import           PostgREST.AppState    (AppState, getConfig)
 import           PostgREST.Config      (AppConfig (..))
-import           PostgREST.Observation (Observation (..),
-                                        isDbListenerBug)
+import           PostgREST.Observation (Observation (..))
 import           PostgREST.Version     (prettyVersion)

 import qualified PostgREST.AppState as AppState
@@ -20,6 +19,7 @@ import qualified PostgREST.Config   as Config
 import           Control.Arrow              ((&&&))
 import           Data.Bitraversable         (bisequence)
 import           Data.Either.Combinators    (whenRight)
+import qualified Data.Text                  as T
 import qualified Database.PostgreSQL.LibPQ  as LibPQ
 import qualified Hasql.Session              as SQL
 import           PostgREST.Config.Database  (queryPgVersion)
@@ -31,12 +31,12 @@ runListener :: AppState -> IO ()
 runListener appState = do
  AppConfig{..} <- getConfig appState
  when configDbChannelEnabled $
-    void . forkIO . void $ retryingListen appState
+    void . forkIO . void $ retryingListen appState False

 -- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
 -- | This function never returns (but can throw) and return type enforces that.
-retryingListen :: AppState -> IO Void
-retryingListen appState = do
+retryingListen :: AppState -> Bool -> IO Void
+retryingListen appState hasDbListenerBug = do
  AppConfig{..} <- AppState.getConfig appState
  let
    dbChannel = toS configDbChannel
@@ -44,7 +44,7 @@ retryingListen appState = do
      AppState.putIsListenerOn appState False
      observer $ DBListenFail dbChannel (Right err)
      when (isDbListenerBug err) $
-        observer DBListenBugHint
+        observer DBListenBugCallQueryFix
      unless configDbPoolAutomaticRecovery $
        killThread mainThreadId

@@ -55,7 +55,7 @@ retryingListen appState = do
      unless (delay == maxDelay) $
        AppState.putNextListenerDelay appState (delay * 2)
      -- loop running the listener
-      retryingListen appState
+      retryingListen appState (isDbListenerBug err)

  -- Execute the listener with with error handling
  handle onError $ do
@@ -68,9 +68,10 @@ retryingListen appState = do
      -- use connection
      \case
        Right db -> do
-          SQL.listen db $ SQL.toPgIdentifier dbChannel
          (pqHost, pqPort) <- SQL.withLibPQConnection db $ bisequence . (LibPQ.host &&& LibPQ.port)
          pgFullName <- SQL.run (queryPgVersion False) db >>= either throwIO (pure . pgvFullName)
+          when hasDbListenerBug $ SQL.run callNotifQueryUsage db >>= either throwIO pure
+          SQL.listen db $ SQL.toPgIdentifier dbChannel

          AppState.putIsListenerOn appState True

@@ -106,3 +107,10 @@ retryingListen appState = do
      AppState.schemaCacheLoader appState

    releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release
+
+    isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e
+
+    -- Used to fix a Postgres bug in the listener, see: https://github.com/PostgREST/postgrest/issues/3147#issuecomment-3494591361
+    -- This query advances the async notification query tail, which solves this issue.
+    callNotifQueryUsage :: SQL.Session ()
+    callNotifQueryUsage = SQL.sql "SELECT pg_notification_queue_usage();"
@@ -1,3 +1,4 @@
+{-# LANGUAGE LambdaCase      #-}
 {-# LANGUAGE RecordWildCards #-}
 {-|
 Module      : PostgREST.Logger
@@ -35,7 +36,14 @@ import PostgREST.Config      (LogLevel (..))
 import PostgREST.Observation
 import PostgREST.Query       (MainQuery (..))

-import Protolude
+import qualified Data.ByteString.Lazy       as LBS
+import qualified Data.Text                  as T
+import qualified Hasql.Connection           as SQL
+import qualified Hasql.Pool.Observation     as SQL
+import           Numeric                    (showFFloat)
+import           PostgREST.Config.PgVersion (pgvName)
+import qualified PostgREST.Error            as Error
+import           Protolude

 data LoggerState = LoggerState
  { stateGetZTime               :: IO ZonedTime  -- ^ Time with time zone used for logs
@@ -110,6 +118,9 @@ observationLogger loggerState logLevel obs = case obs of
  o@PoolRequestFullfilled ->
    when (logLevel >= LogDebug) $ do
      logWithZTime loggerState $ observationMessage o
+  o@PoolFlushed ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessage o
  o@JwtCacheEviction ->
    when (logLevel >= LogDebug) $ do
      logWithZTime loggerState $ observationMessage o
@@ -143,3 +154,117 @@ renderSnippet snippet =
      prepared = False  -- unused
  in
    sql
+
+
+observationMessage :: Observation -> Text
+observationMessage = \case
+  AdminStartObs address ->
+    "Admin server listening on " <> address
+  AppStartObs ver ->
+    "Starting PostgREST " <> T.decodeUtf8 ver <> "..."
+  AppServerAddressObs address ->
+    "API server listening on " <> address
+  DBConnectedObs ver ->
+    "Successfully connected to " <> ver
+  ExitUnsupportedPgVersion pgVer minPgVer ->
+    "Cannot run in this PostgreSQL version (" <> pgvName pgVer <> "), PostgREST needs at least " <> pgvName minPgVer
+  ExitDBNoRecoveryObs ->
+    "Automatic recovery disabled, exiting."
+  ExitDBFatalError ServerAuthError usageErr ->
+    "Failed to establish a connection. " <> jsonMessage usageErr
+  ExitDBFatalError ServerPgrstBug usageErr ->
+    "This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues. " <> jsonMessage usageErr
+  ExitDBFatalError ServerError42P05 usageErr ->
+    "If you are using connection poolers in transaction mode, try setting db-prepared-statements to false. " <> jsonMessage usageErr
+  ExitDBFatalError ServerError08P01 usageErr ->
+    "Connection poolers in statement mode are not supported." <> jsonMessage usageErr
+  SchemaCacheEmptyObs ->
+    T.decodeUtf8 . LBS.toStrict . Error.errorPayload $ Error.NoSchemaCacheError
+  SchemaCacheErrorObs dbSchemas extraPaths usageErr ->
+    "Failed to load the schema cache using "
+      <> "db-schemas=" <> T.intercalate "," (toList dbSchemas)
+      <> " and "
+      <> "db-extra-search-path=" <> T.intercalate "," extraPaths <> ". " <> jsonMessage usageErr
+  SchemaCacheQueriedObs resultTime ->
+    "Schema cache queried in " <> showMillis resultTime  <> " milliseconds"
+  SchemaCacheSummaryObs summary ->
+    "Schema cache loaded " <> summary
+  SchemaCacheLoadedObs resultTime ->
+    "Schema cache loaded in " <> showMillis resultTime <> " milliseconds"
+  ConnectionRetryObs delay ->
+    "Attempting to reconnect to the database in " <> (show delay::Text) <> " seconds..."
+  QueryPgVersionError usageErr ->
+    "Failed to query the PostgreSQL version. " <> jsonMessage usageErr
+  DBListenStart host port fullName channel -> do
+    "Listener connected to " <> fullName <> " on " <> show (fold $ host <> fmap (":" <>) port) <> " and listening for database notifications on the " <> show channel <> " channel"
+  DBListenFail channel listenErr ->
+    "Failed listening for database notifications on the " <> show channel <> " channel. " <>
+      either showListenerConnError showListenerException listenErr
+  DBListenRetry delay ->
+    "Retrying listening for database notifications in " <> (show delay::Text) <> " seconds..."
+  DBListenBugCallQueryFix ->
+    "This is likely a PostgreSQL bug in the notification queue, executing the following to try to solve it: SELECT pg_notification_queue_usage();"
+  DBListenerGotSCacheMsg channel ->
+    "Received a schema cache reload message on the " <> show channel <> " channel"
+  DBListenerGotConfigMsg channel ->
+    "Received a config reload message on the " <> show channel <> " channel"
+  DBListenerConnectionCleanupFail ex ->
+    "Failed during listener connection cleanup: " <> showOnSingleLine '\t' (show ex)
+  QueryObs{} ->
+    mempty -- TODO pending refactor: The logic for printing the query cannot be done here. Join the observationMessage function into observationLogger to avoid this mempty.
+  ConfigReadErrorObs usageErr ->
+    "Failed to query database settings for the config parameters." <> jsonMessage usageErr
+  QueryRoleSettingsErrorObs usageErr ->
+    "Failed to query the role settings. " <> jsonMessage usageErr
+  QueryErrorCodeHighObs usageErr ->
+    jsonMessage usageErr
+  ConfigInvalidObs err ->
+    "Failed reloading config: " <> err
+  ConfigSucceededObs ->
+     "Config reloaded"
+  PoolInit poolSize ->
+     "Connection Pool initialized with a maximum size of " <> show poolSize <> " connections"
+  PoolAcqTimeoutObs usageErr ->
+    jsonMessage usageErr
+  HasqlPoolObs (SQL.ConnectionObservation uuid status) ->
+    "Connection " <> show uuid <> (
+      case status of
+        SQL.ConnectingConnectionStatus   -> " is being established"
+        SQL.ReadyForUseConnectionStatus  -> " is available"
+        SQL.InUseConnectionStatus        -> " is used"
+        SQL.TerminatedConnectionStatus reason -> " is terminated due to " <> case reason of
+          SQL.AgingConnectionTerminationReason          -> "max lifetime"
+          SQL.IdlenessConnectionTerminationReason       -> "max idletime"
+          SQL.ReleaseConnectionTerminationReason        -> "release"
+          SQL.NetworkErrorConnectionTerminationReason _ -> "network error" -- usage error is already logged, no need to repeat the same message.
+    )
+  PoolRequest ->
+    "Trying to borrow a connection from pool"
+  PoolRequestFullfilled ->
+    "Borrowed a connection from the pool"
+  PoolFlushed ->
+    "Database connection pool flushed"
+  JwtCacheLookup _ ->
+    "Looked up a JWT in JWT cache"
+  JwtCacheEviction ->
+    "Evicted entry from JWT cache"
+  TerminationUnixSignalObs signal ->
+    "Received termination unix signal " <> signal
+  WarpServerObs txt ->
+    "Warp server: " <> txt
+  where
+    showMillis :: Double -> Text
+    showMillis x = toS $ showFFloat (Just 1) x ""
+
+    jsonMessage err = T.decodeUtf8 . LBS.toStrict . Error.errorPayload $ Error.PgError False err
+
+
+    showListenerConnError :: SQL.ConnectionError -> Text
+    showListenerConnError = maybe "Connection error" (showOnSingleLine '\t' . T.decodeUtf8)
+
+    showListenerException :: SomeException -> Text
+    showListenerException = showOnSingleLine '\t' . show
+
+
+showOnSingleLine :: Char -> Text -> Text
+showOnSingleLine split txt = T.intercalate " " $ T.filter (/= split) <$> T.lines txt -- the errors from hasql-notifications come intercalated with "\t\n"
@@ -5,7 +5,10 @@ Description : Metrics based on the Observation module. See Observation.hs.
 -}
 module PostgREST.Metrics
  ( init
+  , ConnTrack
+  , ConnStats (..)
  , MetricsState (..)
+  , connectionCounts
  , observationMetrics
  , metricsToText
  ) where
@@ -17,12 +20,18 @@ import Prometheus

 import PostgREST.Observation

-import Protolude
+import           Control.Arrow      ((&&&))
+import           Data.Bitraversable (bisequenceA)
+import           Data.Tuple.Extra   (both)
+import           Data.UUID          (UUID)
+import qualified Focus
+import           Protolude
+import qualified StmHamt.SizedHamt  as SH

 data MetricsState =
  MetricsState {
    poolTimeouts         :: Counter,
-    poolAvailable        :: Gauge,
+    connTrack            :: ConnTrack,
    poolWaiting          :: Gauge,
    poolMaxSize          :: Gauge,
    schemaCacheLoads     :: Vector Label1 Counter,
@@ -36,7 +45,7 @@ init :: Int -> IO MetricsState
 init configDbPoolSize = do
  metricState <- MetricsState <$>
    register (counter (Info "pgrst_db_pool_timeouts_total" "The total number of pool connection timeouts")) <*>
-    register (gauge (Info "pgrst_db_pool_available" "Available connections in the pool")) <*>
+    register (Metric ((identity &&& dbPoolAvailable) <$> connectionTracker)) <*>
    register (gauge (Info "pgrst_db_pool_waiting" "Requests waiting to acquire a pool connection")) <*>
    register (gauge (Info "pgrst_db_pool_max" "Max pool connections")) <*>
    register (vector "status" $ counter (Info "pgrst_schema_cache_loads_total" "The total number of times the schema cache was loaded")) <*>
@@ -46,20 +55,28 @@ init configDbPoolSize = do
    register (counter (Info "pgrst_jwt_cache_evictions_total" "The total number of JWT cache evictions"))
  setGauge (poolMaxSize metricState) (fromIntegral configDbPoolSize)
  pure metricState
+  where
+    dbPoolAvailable = (pure . noLabelsGroup (Info "pgrst_db_pool_available" "Available connections in the pool") GaugeType . calcAvailable <$>) . connectionCounts
+      where
+        calcAvailable = liftA2 (-) connected inUse
+    toSample name labels = Sample name labels . encodeUtf8 . show
+    noLabelsGroup info sampleType = SampleGroup info sampleType . pure . toSample (metricName info) mempty

 -- Only some observations are used as metrics
 observationMetrics :: MetricsState -> ObservationHandler
 observationMetrics MetricsState{..} obs = case obs of
  (PoolAcqTimeoutObs _) -> do
    incCounter poolTimeouts
-  (HasqlPoolObs (SQL.ConnectionObservation _ status)) -> case status of
-     SQL.ReadyForUseConnectionStatus  -> do
-      incGauge poolAvailable
-     SQL.InUseConnectionStatus        -> do
-      decGauge poolAvailable
-     SQL.TerminatedConnectionStatus  _ -> do
-      decGauge poolAvailable
-     SQL.ConnectingConnectionStatus -> pure ()
+  -- Handle pool observations with connection tracking
+  -- this is necessary because it is not possible
+  -- to accurately maintain open/in use conneciton counts
+  -- statelessly based only on pool observation events.
+  -- The reason is that hasql-pool emits TerminatedConnectionStatus
+  -- both for connections successfully established and failed when connecting.
+  -- When receiving TerminatedConnectionStatus we have to find out
+  -- if we can decrement established connection count. To do that we have to track
+  -- established connections.
+  (HasqlPoolObs sqlObs) -> trackConnections connTrack sqlObs
  PoolRequest ->
    incGauge poolWaiting
  PoolRequestFullfilled ->
@@ -77,3 +94,28 @@ observationMetrics MetricsState{..} obs = case obs of

 metricsToText :: IO LBS.ByteString
 metricsToText = exportMetricsAsText
+
+data ConnStats = ConnStats {
+    connected :: Int,
+    inUse     :: Int
+} deriving (Eq, Show)
+
+data ConnTrack = ConnTrack { connTrackConnected :: SH.SizedHamt UUID, connTrackInUse :: SH.SizedHamt UUID }
+
+connectionTracker :: IO ConnTrack
+connectionTracker =  ConnTrack <$> SH.newIO <*> SH.newIO
+
+trackConnections :: ConnTrack -> SQL.Observation -> IO ()
+trackConnections ConnTrack{..} (SQL.ConnectionObservation uuid status) = case status of
+  SQL.ReadyForUseConnectionStatus -> atomically $
+    SH.insert identity uuid connTrackConnected *>
+    SH.focus Focus.delete identity uuid connTrackInUse
+  SQL.TerminatedConnectionStatus _ -> atomically $
+    SH.focus Focus.delete identity uuid connTrackConnected *>
+    SH.focus Focus.delete identity uuid connTrackInUse
+  SQL.InUseConnectionStatus -> atomically $
+    SH.insert identity uuid connTrackInUse
+  _ -> mempty
+
+connectionCounts :: ConnTrack -> IO ConnStats
+connectionCounts = atomically . fmap (uncurry ConnStats) . bisequenceA . both SH.size . (connTrackConnected &&& connTrackInUse)
@@ -1,5 +1,4 @@
 {-# LANGUAGE DeriveGeneric #-}
-{-# LANGUAGE LambdaCase    #-}
 {-|
 Module      : PostgREST.Observation
 Description : This module holds an Observation type which is the core of Observability for PostgREST.
@@ -10,23 +9,14 @@ Description : This module holds an Observation type which is the core of Observa
 module PostgREST.Observation
  ( Observation(..)
  , ObsFatalError(..)
-  , observationMessage
  , ObservationHandler
-  , showOnSingleLine
-  , isDbListenerBug
  ) where

-import qualified Data.ByteString.Lazy       as LBS
-import           Data.List.NonEmpty         (toList)
-import qualified Data.Text                  as T
-import qualified Data.Text.Encoding         as T
 import qualified Hasql.Connection           as SQL
 import qualified Hasql.Pool                 as SQL
 import qualified Hasql.Pool.Observation     as SQL
 import           Network.HTTP.Types.Status  (Status)
-import           Numeric                    (showFFloat)
 import           PostgREST.Config.PgVersion
-import qualified PostgREST.Error            as Error
 import           PostgREST.Query            (MainQuery)

 import Protolude hiding (toList)
@@ -48,7 +38,7 @@ data Observation
  | DBListenStart (Maybe ByteString) (Maybe ByteString) Text Text -- host, port, version string, channel
  | DBListenFail Text (Either SQL.ConnectionError SomeException)
  | DBListenRetry Int
-  | DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147
+  | DBListenBugCallQueryFix
  | DBListenerGotSCacheMsg ByteString
  | DBListenerGotConfigMsg ByteString
  | DBListenerConnectionCleanupFail SomeException
@@ -64,6 +54,7 @@ data Observation
  | HasqlPoolObs SQL.Observation
  | PoolRequest
  | PoolRequestFullfilled
+  | PoolFlushed
  | JwtCacheLookup Bool
  | JwtCacheEviction
  | TerminationUnixSignalObs Text
@@ -73,118 +64,3 @@ data Observation
 data ObsFatalError = ServerAuthError | ServerPgrstBug | ServerError42P05 | ServerError08P01

 type ObservationHandler = Observation -> IO ()
-
-observationMessage :: Observation -> Text
-observationMessage = \case
-  AdminStartObs address ->
-    "Admin server listening on " <> address
-  AppStartObs ver ->
-    "Starting PostgREST " <> T.decodeUtf8 ver <> "..."
-  AppServerAddressObs address ->
-    "API server listening on " <> address
-  DBConnectedObs ver ->
-    "Successfully connected to " <> ver
-  ExitUnsupportedPgVersion pgVer minPgVer ->
-    "Cannot run in this PostgreSQL version (" <> pgvName pgVer <> "), PostgREST needs at least " <> pgvName minPgVer
-  ExitDBNoRecoveryObs ->
-    "Automatic recovery disabled, exiting."
-  ExitDBFatalError ServerAuthError usageErr ->
-    "Failed to establish a connection. " <> jsonMessage usageErr
-  ExitDBFatalError ServerPgrstBug usageErr ->
-    "This is probably a bug in PostgREST, please report it at https://github.com/PostgREST/postgrest/issues. " <> jsonMessage usageErr
-  ExitDBFatalError ServerError42P05 usageErr ->
-    "If you are using connection poolers in transaction mode, try setting db-prepared-statements to false. " <> jsonMessage usageErr
-  ExitDBFatalError ServerError08P01 usageErr ->
-    "Connection poolers in statement mode are not supported." <> jsonMessage usageErr
-  SchemaCacheEmptyObs ->
-    T.decodeUtf8 . LBS.toStrict . Error.errorPayload $ Error.NoSchemaCacheError
-  SchemaCacheErrorObs dbSchemas extraPaths usageErr ->
-    "Failed to load the schema cache using "
-      <> "db-schemas=" <> T.intercalate "," (toList dbSchemas)
-      <> " and "
-      <> "db-extra-search-path=" <> T.intercalate "," extraPaths
-      <> ". " <> jsonMessage usageErr
-  SchemaCacheQueriedObs resultTime ->
-    "Schema cache queried in " <> showMillis resultTime  <> " milliseconds"
-  SchemaCacheSummaryObs summary ->
-    "Schema cache loaded " <> summary
-  SchemaCacheLoadedObs resultTime ->
-    "Schema cache loaded in " <> showMillis resultTime <> " milliseconds"
-  ConnectionRetryObs delay ->
-    "Attempting to reconnect to the database in " <> (show delay::Text) <> " seconds..."
-  QueryPgVersionError usageErr ->
-    "Failed to query the PostgreSQL version. " <> jsonMessage usageErr
-  DBListenStart host port fullName channel -> do
-    "Listener connected to " <> fullName <> " on " <> show (fold $ host <> fmap (":" <>) port) <> " and listening for database notifications on the " <> show channel <> " channel"
-  DBListenFail channel listenErr ->
-    "Failed listening for database notifications on the " <> show channel <> " channel. " <>
-      either showListenerConnError showListenerException listenErr
-  DBListenRetry delay ->
-    "Retrying listening for database notifications in " <> (show delay::Text) <> " seconds..."
-  DBListenBugHint ->
-    "HINT:  This is likely a bug in the notification queue, try executing the following to solve it: select pg_notification_queue_usage();"
-  DBListenerGotSCacheMsg channel ->
-    "Received a schema cache reload message on the " <> show channel <> " channel"
-  DBListenerGotConfigMsg channel ->
-    "Received a config reload message on the " <> show channel <> " channel"
-  DBListenerConnectionCleanupFail ex ->
-    "Failed during listener connection cleanup: " <> showOnSingleLine '\t' (show ex)
-  QueryObs{} ->
-    mempty -- TODO pending refactor: The logic for printing the query cannot be done here. Join the observationMessage function into observationLogger to avoid this mempty.
-  ConfigReadErrorObs usageErr ->
-    "Failed to query database settings for the config parameters." <> jsonMessage usageErr
-  QueryRoleSettingsErrorObs usageErr ->
-    "Failed to query the role settings. " <> jsonMessage usageErr
-  QueryErrorCodeHighObs usageErr ->
-    jsonMessage usageErr
-  ConfigInvalidObs err ->
-    "Failed reloading config: " <> err
-  ConfigSucceededObs ->
-     "Config reloaded"
-  PoolInit poolSize ->
-     "Connection Pool initialized with a maximum size of " <> show poolSize <> " connections"
-  PoolAcqTimeoutObs usageErr ->
-    jsonMessage usageErr
-  HasqlPoolObs (SQL.ConnectionObservation uuid status) ->
-    "Connection " <> show uuid <> (
-      case status of
-        SQL.ConnectingConnectionStatus   -> " is being established"
-        SQL.ReadyForUseConnectionStatus  -> " is available"
-        SQL.InUseConnectionStatus        -> " is used"
-        SQL.TerminatedConnectionStatus reason -> " is terminated due to " <> case reason of
-          SQL.AgingConnectionTerminationReason          -> "max lifetime"
-          SQL.IdlenessConnectionTerminationReason       -> "max idletime"
-          SQL.ReleaseConnectionTerminationReason        -> "release"
-          SQL.NetworkErrorConnectionTerminationReason _ -> "network error" -- usage error is already logged, no need to repeat the same message.
-    )
-  PoolRequest ->
-    "Trying to borrow a connection from pool"
-  PoolRequestFullfilled ->
-    "Borrowed a connection from the pool"
-  JwtCacheLookup _ ->
-    "Looked up a JWT in JWT cache"
-  JwtCacheEviction ->
-    "Evicted entry from JWT cache"
-  TerminationUnixSignalObs signal ->
-    "Received termination unix signal " <> signal
-  WarpServerObs txt ->
-    "Warp server: " <> txt
-  where
-    showMillis :: Double -> Text
-    showMillis x = toS $ showFFloat (Just 1) x ""
-
-    jsonMessage err = T.decodeUtf8 . LBS.toStrict . Error.errorPayload $ Error.PgError False err
-
-
-    showListenerConnError :: SQL.ConnectionError -> Text
-    showListenerConnError = maybe "Connection error" (showOnSingleLine '\t' . T.decodeUtf8)
-
-    showListenerException :: SomeException -> Text
-    showListenerException = showOnSingleLine '\t' . show
-
-
-showOnSingleLine :: Char -> Text -> Text
-showOnSingleLine split txt = T.intercalate " " $ T.filter (/= split) <$> T.lines txt -- the errors from hasql-notifications come intercalated with "\t\n"
-
-isDbListenerBug :: SomeException -> Bool
-isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e
@@ -102,7 +102,7 @@ showSummary (SchemaCache tbls rels routs reps mediaHdlrs tzs _) =
  T.intercalate ", "
  [ show (HM.size tbls)       <> " Relations"
  , show (HM.size rels)       <> " Relationships"
-  , show (HM.size routs)      <> " Functions"
+  , show (HM.size routs)      <> " RPCs"
  , show (HM.size reps)       <> " Domain Representations"
  , show (HM.size mediaHdlrs) <> " Media Type Handlers"
  , show (S.size tzs)         <> " Timezones"
@@ -10,6 +10,7 @@ CONFIGSDIR = BASEDIR / "configs"
 FIXTURES = yaml.load(
    (BASEDIR / "fixtures/fixtures.yaml").read_text(), Loader=yaml.Loader
 )
+NGINX_BIN = shutil.which("nginx")
 POSTGREST_BIN = shutil.which("postgrest")
 SECRET = "reallyreallyreallyreallyverysafe"

@@ -11399,7 +11399,7 @@ $$;
 DROP ROLE IF EXISTS postgrest_test_anonymous;
 CREATE ROLE postgrest_test_anonymous;

-GRANT postgrest_test_anonymous TO :PGUSER;
+GRANT postgrest_test_anonymous TO :"PGUSER";

 GRANT USAGE ON SCHEMA apflora TO postgrest_test_anonymous;
 GRANT USAGE ON SCHEMA fuzzysearch TO postgrest_test_anonymous;
@@ -4,7 +4,7 @@ set check_function_bodies = false; -- to allow conditionals based on the pg vers
 set search_path to public;

 CREATE ROLE postgrest_test_anonymous;
-ALTER ROLE :PGUSER SET pgrst.db_anon_role = 'postgrest_test_anonymous';
+ALTER ROLE :"PGUSER" SET pgrst.db_anon_role = 'postgrest_test_anonymous';

 CREATE ROLE postgrest_test_author;

@@ -21,14 +21,14 @@ alter role postgrest_test_w_superuser_settings set log_min_messages = 'fatal';
 DO $do$BEGIN
  IF (SELECT current_setting('server_version_num')::INT >= 150000) THEN
    ALTER ROLE postgrest_test_w_superuser_settings SET log_min_duration_sample = 12345;
-    GRANT SET ON PARAMETER log_min_duration_sample to postgrest_test_authenticator;
+    GRANT SET ON PARAMETER log_min_duration_sample to "Postgrest_Test_Authenticator";
  END IF;
 END$do$;

 GRANT
  postgrest_test_anonymous, postgrest_test_author,
  postgrest_test_serializable, postgrest_test_repeatable_read,
-  postgrest_test_w_superuser_settings TO :PGUSER;
+  postgrest_test_w_superuser_settings TO :"PGUSER";

 CREATE SCHEMA v1;
 GRANT USAGE ON SCHEMA v1 TO postgrest_test_anonymous;
@@ -57,7 +57,7 @@ $$ language sql;
 create function change_max_rows_config(val int, notify bool default false) returns void as $_$
 begin
  execute format($$
-    alter role postgrest_test_authenticator set pgrst.db_max_rows = %L;
+    alter role "Postgrest_Test_Authenticator" set pgrst.db_max_rows = %L;
  $$, val);
  if notify then
    perform pg_notify('pgrst', 'reload config');
@@ -66,13 +66,13 @@ end $_$ volatile security definer language plpgsql ;

 create function reset_max_rows_config() returns void as $_$
 begin
-  alter role postgrest_test_authenticator reset pgrst.db_max_rows;
+  alter role "Postgrest_Test_Authenticator" reset pgrst.db_max_rows;
 end $_$ volatile security definer language plpgsql ;

 create function change_db_schema_and_full_reload(schemas text) returns void as $_$
 begin
  execute format($$
-    alter role postgrest_test_authenticator set pgrst.db_schemas = %L;
+    alter role "Postgrest_Test_Authenticator" set pgrst.db_schemas = %L;
  $$, schemas);
  perform pg_notify('pgrst', 'reload config');
  perform pg_notify('pgrst', 'reload schema');
@@ -80,14 +80,14 @@ end $_$ volatile security definer language plpgsql ;

 create function v1.reset_db_schema_config() returns void as $_$
 begin
-  alter role postgrest_test_authenticator reset pgrst.db_schemas;
+  alter role "Postgrest_Test_Authenticator" reset pgrst.db_schemas;
  perform pg_notify('pgrst', 'reload config');
  perform pg_notify('pgrst', 'reload schema');
 end $_$ volatile security definer language plpgsql ;

 create function invalid_role_claim_key_reload() returns void as $_$
 begin
-  alter role postgrest_test_authenticator set pgrst.jwt_role_claim_key = 'test';
+  alter role "Postgrest_Test_Authenticator" set pgrst.jwt_role_claim_key = 'test';
  perform pg_notify('pgrst', 'reload config');
 end $_$ volatile security definer language plpgsql ;

@@ -100,7 +100,7 @@ $_$ language sql;

 create function reset_invalid_role_claim_key() returns void as $_$
 begin
-  alter role postgrest_test_authenticator reset pgrst.jwt_role_claim_key;
+  alter role "Postgrest_Test_Authenticator" reset pgrst.jwt_role_claim_key;
  perform pg_notify('pgrst', 'reload config');
 end $_$ volatile security definer language plpgsql ;

@@ -235,12 +235,12 @@ $$ language sql;

 create function change_db_schemas_config() returns void as $_$
 begin
-  alter role postgrest_test_authenticator set pgrst.db_schemas = 'test';
+  alter role "Postgrest_Test_Authenticator" set pgrst.db_schemas = 'test';
 end $_$ volatile security definer language plpgsql;

 create function reset_db_schemas_config() returns void as $_$
 begin
-  alter role postgrest_test_authenticator reset pgrst.db_schemas;
+  alter role "Postgrest_Test_Authenticator" reset pgrst.db_schemas;
 end $_$ volatile security definer language plpgsql ;

 create function test.get_current_schema() returns text as $$
@@ -13,7 +13,7 @@ create table replica.items as select x as id from generate_series(1, 10) x;
 DROP ROLE IF EXISTS postgrest_test_anonymous;
 CREATE ROLE postgrest_test_anonymous;

-GRANT postgrest_test_anonymous TO :PGUSER;
+GRANT postgrest_test_anonymous TO :"PGUSER";

 GRANT USAGE ON SCHEMA replica TO postgrest_test_anonymous;

@@ -0,0 +1,13 @@
+# the PG* variables are replaced by preprocessing, not done by nginx itself
+daemon off;
+pid        ./nginx.pid;
+
+events {}
+
+stream {
+  server {
+    listen unix:$PGPROXYHOST/.s.PGSQL.5432;
+    proxy_timeout $PGPROXY_TIMEOUT;
+    proxy_pass unix:$PGHOST/.s.PGSQL.5432;
+  }
+}
@@ -2,18 +2,20 @@

 import contextlib
 import dataclasses
+import enum
 import os
 import pathlib
 import socket
 import subprocess
 import tempfile
 import time
+import string
 import urllib.parse

 import requests
 import requests_unixsocket

-from config import POSTGREST_BIN, hpctixfile
+from config import POSTGREST_BIN, NGINX_BIN, hpctixfile


 def sleep_until_postgrest_scache_reload():
@@ -35,6 +37,13 @@ class PostgrestTimedOut(Exception):
    "Connecting to PostgREST endpoint timed out."


+class Admin(str, enum.Enum):
+    "Admin endpoint to wait for before yielding a PostgREST process."
+
+    live = "live"
+    ready = "ready"
+
+
 class PostgrestSession(requests_unixsocket.Session):
    "HTTP client session directed at a PostgREST endpoint."

@@ -86,7 +95,7 @@ def run(
    env=None,
    port=None,
    host=None,
-    wait_for_readiness=True,
+    wait_for=Admin.ready,
    wait_max_seconds=1,
    no_pool_connection_available=False,
    no_startup_stdout=True,
@@ -138,8 +147,10 @@ def run(
            process.stdin.write(stdin or b"")
            process.stdin.close()

-            if wait_for_readiness:
+            if wait_for == Admin.ready:
                wait_until_status_code(adminurl + "/ready", wait_max_seconds, 200)
+            elif wait_for == Admin.live:
+                wait_until_status_code(adminurl + "/live", wait_max_seconds, 200)

            if no_startup_stdout:
                process.stdout.read()
@@ -165,6 +176,52 @@ def run(
                process.wait()


+@contextlib.contextmanager
+def run_pgproxy(env=None, proxy_timeout="1s"):
+    "Run nginx as a unix socket proxy for PostgreSQL and expose PGPROXYHOST."
+    env = dict(os.environ if env is None else env)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # build a <tmpdir>/conf/ so `nginx -p` picks the config automatically
+        tmpdir = pathlib.Path(tmpdir)
+        conf_dir = tmpdir / "conf"
+        conf_dir.mkdir(parents=True)
+
+        nginx_env = dict(env)
+        nginx_env["PGPROXYHOST"] = str(tmpdir)
+        nginx_env["PGPROXY_TIMEOUT"] = proxy_timeout
+
+        source_conf = pathlib.Path("test/io/nginx/nginx.conf")
+        out_conf = conf_dir / "nginx.conf"
+        out_conf.write_text(
+            string.Template(source_conf.read_text()).substitute(nginx_env)
+        )
+
+        process = subprocess.Popen(
+            [NGINX_BIN, "-p", str(tmpdir), "-e", "stderr"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            env=nginx_env,
+        )
+
+        if process.poll() is not None:
+            (_, stderr_output) = process.communicate(timeout=1)
+            raise RuntimeError(
+                f"{NGINX_BIN} exited with {process.returncode}: {stderr_output}"
+            )
+
+        try:
+            yield str(tmpdir)
+        finally:
+            process.terminate()
+            try:
+                process.wait(timeout=1)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                process.wait()
+
+
 def freeport(used_ports=None):
    "Find an unused free port on localhost."
    while True:
@@ -167,7 +167,7 @@ def test_fail_with_invalid_password(defaultenv):
    "Connecting with an invalid password should fail without retries."
    uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
    env = {**defaultenv, "PGRST_DB_URI": uri}
-    with run(env=env, wait_for_readiness=False) as postgrest:
+    with run(env=env, wait_for=None) as postgrest:
        exitCode = wait_until_exit(postgrest)
        assert exitCode == 1

@@ -3,16 +3,24 @@
 import os
 import re
 import signal
+import subprocess
 import time
 import pytest

 from config import CONFIGSDIR, FIXTURES, SECRET
-from util import Thread, jwtauthheader, parse_server_timings_header
+from util import (
+    Thread,
+    jwtauthheader,
+    parse_server_timings_header,
+    match_log,
+)
 from postgrest import (
+    Admin,
    freeport,
    is_ipv6,
    reset_statement_timeout,
    run,
+    run_pgproxy,
    set_statement_timeout,
    sleep_until_postgrest_config_reload,
    sleep_until_postgrest_full_reload,
@@ -529,6 +537,39 @@ def test_admin_ready_w_channel(defaultenv):
        assert response.status_code == 200


+def test_listener_query_is_visible_in_pg_stat_activity(defaultenv):
+    "The listener connection should show the LISTEN pgrst statement in pg_stat_activity"
+
+    env = {
+        **defaultenv,
+        "PGRST_DB_CHANNEL_ENABLED": "true",
+        "PGAPPNAME": "listener-query-test",
+    }
+
+    with run(env=env):
+        query = """
+select query
+from pg_stat_activity
+where application_name = 'listener-query-test'
+  and query = 'LISTEN "pgrst"'
+limit 1;
+"""
+        output = subprocess.check_output(
+            [
+                "psql",
+                "--set",
+                "ON_ERROR_STOP=1",
+                "--tuples-only",
+                "--no-align",
+                "-c",
+                query,
+            ],
+            text=True,
+        ).strip()
+
+        assert output == 'LISTEN "pgrst"'
+
+
 def test_admin_ready_wo_channel(defaultenv):
    "Should get a success response from the admin server ready endpoint when the LISTEN channel is disabled"

@@ -679,55 +720,45 @@ def test_log_level(level, defaultenv):
        response = postgrest.session.get("/")
        assert response.status_code == 200

-        output = sorted(postgrest.read_stdout(nlines=7))
+        output = postgrest.read_stdout(nlines=9)

        if level == "crit":
            assert len(output) == 0
        elif level == "error":
-            assert re.match(
-                r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
-                output[0],
+            match_log(
+                output,
+                [r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"'],
            )
            assert len(output) == 1
        elif level == "warn":
-            assert re.match(
-                r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
-                output[0],
-            )
-            assert re.match(
-                r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
-                output[1],
+            match_log(
+                output,
+                [
+                    r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
+                    r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
+                ],
            )
            assert len(output) == 2
        elif level == "info":
-            assert re.match(
-                r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
-                output[0],
-            )
-            assert re.match(
-                r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
-                output[1],
-            )
-            assert re.match(
-                r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
-                output[2],
+            match_log(
+                output,
+                [
+                    r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
+                    r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
+                    r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
+                ],
            )
            assert len(output) == 3
        elif level == "debug":
-            assert re.match(
-                r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
-                output[0],
+            match_log(
+                output,
+                [
+                    r'- - - \[.+\] "GET / HTTP/1.1" 500 \d+ "" "python-requests/.+"',
+                    r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
+                    r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
+                ],
            )
-            assert re.match(
-                r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 \d+ "" "python-requests/.+"',
-                output[1],
-            )
-            assert re.match(
-                r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 \d+ "" "python-requests/.+"',
-                output[2],
-            )
-
-            assert len(output) == 7
+            assert len(output) == 9
            assert any("Connection" and "is available" in line for line in output)
            assert any("Connection" and "is used" in line for line in output)

@@ -1049,7 +1080,7 @@ def test_schema_cache_concurrent_notifications(slow_schema_cache_env):
        int(slow_schema_cache_env["PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP"]) / 1000
    )

-    with run(env=slow_schema_cache_env, wait_for_readiness=False) as postgrest:
+    with run(env=slow_schema_cache_env, wait_for=None) as postgrest:
        time.sleep(2 * internal_sleep + 0.1)  # wait for readiness manually

        # first request, create a function and set a schema cache reload in progress
@@ -1240,7 +1271,7 @@ def test_fail_with_invalid_dbname_and_automatic_recovery_disabled(defaultenv):
        "PGRST_DB_POOL_AUTOMATIC_RECOVERY": "false",
    }

-    with run(env=env, wait_for_readiness=False) as postgrest:
+    with run(env=env, wait_for=None) as postgrest:
        exitCode = wait_until_exit(postgrest)
        assert exitCode == 1

@@ -1364,16 +1395,21 @@ def test_db_error_logging_to_stderr(level, defaultenv, metapostgrest):
        assert response.status_code == 500

        # ensure the message appears on the logs
-        output = sorted(postgrest.read_stdout(nlines=6))
+        output = postgrest.read_stdout(nlines=8)

        if level == "crit":
            assert len(output) == 0
        elif level == "debug":
-            assert " 500 " in output[0]
-            assert "canceling statement due to statement timeout" in output[5]
+            match_log(
+                output,
+                [
+                    r".*canceling statement due to statement timeout.*",
+                    r".*500.*",
+                ],
+            )
        else:
-            assert " 500 " in output[0]
-            assert "canceling statement due to statement timeout" in output[1]
+            assert " 500 " in output[1]
+            assert "canceling statement due to statement timeout" in output[0]

    reset_statement_timeout(metapostgrest, role)

@@ -1524,7 +1560,7 @@ def test_log_error_when_empty_schema_cache_on_startup_to_stderr(defaultenv):
        "PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": "300",
    }

-    with run(env=env, wait_for_readiness=False) as postgrest:
+    with run(env=env, wait_for=None) as postgrest:
        postgrest.wait_until_scache_starts_loading()

        response = postgrest.session.get("/projects")
@@ -1545,7 +1581,7 @@ def test_no_double_schema_cache_reload_on_empty_schema(defaultenv):
        "PGRST_INTERNAL_SCHEMA_CACHE_QUERY_SLEEP": "300",
    }

-    with run(env=env, port=freeport(), wait_for_readiness=False) as postgrest:
+    with run(env=env, port=freeport(), wait_for=None) as postgrest:
        postgrest.wait_until_scache_starts_loading()

        response = postgrest.session.get("/projects")
@@ -1567,18 +1603,17 @@ def test_log_pool_req_observation(level, defaultenv):

    headers = jwtauthheader({"role": "postgrest_test_author"}, SECRET)

-    pool_req = "Trying to borrow a connection from pool"
-    pool_req_fullfill = "Borrowed a connection from the pool"
+    pool_req = r".*Trying to borrow a connection from pool.*"
+    pool_req_fullfill = r".*Borrowed a connection from the pool.*"

    with run(env=env) as postgrest:

        postgrest.session.get("/authors_only", headers=headers)

        if level == "debug":
-            output = postgrest.read_stdout(nlines=5)
-            assert pool_req in output[1]
-            assert pool_req_fullfill in output[4]
-            assert len(output) == 5
+            output = postgrest.read_stdout(nlines=7)
+            assert len(output) == 7
+            match_log(output, [pool_req, pool_req_fullfill])
        elif level == "info":
            output = postgrest.read_stdout(nlines=4)
            assert len(output) == 1
@@ -1628,7 +1663,7 @@ def test_schema_cache_error_observation(defaultenv):
        "PGRST_DB_EXTRA_SEARCH_PATH": "x",
    }

-    with run(env=env, no_startup_stdout=False, wait_for_readiness=False) as postgrest:
+    with run(env=env, no_startup_stdout=False, wait_for=None) as postgrest:
        # TODO: postgrest should exit here, instead it keeps retrying
        # exitCode = wait_until_exit(postgrest)
        # assert exitCode == 1
@@ -1649,7 +1684,7 @@ def test_log_listener_connection_errors(defaultenv):
        "PGRST_DB_CHANNEL_ENABLED": "true",
    }

-    with run(env=env, no_startup_stdout=False, wait_for_readiness=False) as postgrest:
+    with run(env=env, no_startup_stdout=False, wait_for=None) as postgrest:
        output = postgrest.read_stdout(nlines=5)
        assert any(
            'Failed listening for database notifications on the "pgrst" channel. could not translate host name "no_host" to address:'
@@ -1666,7 +1701,7 @@ def test_log_listener_connection_start(defaultenv):
        "PGRST_DB_CHANNEL_ENABLED": "true",
    }

-    with run(env=env, no_startup_stdout=False, wait_for_readiness=True) as postgrest:
+    with run(env=env, no_startup_stdout=False, wait_for=Admin.ready) as postgrest:
        output = postgrest.read_stdout(nlines=10)
        # Check for the listener start message containing host and port
        # Do not check if pg version is displayed properly as it is tricky to test it
@@ -1694,7 +1729,7 @@ def test_db_pre_config_with_pg_reserved_words(defaultenv):
        "PGRST_DB_PRE_CONFIG": "select",  # no "select" function in our fixtures, fail gracefully at startup
    }

-    with run(env=env, no_startup_stdout=False, wait_for_readiness=False) as postgrest:
+    with run(env=env, no_startup_stdout=False, wait_for=None) as postgrest:
        output = postgrest.read_stdout(nlines=8)
        assert any(
            'Failed to query database settings for the config parameters.{"code":"42883","details":null,"hint":"No function matches the given name and argument types. You might need to add explicit type casts.","message":"function select() does not exist"}'
@@ -1775,3 +1810,21 @@ def test_server_timing_transaction_duration(defaultenv, metapostgrest):
        ]

        assert 2000 <= response_dur < 3000
+
+
+def test_positive_pool_metric(defaultenv):
+    "When a network failure is caused on the pg connection, pgrst_db_pool_available stays positive"
+
+    with run_pgproxy(defaultenv, proxy_timeout="1ms") as pgproxyhost:
+        env = {**defaultenv, "PGHOST": pgproxyhost}
+
+        with run(env=env, wait_for=Admin.live) as postgrest:
+            response = postgrest.admin.get("/metrics", timeout=1)
+            assert response.status_code == 200
+
+            metrics = float(
+                re.search(
+                    r"pgrst_db_pool_available (-?\d+(?:\.\d+)?)", response.text
+                ).group(1)
+            )
+            assert metrics >= 0
@@ -1,3 +1,4 @@
+import re
 import threading
 import jwt

@@ -21,6 +22,19 @@ class Thread(threading.Thread):
            raise self._exception


+def match_log(output, matchers):
+    ito = iter(output)
+    itm = iter(matchers)
+    nextMatcher = next(itm, None)
+    while nextMatcher is not None and (line := next(ito, None)) is not None:
+        if re.match(nextMatcher, line) is not None:
+            nextMatcher = next(itm, None)
+    if nextMatcher is not None:
+        raise AssertionError(
+            f"Expected log line matching {nextMatcher} not found in output"
+        )
+
+
 def authheader(token):
    "Bearer token HTTP authorization header."
    return {"Authorization": f"Bearer {token}"}
@@ -1,7 +1,7 @@
 CREATE ROLE postgrest_test_anonymous;
 CREATE ROLE postgrest_test_author;
-GRANT postgrest_test_anonymous TO :PGUSER;
-GRANT postgrest_test_author TO :PGUSER;
+GRANT postgrest_test_anonymous TO :"PGUSER";
+GRANT postgrest_test_author TO :"PGUSER";
 CREATE SCHEMA test;

 -- PUT+PATCH target needs one record and column to modify
@@ -17,10 +17,11 @@ import           PostgREST.SchemaCache     (querySchemaCache)
 import qualified Observation.JwtCache
 import qualified Observation.MetricsSpec

-import ObsHelper
-import PostgREST.Observation (Observation (HasqlPoolObs))
-import Protolude             hiding (toList, toS)
-import Test.Hspec
+import qualified Observation.SchemaCacheSpec
+import           ObsHelper
+import           PostgREST.Observation       (Observation (HasqlPoolObs))
+import           Protolude                   hiding (toList, toS)
+import           Test.Hspec

 main :: IO ()
 main = do
@@ -64,6 +65,8 @@ main = do
      describe "Observation.JwtCacheObs" Observation.JwtCache.spec
    before (initApp baseSchemaCache testCfg) $
      describe "Feature.MetricsSpec" Observation.MetricsSpec.spec
+    before (initApp baseSchemaCache testCfg) $
+      describe "Feature.SchemaCacheSpec" Observation.SchemaCacheSpec.spec

  where
    loadSCache pool conf =
@@ -6,17 +6,21 @@

 module Observation.MetricsSpec where

-import           Data.List             (lookup)
-import           Network.Wai           (Application)
+import           Data.List              (lookup)
+import qualified Hasql.Pool.Observation as SQL
+import           Network.Wai            (Application)
 import           ObsHelper
-import qualified PostgREST.AppState    as AppState
-import           PostgREST.Config      (AppConfig (configDbSchemas))
-import qualified PostgREST.Metrics     as Metrics
+import qualified PostgREST.AppState     as AppState
+import           PostgREST.Config       (AppConfig (configDbSchemas))
+import           PostgREST.Metrics      (ConnStats (..),
+                                         MetricsState (..),
+                                         connectionCounts)
 import           PostgREST.Observation
-import           Prometheus            (getCounter, getVectorWith)
-import           Protolude
-import           Test.Hspec            (SpecWith, describe, it)
-import           Test.Hspec.Wai        (getState)
+import           Prometheus             (getCounter, getVectorWith)
+import           Test.Hspec             (SpecWith, describe, it)
+import           Test.Hspec.Wai         (getState)
+
+import Protolude

 spec :: SpecWith (SpecState, Application)
 spec = describe "Server started with metrics enabled" $ do
@@ -71,9 +75,40 @@ spec = describe "Server started with metrics enabled" $ do
        -- (there should be none but we need to verify that)
        threadDelay $ 1 * sec

+  -- The test verifies we properly count in use connections
+  -- The idea is to fork a worker thread that
+  -- borrows connection from the pool and waits for a signal to release it
+  -- Main thread checks that
+  -- in use connections counter is incremented by worker
+  -- then it signals the worker to release the connection
+  -- and finally verifies that in use connection counter is back to original value
+  it "Should track in use connections" $ do
+    SpecState{specAppState = appState, specMetrics = metrics, specObsChan} <- getState
+    let waitFor = waitForObs specObsChan
+
+    liftIO $ checkState' metrics [
+        -- we expect in use connections to be the same once finished
+        inUseConnections (+ 0)
+      ] $ do
+        signal <- newEmptyMVar
+        -- make sure waiting thread is signaled
+        (`finally` tryPutMVar signal ()) $
+          -- expecting one more connection in use
+          checkState' metrics [
+            inUseConnections (+ 1)
+          ] $ do
+            -- start a thread hanging on a single connection until signaled
+            void $ forkIO $ void $ AppState.usePool appState $ liftIO (readMVar signal)
+            -- main thread waits for ConnectionObservation with InUseConnectionStatus
+            -- after which used connections count should be incremented
+            waitFor (1 * sec) "InUseConnectionStatus" $ \x -> [ o | o@(HasqlPoolObs (SQL.ConnectionObservation _ SQL.InUseConnectionStatus)) <- pure x]
+
+        -- hanging thread was signaled and should return the connection
+        waitFor (1 * sec) "ReadyForUseConnectionStatus" $ \x -> [ o | o@(HasqlPoolObs (SQL.ConnectionObservation _ SQL.ReadyForUseConnectionStatus)) <- pure x]

  where
    -- prometheus-client api to handle vectors is convoluted
    schemaCacheLoads label = expectField @"schemaCacheLoads" $
      fmap (maybe (0::Int) round . lookup label) . (`getVectorWith` getCounter)
+    inUseConnections = expectField @"connTrack" ((inUse <$>) . connectionCounts)
    sec = 1000000
@@ -0,0 +1,49 @@
+{-# LANGUAGE DataKinds           #-}
+{-# LANGUAGE MonadComprehensions #-}
+{-# LANGUAGE NamedFieldPuns      #-}
+module Observation.SchemaCacheSpec where
+
+import           Network.Wai           (Application)
+import           ObsHelper
+import qualified PostgREST.AppState    as AppState
+import           PostgREST.Config      (configDbSchemas)
+import           PostgREST.Observation
+import           Protolude
+import           Test.Hspec            (SpecWith, describe, it)
+import           Test.Hspec.Wai        (getState)
+
+spec :: SpecWith (SpecState, Application)
+spec = describe "Server started with metrics enabled" $ do
+
+  it "Should emit PoolFlushed, SchemaCacheQueriedObs and SchemaCacheLoadedObs when schema cache is reloaded" $ do
+    SpecState{specAppState = appState, specObsChan} <- getState
+    let waitFor = waitForObs specObsChan
+
+    liftIO $ do
+      AppState.schemaCacheLoader appState
+
+      waitFor (1 * sec) "PoolFlushed" $ \x -> [ o | o@PoolFlushed <- pure x ]
+      waitFor (1 * sec) "SchemaCacheQueriedObs" $ \x -> [ o | o@SchemaCacheQueriedObs{} <- pure x ]
+      waitFor (1 * sec) "SchemaCacheLoadedObs" $ \x -> [ o | o@SchemaCacheLoadedObs{} <- pure x ]
+
+
+  it "Should flush pool once when schema reloading retries" $ do
+    SpecState{specAppState = appState, specObsChan} <- getState
+    let waitFor = waitForObs specObsChan
+
+    liftIO $ do
+        AppState.getConfig appState >>= \cfg -> do
+          AppState.putConfig appState $ cfg { configDbSchemas = pure "bad_schema" }
+          AppState.schemaCacheLoader appState
+
+          waitFor (1 * sec) "SchemaCacheErrorObs" $ \x -> [ o | o@SchemaCacheErrorObs{} <- pure x ]
+
+          -- Restore configuration
+          AppState.putConfig appState cfg
+
+        -- Wait for 2 seconds so that retry can happen
+        waitFor (2 * sec) "PoolFlushed" $ \x -> [ o | o@PoolFlushed <- pure x ]
+        waitFor (1 * sec) "SchemaCacheQueriedObs" $ \x -> [ o | o@SchemaCacheQueriedObs{} <- pure x ]
+        waitFor (1 * sec) "SchemaCacheLoadedObs" $ \x -> [ o | o@SchemaCacheLoadedObs{} <- pure x ]
+  where
+    sec = 1000000
@@ -2,4 +2,4 @@ DROP ROLE IF EXISTS postgrest_test_anonymous, postgrest_test_author;
 CREATE ROLE postgrest_test_anonymous;
 CREATE ROLE postgrest_test_author;

-GRANT postgrest_test_anonymous, postgrest_test_author TO :PGUSER;
+GRANT postgrest_test_anonymous, postgrest_test_author TO :"PGUSER";
@@ -4,4 +4,4 @@ CREATE ROLE postgrest_test_default_role;
 CREATE ROLE postgrest_test_author;
 CREATE ROLE postgrest_test_superuser WITH SUPERUSER;

-GRANT postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author, postgrest_test_superuser TO :PGUSER;
+GRANT postgrest_test_anonymous, postgrest_test_default_role, postgrest_test_author, postgrest_test_superuser TO :"PGUSER";
Author	SHA1	Message	Date
steve-chavez	6200fbad58	chore: bump version to 14.12	2026-05-20 12:20:38 -05:00
steve-chavez	b71345ab09	test: adjust test_positive_pool_metric Adjusted so it waits for liveness instead of requiring sleeping	2026-05-20 10:29:12 -05:00
steve-chavez	9be5c41c05	test: change wait_for_readiness to an enum This so we can wait for both readiness and liveness	2026-05-20 10:29:12 -05:00
Michał KłeczekandSteve Chavez	ed2dc1fe86	fix: implement connection tracking in metrics Right now metrics observation handler does not track database connections but updates a single Gauge based on HasqlPoolObs events. This is problematic because Hasql pool reports various connection events in multiple phases. The connection state machine is not simple and to precisely report the number of connections in various states, it is necessary to track their lifecycles. This change adds a ConnTrack data structure and logic to track database connections lifecycles. At the moment it supports "connected" and "inUse" connection counts precisely. The "pgrst_db_pool_available" metric is implemented on top of ConnTrack instead of a simple Gauge.	2026-05-20 10:29:12 -05:00
steve-chavez	f6e99117ab	test: adjust sleep in test_positive_pool_metric Otherwise it's not enough time for the pool metric to reach a negative value.	2026-05-20 10:29:12 -05:00
steve-chavez	0be41a4941	test: negative pgrst_db_pool_available in metrics Proves the failure on https://github.com/PostgREST/postgrest/issues/4622. This doesn't require additional test infra, only nginx. Taking advantage of the `stream {}` context which is also compatible with unix socket besides TCP.	2026-05-20 10:29:12 -05:00
renovate[bot]andWolfgang Walther	ae1e3d6c4d	chore(deps): update codecov/codecov-action action to v6.0.1	2026-05-19 07:04:25 +00:00
Taimoor Zaeem	f2932bffea	test(io): move utility function to util.py The function `match_log` should be in `util.py` so it can be reused in other modules. Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>	2026-05-19 10:01:51 +05:00
Taimoor ZaeemandSteve Chavez	15b83ff4da	docs: add redirect of the moved postgis section In `055921ea`, we missed redirecting the old link to the new one. Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com> (cherry picked from commit `ab43ee1fbb`)	2026-05-15 19:40:45 -05:00
renovate[bot]andWolfgang Walther	a2d76b01b7	chore(deps): update dependency macos to v26	2026-05-15 20:38:43 +00:00
renovate[bot]andTaimoor Zaeem	16d4477c62	chore(deps): update actions/create-github-app-token action to v3.2.0	2026-05-14 10:14:25 +05:00
Taimoor Zaeem	d75d2838fc	ci: catch linkcheck failure The scheduled linkcheck job fails without notification to maintainers. To catch the failure, a github workflow step has been added which comments on issue #4106 so that the maintainers get notified. Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com> (cherry picked from commit `f90d7d7296`)	2026-05-14 09:56:19 +05:00
renovate[bot]andWolfgang Walther	4e08b2748c	chore(deps): update actions/create-github-app-token action to v3.2.0	2026-05-13 21:01:16 +00:00
Taimoor Zaeem	86f03c5ded	docs: move PostGIS under Integrations For better visibility. Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com> (cherry picked from commit `055921eaf5`)	2026-05-13 13:27:10 +05:00
renovate[bot]andWolfgang Walther	eda5a10aa5	chore(deps): update all dependencies	2026-05-12 13:03:04 +00:00
Taimoor ZaeemandWolfgang Walther	51c51ea140	ci: cancel previous runs for tag pipelines Reverts the change in `ad5bb38d70`. https://github.com/orgs/community/discussions/13015 is now solved. Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com> (cherry picked from commit `96f12f83f0`)	2026-05-07 07:10:27 +00:00
Taimoor Zaeem	3824e73c91	nix(chore): add commit prefix in release commit message We should follow our commit convention. Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com> (cherry picked from commit `4c04572783`)	2026-05-07 09:29:56 +05:00
renovate[bot]andWolfgang Walther	d65288dfd6	chore(deps): update korthout/backport-action digest to bf97bcf	2026-05-06 19:47:08 +00:00
Michał KłeczekandTaimoor Zaeem	80ea1f4a1b	refactor: make Observation Logger and Listener modules more cohesive Observation module exports observationMessage and isDbListenerBug functions. The first one is used only in Logger module and the second one - only in Listener. This change moves observationMessage function to Logger module and isDbListenerBug function to Listener module, making all three modules more cohesive and lessening dependencies. It also gives the compiler more opportunities for intra-module inlining and optimization.	2026-05-07 00:36:36 +05:00
renovate[bot]andWolfgang Walther	23b0c8d82a	chore(deps): update ubuntu:resolute docker digest to f3d2860	2026-05-05 15:53:44 +00:00
Taimoor Zaeem	1f69247cc2	bump version to 14.11	2026-05-04 22:59:24 +05:00
Taimoor Zaeem	e25a87bd84	fix: functions count in startup logs not counting overloaded functions Renames the "Functions" in the logs to "RPCs". This clarifies that we log number of callable functions and not the number of SQL objects. Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>	2026-05-04 22:08:06 +05:00
Laurence IslaandTaimoor Zaeem	0502488b70	fix: call `pg_notification_queue_usage()` automatically when the LISTEN channel bug is detected (#4858 ) There's a PostgreSQL bug that doesn't let any listener to register in the DB: https://www.postgresql.org/message-id/flat/CAK98qZ3wZLE-RZJN_Y%2BTFjiTRPPFPBwNBpBi5K5CU8hUHkzDpw%40mail.gmail.com The only workaround is to advance the async notification queue tail, which can be done by executing: "SELECT pg_notification_queue_usage();". Before we just logged a HINT with this suggestion, but now we call that function directly and then let the listener to automatically recover. No automated tests were added here as it would be too complex and this is a PostgreSQL bug. But this was manually tested following the steps on https://github.com/PostgREST/postgrest/pull/4581#issuecomment-3690610592	2026-05-04 22:08:06 +05:00
Michal KleczekandTaimoor Zaeem	a62b6de1d0	fix: Restore showing LISTEN pgrst in pg_stat_activity	2026-05-04 22:08:06 +05:00
Taimoor ZaeemandWolfgang Walther	9baf17aed0	fix: login with uppercase and mixed case role names PostgREST failed when querying role settings where current role name contained uppercase letters. This commit resolves it by quoting the CURRENT_USER. Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>	2026-04-27 19:53:52 +00:00
renovate[bot]andWolfgang Walther	d9eabb2742	chore(deps): update korthout/backport-action action to v4.5	2026-04-27 19:51:56 +00:00
renovate[bot]andWolfgang Walther	7711a86c0c	chore(deps): update ubuntu docker tag to v26	2026-04-24 06:29:44 +00:00
renovate[bot]andWolfgang Walther	d75a2bdce9	chore(deps): update korthout/backport-action action to v4.4.0	2026-04-21 09:05:15 +00:00
steve-chavezandWolfgang Walther	dd1956557d	ci: reinstate macos x86 releases Closes https://github.com/PostgREST/postgrest/issues/4755 (cherry picked from commit `b6c6bb27a8`)	2026-04-19 12:19:17 +00:00
steve-chavez	83e1bebcdc	bump version to 14.10	2026-04-16 13:54:44 -05:00
renovate[bot]andWolfgang Walther	89132e886a	chore(deps): update ubuntu:noble docker digest to c4a8d55	2026-04-16 10:44:03 +00:00
Michal KleczekandTaimoor Zaeem	abd76ca8ac	fix: Flush pool as late as possible during schema cache reloading (#4645 ) retryingSchemaCacheLoad flushes the pool upon every retry before it starts reloading the schema. This is too early as schema reloading might take some time during which new connections might be acquired. The consequence is that: * upon successful schema cache reload we might have some connections created with the old schema cache * we close connections upon each retry and under load we will keep closing and re-opening connections until schema cache load succeeds This change is to make sure we flush the pool only after successful schema cache querying but before loading (so that connections acquired during loading wait for it and do not interfere with timing the loading process).	2026-04-16 11:55:43 +05:00
Michał KłeczekandTaimoor Zaeem	df87ce46ed	add: log pool flushes Emit a dedicated PoolFlushed observation when the DB pool is released during schema cache reload.	2026-04-16 11:18:44 +05:00
renovate[bot]andWolfgang Walther	d369d2c41e	chore(deps): update haskell-actions/setup action to v2.11.0	2026-04-15 18:05:21 +00:00
Wolfgang Walther	9ee366a903	docs: fix broken link	2026-04-15 09:30:05 +02:00
Michał KłeczekandTaimoor Zaeem	01f7f13389	test(refactor): make debug output testing more robust This change introduces a match_log function, that implements lenient matching of log output lines. Tests of log output are retrofited to use the function.	2026-04-15 07:40:16 +05:00
steve-chavez	4922a9feaf	docs: target_session_attrs=read-only in libpq>=14	2026-04-14 11:11:22 -05:00
renovate[bot]andWolfgang Walther	679b957355	chore(deps): update actions/cache action to v5.0.5	2026-04-13 21:04:21 +00:00
renovate[bot]andWolfgang Walther	05df2c5692	chore(deps): update all dependencies	2026-04-12 15:25:03 +00:00