nix: exclude protolude from style checks

Currently, our vendored protolude has many failing checks for
style, lint and hsie. Temporarily excluding it. We should
reinstate these checks later.

Signed-off-by: Taimoor Zaeem <taimoorzaeem@gmail.com>

.cirrus.yml

@@ -0,0 +1,42 @@
+freebsd_instance:
+  image_family: freebsd-14-3
+
+build_task:
+  # Don't change this name without adjusting .github/workflows/build.yaml
+  name: Build FreeBSD (Stack)
+  install_script: pkg install -y postgresql16-client hs-stack git
+
+  only_if: |
+    $CIRRUS_TAG != '' || $CIRRUS_BRANCH == 'main' || $CIRRUS_BRANCH =~ 'v*' ||
+    changesInclude(
+      '.github/workflows/build.yaml',
+      '.github/actions/artifact-from-cirrus/**',
+      '.cirrus.yml',
+      'postgrest.cabal',
+      'stack.yaml*',
+      '**.hs'
+    )
+
+  stack_cache:
+    folders: /.stack
+    fingerprint_script:
+      - echo $CIRRUS_OS
+      - stack --version
+      - md5sum postgrest.cabal
+      - md5sum stack.yaml.lock
+
+  stack_work_cache:
+    folders: .stack-work
+    fingerprint_script:
+      - echo $CIRRUS_OS
+      - stack --version
+      - md5sum postgrest.cabal
+      - md5sum stack.yaml.lock
+      - find main src -type f -iname '*.hs' -exec md5sum "{}" +
+
+  build_script: |
+    stack build -j 1 --local-bin-path . --copy-bins
+    strip postgrest
+
+  bin_artifacts:
+    path: postgrest

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Ignore blame for commit that moved protolude files under src/protolude
+d4949c633e8172d0e4dd8f5c991eaaae6b48fbb0

.github/actions/artifact-from-cirrus/action.yaml

@@ -0,0 +1,119 @@
+name: Artifact from Cirrus
+
+description: Waits for a specific Cirrus CI run to complete, then downloads the artifact and uploads it to the current workflow. This will silently succeed if Cirrus CI did not schedule a task within 2 minutes.
+
+inputs:
+  download:
+    description: Name of Artifact to download from Cirrus CI
+    required: true
+  task:
+    description: Name of Cirrus Task
+    required: true
+  token:
+    description: GitHub Token
+    required: true
+  upload:
+    description: Name of Artifact to upload on GitHub Actions
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: echo "GH_TOKEN=${{ inputs.token }}" >> "$GITHUB_ENV"
+    - name: Wait for Check Suite to be created
+      id: check-suite
+      env:
+        # GITHUB_SHA does weird things for pull request, so we roll our own:
+        COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+      shell: bash
+      run: |
+        get_check_runs_url() {
+          gh api "repos/{owner}/{repo}/commits/${COMMIT}/check-suites" \
+            | jq -r '.check_suites[] | select(.app.slug == "cirrus-ci") | .check_runs_url'
+        }
+        for _ in $(seq 1 12); do
+          check_runs_url="$(get_check_runs_url)"
+          if [ -z "$check_runs_url" ]; then
+            echo "Cirrus CI task has not started, yet. Waiting..."
+            sleep 10
+          else
+            echo "check_runs_url=$check_runs_url" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+        done
+        >&2 echo "Cirrus CI check suite not found. Is Cirrus CI enabled for this repo?"
+    - name: Find task by name
+      id: find-task
+      if: steps.check-suite.outputs.check_runs_url
+      shell: bash
+      run: |
+        get_number_of_tasks() {
+          gh api "${{ steps.check-suite.outputs.check_runs_url }}" \
+            | jq -r '.check_runs | map(select(.name == "${{ inputs.task }}")) | length'
+        }
+        tasks="$(get_number_of_tasks)"
+        case "$tasks" in
+          0)
+            echo "Task not found, assuming it's skipped intentionally..."
+            exit 0
+            ;;
+          1)
+            echo "task_found=1" >> "$GITHUB_OUTPUT"
+            exit 0
+            ;;
+          *)
+            >&2 echo "More than 1 task with the same name found. Don't know what to do..."
+            exit 1
+            ;;
+        esac
+    - name: Wait for Cirrus CI to complete task
+      if: steps.find-task.outputs.task_found
+      shell: bash
+      run: |
+        get_conclusion() {
+          gh api "${{ steps.check-suite.outputs.check_runs_url }}" \
+            | jq -r '.check_runs[] | select(.name == "${{ inputs.task }}" and .status == "completed") | .conclusion'
+        }
+        while true; do
+          conclusion="$(get_conclusion)"
+          if [ -z "$conclusion" ]; then
+            echo "Cirrus CI task has not completed, yet. Waiting..."
+            sleep 30
+          else
+            if [ "$conclusion" == "success" ]; then
+              break
+            else
+              exit 1
+            fi
+          fi
+        done
+    - name: Download artifact from Cirrus CI
+      if: steps.find-task.outputs.task_found
+      id: download
+      shell: bash
+      run: |
+        get_external_id() {
+          gh api "${{ steps.check-suite.outputs.check_runs_url }}" \
+            | jq -er '.check_runs[] | select(.name == "${{ inputs.task }}") | .external_id'
+        }
+        archive="$(mktemp)"
+        artifacts="$(mktemp -d)"
+        until curl --no-progress-meter --fail -o "${archive}" \
+                "https://api.cirrus-ci.com/v1/artifact/task/$(get_external_id)/${{ inputs.download }}.zip"
+        do
+          # This happens when a tag is pushed on the same commit. In this case the
+          # job is immediately marked as "completed" for us, so we end up here after a few
+          # seconds - but the actual Cirrus CI task is still running and didn't produce its artifact, yet.
+          echo "Artifact not found on Cirrus CI, yet. Waiting..."
+          sleep 30
+        done
+        unzip "${archive}" -d "${artifacts}"
+        echo "artifacts=${artifacts}" >> "$GITHUB_OUTPUT"
+    - name: Save artifact to GitHub Actions
+      if: steps.find-task.outputs.task_found
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: ${{ inputs.upload }}
+        path: ${{ steps.download.outputs.artifacts }}
+        if-no-files-found: error

.github/actions/cache-on-main/action.yaml

@@ -18,14 +18,14 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+    - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       if: ${{ startsWith(github.ref, 'refs/heads/') || (inputs.save-prs && startsWith(github.ref, 'refs/pull/')) }}
       with:
         path: ${{ inputs.path }}
         key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.prefix }}-${{ inputs.suffix }}
         restore-keys: |
           ${{ runner.os }}-${{ runner.arch }}-${{ inputs.prefix }}-
-    - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+    - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       if: ${{ !startsWith(github.ref, 'refs/heads/') && !(inputs.save-prs && startsWith(github.ref, 'refs/pull/'))  }}
       with:
         path: ${{ inputs.path }}

.github/actions/run-anywhere/action.yaml

@@ -1,35 +0,0 @@
-name: Run anywhere
-
-description: Runs the same code either in a VM or on the bare machine
-
-inputs:
-  vm:
-    description: Which VM to run on.
-  envs:
-    description: List of relevant environment variables, which might need to be copied into the VM.
-  prepare:
-    description: Code to run in a prepare step, e.g. installing dependencies.
-  run:
-    description: Code to run as the main action.
-    required: true
-
-runs:
-  using: composite
-  steps:
-    - if: ${{ inputs.vm == 'freebsd' }}
-      uses: vmactions/freebsd-vm@83b151f58c6047089f4c80eb5ba2039d158ce093 # v1.5.3
-      with:
-        envs: ${{ inputs.envs }}
-        prepare: ${{ inputs.prepare }}
-        # Work around https://github.com/vmactions/freebsd-vm/issues/59
-        run: |
-          pw user add -n action -m
-          su action -c '${{ inputs.run }}'
-    - if: ${{ inputs.vm == '' }}
-      name: Prepare
-      shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }}
-      run: ${{ inputs.prepare }}
-    - if: ${{ inputs.vm == '' }}
-      name: Run
-      shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }}
-      run: ${{ inputs.run }}

.github/actions/setup-nix/action.yaml

@@ -11,12 +11,12 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35
+    - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
       with:
         nix_conf: |-
           always-allow-substitutes = true
           max-jobs = auto
-    - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
+    - uses: cachix/cachix-action@1eb2ef646ac0255473d23a5907ad7b04ce94065c # v17
       with:
         name: postgrest
         authToken: ${{ inputs.authToken }}

.github/workflows/backport.yaml

@@ -9,7 +9,7 @@ on:
 jobs:
   backport:
     name: Backport
-    runs-on: ubuntu-slim
+    runs-on: ubuntu-24.04
     # It triggers only when PR is already merged on either:
     #
     # - The merge event itself (action != labeled) or
@@ -28,9 +28,9 @@ jobs:
       # This actions creates the github token using the postgrest app secrets
       - name: Create Github App Token
         id: app-token
-        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
         with:
-          client-id: ${{ vars.POSTGREST_CI_APP_ID }}
+          app-id: ${{ vars.POSTGREST_CI_APP_ID }}
           private-key: ${{ secrets.POSTGREST_CI_PRIVATE_KEY }}
           permission-contents: write
           permission-pull-requests: write
@@ -38,15 +38,14 @@ jobs:
 
       # This is required for backport action to cherry-pick the PR
       - name: Fetch PR ref
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          allow-unsafe-pr-checkout: true
           ref: ${{ github.event.pull_request.head.sha }}
           token: ${{ steps.app-token.outputs.token }}
 
       # Backport action that creates the PR with given settings
       - name: Create backport PR
-        uses: korthout/backport-action@2e830a1d0b8269505846ddd407a70876913ad1f8 # v4.6
+        uses: korthout/backport-action@7c3f6cd5843cac11bc59a04a1b7699af93261670 # v4.5
         with:
           github_token: ${{ steps.app-token.outputs.token }}
           pull_description: 'Backport for #${pull_number}.'

.github/workflows/build.yaml

@@ -44,7 +44,7 @@ jobs:
     name: Nix - ${{ matrix.name }} static
     runs-on: ${{ matrix.runs-on }}
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -68,17 +68,12 @@ jobs:
           path: postgrest-docker-${{ matrix.artifact }}.tar.gz
           if-no-files-found: error
 
-      - name: Test static executable with NixOS' VM test
-        # GHA's ARM runner does not support KVM
-        if: runner.arch == 'X64'
-        run: nix-build -A nixpkgs-nixos-test
-
 
   macos:
     name: Nix - MacOS
-    runs-on: macos-26
+    runs-on: macos-15
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -98,64 +93,64 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: FreeBSD x86-64
-            runs-on: ubuntu-24.04
-            vm: freebsd
-            artifact: postgrest-freebsd-x86-64
-            deps: pkg install -y git postgresql16-client hs-stack
-
           - name: MacOS aarch64
             runs-on: macos-14
+            cache: |
+              ~/.stack/pantry
+              ~/.stack/snapshots
+              ~/.stack/stack.sqlite3
             artifact: postgrest-macos-aarch64
             deps: brew link --force libpq
 
           - name: MacOS x86-64
             runs-on: macos-15-intel
+            cache: |
+              ~/.stack/pantry
+              ~/.stack/snapshots
+              ~/.stack/stack.sqlite3
             artifact: postgrest-macos-x86-64
             deps: brew link --force libpq
 
           - name: Windows
             runs-on: windows-2022
+            cache: |
+              C:\sr\pantry
+              C:\sr\snapshots
+              C:\sr\stack.sqlite3
             deps: Add-Content $env:GITHUB_PATH $env:PGBIN
             artifact: postgrest-windows-x86-64
 
     name: Stack - ${{ matrix.name }}
     runs-on: ${{ matrix.runs-on }}
-    env:
-      # Putting .stack in the working directory helps with moving this in and out of the FreeBSD VM.
-      STACK_ROOT: ${{ github.workspace }}/.stack
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - if: ${{ !matrix.vm }}
-        uses: haskell-actions/setup@6037f33647c3f17758a2356c80fc4a53d7e0685d # v2.12.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0
         with:
           # This must match the version in stack.yaml's resolver
           ghc-version: 9.10.3
           enable-stack: true
           stack-no-global: true
           stack-setup-ghc: true
-      - name: Cache .stack
+      - name: Cache ~/.stack
         uses: ./.github/actions/cache-on-main
         with:
-          path: .stack
-          prefix: ${{ matrix.vm }}${{ matrix.vm && '-' }}stack
+          path: ${{ matrix.cache }}
+          prefix: stack
           suffix: ${{ hashFiles('postgrest.cabal', 'stack.yaml.lock') }}
       - name: Cache .stack-work
         uses: ./.github/actions/cache-on-main
         with:
           path: .stack-work
           save-prs: true
-          prefix: ${{ matrix.vm }}${{ matrix.vm && '-' }}stack-work-${{ hashFiles('postgrest.cabal', 'stack.yaml.lock') }}
+          prefix: stack-work-${{ hashFiles('postgrest.cabal', 'stack.yaml.lock') }}
           suffix: ${{ hashFiles('main/**/*.hs', 'src/**/*.hs') }}
+      - name: Install dependencies
+        if: matrix.deps
+        run: ${{ matrix.deps }}
       - name: Build with Stack
-        uses: ./.github/actions/run-anywhere
-        with:
-          vm: ${{ matrix.vm }}
-          envs: STACK_ROOT
-          prepare: ${{ matrix.deps }}
-          run: |
-            stack build --lock-file error-on-write --local-bin-path result --copy-bins
-            strip result/postgrest*
+        run: stack build --lock-file error-on-write --local-bin-path result --copy-bins
+      - name: Strip Executable
+        run: strip result/postgrest*
       - name: Save built executable as artifact
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -166,6 +161,19 @@ jobs:
           if-no-files-found: error
 
 
+  freebsd:
+    name: Stack - FreeBSD from CirrusCI
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ./.github/actions/artifact-from-cirrus
+        with:
+          token: ${{ github.token }}
+          task: Build FreeBSD (Stack)
+          download: bin
+          upload: postgrest-freebsd-x86-64
+
+
   cabal:
     strategy:
       matrix:
@@ -174,8 +182,8 @@ jobs:
     name: Cabal - Linux x86-64 - GHC ${{ matrix.ghc }}
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: haskell-actions/setup@6037f33647c3f17758a2356c80fc4a53d7e0685d # v2.12.0
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0
         with:
           ghc-version: ${{ matrix.ghc }}
       - name: Cache .cabal

.github/workflows/check.yaml

@@ -20,7 +20,7 @@ jobs:
     name: Lint & Style
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -36,7 +36,7 @@ jobs:
     name: Commit
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 100  # fetch history (last 100 commits) instead of default shallow clone history, this is deemed enough for a PR history
       - name: Setup Nix Environment

.github/workflows/ci.yaml

@@ -41,15 +41,16 @@ jobs:
     concurrency:
       # Never tag outdated commits on the main branch by skipping superseded commits
       group: ci-tag-${{ (github.ref == 'refs/heads/main' && github.ref) || github.run_id }}
-      cancel-in-progress: true
+      # TODO: Enable this once https://github.com/orgs/community/discussions/13015 is solved
+      cancel-in-progress: false
     if: vars.RELEASE_ENABLED
-    runs-on: ubuntu-slim
+    runs-on: ubuntu-24.04
     needs:
       - docs
       - test
       - build
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           ssh-key: ${{ secrets.POSTGREST_SSH_KEY }}
       - name: Tag latest commit

.github/workflows/docs.yaml

@@ -28,7 +28,7 @@ jobs:
     name: Build
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
     - name: Setup Nix Environment
       uses: ./.github/actions/setup-nix
       with:
@@ -42,7 +42,7 @@ jobs:
     name: Spellcheck
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
     - name: Setup Nix Environment
       uses: ./.github/actions/setup-nix
       with:

.github/workflows/linkcheck.yaml

@@ -7,37 +7,12 @@ on:
 
 jobs:
   linkcheck:
-    name: Linkcheck
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
     - name: Setup Nix Environment
       uses: ./.github/actions/setup-nix
       with:
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
         tools: docs.linkcheck.bin
-
-    - name: Run Linkcheck
-      id: linkcheck
-      run: postgrest-docs-linkcheck
-
-    # This actions creates the github token using the postgrest app secrets
-    - name: Create Github App Token (Runs only on linkcheck failure)
-      id: app-token
-      if: ${{ failure() && steps.linkcheck.outcome == 'failure' }} # only create the token on linkcheck failure
-      uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
-      with:
-        client-id: ${{ vars.POSTGREST_CI_APP_ID }}
-        private-key: ${{ secrets.POSTGREST_CI_PRIVATE_KEY }}
-        permission-issues: write # required for commenting on issues
-
-    - name: Notify on linkcheck failure by commenting on GH Issue 4106
-      if: ${{ failure() && steps.linkcheck.outcome == 'failure' }}
-      uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
-      with:
-        token: ${{ steps.app-token.outputs.token }}
-        issue-number: 4106
-        body: |
-          **Linkcheck Job Failed!**
-
-          A broken link was detected in the docs. Please check the [failed run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+    - run: postgrest-docs-linkcheck

.github/workflows/release.yaml

@@ -9,7 +9,8 @@ on:
 concurrency:
   # Terminate all previous runs of the same workflow for the same tag.
   group: release-${{ github.ref }}
-  cancel-in-progress: true
+  # TODO: Enable this once https://github.com/orgs/community/discussions/13015 is solved
+  cancel-in-progress: false
 
 jobs:
   build:
@@ -19,15 +20,13 @@ jobs:
       CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
 
-  github:
-    name: GitHub
-    permissions:
-      contents: write
-    runs-on: ubuntu-slim
+  prepare:
+    name: Prepare
+    runs-on: ubuntu-24.04
     needs:
       - build
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Check the version to be released
         run: |
           cabal_version="$(grep -oP '^version:\s*\K.*' postgrest.cabal)"
@@ -49,7 +48,23 @@ jobs:
 
           echo "Relevant extract from CHANGELOG.md:"
           cat CHANGES.md
+      - name: Save CHANGES.md as artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: release-changes
+          path: CHANGES.md
+          if-no-files-found: error
 
+
+  github:
+    name: GitHub
+    permissions:
+      contents: write
+    runs-on: ubuntu-24.04
+    needs:
+      - prepare
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Download all artifacts
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
@@ -101,14 +116,14 @@ jobs:
             gh release edit devel \
               -t devel \
               --verify-tag \
-              -F CHANGES.md \
+              -F artifacts/release-changes/CHANGES.md \
               --prerelease
             gh release upload --clobber devel release-bundle/*
           else
             gh release create "${GITHUB_REF_NAME}" \
               -t "${GITHUB_REF_NAME}" \
               --verify-tag \
-              -F CHANGES.md \
+              -F artifacts/release-changes/CHANGES.md \
               release-bundle/*
           fi
 
@@ -117,13 +132,13 @@ jobs:
     name: Docker Hub
     runs-on: ubuntu-24.04-arm
     needs:
-      - github
+      - prepare
     if: |
       vars.DOCKER_REPO && vars.DOCKER_USER
     env:
       DOCKER_REPO: ${{ vars.DOCKER_REPO }}
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Download aarch64 Docker image
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
@@ -132,8 +147,8 @@ jobs:
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: postgrest-docker-x86-64
-      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
-      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+      - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+      - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
         with:
           username: ${{ vars.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
@@ -163,9 +178,16 @@ jobs:
             echo "Skipping push to 'latest' tag for pre-release..."
           fi
 
+
+  docker-description:
+    name: Docker Hub Description
+    runs-on: ubuntu-24.04
+    if: |
+      vars.DOCKER_REPO && vars.DOCKER_USER &&
+      github.ref == 'refs/tags/devel'
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0
-        if: github.ref == 'refs/tags/devel'
-        name: Docker Hub Description
         with:
           username: ${{ vars.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
@@ -173,54 +195,3 @@ jobs:
           short-description: ${{ github.event.repository.description }}
           readme-filepath: ./docker-hub-readme.md
 
-
-  ghcr:
-    name: GitHub Container Registry
-    runs-on: ubuntu-24.04-arm
-    needs:
-      - github
-    permissions:
-      packages: write
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: Download aarch64 Docker image
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: postgrest-docker-aarch64
-      - name: Download x86-64 Docker image
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: postgrest-docker-x86-64
-      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
-      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Publish images on Docker Hub
-        run: |
-          docker load -i postgrest-docker-aarch64.tar.gz
-          docker tag postgrest:latest "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}-linux-arm64"
-          docker push "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}-linux-arm64"
-
-          docker load -i postgrest-docker-x86-64.tar.gz
-          docker tag postgrest:latest "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}-linux-amd64"
-          docker push "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}-linux-amd64"
-
-          docker manifest create "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}" \
-            "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}-linux-arm64" \
-            "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}-linux-amd64"
-          docker manifest push "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}"
-
-          # Only tag 'latest' for full releases
-          if [ "${GITHUB_REF_NAME}" != "devel" ]; then
-            echo "Pushing to 'latest' tag for full release of ${GITHUB_REF_NAME} ..."
-            docker manifest create "ghcr.io/${GITHUB_REPOSITORY,,}:latest" \
-              "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}-linux-arm64" \
-              "ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_REF_NAME}-linux-amd64"
-            docker manifest push "ghcr.io/${GITHUB_REPOSITORY,,}:latest"
-          else
-            echo "Skipping push to 'latest' tag for pre-release..."
-          fi
-
-

.github/workflows/test.yaml

@@ -25,10 +25,6 @@ on:
       - test/**
       - '!**.md'
 
-defaults:
-  run:
-    shell: bash
-
 concurrency:
   # Terminate all previous runs of the same workflow for pull requests
   group: test-${{ github.head_ref || github.run_id }}
@@ -44,7 +40,7 @@ jobs:
         # https://github.com/actions/runner/issues/241#issuecomment-842566950
         shell: script -qec "bash --noprofile --norc -eo pipefail {0}"
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -56,14 +52,14 @@ jobs:
       - name: Run coverage (IO tests and Spec tests against latest supported PostgreSQL)
         run: postgrest-coverage
       - name: Upload coverage to codecov
-        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
         with:
           files: ./coverage/codecov.json
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Run doctests
         if: always()
-        run: nix-shell --run postgrest-test-doctests
+        run: postgrest-test-doctests
 
       - name: Check the spec tests for idempotence
         if: always()
@@ -74,9 +70,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Latest version is tested via `coverage` above.
-        pgVersion: [pg-14, pg-15, pg-16, pg-17, oriole-18, pg-18]
-    name: ${{ matrix.pgVersion }}
+        pgVersion: [14, 15, 16, 17, 18]
+    name: PG ${{ matrix.pgVersion }}
     runs-on: ubuntu-24.04
     defaults:
       run:
@@ -84,37 +79,37 @@ jobs:
         # https://github.com/actions/runner/issues/241#issuecomment-842566950
         shell: script -qec "bash --noprofile --norc -eo pipefail {0}"
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: tests.testSpec.bin tests.testObservability.bin tests.testIO.bin tests.testBigSchema.bin withTools.${{ matrix.pgVersion }}.bin cabalTools.update.bin
+          tools: tests.testSpec.bin tests.testObservability.bin tests.testIO.bin tests.testBigSchema.bin withTools.pg-${{ matrix.pgVersion }}.bin cabalTools.update.bin
 
       - run: postgrest-cabal-update
 
       - name: Run spec tests
         if: always()
-        run: postgrest-with-${{ matrix.pgVersion }} postgrest-test-spec
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-spec
 
       - name: Run observability tests
         if: always()
-        run: postgrest-with-${{ matrix.pgVersion }} postgrest-test-observability
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-observability
 
       - name: Run IO tests
         if: always()
-        run: postgrest-with-${{ matrix.pgVersion }} postgrest-test-io -vv
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-io -vv
 
       - name: Run IO tests on a big schema
         if: always()
-        run: postgrest-with-${{ matrix.pgVersion }} postgrest-test-big-schema -vv
+        run: postgrest-with-pg-${{ matrix.pgVersion }} postgrest-test-big-schema -vv
 
 
   memory:
     name: Memory
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
@@ -129,20 +124,19 @@ jobs:
 
   loadtest:
     strategy:
-      fail-fast: false
       matrix:
-        kind: ['mixed', 'jwt-cache']
+        kind: ['mixed', 'errors', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
     name: Loadtest
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
       - name: Setup Nix Environment
         uses: ./.github/actions/setup-nix
         with:
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: loadtest.loadtestAgainst.bin loadtest.report.bin loadtest.report-load.bin cabalTools.update.bin
+          tools: loadtest.loadtestAgainst.bin loadtest.report.bin cabalTools.update.bin
 
       - run: postgrest-cabal-update
 
@@ -156,48 +150,21 @@ jobs:
             latest_tag=$(git tag --merged HEAD --sort=-creatordate "v*" | head -n1)
           fi
           postgrest-loadtest-against -k ${{ matrix.kind }} "$TARGET_BRANCH" "$latest_tag"
-
-      - name: Report P50
-        # This step checks whether any red cross indicators (:x:) are present in the step summary.
-        # The loadtest reporter writes them when any of individual steps fails the performance
-        # regression threshold.
-        run: |
-          ! (postgrest-loadtest-report -g ${{ matrix.kind }} -p 50 \
-              | tee "$GITHUB_STEP_SUMMARY" \
-              | grep ':x:')
-
-      - name: Report P0
-        if: always()
-        run: |
-          postgrest-loadtest-report -g ${{ matrix.kind }} -p 0 >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Report P90
-        if: always()
-        run: |
-          postgrest-loadtest-report -g ${{ matrix.kind }} -p 90 >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Report P95
-        if: always()
-        run: |
-          postgrest-loadtest-report -g ${{ matrix.kind }} -p 95 >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Report CPU/MEM
-        if: always()
-        run: |
-          postgrest-loadtest-report-load -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY"
+          postgrest-loadtest-report -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY"
 
   flake:
     strategy:
       fail-fast: false
       matrix:
         runs-on:
+          - macos-15-intel # x86_64-darwin
           - macos-14 # aarch64-darwin
           - ubuntu-24.04 # x86_64-linux
           - ubuntu-24.04-arm # aarch64-linux
     name: Flake Check
     runs-on: ${{ matrix.runs-on }}
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
       - name: Setup Nix Environment

.gitignore

@@ -25,9 +25,5 @@ loadtest
 .history
 .docs-build
 gen_targets.http
-gen_jwks.json
+gen_jwk.json
 gen_private.json
-.pytest_cache
-.ruff_cache
-postgrest-module-graph.png
-.ghc.environment.*

.stylish-haskell.yaml

@@ -200,7 +200,7 @@ steps:
 
 # A common setting is the number of columns (parts of) code will be wrapped
 # to. Different steps take this into account. Default: 80.
-columns: 80
+columns: 70
 
 # By default, line endings are converted according to the OS. You can override
 # preferred format here.

CHANGELOG.md

@@ -4,148 +4,34 @@ All notable changes to this project will be documented in this file. From versio
 
 ## Unreleased
 
-## [16.1] - 2026-08-10
+### Added
 
-### Fixed
-
-- JWT validation uses wrong current time due to a bug in auto-update by @mkleczek in #5159
-
-## [16.0] - 2026-08-07
-
-### Changes
-
-#### HTTP Server
-
-- [Graceful shutdown](https://docs.postgrest.org/en/v16/references/http_server.html#graceful-shutdown) by @mkleczek, @Vlix in #4702
-
-- [server-reuseport](https://docs.postgrest.org/en/v16/references/configuration.html#server-reuseport) allows starting multiple PostgREST instances using the same port on supported platforms by @mkleczek in #4703, #4694
-
-#### Performance
-
-- Optimize schema cache domain type resolution by using [pg_basetype](https://www.postgresql.org/docs/current/functions-info.html#FUNCTIONS-INFO-CATALOG) on PostgreSQL 17+ by @joelonsql in #4567
-
-- [Prefer: count=exact](https://docs.postgrest.org/en/v16/references/api/pagination_count.html#exact-count) no longer does a double count on requests that do not use ranges or `db-max-rows` by @laurenceisla in #3957
-
-- [Prefer: timezone](https://docs.postgrest.org/en/v16/references/api/preferences.html#prefer-timezone) no longer requires the schema cache by @steve-chavez in #5100
-  + Previously this required caching [pg_timezone_names](https://www.postgresql.org/docs/current/view-pg-timezone-names.html) which was slow in some systems
-
-#### Integrations
-
-- PostgREST is now tested to work with [OrioleDB](https://github.com/orioledb/orioledb/) in #4845 by @wolfgangwalther
-  + See [our guide for running OrioleDB on NixOS](https://docs.postgrest.org/en/v16/integrations/nixos.html)
-
-#### JWT
-
-- [JWT Role Extraction](https://docs.postgrest.org/en/v16/references/auth.html#jwt-role-extract) is now more flexible, supporting the standard JSON Path defined in RFC 9535 by @taimoorzaeem in #4984
-
-#### API
-
-- [Prefer: timezone](https://docs.postgrest.org/en/v16/references/api/preferences.html#timezone) now supports numeric offsets like `05:00` or `-4` by @steve-chavez in #5100
-
-- Fix unexpected results when embedding and filtering the same table more than once by @laurenceisla in #4075
-  + You need to set [url-use-legacy-target-names](https://docs.postgrest.org/en/v16/references/configuration.html#url-use-legacy-target-names) to `false`.
-
-- Deprecate filters, orders and limits with the name of an embedded table when it has an alias by @steve-chavez, @laurenceisla in #4075
-  + e.g. `?select=alias:table(*)&table.id=eq.1` will not be possible anymore, use `?select=alias:table(*)&alias.id=eq.1` instead.
-  + You will see a warning in the logs and a `Warning` header on the client response when this happens.
-  + You can disable this behavior now by setting `url-use-legacy-target-names = false`.
-
-- Add `Vary` header to responses by @develop7 in #4609
-
-- Fix automatic transaction retries on `40001 (serialization_failure)` errors to prevent replication lag by @laurenceisla in #3673
-
-#### Observability
-
-- [GHC runtime metrics](https://docs.postgrest.org/en/v16/references/observability.html#ghc-runtime-metrics) by @mkleczek in #4862
-- [client-error-verbosity](https://docs.postgrest.org/en/v16/references/configuration.html#client-error-verbosity) to customize responses error verbosity by @taimoorzaeem in #4088, #3980, #3824
-- [log-level](https://docs.postgrest.org/en/v16/references/configuration.html#log-level) config is now reloadable by @taimoorzaeem in #5113
 - Log error when `db-schemas` config contains schema `pg_catalog` or `information_schema` by @taimoorzaeem in #4359
+- Add string slicing operator for `jwt-role-claim-key` by @taimoorzaeem in #4599
+- Optimize requests with `Prefer: count=exact` that do not use ranges or `db-max-rows` by @laurenceisla in #3957
+  + Removed unnecessary double count when building the `Content-Range`.
+- Add config `client-error-verbosity` to customize error verbosity by @taimoorzaeem in #4088, #3980, #3824
+- Add `Vary` header to responses by @develop7 in #4609
+- Add config `db-timezone-enabled` for optional querying of timezones by @taimoorzaeem in #4751
 - Log schema cache queries timings on `log-level=debug` by @steve-chavez in #4805
 
-#### Admin Server
+### Fixed
 
-- [admin-server-unix-socket](https://docs.postgrest.org/en/v16/references/configuration.html#admin-server-unix-socket)/[admin-server-unix-socket-mode](https://docs.postgrest.org/en/v16/references/configuration.html#admin-server-unix-socket-mode) to run the admin server on a unix socket by @wolfgangwalther in #5003
-- Fix responding with `Something went wrong` on Admin server when under EMFILE by @mkleczek in #5077
+- Shutdown should wait for in flight requests by @mkleczek in #4702
+- Fix login with uppercase and mixed case role names by @taimoorzaeem in #4678
+- Remove automatic transaction retries on `40001 (serialization_failure)` errors to prevent replication lag by @laurenceisla in #3673
+- Fix unexpected results when embedding and filtering the same table more than once by @laurenceisla in #4075
 
-#### Deployment
-
-- Make executable for aarch64-linux static instead of Ubuntu-based by @wolfgangwalther in #4193
-- Docker image for aarch64-linux is now built from scratch instead of being Ubuntu-based by @wolfgangwalther in #4193
-- Besides Docker Hub, docker images are now published to Github Container Registry by @wolfgangwalther in #2836
-
-#### Schema Cache
-
-- Fix requests failing when the schema cache fails to reload, when this happens PostgREST will continue serving requests in "best effort" by @mkleczek in #4873 #4869
-- Fix reporting 503s errors unnecessarily while the schema cache is loading at startup by @mkleczek in #4880
-- Fix schema cache dump missing RPC transaction isolation level by @taimoorzaeem in #5079
-
-#### Listener
-
-- Fix config `db-channel-enabled` not reloading by @taimoorzaeem in #4894
-
-### Migration to v16
+### Changed
 
 - Drop support for PostgreSQL EOL version 13 by @wolfgangwalther in #4193
-  + PostgreSQL 13 end of life was on 2025 ([ref](https://www.postgresql.org/support/versioning/))
-  + Upgrade your PostgreSQL version to at least 14 to use this new PostgREST version.
-
-- Fail at startup when `db-schemas` contains schema `pg_catalog` or `information_schema` by @taimoorzaeem in #4359
-  + Previously it failed at runtime with `PGRST205` on requests related to these schemas.
-  + Remove `pg_catalog` and `information_schema` from `db-schemas`.
-
-- `Prefer: timezone` no longer complies with `handling=lenient` and instead always fails by @steve-chavez in #5128
-  + Supporting this required caching `pg_timezone_names`, which was expensive.
-  + Ensure your requests always have a valid timezone.
-
-- `jwt-role-claim-key` no longer uses the JSPath DSL and instead uses JSON Path by @taimoorzaeem in #4984
-  + Now all config values must start with `$` character.
-     Example: `.roles.read` -> `$.roles.read`
-  + Keys with special characters, with the exception of `_` char must be quoted.
-      Example: `.roles.write-role` -> `$.roles["write-role"]`
-  + String comparison operators (`^==`, `==^` and `*==`) are replaced with regular expression search.
-      Example: `.roles[?(@ ^== "postgrest_test_")]` -> `$.roles[?search(@, "^postgrest_test_")]`
-  + Update the `jwt-role-claim-key` value accoring to the above rules. Also see the syntax reference: [RFC 9535](https://www.rfc-editor.org/rfc/rfc9535.html#name-jsonpath-syntax-and-semanti).
-
-## [14.16] - 2026-07-27
-
-### Fixed
-
-- Fix admin server crashing without a way to recover by @taimoorzaeem in #5096
-
-## [14.15] - 2026-07-13
-
-### Fixed
-
-- Fix admin server dying silently by @Vlix, @mkleczek, @steve-chavez in #5012
-
-## [14.14] - 2026-06-29
-
-### Fixed
-
-- Fix admin server not logging cause of failure by @taimoorzaeem in #5012
-
-## [14.13] - 2026-06-04
-
-### Fixed
-
-- Fix connection retrying message in `PGRST000` error by @netqo in #4980
-  + Remove redundant "Retrying the connection." from message because it is logged separately
-- Fix request failures when `work_mem` is set on a role by @laurenceisla in #4955
-
-## [14.12] - 2026-05-20
-
-### Fixed
-
-- Fix race condition in pool_available metric causing negative values during network instability by @mkleczek in #4622
-
-## [14.11] - 2026-05-04
-
-### Fixed
-
-- Fix login with uppercase and mixed case role names by @taimoorzaeem in #4678
-- Restore Listener query shape so it can be found in `pg_stat_activity` by @mkleczek in #4857 #4859
-- The LISTEN channel now automatically recovers when it stops working due to a PostgreSQL bug @laurenceisla in #3147
-- Fix misleading "Functions" name on schema cache summary in startup logs by @taimoorzaeem in #4821
+- All responses now include a `Vary` header by @develop7 in #4609
+- Log error when `db-schemas` config contains schema `pg_catalog` or `information_schema` by @taimoorzaeem in #4359
+  + Now fails at startup. Prior to this, it failed with `PGRST205` on requests related to these schemas.
+- Build a static executable for aarch64-linux by @wolfgangwalther in #4193
+- Build the minimal docker image for aarch64-linux by @wolfgangwalther in #4193
+- The name of an embedded table can no longer be used in filters if it has an alias by @laurenceisla in #4075
+  + e.g. `?select=alias:table(*)&table.id=eq.1` is not possible anymore, use `?select=alias:table(*)&alias.id=eq.1` instead.
 
 ## [14.10] - 2026-04-16
 

CONTRIBUTING.md

@@ -40,8 +40,6 @@ Check the [development docs](https://github.com/PostgREST/postgrest/blob/main/ni
 
 * All fixes or features must have a test proving the improvement.
 
-* All features must document the new behavior. Critical fixes that introduce new behavior must be documented too.
-
 * All code must also pass a [linter](http://community.haskell.org/~ndm/hlint/) and [styler](https://github.com/jaspervdj/stylish-haskell)
   with no warnings. This helps enforce a uniform style for all committers. Continuous integration will check this as well on every
   pull request. There are useful tools in the nix-shell that help with checking this locally. You can run `postgrest-check` to do this manually but

cabal.project

@@ -2,6 +2,3 @@ packages: postgrest.cabal
 tests: true
 allow-newer:
   hasql:postgresql-libpq
-
--- https://github.com/martijnbastiaan/doctest-parallel/blob/main/example/README.md#cabalproject
-write-ghc-environment-files: always

cabal.project.freeze

@@ -1 +1 @@
-index-state: hackage.haskell.org 2026-08-10T16:58:32Z
+index-state: hackage.haskell.org 2026-04-18T18:42:36Z

default.nix

@@ -52,20 +52,11 @@ let
 
   postgresqlVersions =
     [
-      { name = "pg-19"; postgresql = pkgs.postgresql_19.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
       { name = "pg-18"; postgresql = pkgs.postgresql_18.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
       { name = "pg-17"; postgresql = pkgs.postgresql_17.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
       { name = "pg-16"; postgresql = pkgs.postgresql_16.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
       { name = "pg-15"; postgresql = pkgs.postgresql_15.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
       { name = "pg-14"; postgresql = pkgs.postgresql_14.withPackages (p: [ p.postgis p.pg_safeupdate ]); }
-      {
-        name = "oriole-18";
-        postgresql = pkgs.orioledb.withPackages (p: [ p.postgis p.pg_safeupdate ]);
-        config = "
-          default_table_access_method = 'orioledb'
-          shared_preload_libraries = 'orioledb, pg_stat_statements'
-        ";
-      }
     ];
 
   haskellPackages = pkgs.haskell.packages."${compiler}";
@@ -86,16 +77,6 @@ let
     "-f dev --test-show-detail=direct";
 
   inherit (pkgs.haskell) lib;
-
-  nixos-lib = import (pkgs.path + "/nixos/lib") { };
-  runTest = postgrest: test: (nixos-lib.runTest {
-    hostPkgs = pkgs;
-    # Replace the top-level `pkgs.postgrest` attribute with our current version on this branch.
-    defaults.nixpkgs.overlays = [ (_: _: { inherit postgrest; }) ];
-    # Speeds up evaluation a little bit; documentation is really not required for tests.
-    defaults.documentation.enable = pkgs.lib.mkDefault false;
-    imports = [ test ];
-  }).config.result;
 in
 rec {
   inherit nixpkgs pkgs;
@@ -139,7 +120,7 @@ rec {
 
   # Development tools.
   devTools =
-    pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie; };
+    pkgs.callPackage nix/tools/devTools.nix { inherit tests style devCabalOptions hsie withTools; };
 
   # Documentation tools.
   docs =
@@ -181,7 +162,4 @@ rec {
   # Docker images and loading script.
   docker =
     pkgs.callPackage nix/tools/docker { postgrest = postgrestStatic; };
-
-  # NixOS VM tests
-  nixpkgs-nixos-test = runTest postgrestStatic (pkgs.path + "/nixos/tests/postgrest.nix");
 }

docs/_diagrams/uml/arch.uml

@@ -87,8 +87,8 @@ url of Authorization is [[../explanations/db_authz.html]]
 url of CLI is [[../references/cli.html#cli]]
 url of "Connection Pool" is [[../references/connection_pool.html]]
 url of Config is [[../references/configuration.html#configuration]]
-url of HTTPADMIN is [[../references/http_server.html]]
-url of HTTPAPI is [[../references/http_server.html]]
+url of HTTPADMIN is [[../explanations/architecture.html#http]]
+url of HTTPAPI is [[../explanations/architecture.html#http]]
 url of Listener is [[../references/listener.html#listener]]
 url of Proxy is [[../explanations/nginx.html]]
 url of "Schema Cache" is [[../references/schema_cache.html#schema-cache]]

docs/conf.py

@@ -49,7 +49,7 @@ source_suffix = ".rst"
 master_doc = "index"
 
 # This is overridden by readthedocs with the version tag anyway
-version = "16"
+version = "devel"
 # To avoid repetition in <title> we set this to an empty string.
 release = ""
 
@@ -300,7 +300,6 @@ linkcheck_ignore = [
     r"https://www.patreon.com/postgrest",
     r"https://blog.frankel.ch/poor-man-api",
     r"https://www.cybertec-postgresql.com/.*",
-    r"https://stackoverflow.com/.*",
     # Odd SSL error
     r"https://www.dripdepot.com",
     r"https://www.euronodes.com",

docs/ecosystem.rst

@@ -83,7 +83,6 @@ Extensions
 Client-Side Libraries
 ---------------------
 
-* `efcore-postgrest <https://github.com/pedro-gilmora/EF.PostgREST.Provider>`_ - C#
 * `postgrest-csharp <https://github.com/supabase-community/postgrest-csharp>`_ - C#
 * `postgrest-dart <https://github.com/supabase/postgrest-dart>`_ - Dart
 * `postgrest-ex <https://github.com/supabase-community/postgrest-ex>`_ - Elixir

docs/explanations/architecture.rst

@@ -31,60 +31,65 @@ This section talks briefly about various important modules.
 Main
 ----
 
-The starting point of the program is `Main.hs <https://github.com/PostgREST/postgrest/blob/v16/src/executable/Main.hs>`_.
+The starting point of the program is `Main.hs <https://github.com/PostgREST/postgrest/blob/main/main/Main.hs>`_.
 
 CLI
 ---
 
-Main then calls `CLI.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/CLI.hs>`_, which is in charge of :ref:`cli`.
+Main then calls `CLI.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/CLI.hs>`_, which is in charge of :ref:`cli`.
 
 App
 ---
 
-`App.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/App.hs>`_ is then in charge of composing the different modules.
+`App.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/App.hs>`_ is then in charge of composing the different modules.
 
 Auth
 ----
 
-`Auth.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/Auth.hs>`_ is in charge  of :ref:`authn`.
+`Auth.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/Auth.hs>`_ is in charge  of :ref:`authn`.
 
 Api Request
 -----------
 
-`ApiRequest.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/ApiRequest.hs>`_ is in charge of parsing the URL query string (following PostgREST syntax), the request headers, and the request body.
+`ApiRequest.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/ApiRequest.hs>`_ is in charge of parsing the URL query string (following PostgREST syntax), the request headers, and the request body.
 
 A request might be rejected at this level if it's invalid. For example when providing an unknown media type to PostgREST or using an unknown HTTP method.
 
 Plan
 ----
 
-Using the Schema Cache, `Plan.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/Plan.hs>`_ generates an internal AST, filling out-of-band SQL details (like an ``ON CONFLICT (pk)`` clause) required to complete the user request.
+Using the Schema Cache, `Plan.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/Plan.hs>`_ generates an internal AST, filling out-of-band SQL details (like an ``ON CONFLICT (pk)`` clause) required to complete the user request.
 
 A request might be rejected at this level if it's invalid. For example when doing resource embedding on a nonexistent resource.
 
 Query
 -----
 
-`Query.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/Query.hs>`_ generates the SQL queries (parametrized and prepared) required to satisfy the user request.
+`Query.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/Query.hs>`_ generates the SQL queries (parametrized and prepared) required to satisfy the user request.
 
 Only at this stage a connection from the pool might be used.
 
 Schema Cache
 ------------
 
-`SchemaCache.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/SchemaCache.hs>`_ is in charge of :ref:`schema_cache`.
+`SchemaCache.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/SchemaCache.hs>`_ is in charge of :ref:`schema_cache`.
 
 Config
 ------
 
-`Config.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/Config.hs>`_ is in charge of :ref:`configuration`.
+`Config.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/Config.hs>`_ is in charge of :ref:`configuration`.
 
 Admin
 -----
 
-`Admin.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/Admin.hs>`_ is in charge of the :ref:`admin_server`.
+`Admin.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/Admin.hs>`_ is in charge of the :ref:`admin_server`.
+
+HTTP
+----
+
+The HTTP server is provided by `Warp <https://aosabook.org/en/posa/warp.html>`_.
 
 Listener
 --------
 
-`Reload.hs <https://github.com/PostgREST/postgrest/blob/v16/src/library/PostgREST/AppState/Reload.hs>`_ is in charge of the :ref:`listener`.
+`Listener.hs <https://github.com/PostgREST/postgrest/blob/main/src/PostgREST/Listener.hs>`_ is in charge of the :ref:`listener`.

docs/how-tos/debugging-performance-with-pg-stat-statements.rst

@@ -1,62 +0,0 @@
-.. _debugging_performance_pg_stat_statements:
-
-Debugging Performance with pg_stat_statements
-=============================================
-
-This how-to shows how to get a query identifier through PostgREST and then use it to inspect the same query in ``pg_stat_statements``.
-
-.. important::
-
-  - :ref:`db-plan-enabled` must be enabled in PostgREST.
-  - PostgreSQL 14 or newer with ``pg_stat_statements`` available.
-
-Get the Query Identifier from PostgREST
----------------------------------------
-
-Request the plan in JSON format with the ``verbose`` option:
-
-.. code-block:: bash
-
-   curl "http://localhost:3000/projects?select=id,name&order=id" \
-     -H "Accept: application/vnd.pgrst.plan+json; options=verbose"
-
-The response will contain a top-level ``Query Identifier`` field:
-
-.. code-block:: json
-
-   [
-     {
-       "Plan": {
-         "Node Type": "Aggregate"
-       },
-       "Query Identifier": -432192689578025496
-     }
-   ]
-
-Look up the query in pg_stat_statements
----------------------------------------
-
-Use that identifier against ``pg_stat_statements``:
-
-.. code-block:: postgres
-
-   select
-     calls,
-     total_exec_time,
-     mean_exec_time,
-     rows,
-     query
-   from pg_stat_statements
-   where queryid = -432192689578025496;
-
-.. csv-table::
-   :header: "calls", "total_exec_time", "mean_exec_time", "rows", "query"
-
-   "13", "0.6355850000000001", "0.04889115384615385", "13", "WITH pgrst_source AS (...)"
-
-This lets you correlate a PostgREST request with PostgreSQL runtime statistics such as:
-
-- how often the query ran
-- total and average execution time
-- how many rows it produced
-- the normalized SQL text recorded by PostgreSQL

docs/how-tos/working-with-postgresql-data-types.rst

@@ -318,6 +318,144 @@ You can insert a new product using a JSON object for the ``extra_info`` column:
 
 To query and filter the data see :ref:`json_columns` for a complete reference.
 
+.. _ww_postgis:
+
+PostGIS
+-------
+
+You can use the string representation for `PostGIS <https://postgis.net/>`_ data types such as ``geometry`` or ``geography`` (you need to `install PostGIS <https://postgis.net/documentation/getting_started/>`_ first).
+
+.. code-block:: postgres
+
+  -- Activate the postgis module in the current database
+  create extension if not exists postgis;
+
+  create table coverage (
+    id int primary key,
+    name text unique,
+    area geometry
+  );
+
+To add areas in polygon format, you can use string representation:
+
+.. code-block:: bash
+
+  curl "http://localhost:3000/coverage" \
+    -X POST -H "Content-Type: application/json" \
+    -d @- << EOF
+    [
+      { "id": 1, "name": "small", "area": "SRID=4326;POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))" },
+      { "id": 2, "name": "big", "area": "SRID=4326;POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))" }
+    ]
+  EOF
+
+Now, when you request the information, PostgREST will automatically cast the ``area`` column into a ``Polygon`` geometry type. Although this is useful, you may need the whole output to be in `GeoJSON <https://geojson.org/>`_ format out of the box, which can be done by including the ``Accept: application/geo+json`` in the request. This will work for PostGIS versions 3.0.0 and up and will return the output as a `FeatureCollection Object <https://www.rfc-editor.org/rfc/rfc7946#section-3.3>`_:
+
+.. code-block:: bash
+
+  curl "http://localhost:3000/coverage" \
+    -H "Accept: application/geo+json"
+
+.. code-block:: json
+
+  {
+    "type": "FeatureCollection",
+    "features": [
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Polygon",
+          "coordinates": [
+            [[0,0],[1,0],[1,1],[0,1],[0,0]]
+          ]
+        },
+        "properties": {
+          "id": 1,
+          "name": "small"
+        }
+      },
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Polygon",
+          "coordinates": [
+            [[0,0],[10,0],[10,10],[0,10],[0,0]]
+          ]
+        },
+        "properties": {
+          "id": 2,
+          "name": "big"
+        }
+      }
+    ]
+  }
+
+If you need to add an extra property, like the area in square units by using ``st_area(area)``, you could add a generated column to the table and it will appear in the ``properties`` key of each ``Feature``.
+
+.. code-block:: postgres
+
+  alter table coverage
+    add square_units double precision generated always as ( st_area(area) ) stored;
+
+In the case that you are using older PostGIS versions, then creating a function is your best option:
+
+.. code-block:: postgres
+
+  create or replace function coverage_geo_collection() returns json as $$
+    select
+      json_build_object(
+        'type', 'FeatureCollection',
+        'features', json_agg(
+          json_build_object(
+            'type', 'Feature',
+            'geometry', st_AsGeoJSON(c.area)::json,
+            'properties', json_build_object('id', c.id, 'name', c.name)
+          )
+        )
+      )
+    from coverage c;
+  $$ language sql;
+
+Now this query will return the same results:
+
+.. code-block:: bash
+
+  curl "http://localhost:3000/rpc/coverage_geo_collection"
+
+.. code-block:: json
+
+  {
+    "type": "FeatureCollection",
+    "features": [
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Polygon",
+          "coordinates": [
+            [[0,0],[1,0],[1,1],[0,1],[0,0]]
+          ]
+        },
+        "properties": {
+          "id": 1,
+          "name": "small"
+        }
+      },
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Polygon",
+          "coordinates": [
+            [[0,0],[10,0],[10,10],[0,10],[0,0]]
+          ]
+        },
+        "properties": {
+          "id": 2,
+          "name": "big"
+        }
+      }
+    ]
+  }
+
 Ranges
 ------
 
@@ -471,20 +609,3 @@ You can use other comparative filters and also all the `PostgreSQL special date/
       "due_date": "2022-02-27T06:00:00-05:00"
     }
   ]
-
-.. raw:: html
-
-  <script type="text/javascript">
-    let hash = window.location.hash;
-
-    const redirects = {
-      // PostGIS
-      '#postgis': '../integrations/postgis.html#postgis',
-    };
-
-    let willRedirectTo = redirects[hash];
-
-    if (willRedirectTo) {
-      window.location.href = willRedirectTo;
-    }
-  </script>

docs/index.rst

@@ -119,14 +119,11 @@ Releases
 PostgREST follows ``MAJOR.PATCH`` two-part versioning:
 
 - ``MAJOR``: feature release, may deprecate or remove things.
-- ``PATCH``: fix/security release only, no features and no behavior changes.
+- ``PATCH``: fix/security release only; no features, no behavior changes.
 
-MAJOR releases are published twice a year, with their scope and target dates tracked through `GitHub milestones <https://github.com/PostgREST/postgrest/milestones>`_.
-PATCH releases are published on an as-needed basis.
+Starting from ``v14.0``, only even-numbered MAJOR versions will be released, reserving odd-numbered MAJOR versions for development.
 
-Starting from ``v14.0``, only even-numbered MAJOR versions are released, reserving odd-numbered MAJOR versions for development.
-
-All releases are published on `PostgREST's GitHub release page <https://github.com/PostgREST/postgrest/releases>`_, along with the corresponding upgrade guides.
+All the releases are published on `PostgREST's GitHub release page <https://github.com/PostgREST/postgrest/releases>`_.
 
 Tutorials
 ---------

docs/integrations/nixos.rst

@@ -1,36 +0,0 @@
-NixOS
-=====
-
-Nixpkgs contains a `NixOS module to run PostgREST <https://search.nixos.org/options?channel=unstable&query=services.postgrest&type=options>`_, which can be enabled with ``services.postgrest.enable = true``.
-
-A PostgreSQL server can be enabled on the same machine with ``services.postgresql.enable = true``. Connections will use the name of the system user as user and database names by default, in this case ``postgrest``.
-
-A minimal example could look like this:
-
-.. code-block:: nix
-
-  {
-    pkgs,
-    ...
-  }:
-
-  {
-    services.postgresql = {
-      enable = true;
-      initialScript = pkgs.writeText "init.sql" ''
-        CREATE ROLE postgrest LOGIN NOINHERIT;
-        CREATE ROLE anon ROLE postgrest;
-      '';
-    };
-
-    services.postgrest = {
-      enable = true;
-      settings.db-anon-role = "anon";
-      settings.db-uri.dbname = "postgres";
-    };
-  }
-
-This will expose the PostgREST server on localhost on the NixOS machine and allow anonymous access.
-
-.. tip::
-  NixOS also allows to quickly spin up different PostgreSQL versions or even forks this way. For example, to test the current beta version of `OrioleDB <https://www.orioledb.com>`_, use ``services.postgresql.package = pkgs.orioledb``.

docs/integrations/postgis.rst

@@ -1,154 +0,0 @@
-.. _ww_postgis:
-
-PostGIS
-=======
-
-To work with `PostGIS <https://postgis.net/>`_ data types such as ``geometry`` or ``geography``, you'll need to `install PostGIS <https://postgis.net/documentation/getting_started/>`_ first.
-
-.. code-block:: postgres
-
-  -- Activate the postgis module in the current database
-  create extension if not exists postgis;
-
-  create table coverage (
-    id int primary key,
-    name text unique,
-    area geometry
-  );
-
-  insert into coverage (id, name, area) values
-    (1, 'small', ST_GeomFromText('POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))',4326)),
-    (2, 'big', ST_GeomFromText('POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))', 4326);
-
-.. _application/geo+json:
-
-``application/geo+json``
-------------------------
-
-PostgREST supports the `standard <https://www.iana.org/assignments/media-types/application/geo+json>`_ ``application/geo+json`` media type which can be used to get the output in `GeoJSON <https://geojson.org/>`_ format. This will work for PostGIS versions 3.0.0 and up and will return the output as a `FeatureCollection Object <https://www.rfc-editor.org/rfc/rfc7946#section-3.3>`_:
-
-.. code-block:: bash
-
-  curl "http://localhost:3000/coverage" \
-    -H "Accept: application/geo+json"
-
-.. code-block:: json
-
-  {
-    "type": "FeatureCollection",
-    "features": [
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Polygon",
-          "coordinates": [
-            [[0,0],[1,0],[1,1],[0,1],[0,0]]
-          ]
-        },
-        "properties": {
-          "id": 1,
-          "name": "small"
-        }
-      },
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Polygon",
-          "coordinates": [
-            [[0,0],[10,0],[10,10],[0,10],[0,0]]
-          ]
-        },
-        "properties": {
-          "id": 2,
-          "name": "big"
-        }
-      }
-    ]
-  }
-
-Using generated columns
------------------------
-
-If you need to add an extra property, like the area in square units by using ``st_area(area)``, you could add a generated column to the table and it will appear in the ``properties`` key of each ``Feature``.
-
-.. code-block:: postgres
-
-  alter table coverage
-    add square_units double precision generated always as ( st_area(area) ) stored;
-
-In the case that you are using older PostGIS versions, then creating a function is your best option:
-
-.. code-block:: postgres
-
-  create or replace function coverage_geo_collection() returns json as $$
-    select
-      json_build_object(
-        'type', 'FeatureCollection',
-        'features', json_agg(
-          json_build_object(
-            'type', 'Feature',
-            'geometry', st_AsGeoJSON(c.area)::json,
-            'properties', json_build_object('id', c.id, 'name', c.name)
-          )
-        )
-      )
-    from coverage c;
-  $$ language sql;
-
-Now this query will return the same results:
-
-.. code-block:: bash
-
-  curl "http://localhost:3000/rpc/coverage_geo_collection"
-
-.. code-block:: json
-
-  {
-    "type": "FeatureCollection",
-    "features": [
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Polygon",
-          "coordinates": [
-            [[0,0],[1,0],[1,1],[0,1],[0,0]]
-          ]
-        },
-        "properties": {
-          "id": 1,
-          "name": "small"
-        }
-      },
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Polygon",
-          "coordinates": [
-            [[0,0],[10,0],[10,10],[0,10],[0,0]]
-          ]
-        },
-        "properties": {
-          "id": 2,
-          "name": "big"
-        }
-      }
-    ]
-  }
-
-Using string representation
----------------------------
-
-To insert areas in polygon format, you can use string representation:
-
-.. code-block:: bash
-
-  curl "http://localhost:3000/coverage" \
-    -X POST -H "Content-Type: application/json" \
-    -d @- << EOF
-    [
-      { "id": 3, "name": "strip", "area": "SRID=4326;POLYGON((0 0, 50 0, 50 2, 0 2, 0 0))" },
-      { "id": 4, "name": "diamond", "area": "SRID=4326;POLYGON((5 0, 10 5, 5 10, 0 5, 5 0))" }
-    ]
-  EOF
-
-PostgREST will automatically cast the ``area`` column into a ``Polygon`` geometry type. 

docs/postgrest.dict

@@ -1,6 +1,5 @@
 personal_ws-1.1 en 0 utf-8
 api
-autoscaling
 API's
 APIs
 APISIX
@@ -31,11 +30,11 @@ CSV
 durations
 DDL
 DOM
+DSL
 DevOps
 Dramatiq
 dockerize
 enum
-ECS
 Enums
 Entra
 eq
@@ -46,9 +45,7 @@ filename
 FreeBSD
 fts
 fullstack
-GC
 GeoJSON
-GHC
 Github
 Google
 grantor
@@ -77,6 +74,7 @@ isdistinct
 JS
 js
 JSON
+JSPath
 JWK
 JWT
 jwt
@@ -99,7 +97,6 @@ namespaced
 Nanos
 neq
 nginx
-NixOS
 nixpkgs
 npm
 nxl
@@ -145,13 +142,11 @@ Redux
 refactor
 reloadable
 Reloadable
-reuseport
 requester's
 RESTful
 RLS
 RPC
 RSA
-RTS
 safeupdate
 savepoint
 schemas
@@ -205,4 +200,4 @@ webuser
 wfts
 www
 debouncing
-deduplicates
+deduplicates

docs/references/admin_server.rst

@@ -3,9 +3,7 @@
 Admin Server
 ############
 
-PostgREST provides an admin server that can be enabled by setting :ref:`admin-server-port` or :ref:`admin-server-unix-socket`.
-
-Multiple PostgREST instances can share the same public API host and port when :ref:`server-reuseport` is enabled. Admin ports are not shared: give each instance a different :ref:`admin-server-port`, otherwise the new instance will fail to start.
+PostgREST provides an admin server that can be enabled by setting :ref:`admin-server-port`.
 
 .. _health_check:
 
@@ -74,4 +72,5 @@ Provides the ``schema_cache`` endpoint that prints the runtime :ref:`schema_cach
     "dbRepresentations": ["..."],
     "dbRoutines": ["..."],
     "dbTables": ["..."],
+    "dbTimezones": ["..."]
   }

docs/references/api/preferences.rst

@@ -62,12 +62,8 @@ The server ignores unrecognized or unfulfillable preferences by default. You can
 Timezone
 ========
 
-.. important::
+The ``timezone`` preference allows you to change the `PostgreSQL timezone <https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-TIMEZONE>`_. It accepts all time zones in `pg_timezone_names <https://www.postgresql.org/docs/current/view-pg-timezone-names.html>`_.
 
-  ``handling=lenient`` is ignored for ``timezone``. Invalid time zones always return an error.
-
-The ``timezone`` preference allows you to change the `PostgreSQL timezone <https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-TIMEZONE>`_.
-It accepts all time zones in `pg_timezone_names <https://www.postgresql.org/docs/current/view-pg-timezone-names.html>`_ and numeric offsets.
 
 .. code-block:: bash
 
@@ -88,40 +84,43 @@ It accepts all time zones in `pg_timezone_names <https://www.postgresql.org/docs
     {"t":"2023-10-18T09:37:59.611-07:00"}
   ]
 
-Offsets are also accepted:
-
-.. code-block:: bash
-
-  curl -i "http://localhost:3000/timestamps" \
-    -H "Prefer: timezone=05:30"
-
-.. code-block:: http
-
-  HTTP/1.1 200 OK
-  Content-Type: application/json; charset=utf-8
-  Preference-Applied: timezone=05:30
-
-.. code-block:: json
-
-  [
-    {"t":"2023-10-18T17:07:59.611+05:30"},
-    {"t":"2023-10-18T19:07:59.611+05:30"},
-    {"t":"2023-10-18T21:07:59.611+05:30"}
-  ]
-
-You can also use negative offsets like ``-03:00``.
-
-For an invalid time zone, PostgREST returns a database error.
+For an invalid time zone, PostgREST returns values with the default time zone (configured on ``postgresql.conf`` or as a setting on the :ref:`authenticator <roles>`).
 
 .. code-block:: bash
 
   curl -i "http://localhost:3000/timestamps" \
     -H "Prefer: timezone=Jupiter/Red_Spot"
 
+.. code-block:: http
+
+  HTTP/1.1 200 OK
+  Content-Type: application/json; charset=utf-8
+
+.. code-block:: json
+
+  [
+    {"t":"2023-10-18T12:37:59.611+00:00"},
+    {"t":"2023-10-18T14:37:59.611+00:00"},
+    {"t":"2023-10-18T16:37:59.611+00:00"}
+  ]
+
+Note that there's no ``Preference-Applied`` in the response.
+
+However, with ``handling=strict``, an invalid time zone preference will throw an :ref:`error <pgrst122>`.
+
+.. code-block:: bash
+
+  curl -i "http://localhost:3000/timestamps" \
+    -H "Prefer: handling=strict, timezone=Jupiter/Red_Spot"
+
 .. code-block:: http
 
   HTTP/1.1 400 Bad Request
 
+.. note::
+
+  This feature requires querying `pg_timezone_names <https://www.postgresql.org/docs/current/view-pg-timezone-names.html>`_ during :ref:`schema_cache` load. If this is not desired, you can disable the feature with :ref:`db-timezone-enabled`.
+
 .. _prefer_return:
 
 Return Representation

docs/references/api/resource_representation.rst

@@ -51,7 +51,7 @@ Builtin handlers are offered for common standard media types.
 
 * ``text/csv`` and ``application/json``, for all API endpoints. See :ref:`tables_views` and :ref:`functions`.
 * ``application/openapi+json``, for the root endpoint. See :ref:`open-api`.
-* ``application/geo+json``, see :ref:`application/geo+json`.
+* ``application/geo+json``, see :ref:`ww_postgis`.
 * ``*/*``, resolves to ``application/json`` for API endpoints and to ``application/openapi+json`` for the root endpoint.
 
 The following vendor media types handlers are also supported.

docs/references/auth.rst

@@ -224,31 +224,56 @@ It's recommended to leave the JWT cache enabled as our load tests indicate ~20%
 JWT Role Extraction
 -------------------
 
-A JSON Path (`RFC 9535 <https://www.rfc-editor.org/rfc/rfc9535.html>`_) can be specified for the location of the :code:`role` key in the JWT claims. It's configured by :ref:`jwt-role-claim-key`. This can be used to consume a JWT provided by a third party service like Auth0, Okta, Microsoft Entra or Keycloak.
+A JSPath DSL that specifies the location of the :code:`role` key in the JWT claims. It's configured by :ref:`jwt-role-claim-key`. This can be used to consume a JWT provided by a third party service like Auth0, Okta, Microsoft Entra or Keycloak.
 
-You can quickly try out JSON Path by visiting https://serdejsonpath.live.
+The DSL follows the `JSONPath <https://goessner.net/articles/JsonPath/>`_ expression grammar with extended string comparison operators. Supported operators are:
+
+- ``==`` selects the first array element that exactly matches the right operand
+- ``!=`` selects the first array element that does not match the right operand
+- ``^==`` selects the first array element that starts with the right operand
+- ``==^`` selects the first array element that ends with the right operand
+- ``*==`` selects the first array element that contains the right operand
+
+The selected role value can also be sliced using the slice operator ``[a:b]``. It is similar to `slice operator in python <https://docs.python.org/3/library/functions.html#slice>`_. Negative index values are also supported. The syntax is as:
+
+- ``[a:b]`` take slice from index ``a`` up to ``b``
+- ``[a:]`` take slice from index ``a`` to end
+- ``[:b]`` take slice from start to index ``b``
+- ``[:]`` select everything, no slicing
+
+.. important::
+
+  Make sure that you are not taking a slice where the start index comes after the end index like ``[11:2]``. The result of this would be empty string and so no role would get selected.
 
 Usage examples:
 
   .. code:: bash
 
     # {"postgrest":{"roles": ["other", "author"]}}
-    jwt-role-claim-key = "$$.postgrest.roles[1]"
+    # the DSL accepts characters that are alphanumerical or one of "_$@" as keys
+    jwt-role-claim-key = ".postgrest.roles[1]"
 
     # {"https://www.example.com/role": { "key": "author" }}
-    # non-alphanumerical characters can go inside single quotes
-    jwt-role-claim-key = "$$['https://www.example.com/role'].key"
+    # non-alphanumerical characters can go inside quotes(escaped in the config value)
+    jwt-role-claim-key = ".\"https://www.example.com/role\".key"
 
     # {"postgrest":{"roles": ["other", "author"]}}
-    # filter based on equality or regular expression
-    jwt-role-claim-key = "$$.postgrest.roles[?(@ == 'author')]"
-    jwt-role-claim-key = "$$.postgrest.roles[?search(@, '^au')]"
+    # `@` represents the current element in the array
+    # all the these match the string "author"
+    jwt-role-claim-key = ".postgrest.roles[?(@ == \"author\")]"
+    jwt-role-claim-key = ".postgrest.roles[?(@ != \"other\")]"
+    jwt-role-claim-key = ".postgrest.roles[?(@ ^== \"aut\")]"
+    jwt-role-claim-key = ".postgrest.roles[?(@ ==^ \"hor\")]"
+    jwt-role-claim-key = ".postgrest.roles[?(@ *== \"utho\")]"
+
+    # {"postgrest":{"wlcg": ["/groupa", "/groupb/"]}}
+    # skip the "/" character using slice operator
+    jwt-role-claim-key = ".postgrest.wlcg[0][1:]"
+    jwt-role-claim-key = ".postgrest.wlcg[1][1:-1]"
 
 .. note::
 
-  - If JSON Path query returns multiple values, the first one gets selected.
-  - Only when using the :ref:`file_config`, all ``$`` characters in the value must be escaped with an additional ``$`` char. For :ref:`env_variables_config` and :ref:`in_db_config`, only use a single ``$`` char.
-  - In our implementation, only the `search()` function from `JSON Path Functions <https://www.rfc-editor.org/rfc/rfc9535.html#name-function-extensions>`_ is available for filtering.
+  The string comparison operators are implemented as a custom extension to the JSPath and does not strictly follow the `RFC 9535 <https://www.rfc-editor.org/rfc/rfc9535.html>`_.
 
 JWT Security
 ------------

docs/references/configuration.rst

@@ -176,46 +176,6 @@ admin-server-port
 
   Specifies the port for the :ref:`admin_server`. Cannot be equal to :ref:`server-port`.
 
-.. _admin-server-unix-socket:
-
-admin-server-unix-socket
-------------------------
-
-  =============== =================================
-  **Type**        String
-  **Default**     `n/a`
-  **Reloadable**  N
-  **Environment** PGRST_ADMIN_SERVER_UNIX_SOCKET
-  **In-Database** `n/a`
-  =============== =================================
-
-  `Unix domain socket <https://en.wikipedia.org/wiki/Unix_domain_socket>`_ where to bind the :ref:`admin_server`.
-  If specified, this takes precedence over :ref:`admin-server-port`. Example:
-
-  .. code:: bash
-
-    admin-server-unix-socket = "/tmp/pgrst-admin.sock"
-
-.. _admin-server-unix-socket-mode:
-
-admin-server-unix-socket-mode
------------------------------
-
-  =============== ===================================
-  **Type**        String
-  **Default**     660
-  **Reloadable**  N
-  **Environment** PGRST_ADMIN_SERVER_UNIX_SOCKET_MODE
-  **In-Database** `n/a`
-  =============== ===================================
-
-  `Unix file mode <https://en.wikipedia.org/wiki/File_system_permissions>`_ to be set for the socket specified in :ref:`admin-server-unix-socket`
-  Needs to be a valid octal between 600 and 777.
-
-  .. code:: bash
-
-    admin-server-unix-socket-mode = "660"
-
 .. _app.settings.*:
 
 app.settings.*
@@ -580,6 +540,21 @@ db-schemas
 
   The list of database schemas to expose to clients. See :ref:`schemas`.
 
+.. _db-timezone-enabled:
+
+db-timezone-enabled
+-------------------
+
+  =============== =================================
+  **Type**        Boolean
+  **Default**     True
+  **Reloadable**  Y
+  **Environment** PGRST_DB_TIMEZONE_ENABLED
+  **In-Database** pgrst.db_timezone_enabled
+  =============== =================================
+
+  Enables the use of :ref:`prefer_timezone` preference header. Disabled when set to ``false``.
+
 .. _db-tx-end:
 
 db-tx-end
@@ -679,7 +654,7 @@ jwt-role-claim-key
 
   =============== =================================
   **Type**        String
-  **Default**     $.role
+  **Default**     .role
   **Reloadable**  Y
   **Environment** PGRST_JWT_ROLE_CLAIM_KEY
   **In-Database** pgrst.jwt_role_claim_key
@@ -689,10 +664,6 @@ jwt-role-claim-key
 
   See :ref:`jwt_role_extract` on how to specify key paths and usage examples.
 
-.. warning::
-
-  Only when using :ref:`file_config`, the ``$`` char needs to be escaped, so use ``$$`` and PostgREST will interpret it as a single ``$`` character.
-
 .. _jwt-secret:
 
 jwt-secret
@@ -752,7 +723,7 @@ log-level
   =============== =================================
   **Type**        String
   **Default**     error
-  **Reloadable**  Y
+  **Reloadable**  N
   **Environment** PGRST_LOG_LEVEL
   **In-Database** `n/a`
   =============== =================================
@@ -928,50 +899,6 @@ server-port
 
   The TCP port to bind the web server. Use ``0`` to automatically assign a port.
 
-.. _server-reuseport:
-
-server-reuseport
-----------------
-
-  =============== =================================
-  **Type**        Bool
-  **Default**     false
-  **Reloadable**  N
-  **Environment** PGRST_SERVER_REUSEPORT
-  **In-Database** `n/a`
-  =============== =================================
-
-  Enables ``SO_REUSEPORT`` on the TCP server socket. This allows multiple
-  PostgREST processes to bind to the same :ref:`server-host` and
-  :ref:`server-port` when the operating system supports it.
-  
-  For example, two PostgREST processes can use the same configuration:
-
-  .. code:: ini
-
-    server-host = "127.0.0.1"
-    server-port = 3000
-    server-reuseport = true
-
-  New connections are then distributed by the operating system between the
-  running PostgREST processes. This can be used to start a replacement process
-  before stopping the old one, or to run several PostgREST processes behind one
-  port.
-
-  If ``server-reuseport`` is disabled, starting another PostgREST process on
-  the same host and port will fail with the usual address-in-use error.
-
-  Enabling this setting on an operating system that does not support
-  ``SO_REUSEPORT`` is a configuration error. PostgREST will fail to start
-  instead of falling back to a normal TCP socket.
-
-  When running multiple PostgREST instances on the same :ref:`server-port`, use
-  a different ``admin-server-port`` for each instance. Admin ports are not shared
-  between instances, so readiness checks always target one specific PostgREST
-  instance.
-
-  This setting does not apply when :ref:`server-unix-socket` is used.
-
 .. _server-trace-header:
 
 server-trace-header
@@ -1042,37 +969,3 @@ server-unix-socket-mode
   .. code:: bash
 
     server-unix-socket-mode = "660"
-
-.. _url-use-legacy-target-names:
-
-url-use-legacy-target-names
----------------------------
-
-  =============== =================================
-  **Type**        Boolean
-  **Default**     True
-  **Reloadable**  Y
-  **Environment** PGRST_URL_USE_LEGACY_TARGET_NAMES
-  **In-Database** pgrst.url_use_legacy_target_names
-  =============== =================================
-
-  When active, it allows using the the name of an embedded table in filters, orders or limits even if it has an alias:
-
-  .. code:: bash
-
-    curl "http://localhost:3000/table?select=alias:target(*)&target.order=id" -i
-
-  .. code:: text
-
-    Warning: 299 PostgRESTv16 "Embedded resource was referenced by relation name even though it has an alias. This is deprecated and will stop working in a future release. Update `target` to `alias` in query string filters, orders or limits."
-    [...]
-
-  Note that the response includes a deprecation message in the ``Warning`` header.
-  This will also show in the PostgREST logs:
-
-  .. code::
-
-    28/May/2026:20:33:22 -0500: WARNING: Embedded resource was referenced by relation name even though it has an alias. This is deprecated and will stop working in a future release.
-    28/May/2026:20:33:22 -0500: Update filters, orders or limits that use `target` to `alias` in `GET /table?select=alias:target(*)&target.order=id`
-
-  This feature will be removed in a future release, so you should start using the ``alias`` in these cases.

docs/references/connection_pool.rst

@@ -47,8 +47,6 @@ Under a busy system, the :ref:`db-pool-max-idletime` won't be reached and the co
 To avoid this problem and save resources, a connection max lifetime (:ref:`db-pool-max-lifetime`) is enforced.
 After the max lifetime is reached, connections from the pool will be released and new ones will be created. This doesn't affect running requests, only unused connections will be released.
 
-.. _pool_timeout:
-
 Acquisition Timeout
 -------------------
 

docs/references/errors.rst

@@ -199,7 +199,7 @@ Related to the HTTP request elements.
 |               |             | :ref:`switching schemas <multiple-schemas>` is not present  |
 | PGRST106      |             | in the :ref:`db-schemas` configuration variable.            |
 +---------------+-------------+-------------------------------------------------------------+
-| .. _pgrst107: | 406         | The ``Accept`` media type sent in the request is invalid.   |
+| .. _pgrst107: | 415         | The ``Content-Type`` sent in the request is invalid.        |
 |               |             |                                                             |
 | PGRST107      |             |                                                             |
 +---------------+-------------+-------------------------------------------------------------+

docs/references/http_server.rst

@@ -1,18 +0,0 @@
-.. _http_server:
-
-HTTP Server
-###########
-
-The HTTP server is provided by `Warp <https://aosabook.org/en/posa/warp.html>`_.
-
-Graceful shutdown
------------------
-
-PostgREST uses Warp's graceful shutdown, when a ``SIGTERM`` is received:
-
-- It stops accepting new requests.
-- Allows requests that are already in progress to finish.
-- Closes idle ``Keep-Alive`` connections instead of waiting for them to expire.
-- Responses sent during shutdown indicate that the connection should not be reused (e.g. for HTTP/1.x, it sends ``Connection: close``).
-
-This allows PostgREST to shut down promptly without interrupting in-flight requests. Useful for zero-downtime upgrades and autoscaling/load-balancing under cloud environments (AWS ECS, Kubernetes).

docs/references/observability.rst

@@ -39,7 +39,7 @@ For diagnostic information about the server itself, PostgREST logs to ``stderr``
    06/May/2024:08:16:11 -0500: Listening for database notifications on the "pgrst" channel
    06/May/2024:08:16:11 -0500: Config reloaded
    06/May/2024:08:16:11 -0500: Schema cache queried in 3.8 milliseconds
-   06/May/2024:08:16:11 -0500: Schema cache loaded 15 Relations, 8 Relationships, 8 RPCs, 0 Domain Representations, 4 Media Type Handlers
+   06/May/2024:08:16:11 -0500: Schema cache loaded 15 Relations, 8 Relationships, 8 Functions, 0 Domain Representations, 4 Media Type Handlers
    06/May/2024:14:11:27 -0500: Received a config reload message on the "pgrst" channel
    06/May/2024:14:11:27 -0500: Config reloaded
 
@@ -238,45 +238,6 @@ pgrst_jwt_cache_evictions_total
 
 The total number of JWT cache evictions.
 
-GHC Runtime Metrics
--------------------
-
-PostgREST can also expose GHC runtime system metrics. These use the ``ghc_*``
-prefix and include
-`GHC RTS statistics <https://ghc.gitlab.haskell.org/ghc/doc/libraries/base-4.22.0.0-inplace/GHC-Stats.html#g:1>`_
-for runtime allocation, garbage collection, memory, and CPU/elapsed time.
-
-These are useful for monitoring PostgREST process health and diagnosing memory
-pressure or GC behavior.
-
-To expose these metrics, enable GHC RTS statistics when starting PostgREST:
-
-.. code-block:: bash
-
-  postgrest +RTS -T -RTS
-
-When enabled, the admin ``/metrics`` endpoint includes samples such as:
-
-.. code-block:: text
-
-  # HELP ghc_gcs_total Total number of GCs
-  # TYPE ghc_gcs_total counter
-  ghc_gcs_total 1
-  # HELP ghc_allocated_bytes_total Total bytes allocated
-  # TYPE ghc_allocated_bytes_total counter
-  ghc_allocated_bytes_total 12345678
-
-Other available GHC runtime metrics include:
-
-- ``ghc_gcs_total``
-- ``ghc_major_gcs_total``
-- ``ghc_allocated_bytes_total``
-- ``ghc_max_live_bytes``
-- ``ghc_max_mem_in_use_bytes``
-- ``ghc_mutator_cpu_seconds_total``
-- ``ghc_gc_cpu_seconds_total``
-- ``ghc_elapsed_seconds_total``
-
 Traces
 ======
 
@@ -430,8 +391,6 @@ By default the plan is assumed to generate the JSON representation of a resource
 
 The other available parameters are ``analyze``, ``verbose``, ``settings``, ``buffers`` and ``wal``, which correspond to the `EXPLAIN command options <https://www.postgresql.org/docs/current/sql-explain.html>`_. To use the ``analyze`` and ``wal`` parameters for example, you would add them like ``Accept: application/vnd.pgrst.plan; options=analyze|wal``.
 
-For a workflow that takes the ``Query Identifier`` from a verbose PostgREST plan and uses it to inspect the same query in ``pg_stat_statements``, see :ref:`debugging_performance_pg_stat_statements`.
-
 Note that akin to the EXPLAIN command, the changes will be committed when using the ``analyze`` option. To avoid this, you can use the :ref:`db-tx-end` and the ``Prefer: tx=rollback`` header.
 
 Securing the Execution Plan

docs/references/schema_cache.rst

@@ -24,7 +24,7 @@ You can do this with UNIX signals or with PostgreSQL notifications. It's also po
 
 .. note::
 
-  - If the schema cache fails to reload (e.g. due to a ``statement_timeout`` or :ref:`pool timeout <pool_timeout>`), PostgREST will continue serving requests in a "best effort" basis.
+  - Requests will wait until the schema cache reload is done. This to prevent client errors due to an stale schema cache.
   - If you are using the :ref:`in_db_config`, a schema cache reload will :ref:`reload the configuration<config_reloading>` as well.
 
 .. _schema_reloading_signals:

docs/tutorials/tut1.rst

@@ -172,7 +172,7 @@ Go back to :ref:`tut1_step3` and change the payload to
 
 .. code-block:: bash
 
-  payload=$(echo -n "{\"role\":\"todo_user\",\"exp\":123456789}" | _base64)
+  payload=$(echo -n "{\"role\":\"todo_user\",\"exp\":\"123456789\"}" | _base64)
 
   echo -n "$header.$payload.$signature"
 

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1784115452,
-        "narHash": "sha256-BoYPdqk6jlKXy+DyUzyGV/CtRGfAhk2MmIgBhsemTGI=",
+        "lastModified": 1776949667,
+        "narHash": "sha256-GMSVw35Q+294GlrTUKlx087E31z7KurReQ1YHSKp5iw=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "35d3407a3816f3b341d8cf1d60abaf2b7b8166ac",
+        "rev": "01fbdeef22b76df85ea168fbfe1bfd9e63681b30",
         "type": "github"
       },
       "original": {

nix/README.md

@@ -70,16 +70,55 @@ The PostgREST utilities available in `nix-shell` all have names that begin with
 `<tab>`) in `nix-shell` to see all that are available:
 
 ```bash
+# Note: The utilities listed here might not be up to date.
 [nix-shell]$ postgrest-<tab>
-postgrest-build
-postgrest-cabal-update
-postgrest-check
-postgrest-clean
-postgrest-commitlint
+postgrest-build                   postgrest-parallel-curl
+postgrest-check                   postgrest-profiled-run
+postgrest-clean                   postgrest-push-cachix
+postgrest-commitlint              postgrest-release
+postgrest-coverage                postgrest-repl
+postgrest-coverage-draft-overlay  postgrest-run
+postgrest-docs-build              postgrest-style
+postgrest-docs-check              postgrest-style-check
+postgrest-docs-dictcheck          postgrest-test-big-schema
+postgrest-docs-linkcheck          postgrest-test-doctests
+postgrest-docs-render             postgrest-test-io
+postgrest-docs-serve              postgrest-test-memory
+postgrest-docs-spellcheck         postgrest-test-replica
+postgrest-dump-minimal-imports    postgrest-test-spec
+postgrest-dump-schema             postgrest-test-spec-idempotence
+postgrest-gen-ctags               postgrest-watch
+postgrest-gen-jwt                 postgrest-with-all
+postgrest-gen-secret              postgrest-with-git
+postgrest-git-hooks               postgrest-with-pgrst
+postgrest-hsie-graph-modules      postgrest-with-pg-14
+postgrest-hsie-graph-symbols      postgrest-with-pg-15
+postgrest-hsie-minimal-imports    postgrest-with-pg-16
+postgrest-lint                    postgrest-with-pg-17
+postgrest-loadtest                postgrest-with-pg-18
+postgrest-loadtest-against        postgrest-with-slow-pg
+postgrest-loadtest-report         postgrest-with-slow-postgrest
+postgrest-nixpkgs-upgrade
 ...
+
+[nix-shell]$
+
 ```
 
-Most of these commands provide a `--help` output, make sure to check it out.
+The `docker` module has large dependencies to be build before the shell becomes
+available, which could take an especially long time if the cachix binary cache
+is not used. You can activate it by passing a flag to `nix-shell` with
+`nix-shell --arg docker true`. This will make the respective utilities available:
+
+```bash
+$ nix-shell --arg docker true
+[nix-shell]$ postgrest-docker-<tab>
+postgrest-docker-load
+...
+
+```
+
+Note that `postgrest-docker-load` is now also available.
 
 To run one-off commands, you can also use `nix-shell --run <command>`, which
 will launch the Nix shell, run that one command and exit. Note that the tab
@@ -95,6 +134,16 @@ $ nix-shell --run "postgrest-foo --bar"
 
 ```
 
+A third option is to install utilities that you use very often locally:
+
+```bash
+$ nix-env -f default.nix -iA devTools
+
+# `postgrest-style` can now be run directly:
+$ postgrest-style
+
+```
+
 If you use `nix-shell` very often, you might like to use
 https://github.com/xzfc/cached-nix-shell, which skips evaluating all our Nix
 expressions if nothing changed, reducing startup time for the shell
@@ -160,7 +209,13 @@ The loadtests ensure that performance doesn't drop on a change. Underlyingly the
 [nix-shell]$ postgrest-loadtest
 
 # You can loadtest comparing to a different branch
-[nix-shell]$ postgrest-loadtest-against main
+[nix-shell]$ postgrest-loadtest-against master
+
+# You can simulate latency client/postgrest and postgrest/database
+[nix-shell]$ PGRST_DELAY=5ms PGDELAY=5ms postgrest-loadtest
+
+# You can build postgrest directly with cabal for faster iteration
+[nix-shell]$ PGRST_BUILD_CABAL=1 postgrest-loadtest
 
 # Produce a markdown report to be used on CI
 [nix-shell]$ postgrest-loadtest-report

nix/hsie/Main.hs

@@ -37,7 +37,8 @@ import Data.Text                  (Text)
 import GHC.Driver.Errors.Types    (GhcMessage)
 import GHC.Generics               (Generic)
 import GHC.Hs.Extension           (GhcPs)
-import GHC.Types.Error            (Messages, defaultDiagnosticOpts, getMessages)
+import GHC.Types.Error            (Messages, defaultDiagnosticOpts,
+                                   getMessages)
 import GHC.Types.Name.Occurrence  (occNameString)
 import GHC.Types.Name.Reader      (rdrNameOcc)
 import GHC.Unit.Module            (moduleNameString)

nix/hsie/README.md

@@ -5,10 +5,10 @@ project. It's available in PostgREST's `nix-shell` by default.
 
 ## Dumping imports
 
-Given source code in the directories `src/library` and `src/executable`, for example, you can run:
+Given source code in the directories `src` and `main`, for example, you can run:
 
 ```
-hsie dump-imports src/library src/executable
+hsie dump-imports src main
 ```
 
 This dumps all imports of the modules in the given directory to a CSV file,
@@ -18,7 +18,7 @@ To dump to a JSON file (e.g., to further process with `jq`), add the `--json`
 flag:
 
 ```
-hsie dump-imports --json src/library src/executable
+hsie dump-imports --json src main
 ```
 
 ## Graphing imports
@@ -27,7 +27,7 @@ The tool can generate `graphviz` graphs of module and symbol imports by printing
 a file to `stdout` that can directly be rendered with `dot`:
 
 ```
-hsie graph-modules src/library src/executable | dot -Tpng -o modules.png
+hsie graph-modules src main | dot -Tpng -o modules.png
 ```
 
 The command `graph-modules` prints a graph of which modules insert which other
@@ -39,7 +39,7 @@ To check whether modules are imported under consistent aliases in your project,
 run:
 
 ```
-hsie check-aliases src/library src/executable
+hsie check-aliases main src
 ```
 
 This will exit with a non-zero exit code if any inconsistent aliases are found.
@@ -48,13 +48,13 @@ The following command checks whether any modules are imported as wildcards, i.e.
 not qualified and without specifying symbols.
 
 ```
-hsie check-wildcards src/library src/executable
+hsie check-wildcards main src
 ```
 
 To whitelist certain modules to be imported as wildcards, use `--ok`:
 
 ```
-hsie check-wildcards src/library src/executable --ok Protolude --ok Test.Module
+hsie check-wildcards main src --ok Protolude --ok Test.Module
 ```
 
 ## Current limitations

nix/overlays/haskell-packages.nix

@@ -49,26 +49,6 @@ let
       # Before upgrading fuzzyset to 0.3, check: https://github.com/PostgREST/postgrest/issues/3329
       fuzzyset = prev.fuzzyset_0_2_4;
 
-      # TODO: Remove once available in nixpkgs
-      auto-update =
-        prev.callHackageDirect
-          {
-            pkg = "auto-update";
-            ver = "0.2.7";
-            sha256 = "sha256-fHX/OqF/cB9rbpGpLUtA29bcEJS43HUWHcK55yUxKoo=";
-          }
-          { };
-
-      # TODO: Remove once available in nixpkgs
-      aeson-jsonpath =
-        prev.callHackageDirect
-          {
-            pkg = "aeson-jsonpath";
-            ver = "0.4.2.0";
-            sha256 = "sha256-K+3brf1zjSSjojtSCXFrip5rrP7AO/S4zndAxAnvEfc=";
-          }
-          { };
-
       http2 =
         prev.callHackageDirect
           {
@@ -97,14 +77,13 @@ let
           { };
 
       warp =
-        lib.dontCheck
-          (prev.callHackageDirect
-            {
-              pkg = "warp";
-              ver = "3.4.14";
-              sha256 = "sha256-RnoOUlC6dOP0sK/tYAJCX1oLzVFG1GILUY+yVbmvW8Y=";
-            }
-            { });
+        lib.dontCheck (prev.callHackageDirect
+          {
+            pkg = "warp";
+            ver = "3.4.13";
+            sha256 = "sha256-jmr8kpeSPDkOhT0i9PhozZapX4nUs92cOX7POAGb7/M=";
+          }
+          { });
     };
 in
 {

nix/tools/cabalTools.nix

@@ -51,7 +51,7 @@ let
         docs = "Run PostgREST after building it interactively with cabal-install";
         args =
           [
-            "ARG_USE_ENV([PGRST_DB_ANON_ROLE], [], [PostgREST anonymous role. (default: 'postgrest_test_anonymous')])"
+            "ARG_USE_ENV([PGRST_DB_ANON_ROLE], [postgrest_test_anonymous], [PostgREST anonymous role])"
             "ARG_USE_ENV([PGRST_DB_POOL], [1], [PostgREST pool size])"
             "ARG_USE_ENV([PGRST_DB_POOL_ACQUISITION_TIMEOUT], [1], [PostgREST pool timeout])"
             "ARG_USE_ENV([PGRST_JWT_SECRET], [reallyreallyreallyreallyverysafe], [PostgREST JWT secret])"
@@ -62,10 +62,6 @@ let
         withEnv = postgrest.env;
       }
       ''
-        # when there's a default, argbash conflates empty string with unset, so we do this workaround to be able to do `PGRST_DB_ANON_ROLE="" <command>` for manual testing
-        if [[ ! ''${PGRST_DB_ANON_ROLE+x} ]]; then
-          PGRST_DB_ANON_ROLE="postgrest_test_anonymous"
-        fi
         export PGRST_DB_ANON_ROLE
         export PGRST_DB_POOL
         export PGRST_DB_POOL_ACQUISITION_TIMEOUT
@@ -84,7 +80,7 @@ let
         docs = "Run a profiled build of postgREST. This will generate a postgrest.prof file that can be used to do optimization.";
         args =
           [
-            "ARG_USE_ENV([PGRST_DB_ANON_ROLE], [], [PostgREST anonymous role. (default: 'postgrest_test_anonymous')])"
+            "ARG_USE_ENV([PGRST_DB_ANON_ROLE], [postgrest_test_anonymous], [PostgREST anonymous role])"
             "ARG_USE_ENV([PGRST_DB_POOL], [1], [PostgREST pool size])"
             "ARG_USE_ENV([PGRST_DB_POOL_ACQUISITION_TIMEOUT], [1], [PostgREST pool timeout])"
             "ARG_USE_ENV([PGRST_JWT_SECRET], [reallyreallyreallyreallyverysafe], [PostgREST JWT secret])"
@@ -94,10 +90,6 @@ let
         withEnv = postgrest.env;
       }
       ''
-        # when there's a default, argbash conflates empty string with unset, so we do this workaround to be able to do `PGRST_DB_ANON_ROLE="" <command>` for manual testing
-        if [[ ! ''${PGRST_DB_ANON_ROLE+x} ]]; then
-          PGRST_DB_ANON_ROLE="postgrest_test_anonymous"
-        fi
         export PGRST_DB_ANON_ROLE
         export PGRST_DB_POOL
         export PGRST_DB_POOL_ACQUISITION_TIMEOUT

nix/tools/devTools.nix

@@ -5,13 +5,15 @@
 , curl
 , devCabalOptions
 , entr
-, fd
+, git
 , graphviz
 , hsie
 , nix
+, silver-searcher
 , stdenv
 , style
 , tests
+, withTools
 , haskellPackages
 , ctags
 , openssl
@@ -39,7 +41,7 @@ let
       }
       ''
         while true; do
-          (! ${fd}/bin/fd -H -E .git | ${entr}/bin/entr -dr "$_arg_command" "''${_arg_leftovers[@]}")
+          (! ${silver-searcher}/bin/ag -l . | ${entr}/bin/entr -dr "$_arg_command" "''${_arg_leftovers[@]}")
         done
       '';
 
@@ -89,6 +91,156 @@ let
         ${style}/bin/postgrest-style-check
       '';
 
+  gitHooks =
+    let
+      name = "postgrest-git-hooks";
+    in
+    checkedShellScript
+      {
+        inherit name;
+        docs =
+          ''
+            Enable or disable git pre-commit and pre-push hooks.
+
+            Basic is faster and will only run:
+              - pre-commit: postgrest-style
+              - pre-push: postgrest-lint
+
+            Full takes a lot more time and will run:
+              - pre-commit: postgrest-style && postgrest-lint
+              - pre-push: postgrest-check
+
+            Changes made by postgrest-style will be staged automatically.
+
+            Example usage:
+              postgrest-git-hooks disable
+              postgrest-git-hooks enable basic
+              postgrest-git-hooks enable full
+
+            The "run" operation and "--hook" argument are only used internally.
+          '';
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([operation], [Operation])"
+            "ARG_TYPE_GROUP_SET([OPERATION], [OPERATION], [operation], [disable,enable,run])"
+            "ARG_POSITIONAL_SINGLE([mode], [Mode], [basic])"
+            "ARG_TYPE_GROUP_SET([MODE], [MODE], [mode], [basic,full])"
+            "ARG_OPTIONAL_SINGLE([hook], , [Hook], [pre-commit])"
+            "ARG_TYPE_GROUP_SET([HOOK], [HOOK], [hook], [pre-commit,pre-push])"
+          ];
+        positionalCompletion =
+          ''
+            if test "$prev" == "${name}"; then
+              COMPREPLY=( $(compgen -W "enable disable" -- "$cur") )
+            elif test "$prev" == "enable" || test "$prev" == "disable"; then
+              COMPREPLY=( $(compgen -W "basic full" -- "$cur") )
+            fi
+          '';
+        workingDir = "/";
+      }
+      ''
+        if [ run != "$_arg_operation" ]; then
+          # Remove all hooks first and ignore failures because the file might be missing.
+          # This assumes that we're only adding lines that include "postgrest-git-hooks"
+          # to the hook file.
+          sed -i -e '/postgrest-git-hooks/d' .git/hooks/pre-{commit,push} 2> /dev/null || true
+
+          if [ disable != "$_arg_operation" ]; then
+            # The nix-shell && + nix-shell || pattern makes sure we can run the hook
+            # in a pure nix-shell, where nix-shell itself is not available, too.
+
+            # The $(nix-shell --run "command -v ...") pattern ensures we only need to enable
+            # the hooks once and still run the latest of our hook scripts, even when we
+            # update them in the repo.
+
+            echo 'command -v nix-shell > /dev/null || postgrest-git-hooks --hook=pre-commit run' "$_arg_mode" \
+              >> .git/hooks/pre-commit
+            # shellcheck disable=SC2016
+            echo 'command -v nix-shell > /dev/null && $(nix-shell --quiet -Q --run "command -v postgrest-git-hooks") --hook=pre-commit run' "$_arg_mode" \
+             >> .git/hooks/pre-commit
+            chmod +x .git/hooks/pre-commit
+
+            echo 'command -v nix-shell > /dev/null || postgrest-git-hooks --hook=pre-push run' "$_arg_mode" \
+              >> .git/hooks/pre-push
+            # shellcheck disable=SC2016
+            echo 'command -v nix-shell > /dev/null && $(nix-shell --quiet -Q --run "command -v postgrest-git-hooks") --hook=pre-push run' "$_arg_mode" \
+             >> .git/hooks/pre-push
+            chmod +x .git/hooks/pre-push
+          fi
+        else
+          # When run from a git hook, the GIT_ environment variables conflict with our withGit helper.
+          # The following unsets all GIT_ variables.
+          unset "''${!GIT_@}"
+
+          # shellcheck disable=SC2329
+          function restore () {
+            ref="$(git stash list --format=format:%gD --grep "$1" -n1)"
+            # this will avoid merge conflicts when applying the stash
+            ${git}/bin/git restore --source="$ref" .
+            # restore untracked files, too. could fail with no files
+            if [ "$(git show --numstat --format=oneline "$ref^3" | wc -l)" -gt 1 ]; then
+              ${git}/bin/git restore --overlay --source="$ref^3" .
+            fi
+            ${git}/bin/git stash drop "$ref"
+          }
+
+          case "$_arg_mode" in
+            basic)
+              case "$_arg_hook" in
+                pre-commit)
+                  # To be able to automatically add only changes from postgrest-style to the staging area,
+                  # we need to run postgrest-style twice. Otherwise we'd risk merge conflicts when popping
+                  # the stash afterwards.
+                  ${style}/bin/postgrest-style
+
+                  stash="postgrest-git-hooks-$RANDOM"
+                  ${git}/bin/git stash push --include-untracked --keep-index -m "$stash"
+                  if [ "$(git stash list --grep $stash)" ]; then
+                    # Only create the stash pop trap, if we actually created a stash.
+                    # Otherwise stash pop will cause havoc.
+                    trap 'restore "$stash"' EXIT
+                  fi
+
+                  ${style}/bin/postgrest-style
+                  ${git}/bin/git add .
+                  ;;
+                pre-push)
+                  # Create a clean working tree without any uncommitted changes.
+                  ${withTools.withGit} HEAD ${style}/bin/postgrest-lint
+                  ;;
+              esac
+              ;;
+            full)
+              case "$_arg_hook" in
+                pre-commit)
+                  # To be able to automatically add only changes from postgrest-style to the staging area,
+                  # we need to run postgrest-style twice. Otherwise we'd risk merge conflicts when popping
+                  # the stash afterwards.
+                  ${style}/bin/postgrest-style
+
+                  stash="postgrest-git-hooks-$RANDOM"
+                  ${git}/bin/git stash push --include-untracked --keep-index -m "$stash"
+                  if [ "$(git stash list --grep $stash)" ]; then
+                    # Only create the stash pop trap, if we actually created a stash.
+                    # Otherwise stash pop will cause havoc.
+                    trap 'restore "$stash"' EXIT
+                  fi
+
+                  ${style}/bin/postgrest-style
+                  ${git}/bin/git add .
+
+                  ${style}/bin/postgrest-lint
+                  ;;
+                pre-push)
+                  # Create a clean working tree without any uncommitted changes.
+                  ${withTools.withGit} HEAD ${check}
+                  ;;
+              esac
+              ;;
+          esac
+        fi
+      '';
+
   dumpMinimalImports =
     checkedShellScript
       {
@@ -129,10 +281,10 @@ let
       {
         name = "postgrest-hsie-graph-modules";
         docs = "Create a PNG graph of modules imported within the codebase.";
-        args = [ "ARG_OPTIONAL_SINGLE([outfile], [o], [Output filename], [postgrest-module-graph.png])" ];
+        args = [ "ARG_POSITIONAL_SINGLE([outfile], [Output filename])" ];
       }
       ''
-        ${hsie} graph-modules src/library src/executable | ${graphviz}/bin/dot -Tpng -o "$_arg_outfile"
+        ${hsie} graph-modules main src | ${graphviz}/bin/dot -Tpng -o "$_arg_outfile"
       '';
 
   hsieGraphSymbols =
@@ -243,6 +395,7 @@ buildToolbox
     inherit
       check
       dumpMinimalImports
+      gitHooks
       hsieGraphModules
       hsieGraphSymbols
       hsieMinimalImports

nix/tools/docs.nix

@@ -122,8 +122,6 @@ let
         workingDir = "/docs";
       }
       ''
-        echo "Checking spelling mistakes..."
-
         export LC_ALL=C
 
         FILES=$(find . -type f -iname '*.rst' | tr '\n' ' ')
@@ -146,8 +144,6 @@ let
         workingDir = "/docs";
       }
       ''
-        echo "Detecting obsolete dictionary entries..."
-
         export LC_ALL=C
 
         FILES=$(find . -type f -iname '*.rst' | tr '\n' ' ')

nix/tools/gen_rsa_materials.py

@@ -0,0 +1,52 @@
+# Generate RSA JWK/public material for loadtests.
+
+import argparse
+import sys
+from pathlib import Path
+
+import jwcrypto.jwk as jwk
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate RSA JWK/private key pair for loadtests"
+    )
+    parser.add_argument(
+        "--rsa",
+        dest="jwk_path",
+        metavar="JWK_PATH",
+        type=Path,
+        required=True,
+        help="Path to write the RSA JWK file",
+    )
+    parser.add_argument(
+        "--private-key",
+        dest="private_key_path",
+        metavar="PRIVATE_KEY_PATH",
+        type=Path,
+        required=True,
+        help="Path to write the RSA private key file",
+    )
+
+    args = parser.parse_args()
+
+    key = jwk.JWK.generate(kty="RSA", size=4096)
+    private_jwk, public_jwk = key.export_private(), key.export_public()
+
+    try:
+        args.jwk_path.write_text(public_jwk)
+        print(f"Created RSA JWK on {args.jwk_path}")
+    except OSError as e:
+        print(f"Error writing to {args.jwk_path}:{e}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        args.private_key_path.write_text(private_jwk)
+        print(f"Created private key on {args.private_key_path}")
+    except OSError as e:
+        print(f"Error writing to {args.private_key_path}:{e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

nix/tools/generate_targets.py

@@ -12,39 +12,81 @@
 # from an array
 import time
 import argparse
+import subprocess
+import sys
 import random
-import jwcrypto.jwt as jwt
+import jwt
+from typing import Optional
 from pathlib import Path
+from enum import Enum
 
 URL = "http://postgrest"
 
-secret_key = "reallyreallyreallyreallyverysafe"
+secret_key = b"reallyreallyreallyreallyverysafe"
 
 
-def generate_target(
+def generate_jwt(
     now: int,
-    key: jwt.JWK,
-) -> list[str]:
-    """Generate a target using an HS256 or RS256 JWT"""
-    headers = {
+    exp_inc: Optional[int],
+    rsa_private_key: Optional[jwt.algorithms.RSAAlgorithm],
+) -> str:
+    """Generate an HS256 or RS256 JWT"""
+    payload = {
         "sub": f"user_{random.getrandbits(32)}",
         "iat": now,
-    }
-
-    claims = {
         "role": "postgrest_test_author",
     }
 
-    headers["alg"] = "RS256" if key.get("kty") == "RSA" else "HS256"
+    if exp_inc is not None:
+        payload["exp"] = now + exp_inc
 
-    token = jwt.JWT(headers, claims)
-    token.make_signed_token(key)
+    if rsa_private_key is None:
+        key = secret_key
+        alg = "HS256"
+    else:
+        key = rsa_private_key
+        alg = "RS256"
+    return jwt.encode(payload, key, alg)
 
-    return [
-        f"OPTIONS {URL}/authors_only?{headers["alg"]}",
-        f"Authorization: Bearer {token.serialize()}",
-        "",  # blank line to separate requests
-    ]
+
+HTTP_METHODS = (
+    "GET",
+    "OPTIONS",
+)
+
+HttpMethod = Enum(
+    "HttpMethod",
+    {method: method for method in HTTP_METHODS},
+    type=str,
+    module=__name__,
+)
+
+
+def append_targets(lines: list[str], token: str, http_method: HttpMethod):
+    lines.append(f"{http_method.value} {URL}/authors_only")
+    lines.append(f"Authorization: Bearer {token}")
+    lines.append("")  # blank line to separate requests
+
+
+# we use this to chain commands on loadtest.nix
+def run_command(command: list[str]):
+    if not command:
+        return
+
+    if command[0] == "--":
+        command = command[1:]
+
+    if not command:
+        return
+
+    try:
+        subprocess.run(command, check=True)
+    except subprocess.CalledProcessError as exc:
+        print(
+            f"Error executing command {' '.join(command)}: {exc}",
+            file=sys.stderr,
+        )
+        sys.exit(exc.returncode)
 
 
 def main():
@@ -52,47 +94,133 @@ def main():
         description="Generate Vegeta targets with unique JWTs"
     )
     parser.add_argument(
-        "generated_path",
-        metavar="GENERATED_PATH",
-        help="Path to write the generated files",
+        "targets_path",
+        metavar="TARGETS_PATH",
+        help="Path to write the generated targets file",
+    )
+    parser.add_argument(
+        "--private-key",
+        dest="private_key_path",
+        metavar="PRIVATE_KEY_PATH",
         type=Path,
+        default=None,
+        help="Path to the RSA private key file (required when --rsa is used)",
+    )
+    parser.add_argument(
+        "--worst",
+        dest="worst",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        help="Generate worst case targets for a JWT cache",
+    )
+    parser.add_argument(
+        "--rsa",
+        dest="jwk_path",
+        metavar="JWK_PATH",
+        type=Path,
+        default=None,
+        help="Path to an existing RSA JWK file used for signing tokens",
+    )
+    parser.add_argument(
+        "--method",
+        dest="http_method",
+        choices=list(HTTP_METHODS),
+        required=True,
+        help="HTTP method for the vegeta targets",
+    )
+    parser.add_argument(
+        "command",
+        nargs=argparse.REMAINDER,
+        help="Command (and arguments) to run after generating the targets",
     )
 
     args = parser.parse_args()
 
-    targets_path = args.generated_path / "gen_targets.http"
+    rsa_private_key: Optional[jwt.algorithms.RSAAlgorithm] = None
 
-    hs = jwt.JWK.from_password(secret_key)
-    rsa = jwt.JWK.generate(kty="RSA", size=4096)
+    is_hs = args.jwk_path is None
 
-    jwks = jwt.JWKSet()
-    jwks.add(hs)
-    jwks.add(rsa)
+    http_method = HttpMethod(args.http_method)
 
-    jwks_path = args.generated_path / "gen_jwks.json"
+    nsamples = 1000
 
-    # Technically, this exports the private keys, because HS does not have the concept
-    # of a public key. This is not a problem for tests, though, PostgREST can verify
-    # tokens with the private key just as well.
-    jwks_path.write_text(jwks.export())
-    print(f"Created JWKSet on {jwks_path}")
+    if is_hs:
+        ntargets = 200000
+    else:
+        # The asymmetric targets take too long to compute so we reduce them
+        ntargets = 50000
 
-    ntargets = 1000
+    if not is_hs:
+        if args.private_key_path is None:
+            parser.error("--rsa requires the --private-key option")
+        try:
+            private_key_data = args.private_key_path.read_text()
+        except OSError as e:
+            err = (
+                f"Error reading RSA private key from {args.private_key_path}: "
+                f"{e}. Generate RSA materials first with gen_rsa_materials.py."
+            )
+            print(err, file=sys.stderr)
+            sys.exit(1)
+
+        try:
+            rsa_private_key = jwt.algorithms.RSAAlgorithm.from_jwk(private_key_data)
+        except Exception as exc:  # broad exception to capture parsing errors
+            err = (
+                f"Error loading RSA private key from {args.private_key_path}: " f"{exc}"
+            )
+            print(err, file=sys.stderr)
+            sys.exit(1)
 
     print(f"Generating {ntargets} targets...")
 
-    now = int(time.time())
+    start_time = time.time()
+
+    now = int(start_time)
 
     lines = []
 
-    for i in range(ntargets):
-        target = generate_target(now, hs if i % 2 == 0 else rsa)
-        lines.extend(target)
+    # We want to ensure 401 Unauthorized responses don't happen during
+    # JWT validation, this can happen when the jwt `exp` is too short.
+    # At the same time, we want to ensure the `exp` is not too big,
+    # so expires will occur and postgREST needs to
+    # clean cached expired JWTs
+    if args.worst:
+        # estimated time takes to build and run postgrest itself
+        build_run_postgrest_time = 2
 
-    with open(targets_path, "w") as f:
-        f.write("\n".join(lines))
+        # estimated time it takes to generate the targets file
+        # the division numbers are tuned by hand
+        if is_hs:  # hs generation is much faster
+            gen_time = ntargets // 66666
+        else:  # asymmetric is slower so the time is higher
+            gen_time = ntargets // 220
 
+        # estimated exp time so some JWTs will expire
+        inc = build_run_postgrest_time + gen_time
+
+        for i in range(ntargets):
+            token = generate_jwt(now, inc + i // 1000, rsa_private_key)
+            append_targets(lines, token, http_method)
+
+    else:
+        tokens = [generate_jwt(now, None, rsa_private_key) for _ in range(nsamples)]
+        for i in range(ntargets):
+            token = random.choice(tokens)
+            append_targets(lines, token, http_method)
+
+    try:
+        with open(args.targets_path, "w") as f:
+            f.write("\n".join(lines))
+    except IOError as e:
+        print(f"Error writing to {args.targets_path}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    elapsed = time.time() - start_time
     print(f"Created {ntargets} targets", end=" ")
+    print(f"in {args.targets_path} ({elapsed:.2f}s)")
+
+    run_command(args.command)
 
 
 if __name__ == "__main__":

nix/tools/loadtest.nix

@@ -1,11 +1,7 @@
 { buildToolbox
 , checkedShellScript
-, git
 , jq
-, libfaketime
-, python3
 , python3Packages
-, runCommand
 , vegeta
 , withTools
 , writers
@@ -46,18 +42,24 @@ let
         docs = "Run the vegeta loadtests with PostgREST.";
         args = [
           "ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])"
+          "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])"
           "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
-          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-cache,jwt-cache-worst])"
+          "ARG_OPTIONAL_SINGLE([method],, [HTTP method used for the jwt loadtests], [OPTIONS])"
+          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,errors,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
+          "ARG_TYPE_GROUP_SET([METHOD], [METHOD], [method], [OPTIONS,GET])"
           "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
           "ARG_LEFTOVERS([additional vegeta arguments])"
         ];
         workingDir = "/";
       }
       ''
+        # previously required settings to make this work with older branches
         export PGRST_DB_ANON_ROLE="postgrest_test_anonymous"
+        export PGRST_DB_URI="postgresql://"
+        export PGRST_DB_SCHEMAS="test"
+
         export PGRST_DB_CONFIG="false"
         export PGRST_DB_POOL="1"
-        export PGRST_DB_SCHEMAS="test"
         export PGRST_DB_TX_END="rollback-allow-override"
         export PGRST_LOG_LEVEL="crit"
         export PGRST_JWT_SECRET="reallyreallyreallyreallyverysafe"
@@ -66,29 +68,96 @@ let
         abs_output="$(realpath "$_arg_output")"
 
         case "$_arg_kind" in
-          jwt-cache)
-            export PGRST_JWT_SECRET="@${generatedTargets}/gen_jwks.json"
+          jwt-hs)
+            export PGRST_JWT_CACHE_MAX_ENTRIES="0"
 
             # shellcheck disable=SC2145
-            ${withTools.withPg} -f test/load/fixtures.sql \
-            ${withTools.withPgrst} --faketime '2000-01-01 00:00:00' -m "$_arg_monitor" \
-            sh -c "cd test/load && \
-            ${runner} -targets ${generatedTargets}/gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
+            ${withGenTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http \
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ;;
+
+          jwt-hs-cache)
+            # shellcheck disable=SC2145
+            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
+            ${withGenTargets} --method "$_arg_method" "$_arg_testdir"/gen_targets.http \
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ;;
+
+          jwt-hs-cache-worst)
+            # shellcheck disable=SC2145
+            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
+            ${withGenTargets} --method "$_arg_method" --worst "$_arg_testdir"/gen_targets.http \
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ;;
+
+          jwt-rsa)
+            export PGRST_JWT_CACHE_MAX_ENTRIES="0"
+
+            ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
+
+            # shellcheck disable=SC2145
+            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
+            ${withGenTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ;;
+
+          jwt-rsa-cache)
+            ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
+
+            # shellcheck disable=SC2145
+            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
+            ${withGenTargets} --method "$_arg_method" --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ;;
+
+          jwt-rsa-cache-worst)
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
+
+            ${genRsaMaterials} --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json
+            export PGRST_JWT_SECRET="@$_arg_testdir/gen_jwk.json"
+
+            # shellcheck disable=SC2145
+            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
+            ${withGenTargets} --method "$_arg_method" --worst --rsa="$_arg_testdir"/gen_jwk.json --private-key="$_arg_testdir"/gen_private.json "$_arg_testdir"/gen_targets.http \
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ;;
+
+          mixed)
+            # shellcheck disable=SC2145
+            ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
+            ${withTools.withPgrst} -m "$_arg_monitor" \
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
 
           # here we sleep purposefully to check how much memory does the schema cache consume in the final report
-          mixed)
+          errors)
             # shellcheck disable=SC2145
-            ${withTools.withPg} -f test/load/fixtures.sql \
+            ${withTools.withPg} -f "$_arg_testdir"/errors.sql \
             ${withTools.withPgrst} --timeout 2 --sleep 5 -m "$_arg_monitor" \
-            sh -c "cd test/load && \
-            ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -targets errors.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
         esac
 
         ${vegeta}/bin/vegeta report -type=text "$_arg_output"
 
-        if [ "$_arg_kind" != "mixed" ]; then
+        if [ "$_arg_kind" != "errors" ]; then
           # fail in case 401 happened on jwt loadtests
           unauthorized_count="$(${vegeta}/bin/vegeta report -type=json "$_arg_output" \
             | ${jq}/bin/jq -r '.status_codes["401"] // 0')"
@@ -129,9 +198,6 @@ let
             Run the vegeta loadtest against every target branch and HEAD:
             - once on the every <target-#> branch
             - once in the current worktree
-
-            Note that the Nix tooling is always taken from the HEAD branch, while the PostgREST binary is taken from the target branch.
-            For a discussion on why this is set up like this, see https://github.com/PostgREST/postgrest/pull/5013#discussion_r3431508441.
           '';
         args = [
           "ARG_POSITIONAL_INF([target], [Commit-ish reference to compare with], 1)"
@@ -146,40 +212,8 @@ let
         workingDir = "/";
       }
       ''
-        # Build postgrest for every target and HEAD.
-        # Keeps a reference to the postgrest binary and faketime lib for every branch to run later.
-        declare -A pgrst faketime
-        for tgt in "''${_arg_target[@]}" HEAD; do
-          # not using withTmpDir here, because we don't want to keep the directory on error
-          tmpdir="$(mktemp -d)"
-          trap 'rm -rf "$tmpdir"' EXIT
-
-          ${git}/bin/git worktree add -f "$tmpdir" "$tgt" > /dev/null
-          pushd "$tmpdir" > /dev/null
-
-          build_start=$SECONDS
-          echo -n "${name}: Building postgrest (nix) on $tgt... "
-          # Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet.
-          nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > build.log 2>&1 || {
-            echo "failed, output:"
-            cat build.log
-            exit 1
-          }
-          pgrst[$tgt]="$(nix-build --no-out-link -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage')/bin/postgrest"
-          # To avoid glibc mismatches with back-branches, we need to take libfaketime from the target branch.
-          faketime[$tgt]="$(nix-build --no-out-link -A pkgs.libfaketime)/lib/libfaketime.so.1"
-          build_end=$((SECONDS - build_start))
-          printf "done in %ss.\n" "$build_end"
-
-          popd > /dev/null
-          ${git}/bin/git worktree remove -f "$tmpdir" > /dev/null
-          rm -rf "$tmpdir"
-        done
-
-        # Run loadtest for every target and HEAD.
-        # Running the tests is separated from building them to reduce the chances of
-        # other processes skewing the results between two runs.
-        for tgt in "''${_arg_target[@]}" HEAD; do
+        # run loadtest for every target
+        for tgt in "''${_arg_target[@]}"; do
 
         cat << EOF
 
@@ -187,7 +221,12 @@ let
 
         EOF
 
-        FAKETIME_LIB="''${faketime[$tgt]}" PGRST_CMD="''${pgrst[$tgt]}" ${loadtest} -k "$_arg_kind" -m "loadtest/$tgt.csv" --output "loadtest/$tgt.bin"
+        # Runs the test files from the current working tree
+        # to make sure both tests are run with the same files.
+        # Save the results in the current working tree, too,
+        # otherwise they'd be lost in the temporary working tree
+        # created by withTools.withGit.
+        ${withTools.withGit} "$tgt" ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/$tgt.csv" --output "$PWD/loadtest/$tgt.bin" --testdir "$PWD/test/load"
 
         cat << EOF
 
@@ -196,6 +235,22 @@ let
         EOF
 
         done
+
+        # run loadtest once on HEAD
+
+        cat << EOF
+
+        Running "$_arg_kind" loadtest on HEAD...
+
+        EOF
+
+        ${loadtest} -k "$_arg_kind" -m "$PWD/loadtest/head.csv" --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/load"
+
+        cat << EOF
+
+        Done running on HEAD.
+
+        EOF
       '';
 
   reporter =
@@ -205,14 +260,12 @@ let
         docs = "Create a named json report for a single result file.";
         args = [
           "ARG_POSITIONAL_SINGLE([file], [Filename of result to create report for])"
-          "ARG_OPTIONAL_SINGLE([percentile], [p], [Percentile to report latency for], 50)"
           "ARG_LEFTOVERS([additional vegeta arguments])"
         ];
         workingDir = "/";
       }
       ''
-        ${vegeta}/bin/vegeta encode "$_arg_file" \
-          | ${jq}/bin/jq --arg percentile "$_arg_percentile" --slurp 'map(select(.url != "")) | group_by("\(.code) \(.method) \(.url)") | map({("\(.[0].code) \(.[0].method) \(.[0].url)" | sub("http://postgrest";"")): map(.latency) | sort | .[(length-1) * ($percentile | tonumber) / 100 | floor] / 10e3 }) | .[]' \
+        ${vegeta}/bin/vegeta report -type=json "$_arg_file" \
           | ${jq}/bin/jq --arg branch "$(basename "$_arg_file" .bin)" '. + {branch: $branch}'
       '';
 
@@ -225,31 +278,12 @@ let
         import sys
         import pandas as pd
 
-
-        def evaluate_change(df):
-            try:
-                return ((df['HEAD'] / df['main'] - 1) * 100) \
-                  .map(lambda r: "{icon} {ratio:.1f} %".format(
-                    ratio=r,
-                    # Hardcoded failure threshold for CI is 5% here.
-                    icon="" if r < 5 else ":x:"
-                  ))
-            except KeyError:
-                return None
-
-
         pd.read_json(sys.stdin) \
-          .rename(columns={'latency': sys.argv[1]}) \
-          .set_index(sys.argv[1]) \
-          .drop(['branch']) \
+          .set_index('param') \
+          .drop(['branch', 'earliest', 'end', 'latest']) \
+          .fillna("") \
           .convert_dtypes() \
-          .assign(change=evaluate_change) \
-          .to_markdown(
-            sys.stdout,
-            floatfmt='.1f',
-            colglobalalign='right',
-            colalign=('left',)
-          )
+          .to_markdown(sys.stdout, floatfmt='.0f')
       '';
 
 
@@ -260,46 +294,41 @@ let
         docs = "Create a report of all loadtest reports as markdown.";
         args = [
           "ARG_OPTIONAL_SINGLE([group], [g], [Marker to group results])"
-          "ARG_OPTIONAL_SINGLE([percentile], [p], [Percentile to report latency for], 50)"
         ];
         workingDir = "/";
       }
       ''
-        echo -e "## Loadtest results $_arg_group (P$_arg_percentile)\n"
+        marker=''${_arg_group:+"($_arg_group)"}
 
-        find loadtest -type f -iname '*.bin' -exec ${reporter} -p "$_arg_percentile" {} \; \
-          | ${jq}/bin/jq '[paths(scalars) as $path | {latency: $path | join("."), (.branch): getpath($path)}]' \
-          | ${jq}/bin/jq --slurp 'flatten | group_by(.latency) | map(add)' \
-          | ${toMarkdown} "P$_arg_percentile latency [μs]"
-      '';
+        echo -e "## Loadtest results $marker\n"
 
-  report-load =
-    checkedShellScript
-      {
-        name = "postgrest-loadtest-report-load";
-        docs = "Create a report of all CPU/MEM usage as markdown.";
-        args = [
-          "ARG_OPTIONAL_SINGLE([group], [g], [Marker to group results])"
-        ];
-        workingDir = "/";
-      }
-      ''
-        echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM usage $_arg_group\n"
+        find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \
+          | ${jq}/bin/jq '[paths(scalars) as $path | {param: $path | join("."), (.branch): getpath($path)}]' \
+          | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \
+          | ${toMarkdown}
+
+        echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM usage $marker\n"
 
         find loadtest -type f -iname '*.csv' \
           | sort -m \
           | ${mergeMonitorResults}
       '';
 
-  generatedTargets =
-    runCommand "postgrest-loadtest-targets"
+  withGenTargets =
+    writers.writePython3 "postgrest-with-gen-loadtest-targets"
       {
-        nativeBuildInputs = [ (python3.withPackages (pyps: [ pyps.jwcrypto ])) ];
+        libraries = [ python3Packages.pyjwt python3Packages.jwcrypto ];
+        doCheck = false; # postgrest-style conflicts with this
       }
-      ''
-        mkdir -p "$out"
-        ${libfaketime}/bin/faketime '2000-01-01 00:00:00' python3 ${./generate_targets.py} "$out"
-      '';
+      (builtins.readFile ./generate_targets.py);
+
+  genRsaMaterials =
+    writers.writePython3 "postgrest-gen-rsa-materials"
+      {
+        libraries = [ python3Packages.jwcrypto ];
+        doCheck = false; # postgrest-style conflicts with this
+      }
+      (builtins.readFile ./gen_rsa_materials.py);
 
   mergeMonitorResults =
     writers.writePython3 "postgrest-merge-monitor-results"
@@ -310,5 +339,5 @@ let
 in
 buildToolbox {
   name = "postgrest-loadtest";
-  tools = { inherit loadtest loadtestAgainst report report-load; };
+  tools = { inherit loadtest loadtestAgainst report; };
 }

nix/tools/nixpkgsTools.nix

@@ -17,7 +17,7 @@ let
         nix flake update
 
         echo "# This file is auto-generated by postgrest-nixpkgs-upgrade" > docs/requirements.txt
-        cat "$(nix-build --no-out-link -A docs.requirements)" >> docs/requirements.txt
+        cat "$(nix-build -A docs.requirements)" >> docs/requirements.txt
       '';
 
 in

nix/tools/release.nix

@@ -7,6 +7,7 @@ let
       {
         name = "postgrest-release";
         docs = "Patch postgrest.cabal, CHANGELOG.md, commit and push all in one go.";
+        args = [ "ARG_OPTIONAL_BOOLEAN([major], [m], [Bump to new major version (only applies on main branch).])" ];
         workingDir = "/";
       }
       ''
@@ -19,6 +20,7 @@ let
         git diff --exit-code HEAD postgrest.cabal > /dev/null
         trap "" ERR
 
+        # TODO: Support C+D bumps when implementing hackage releases
         bump () {
           current_version="$(grep -oP '^version:\s*\K.*' postgrest.cabal)"
           # shellcheck disable=SC2034
@@ -60,7 +62,7 @@ let
         git add CHANGELOG.md > /dev/null
 
         echo "Committing ..."
-        git commit -m "chore: bump version to $new_version" > /dev/null
+        git commit -m "bump version to $new_version" > /dev/null
 
         if [[ "$current_branch" == "main" ]]; then
           bump devel
@@ -72,7 +74,7 @@ let
           git branch "v$A"
 
           echo "Committing (devel bump)..."
-          git commit -m "chore: bump version to $new_version" > /dev/null
+          git commit -m "bump version to $new_version" > /dev/null
         fi
 
         trap "echo Remote not found. Please push manually ..." ERR

nix/tools/style.nix

@@ -3,13 +3,13 @@
 , buildToolbox
 , checkedShellScript
 , deadnix
-, fd
 , git
 , hlint
 , hsie
 , nixpkgs-fmt
 , python3Packages
 , ruff
+, silver-searcher
 , statix
 , stylish-haskell
 , writeText
@@ -21,19 +21,20 @@ let
         name = "postgrest-style";
         docs = "Automatically format Haskell, Nix and Python files.";
         workingDir = "/";
-        withTmpDir = true;
       }
       ''
         # Format Nix files
         ${statix}/bin/statix fix
-        ${nixpkgs-fmt}/bin/nixpkgs-fmt .
+        ${nixpkgs-fmt}/bin/nixpkgs-fmt . > /dev/null 2> /dev/null
 
         # Format Haskell files
-        ${fd}/bin/fd '\.l?hs$' \
+        # --vimgrep fixes a bug in ag: https://github.com/ggreer/the_silver_searcher/issues/753
+        # TODO: fix style issues in src/protolude and include it
+        ${silver-searcher}/bin/ag -l --vimgrep -g '\.l?hs$' --ignore-dir=src/protolude . \
           | xargs ${stylish-haskell}/bin/stylish-haskell -i
 
         # Format Python files
-        TMPDIR="$tmpdir" ${black}/bin/black .
+        ${black}/bin/black . 2> /dev/null
       '';
 
   # Script to check whether any uncommitted changes result from postgrest-style
@@ -82,17 +83,19 @@ let
 
         # ruff has gaps in scanning for unused code, so we use vulture
         echo "Scanning python files for unused code..."
-        ${fd}/bin/fd '\.l?py$' \
+        ${silver-searcher}/bin/ag -l --vimgrep -g '\.l?py$' . \
           | xargs ${python3Packages.vulture}/bin/vulture --exclude docs/conf.py --min-confidence 80
 
         echo "Linting python files..."
         ${ruff}/bin/ruff check .
 
         echo "Checking consistency of import aliases in Haskell code..."
-        ${hsie} check-aliases src/library src/executable
+        ${hsie} check-aliases main src/PostgREST
 
         echo "Linting Haskell files..."
-        ${fd}/bin/fd '\.l?hs$' \
+        # --vimgrep fixes a bug in ag: https://github.com/ggreer/the_silver_searcher/issues/753
+        # TODO: fix lint issues in src/protolude and include it
+        ${silver-searcher}/bin/ag -l --vimgrep -g '\.l?hs$' --ignore-dir=src/protolude . \
           | xargs ${hlint}/bin/hlint --hint=${hlintConfig}
       '';
 

nix/tools/tests.nix

@@ -9,7 +9,6 @@
 , hpc-codecov
 , jq
 , lib
-, nginx
 , postgrest
 , python3
 , runtimeShell
@@ -55,6 +54,8 @@ let
         withEnv = postgrest.env;
       }
       ''
+        # This makes nix-env -iA tests.doctests.bin work.
+        export NIX_GHC=${postgrest.env.NIX_GHC}
         ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:doctests
       '';
 
@@ -92,7 +93,6 @@ let
         args = [ "ARG_LEFTOVERS([pytest arguments])" ];
         workingDir = "/";
         withEnv = postgrest.env;
-        withPath = [ nginx ];
       }
       ''
         ${cabal-install}/bin/cabal v2-build ${devCabalOptions} exe:postgrest
@@ -155,7 +155,6 @@ let
         redirectTixFiles = false;
         withEnv = postgrest.env;
         withTmpDir = true;
-        withPath = [ nginx ];
       }
       (
         # required for `hpc markup` in CI; glibcLocales is not available e.g. on Darwin

nix/tools/withTools.nix

@@ -1,8 +1,8 @@
 { buildToolbox
 , checkedShellScript
 , curl
+, git
 , lib
-, libfaketime
 , postgresqlVersions
 , postgrest
 , python3Packages
@@ -11,15 +11,9 @@
 }:
 let
   withTmpDb =
-    { name, postgresql, config ? "" }:
+    { name, postgresql }:
     let
       commandName = "postgrest-with-${name}";
-      postgresqlConf = writeText "postgresql.conf" ("
-        autovacuum = false
-        listen_addresses = ''
-        log_statement = all
-        shared_preload_libraries=pg_stat_statements
-      " + config);
     in
     checkedShellScript
       {
@@ -78,10 +72,6 @@ let
           TZ=$PGTZ initdb --no-locale --encoding=UTF8 --nosync -U postgres --auth=trust \
             >> "$setuplog"
 
-          # Append our own config to the one initdb created to avoid replacing
-          # default values created by the latter.
-          cat ${postgresqlConf} >> "$tmpdir/db/postgresql.conf"
-
           log "Starting the database cluster..."
 
           # Instead of listening on a local port, we will listen on a unix domain socket.
@@ -90,7 +80,7 @@ let
           # On MacOS, it's 104 chars
           # See: https://serverfault.com/questions/641347/check-if-a-path-exceeds-maximum-for-unix-domain-socket
 
-          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c hba_file=$HBA_FILE -k $PGHOST " \
+          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
             >> "$setuplog"
 
           log "Creating a minimally privileged $PGUSER connection role..."
@@ -116,7 +106,7 @@ let
             log "Starting replica on $replica_host"
 
             # We set a low max_standby_streaming_delay to make the replication conflict fail faster in tests (otherwise it waits for the default 30s)
-            pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c hba_file=$HBA_FILE -k $replica_host -c max_standby_streaming_delay=\"3s\" " \
+            pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \
               >> "$setuplog"
 
             >&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres"
@@ -145,7 +135,6 @@ let
           load_start=$SECONDS
           >&2 printf "${commandName}: Loading fixtures under the postgres role..."
           psql -U postgres -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
-          psql -U postgres -v ON_ERROR_STOP=1 -c "VACUUM ANALYZE;" >> "$setuplog"
           load_end=$((SECONDS - load_start))
           >&2 printf " done in %ss. Running command...\n" "$load_end"
         fi
@@ -196,6 +185,59 @@ let
 
   withPg = withTmpDb (builtins.head postgresqlVersions);
 
+  withGit =
+    let
+      name = "postgrest-with-git";
+    in
+    checkedShellScript
+      {
+        inherit name;
+        docs =
+          ''
+            Create a new worktree of the postgrest repo in a temporary directory and
+            check out <commit>, then run <command> with arguments inside the temporary folder.
+          '';
+        args =
+          [
+            "ARG_POSITIONAL_SINGLE([commit], [Commit-ish reference to run command with])"
+            "ARG_POSITIONAL_SINGLE([command], [Command to run])"
+            "ARG_LEFTOVERS([command arguments])"
+          ];
+        positionalCompletion =
+          ''
+            if test "$prev" == "${name}"; then
+              __gitcomp_nl "$(__git_refs)"
+            else
+              _command_offset 2
+            fi
+          '';
+        workingDir = "/";
+      }
+      ''
+        # not using withTmpDir here, because we don't want to keep the directory on error
+        tmpdir="$(mktemp -d)"
+        trap 'rm -rf "$tmpdir"' EXIT
+
+        ${git}/bin/git worktree add -f "$tmpdir" "$_arg_commit" > /dev/null
+
+        cd "$tmpdir"
+        ("$_arg_command" "''${_arg_leftovers[@]}")
+
+        ${git}/bin/git worktree remove -f "$tmpdir" > /dev/null
+      '';
+
+  legacyConfig =
+    writeText "legacy.conf"
+      ''
+        # Using this config file to support older postgrest versions for `postgrest-loadtest-against`
+        db-uri="$(PGRST_DB_URI)"
+        db-schema="$(PGRST_DB_SCHEMAS)"
+        db-anon-role="$(PGRST_DB_ANON_ROLE)"
+        db-pool="$(PGRST_DB_POOL)"
+        server-unix-socket="$(PGRST_SERVER_UNIX_SOCKET)"
+        log-level="$(PGRST_LOG_LEVEL)"
+      '';
+
   waitForPgrstReady =
     checkedShellScript
       {
@@ -248,12 +290,10 @@ let
           [
             "ARG_POSITIONAL_SINGLE([command], [Command to run])"
             "ARG_LEFTOVERS([command arguments])"
-            "ARG_OPTIONAL_SINGLE([faketime], [f], [Fake the system time when starting PostgREST. This is useful to test expiry of JWT, for example in loadtests])"
             "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])"
             "ARG_OPTIONAL_SINGLE([timeout], [t], [Maximum time to wait for PostgREST to be ready], [5])"
             "ARG_OPTIONAL_SINGLE([sleep], [s],   [Sleep time after PostgREST is ready, this is useful for monitoring])"
-            "ARG_USE_ENV([FAKETIME_LIB], [${libfaketime}/lib/libfaketime.so.1], [Faketime Library to preload])"
-            "ARG_USE_ENV([PGRST_CMD], [postgrest-run], [PostgREST executable to run])"
+            "ARG_USE_ENV([PGRST_CMD], [], [PostgREST executable to run])"
           ];
         positionalCompletion = "_command";
         workingDir = "/";
@@ -263,23 +303,32 @@ let
       ''
         export PGRST_SERVER_UNIX_SOCKET="$tmpdir"/postgrest.socket
 
-        if [ "''${PGRST_CMD}" == "postgrest-run" ]; then
+        if [ -z "''${PGRST_CMD:-}" ]; then
+          rm -f result
           build_start=$SECONDS
-          echo -n "${commandName}: Building postgrest (cabal)... "
-          postgrest-build
+          if [ -z "''${PGRST_BUILD_CABAL:-}" ]; then
+            echo -n "${commandName}: Building postgrest (nix)... "
+            # Using lib.getBin to also make this work with older checkouts, where .bin was not a thing, yet.
+            nix-build -E 'with import ./. {}; pkgs.lib.getBin postgrestPackage' > "$tmpdir"/build.log 2>&1 || {
+              echo "failed, output:"
+              cat "$tmpdir"/build.log
+              exit 1
+            }
+            PGRST_CMD=$(echo ./result*/bin/postgrest)
+          else
+            echo -n "${commandName}: Building postgrest (cabal)... "
+            postgrest-build
+            PGRST_CMD=postgrest-run
+          fi
           build_end=$((SECONDS - build_start))
           printf "done in %ss.\n" "$build_end"
         fi
 
-        ver=$($PGRST_CMD --version)
+        ver=$($PGRST_CMD ${legacyConfig} --version)
 
         echo -n "${commandName}: Starting $ver... "
 
-        if [[ -n "$_arg_faketime" ]]; then
-          LD_PRELOAD="$FAKETIME_LIB" FAKETIME="$_arg_faketime" "$PGRST_CMD" > "$tmpdir"/run.log 2>&1 &
-        else
-          $PGRST_CMD > "$tmpdir"/run.log 2>&1 &
-        fi
+        $PGRST_CMD ${legacyConfig} > "$tmpdir"/run.log 2>&1 &
         pid=$!
         # shellcheck disable=SC2329
         cleanup() {
@@ -329,6 +378,7 @@ buildToolbox
   name = "postgrest-with";
   tools = {
     inherit
+      withGit
       withPgAll
       withPgrst;
   } // builtins.listToAttrs (

postgrest.cabal

@@ -1,6 +1,6 @@
 cabal-version:      3.0
 name:               postgrest
-version:            16.1
+version:            15
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                     for tables, views, and functions, supporting all HTTP methods that security
@@ -38,16 +38,13 @@ flag hpc
   description: Enable HPC (dev only)
 
 library
-  default-language:   GHC2021
+  default-language:   Haskell2010
   default-extensions: OverloadedStrings
                       NoImplicitPrelude
-  hs-source-dirs:     src/library
+  hs-source-dirs:     src
   exposed-modules:    PostgREST.Admin
                       PostgREST.App
                       PostgREST.AppState
-                      PostgREST.AppState.Pool
-                      PostgREST.AppState.Reload
-                      PostgREST.AppState.Types
                       PostgREST.Auth
                       PostgREST.Auth.Jwt
                       PostgREST.Auth.JwtCache
@@ -70,9 +67,9 @@ library
                       PostgREST.SchemaCache.Table
                       PostgREST.Error
                       PostgREST.Error.Types
+                      PostgREST.Listener
                       PostgREST.Logger
                       PostgREST.MainTx
-                      PostgREST.Logger.Apache
                       PostgREST.MediaType
                       PostgREST.Metrics
                       PostgREST.Network
@@ -105,8 +102,7 @@ library
                     , HTTP                      >= 4000.3.7 && < 4000.5
                     , Ranged-sets               >= 0.3 && < 0.6
                     , aeson                     >= 2.0.3 && < 2.3
-                    , aeson-jsonpath            >= 0.4.2 && < 0.5
-                    , auto-update               >= 0.2.7 && < 0.3
+                    , auto-update               >= 0.1.4 && < 0.3
                     , base64-bytestring         >= 1 && < 1.3
                     , bytestring                >= 0.10.8 && < 0.13
                     , case-insensitive          >= 1.2 && < 1.3
@@ -120,7 +116,6 @@ library
                     , directory                 >= 1.2.6 && < 1.4
                     , either                    >= 4.4.1 && < 5.1
                     , extra                     >= 1.7.0 && < 2.0
-                    , fast-logger               >= 3.2.0 && < 3.3
                     , fuzzyset                  >= 0.2.4 && < 0.3
                     , hasql                     >= 1.9 && <= 1.9.3.1
                     , hasql-dynamic-statements  >= 0.3.1 && <= 0.3.1.8
@@ -141,8 +136,7 @@ library
                     , parsec                    >= 3.1.11 && < 3.2
                     , postgresql-libpq          >= 0.10
                     , prometheus-client         >= 1.1.1 && < 1.2.0
-                    , prometheus-metrics-ghc    >= 1.0.1.2 && < 1.2
-                    , protolude                 >= 0.3.1 && < 0.4
+                    , protolude
                     , regex-tdfa                >= 1.2.2 && < 1.4
                     , retry                     >= 0.7.4 && < 0.10
                     , scientific                >= 0.3.4 && < 0.4
@@ -152,27 +146,27 @@ library
                     , time                      >= 1.6 && < 1.15
                     , unordered-containers      >= 0.2.8 && < 0.3
                     , unix-compat               >= 0.5.4 && < 0.8
+                    , vault                     >= 0.3.1.5 && < 0.4
                     , vector                    >= 0.11 && < 0.14
                     , wai                       >= 3.2.1 && < 3.3
                     , wai-cors                  >= 0.2.5 && < 0.3
                     , wai-extra                 >= 3.1.8 && < 3.2
                     -- We already depend on wai-logger >= 2.3.7 indirectly via wai-extra,
                     -- but we want to depend on 2.4.0 which fixes 'unknownSocket' log output
-                    -- for unix sockets; this is tested in test/io/test_log.py. See
+                    -- for unix sockets; this is tested in test/io/test_io.py. See
                     -- https://github.com/kazu-yamamoto/logger/commit/3a71ca70afdbb93d4ecf0083eeba1fbbbcab3fc3
                     , wai-logger                >= 2.4.0
-                    , warp                      >= 3.4.14 && < 3.5
+                    , warp                      >= 3.4.13 && < 3.5
                     , stm                       >= 2.5 && < 3
                     , stm-hamt                  >= 1.2 && < 2
                     , focus                     >= 1.0 && < 2
                     , some                      >= 1.0.4.1 && < 2
-                    , uuid                      >= 1.3 && < 2
                       -- -fno-spec-constr may help keep compile time memory use in check,
                       --   see https://gitlab.haskell.org/ghc/ghc/issues/16017#note_219304
                       -- -optP-Wno-nonportable-include-path
                       --   prevents build failures on case-insensitive filesystems (macos),
                       --   see https://github.com/commercialhaskell/stack/issues/3918
-  ghc-options:        -j -Werror -Wall -fwarn-identities
+  ghc-options:        -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
 
   if flag(dev)
@@ -181,27 +175,70 @@ library
       ghc-options: -fhpc -hpcdir .hpc
   else
     ghc-options: -O2
-    if impl(ghc >= 9.12)
-      -- Makes GHC consider cross-module specialization for polymorphic functions
-      -- without explicitly needing to add INLINE, INLINABLE or SPECIALIZE pragmas.
-      -- Slightly increases the binary size but improves performance considerably.
-      ghc-options: -fexpose-overloaded-unfoldings -fspecialise-aggressively
 
   if !os(windows)
     build-depends:
       unix
 
+library protolude
+  visibility:         private
+  default-language:   Haskell2010
+  default-extensions: NoImplicitPrelude
+                      FlexibleContexts
+                      MultiParamTypeClasses
+                      OverloadedStrings
+  hs-source-dirs:     src/protolude
+  exposed-modules:    Protolude
+                      Protolude.Applicative
+                      Protolude.Base
+                      Protolude.Bifunctor
+                      Protolude.Bool
+                      Protolude.CallStack
+                      Protolude.Conv
+                      Protolude.ConvertText
+                      Protolude.Debug
+                      Protolude.Either
+                      Protolude.Error
+                      Protolude.Exceptions
+                      Protolude.Functor
+                      Protolude.List
+                      Protolude.Monad
+                      Protolude.Panic
+                      Protolude.Partial
+                      Protolude.Safe
+                      Protolude.Semiring
+                      Protolude.Show
+                      Protolude.Unsafe
+  build-depends:      array                >= 0.4 && < 0.6
+                    , async                >= 2.0 && < 2.3
+                    , base                 >= 4.6 && < 4.22
+                    , bytestring           >= 0.10.8 && < 0.13
+                    , containers           >= 0.5.7 && < 0.8
+                    , deepseq              >= 1.3 && < 1.6
+                    , ghc-prim             >= 0.3 && < 0.14
+                    , hashable             >= 1.2 && < 1.6
+                    , mtl                  >= 2.1 && < 2.4
+                    , mtl-compat           >= 0.2 && < 0.3
+                    , stm                  >= 2.5 && < 3
+                    , text                 >= 1.2.2 && < 2.2
+                    , transformers         >= 0.2 && < 0.7
+                    , transformers-compat  >= 0.4 && < 0.8
+                      -- Protolude has some partial functions, so
+                      -- it is fine to disable that specific warning
+  ghc-options:        -Werror -Wall -fwarn-identities -Wno-x-partial
+                      -fno-spec-constr -optP-Wno-nonportable-include-path
+
 executable postgrest
-  default-language:   GHC2021
+  default-language:   Haskell2010
   default-extensions: OverloadedStrings
                       NoImplicitPrelude
-  hs-source-dirs:     src/executable
+  hs-source-dirs:     main
   main-is:            Main.hs
   build-depends:      base                >= 4.9 && < 4.22
                     , containers          >= 0.5.7 && < 0.8
                     , postgrest
-                    , protolude           >= 0.3.1 && < 0.4
-  ghc-options:        -j -threaded -rtsopts "-with-rtsopts=-N -I0 -qg"
+                    , protolude
+  ghc-options:        -threaded -rtsopts "-with-rtsopts=-N -I0 -qg"
                       -O2 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
 
@@ -216,7 +253,7 @@ executable postgrest
 
 test-suite spec
   type:               exitcode-stdio-1.0
-  default-language:   GHC2021
+  default-language:   Haskell2010
   default-extensions: OverloadedStrings
                       QuasiQuotes
                       NoImplicitPrelude
@@ -226,12 +263,10 @@ test-suite spec
                       Feature.Auth.AudienceJwtSecretSpec
                       Feature.Auth.AuthSpec
                       Feature.Auth.BinaryJwtSecretSpec
-                      Feature.Auth.JwtCacheSpec
                       Feature.Auth.NoAnonSpec
                       Feature.Auth.NoJwtSecretSpec
                       Feature.ConcurrentSpec
                       Feature.CorsSpec
-                      Feature.HttpHeaderSpec
                       Feature.ExtraSearchPathSpec
                       Feature.NoSuperuserSpec
                       Feature.ObservabilitySpec
@@ -260,7 +295,6 @@ test-suite spec
                       Feature.Query.Preferences.HandlingSpec
                       Feature.Query.Preferences.MaxAffectedSpec
                       Feature.Query.Preferences.TimezoneSpec
-                      Feature.Query.PreparedStatementsSpec
                       Feature.Query.QueryLimitedSpec
                       Feature.Query.QuerySpec
                       Feature.Query.RangeSpec
@@ -299,15 +333,14 @@ test-suite spec
                     , postgrest
                     , process           >= 1.4.2 && < 1.7
                     , prometheus-client >= 1.1.1 && < 1.2.0
-                    , protolude         >= 0.3.1 && < 0.4
+                    , protolude
                     , regex-tdfa        >= 1.2.2 && < 1.4
                     , scientific        >= 0.3.4 && < 0.4
                     , text              >= 1.2.2 && < 2.2
-                    , time              >= 1.6 && < 1.15
                     , transformers-base >= 0.4.4 && < 0.5
                     , wai               >= 3.2.1 && < 3.3
                     , wai-extra         >= 3.0.19 && < 3.2
-  ghc-options:        -j -threaded -O0 -Werror -Wall -fwarn-identities
+  ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
                       -fno-warn-missing-signatures
                       -fwrite-ide-info
@@ -316,7 +349,7 @@ test-suite spec
 
 test-suite observability
   type:               exitcode-stdio-1.0
-  default-language:   GHC2021
+  default-language:   Haskell2010
   default-extensions: OverloadedStrings
                       QuasiQuotes
                       NoImplicitPrelude
@@ -339,10 +372,10 @@ test-suite observability
                     , jose-jwt          >= 0.9.6 && < 0.11
                     , postgrest
                     , prometheus-client >= 1.1.1 && < 1.2.0
-                    , protolude         >= 0.3.1 && < 0.4
+                    , protolude
                     , text              >= 1.2.2 && < 2.2
                     , wai               >= 3.2.1 && < 3.3
-  ghc-options:        -j -threaded -O0 -Werror -Wall -fwarn-identities
+  ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
                       -fwrite-ide-info
                       -- https://github.com/PostgREST/postgrest/issues/387
@@ -350,12 +383,15 @@ test-suite observability
 
 test-suite doctests
   type:               exitcode-stdio-1.0
-  default-language:   GHC2021
+  default-language:   Haskell2010
+  default-extensions: OverloadedStrings
+                      NoImplicitPrelude
   hs-source-dirs:     test/doc
   main-is:            Main.hs
   build-depends:      base              >= 4.9 && < 4.22
-                    , doctest-parallel  >= 0.4
+                    , doctest           >= 0.8
                     , postgrest
                     , pretty-simple
-  ghc-options:        -j -threaded -O0 -Werror -Wall -fwarn-identities
+                    , protolude
+  ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path

src/PostgREST/Admin.hs

@@ -7,10 +7,11 @@ import qualified Network.HTTP.Types.Status as HTTP
 import qualified Network.Wai               as Wai
 import qualified Network.Wai.Handler.Warp  as Warp
 
-import Control.Monad.Extra (whenJust)
+import Control.Monad.Extra       (whenJust)
+import Network.Socket            hiding (addrFamily)
+import Network.Socket.ByteString
 
-import PostgREST.AppState    (AppState, getConfig, killApp)
-import PostgREST.Config      (AppConfig (..))
+import PostgREST.AppState    (AppState)
 import PostgREST.MediaType   (MediaType (..), toContentType)
 import PostgREST.Metrics     (metricsToText)
 import PostgREST.Network     (resolveSocketToAddress)
@@ -21,39 +22,30 @@ import qualified PostgREST.AppState as AppState
 import qualified Network.Socket as NS
 import           Protolude
 
-runAdmin :: AppState -> Maybe NS.Socket -> IO Bool -> Warp.Settings -> IO ()
-runAdmin appState maybeAdminSocket checkMainAppLive settings = do
-  conf <- getConfig appState
+runAdmin :: AppState -> Maybe NS.Socket -> NS.Socket -> Warp.Settings -> IO ()
+runAdmin appState maybeAdminSocket socketREST settings = do
   whenJust maybeAdminSocket $ \adminSocket -> do
     address <- resolveSocketToAddress adminSocket
-    void . forkIO $ handle onError $
-      Warp.runSettingsSocket (adminServerSettings conf address) adminSocket adminApp
+    observer $ AdminStartObs address
+    void . forkIO $ Warp.runSettingsSocket settings adminSocket adminApp
   where
-    adminApp = admin appState checkMainAppLive
+    adminApp = admin appState socketREST
     observer = AppState.getObserver appState
-    adminServerSettings config addr=
-      settings
-        & Warp.setBeforeMainLoop (observer $ AdminStartObs addr)
-        & maybe identity Warp.setPort (configAdminServerPort config)
-
-    onError ex = do
-      observer $ AdminServerCrashedObs ex
-      killApp appState -- Admin server crash is deemed unrecoverable, so we kill postgrest
 
 -- | PostgREST admin application
-admin :: AppState.AppState -> IO Bool -> Wai.Application
-admin appState checkMainAppLive req respond = do
-  isMainAppLive <- checkMainAppLive
+admin :: AppState.AppState -> NS.Socket -> Wai.Application
+admin appState socketREST req respond  = do
+  isMainAppReachable  <- isRight <$> reachMainApp socketREST
   isLoaded <- AppState.isLoaded appState
   isPending <- AppState.isPending appState
 
   case Wai.pathInfo req of
     ["live"] ->
-      respond $ Wai.responseLBS (if isMainAppLive then HTTP.status200 else HTTP.status500) [] mempty
+      respond $ Wai.responseLBS (if isMainAppReachable then HTTP.status200 else HTTP.status500) [] mempty
     ["ready"] ->
       let
-        status | isPending              = HTTP.status503
-               | not isMainAppLive      = HTTP.status500
+        status | not isMainAppReachable = HTTP.status500
+               | isPending              = HTTP.status503
                | isLoaded               = HTTP.status200
                | otherwise              = HTTP.status500
       in
@@ -66,3 +58,18 @@ admin appState checkMainAppLive req respond = do
       respond $ Wai.responseLBS HTTP.status200 [toContentType MTTextPlain] mets -- Content-Type is required for prometheus compliance
     _ ->
       respond $ Wai.responseLBS HTTP.status404 [] mempty
+
+-- Try to connect to the main app socket
+-- Note that it doesn't even send a valid HTTP request, we just want to check that the main app is accepting connections
+reachMainApp :: Socket -> IO (Either IOException ())
+reachMainApp appSock = do
+  sockAddr <- getSocketName appSock
+  sock <- socket (addrFamily sockAddr) Stream defaultProtocol
+  try $ do
+    connect sock sockAddr
+    withSocketsDo $ bracket (pure sock) close sendEmpty
+  where
+    sendEmpty sock = void $ send sock mempty
+    addrFamily (SockAddrInet _ _) = AF_INET
+    addrFamily (SockAddrInet6 {}) = AF_INET6
+    addrFamily (SockAddrUnix _)   = AF_UNIX

src/PostgREST/ApiRequest.hs

@@ -8,7 +8,6 @@ module PostgREST.ApiRequest
   ( ApiRequest(..)
   , userApiRequest
   , userPreferences
-  , userBearerAuth
   ) where
 
 import qualified Data.CaseInsensitive as CI
@@ -17,28 +16,32 @@ import qualified Data.List.NonEmpty   as NonEmptyList
 import qualified Data.Set             as S
 import qualified Data.Text.Encoding   as T
 
-import Data.List                       (lookup)
-import Data.Ranged.Ranges              (emptyRange, rangeIntersection,
-                                        rangeIsEmpty)
-import Network.HTTP.Types.Header       (RequestHeaders, hAuthorization, hCookie)
-import Network.Wai                     (Request (..))
-import Network.Wai.Middleware.HttpAuth (extractBearerAuth)
-import Network.Wai.Parse               (parseHttpAccept)
-import Web.Cookie                      (parseCookies)
+import Data.List                 (lookup)
+import Data.Ranged.Ranges        (emptyRange, rangeIntersection,
+                                  rangeIsEmpty)
+import Network.HTTP.Types.Header (RequestHeaders, hCookie)
+import Network.Wai               (Request (..))
+import Network.Wai.Parse         (parseHttpAccept)
+import Web.Cookie                (parseCookies)
 
 import PostgREST.ApiRequest.Payload      (getPayload)
 import PostgREST.ApiRequest.QueryParams  (QueryParams (..))
 import PostgREST.ApiRequest.Types        (Action (..), DbAction (..),
-                                          InvokeMethod (..), Mutation (..),
-                                          Payload (..), RequestBody,
-                                          Resource (..))
-import PostgREST.Config                  (AppConfig (..), OpenAPIMode (..))
-import PostgREST.Error                   (ApiRequestError (..), RangeError (..))
+                                          InvokeMethod (..),
+                                          Mutation (..), Payload (..),
+                                          RequestBody, Resource (..))
+import PostgREST.Config                  (AppConfig (..),
+                                          OpenAPIMode (..))
+import PostgREST.Config.Database         (TimezoneNames)
+import PostgREST.Error                   (ApiRequestError (..),
+                                          RangeError (..))
 import PostgREST.MediaType               (MediaType (..))
 import PostgREST.RangeQuery              (NonnegRange, allRange,
-                                          convertToLimitZeroRange, hasLimitZero,
+                                          convertToLimitZeroRange,
+                                          hasLimitZero,
                                           rangeRequested)
-import PostgREST.SchemaCache.Identifiers (FieldName, QualifiedIdentifier (..),
+import PostgREST.SchemaCache.Identifiers (FieldName,
+                                          QualifiedIdentifier (..),
                                           Schema)
 
 import qualified PostgREST.ApiRequest.Preferences as Preferences
@@ -108,12 +111,8 @@ userApiRequest conf prefs req reqBody = do
     actIsInvokeSafe x = case x of {ActDb (ActRoutine _  (InvRead _)) -> True; _ -> False}
 
 -- | Parses the Prefer header
-userPreferences :: AppConfig -> Request -> Preferences.Preferences
-userPreferences conf req = Preferences.fromHeaders (configDbTxAllowOverride conf) $ requestHeaders req
-
--- | Obtains the Bearer Auth
-userBearerAuth :: Request -> Maybe ByteString
-userBearerAuth req = extractBearerAuth =<< lookup hAuthorization (requestHeaders req)
+userPreferences :: AppConfig -> Request -> TimezoneNames -> Preferences.Preferences
+userPreferences conf req timezones = Preferences.fromHeaders (configDbTxAllowOverride conf) timezones $ requestHeaders req
 
 getResource :: AppConfig -> [Text] -> Either ApiRequestError Resource
 getResource AppConfig{configOpenApiMode, configDbRootSpec} = \case

src/PostgREST/ApiRequest/Preferences.hs

@@ -21,21 +21,20 @@ module PostgREST.ApiRequest.Preferences
   , shouldCount
   , shouldExplainCount
   , prefAppliedHeader
-  , toHeaderValue
   ) where
 
 import qualified Data.ByteString.Char8     as BS
 import qualified Data.Map                  as Map
+import qualified Data.Set                  as S
 import qualified Network.HTTP.Types.Header as HTTP
 
+import PostgREST.Config.Database (TimezoneNames)
+
 import Protolude
 
 -- $setup
 -- Setup for doctests
--- >>> :set -XStandaloneDeriving
 -- >>> import Text.Pretty.Simple (pPrint)
--- >>> import qualified Data.Set as S
--- >>> import Protolude
 -- >>> deriving instance Show PreferResolution
 -- >>> deriving instance Show PreferRepresentation
 -- >>> deriving instance Show PreferCount
@@ -63,8 +62,10 @@ data Preferences
 -- |
 -- Parse HTTP headers based on RFC7240[1] to identify preferences.
 --
+-- >>> let sc = S.fromList ["America/Los_Angeles"]
+--
 -- One header with comma-separated values can be used to set multiple preferences:
--- >>> pPrint $ fromHeaders True [("Prefer", "resolution=ignore-duplicates, count=exact, timezone=America/Los_Angeles, max-affected=100")]
+-- >>> pPrint $ fromHeaders True sc [("Prefer", "resolution=ignore-duplicates, count=exact, timezone=America/Los_Angeles, max-affected=100")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
 --     , preferRepresentation = Nothing
@@ -81,7 +82,7 @@ data Preferences
 --
 -- Multiple headers can also be used:
 --
--- >>> pPrint $ fromHeaders True [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null"), ("Prefer", "handling=lenient"), ("Prefer", "invalid"), ("Prefer", "max-affected=5999")]
+-- >>> pPrint $ fromHeaders True sc [("Prefer", "resolution=ignore-duplicates"), ("Prefer", "count=exact"), ("Prefer", "missing=null"), ("Prefer", "handling=lenient"), ("Prefer", "invalid"), ("Prefer", "max-affected=5999")]
 -- Preferences
 --     { preferResolution = Just IgnoreDuplicates
 --     , preferRepresentation = Nothing
@@ -97,13 +98,13 @@ data Preferences
 --
 -- If a preference is set more than once, only the first is used:
 --
--- >>> preferTransaction $ fromHeaders True [("Prefer", "tx=commit, tx=rollback")]
+-- >>> preferTransaction $ fromHeaders True sc [("Prefer", "tx=commit, tx=rollback")]
 -- Just Commit
 --
 -- This is also the case across multiple headers:
 --
 -- >>> :{
---   preferResolution . fromHeaders True $
+--   preferResolution . fromHeaders True sc $
 --     [ ("Prefer", "resolution=ignore-duplicates")
 --     , ("Prefer", "resolution=merge-duplicates")
 --     ]
@@ -113,7 +114,7 @@ data Preferences
 --
 -- Preferences can be separated by arbitrary amounts of space, lower-case header is also recognized:
 --
--- >>> pPrint $ fromHeaders True [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default, handling=strict, anything")]
+-- >>> pPrint $ fromHeaders True sc [("prefer", "count=exact,    tx=commit   ,return=representation , missing=default, handling=strict, anything")]
 -- Preferences
 --     { preferResolution = Nothing
 --     , preferRepresentation = Just Full
@@ -126,8 +127,8 @@ data Preferences
 --     , invalidPrefs = [ "anything" ]
 --     }
 --
-fromHeaders :: Bool -> [HTTP.Header] -> Preferences
-fromHeaders allowTxDbOverride headers =
+fromHeaders :: Bool -> TimezoneNames -> [HTTP.Header] -> Preferences
+fromHeaders allowTxDbOverride acceptedTzNames headers =
   Preferences
     { preferResolution     = parsePrefs [MergeDuplicates, IgnoreDuplicates]
     , preferRepresentation = parsePrefs [Full, None, HeadersOnly]
@@ -135,7 +136,7 @@ fromHeaders allowTxDbOverride headers =
     , preferTransaction    = if allowTxDbOverride then parsePrefs [Commit, Rollback] else Nothing
     , preferMissing        = parsePrefs [ApplyDefaults, ApplyNulls]
     , preferHandling       = parsePrefs [Strict, Lenient]
-    , preferTimezone       = PreferTimezone <$> timezonePref
+    , preferTimezone       = if isTimezonePrefAccepted then PreferTimezone <$> timezonePref else Nothing
     , preferMaxAffected    = PreferMaxAffected <$> maxAffectedPref
     , invalidPrefs         = filter isUnacceptable prefs
     }
@@ -155,11 +156,12 @@ fromHeaders allowTxDbOverride headers =
     listStripPrefix prefix prefList = listToMaybe $ mapMaybe (BS.stripPrefix prefix) prefList
 
     timezonePref = listStripPrefix "timezone=" prefs
+    isTimezonePrefAccepted = ((S.member . decodeUtf8 <$> timezonePref) <*> pure acceptedTzNames) == Just True
 
     maxAffectedPref = listStripPrefix "max-affected=" prefs >>= readMaybe . BS.unpack
 
     isUnacceptable p = p `notElem` acceptedPrefs &&
-                       isNothing (BS.stripPrefix "timezone=" p) &&
+                       (isNothing (BS.stripPrefix "timezone=" p) ||  not isTimezonePrefAccepted) &&
                        isNothing (BS.stripPrefix "max-affected=" p)
 
     parsePrefs :: ToHeaderValue a => [a] -> Maybe a

src/PostgREST/ApiRequest/QueryParams.hs

@@ -4,22 +4,12 @@
 --
 -- This module is in charge of parsing all the querystring values in an url, e.g.
 -- the select, id, order in `/projects?select=id,name&id=eq.1&order=id,name.desc`.
-{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE LambdaCase    #-}
+{-# LANGUAGE TupleSections #-}
 module PostgREST.ApiRequest.QueryParams
   ( parse
   , QueryParams(..)
-  , pFieldForest
-  , pFieldName
-  , pFieldSelect
-  , pJsonPath
-  , pLogicTree
-  , pOpExpr
-  , pOrder
-  , pRelationSelect
-  , pRequestFilter
   , pRequestRange
-  , pSingleVal
-  , pSpreadRelationSelect
   ) where
 
 import qualified Data.ByteString.Char8         as BS
@@ -38,38 +28,40 @@ import Data.List                     (init, last)
 import Data.Ranged.Boundaries        (Boundary (..))
 import Data.Ranged.Ranges            (Range (..))
 import Data.Tree                     (Tree (..))
-import Text.Parsec.Error             (errorMessages, showErrorMessages)
-import Text.ParserCombinators.Parsec (GenParser, ParseError, Parser, anyChar,
-                                      between, char, choice, digit, eof,
-                                      errorPos, letter, lookAhead, many1,
-                                      noneOf, notFollowedBy, oneOf, optionMaybe,
-                                      sepBy, sepBy1, string, try, (<?>))
+import Text.Parsec.Error             (errorMessages,
+                                      showErrorMessages)
+import Text.ParserCombinators.Parsec (GenParser, ParseError, Parser,
+                                      anyChar, between, char, choice,
+                                      digit, eof, errorPos, letter,
+                                      lookAhead, many1, noneOf,
+                                      notFollowedBy, oneOf,
+                                      optionMaybe, sepBy, sepBy1,
+                                      string, try, (<?>))
 
-import PostgREST.RangeQuery              (NonnegRange, allRange, rangeGeq,
-                                          rangeLimit, rangeOffset,
-                                          restrictRange)
+import PostgREST.RangeQuery              (NonnegRange, allRange,
+                                          rangeGeq, rangeLimit,
+                                          rangeOffset, restrictRange)
 import PostgREST.SchemaCache.Identifiers (FieldName)
 
-import PostgREST.ApiRequest.Types (AggregateFunction (..), EmbedParam (..),
-                                   EmbedPath, Field, Filter (..),
-                                   FtsOperator (..), Hint, IsVal (..),
-                                   JoinType (..), JsonOperand (..),
-                                   JsonOperation (..), JsonPath, ListVal,
-                                   LogicOperator (..), LogicTree (..),
-                                   OpExpr (..), OpQuantifier (..),
-                                   Operation (..), OrderDirection (..),
+import PostgREST.ApiRequest.Types (AggregateFunction (..),
+                                   EmbedParam (..), EmbedPath, Field,
+                                   Filter (..), FtsOperator (..),
+                                   Hint, IsVal (..), JoinType (..),
+                                   JsonOperand (..),
+                                   JsonOperation (..), JsonPath,
+                                   ListVal, LogicOperator (..),
+                                   LogicTree (..), OpExpr (..),
+                                   OpQuantifier (..), Operation (..),
+                                   OrderDirection (..),
                                    OrderNulls (..), OrderTerm (..),
-                                   QuantOperator (..), SelectItem (..),
+                                   QuantOperator (..),
+                                   SelectItem (..),
                                    SimpleOperator (..), SingleVal)
 
 import PostgREST.Error (QPError (..))
 
 import Protolude hiding (Sum, try)
 
--- $setup
--- >>> import qualified Text.ParserCombinators.Parsec as P
--- >>> import Protolude hiding (Sum, try)
-
 data QueryParams =
   QueryParams
     { qsCanonical      :: ByteString

src/PostgREST/ApiRequest/Types.hs

@@ -42,7 +42,8 @@ module PostgREST.ApiRequest.Types
 import qualified Data.ByteString.Lazy as LBS
 import qualified Data.Set             as S
 
-import PostgREST.SchemaCache.Identifiers (FieldName, QualifiedIdentifier (..),
+import PostgREST.SchemaCache.Identifiers (FieldName,
+                                          QualifiedIdentifier (..),
                                           Schema)
 
 import Protolude

src/PostgREST/App.hs

@@ -0,0 +1,288 @@
+{-|
+Module      : PostgREST.App
+Description : PostgREST main application
+
+This module is in charge of mapping HTTP requests to PostgreSQL queries.
+Some of its functionality includes:
+
+- Mapping HTTP request methods to proper SQL statements. For example, a GET request is translated to executing a SELECT query in a read-only TRANSACTION.
+- Producing HTTP Headers according to RFCs.
+- Content Negotiation
+-}
+{-# LANGUAGE RecordWildCards     #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE ViewPatterns        #-}
+module PostgREST.App
+  ( postgrest
+  , run
+  ) where
+
+
+import GHC.IO.Exception (IOErrorType (..))
+import System.IO.Error  (ioeGetErrorType)
+
+import Control.Monad.Except     (liftEither)
+import Control.Monad.Extra      (whenJust)
+import Data.Either.Combinators  (mapLeft, whenLeft)
+import Data.Maybe               (fromJust)
+import Data.String              (IsString (..))
+import Network.Wai.Handler.Warp (defaultSettings, setHost,
+                                 setOnException, setPort,
+                                 setServerName)
+
+import qualified Data.Text.Encoding       as T
+import qualified Network.Wai              as Wai
+import qualified Network.Wai.Handler.Warp as Warp
+
+import qualified PostgREST.Admin      as Admin
+import qualified PostgREST.ApiRequest as ApiRequest
+import qualified PostgREST.AppState   as AppState
+import qualified PostgREST.Auth       as Auth
+import qualified PostgREST.Cors       as Cors
+import qualified PostgREST.Error      as Error
+import qualified PostgREST.Listener   as Listener
+import qualified PostgREST.Logger     as Logger
+import qualified PostgREST.MainTx     as MainTx
+import qualified PostgREST.Plan       as Plan
+import qualified PostgREST.Query      as Query
+import qualified PostgREST.Response   as Response
+import qualified PostgREST.Unix       as Unix (installSignalHandlers)
+
+import PostgREST.ApiRequest           (ApiRequest (..))
+import PostgREST.AppState             (AppState)
+import PostgREST.Auth.Types           (AuthResult (..))
+import PostgREST.Config               (AppConfig (..), LogLevel (..))
+import PostgREST.Error                (Error)
+import PostgREST.Network              (resolveSocketToAddress)
+import PostgREST.Observation          (Observation (..))
+import PostgREST.Response.Performance (ServerTiming (..),
+                                       serverTimingHeader)
+import PostgREST.SchemaCache          (SchemaCache (..))
+import PostgREST.TimeIt               (timeItT)
+import PostgREST.Version              (docsVersion, prettyVersion)
+
+import qualified Data.ByteString.Char8     as BS
+import qualified Data.List                 as L
+import           Data.Streaming.Network    (bindPortTCP,
+                                            bindRandomPortTCP)
+import qualified Data.Text                 as T
+import qualified Network.HTTP.Types        as HTTP
+import qualified Network.HTTP.Types.Header as HTTP (hVary)
+import qualified Network.Socket            as NS
+import           PostgREST.Unix            (createAndBindDomainSocket)
+import           Protolude                 hiding (Handler)
+
+type Handler = ExceptT Error
+
+run :: AppState -> IO ()
+run appState = do
+  conf@AppConfig{..} <- AppState.getConfig appState
+
+  AppState.schemaCacheLoader appState -- Loads the initial SchemaCache
+  (mainSocket, adminSocket) <- initSockets conf
+  let closeSockets = do
+        whenJust adminSocket NS.close
+        NS.close mainSocket
+  Unix.installSignalHandlers observer closeSockets (AppState.schemaCacheLoader appState) (AppState.readInDbConfig False appState)
+
+  Listener.runListener appState
+
+  Admin.runAdmin appState adminSocket mainSocket (serverSettings conf)
+
+  let app = postgrest configLogLevel appState (AppState.schemaCacheLoader appState)
+
+  do
+    address <- resolveSocketToAddress mainSocket
+    observer $ AppServerAddressObs address
+
+  Warp.runSettingsSocket (serverSettings conf & setOnException onWarpException) mainSocket app
+  where
+    observer = AppState.getObserver appState
+
+    onWarpException :: Maybe Wai.Request -> SomeException -> IO ()
+    onWarpException _ ex =
+      when (shouldDisplayException ex) $
+        observer $ WarpServerObs $ show ex
+
+    -- Similar to wai defaultShouldDisplayException in
+    -- https://github.com/yesodweb/wai//blob/8c3882c60f6abe043889fc20c7efd3fa9747fa4a/warp/Network/Wai/Handler/Warp/Settings.hs#L251-L258
+    -- but without omitting AsyncException since it's important to log for ThreadKilled, StackOverflow and other cases.
+    -- We want to reuse this to avoid flooding the logs for some transient failure cases.
+    shouldDisplayException :: SomeException -> Bool
+    shouldDisplayException se
+        | Just (_ :: Warp.InvalidRequest) <- fromException se = False
+        | Just (ioeGetErrorType -> et) <- fromException se, et == ResourceVanished || et == InvalidArgument = False
+        | otherwise = True
+
+serverSettings :: AppConfig -> Warp.Settings
+serverSettings AppConfig{..} =
+  defaultSettings
+    & setHost (fromString $ toS configServerHost)
+    & setPort configServerPort
+    & setServerName ("postgrest/" <> prettyVersion)
+
+-- | PostgREST application
+postgrest :: LogLevel -> AppState.AppState -> IO () -> Wai.Application
+postgrest logLevel appState connWorker =
+  traceHeaderMiddleware appState .
+  Cors.middleware appState .
+  Auth.middleware appState .
+  Logger.middleware logLevel Auth.getRole $
+    -- fromJust can be used, because the auth middleware will **always** add
+    -- some AuthResult to the vault.
+    \req respond -> do
+      appConf@AppConfig{..} <- AppState.getConfig appState -- the config must be read again because it can reload
+      case fromJust $ Auth.getResult req of
+        Left err -> respond $ Error.errorResponseFor configClientErrorVerbosity err
+        Right authResult -> do
+          maybeSchemaCache <- AppState.getSchemaCache appState
+
+          let
+            eitherResponse :: IO (Either Error Wai.Response)
+            eitherResponse =
+              runExceptT $ postgrestResponse appState appConf maybeSchemaCache authResult req
+
+          response <- either (Error.errorResponseFor configClientErrorVerbosity) identity <$> eitherResponse
+          -- Launch the connWorker when the connection is down. The postgrest
+          -- function can respond successfully (with a stale schema cache) before
+          -- the connWorker is done. However, when there's an empty schema cache
+          -- postgrest responds with the error `PGRST002`; this means that the schema
+          -- cache is still loading, so we don't launch the connWorker here because
+          -- it would duplicate the loading process, e.g. https://github.com/PostgREST/postgrest/issues/3704
+          -- TODO: this process may be unnecessary when the Listener is enabled. Revisit once https://github.com/PostgREST/postgrest/issues/1766 is done
+          when (isServiceUnavailable response && isJust maybeSchemaCache) connWorker
+          resp <- do
+            delay <- AppState.getNextDelay appState
+            return $ addRetryHint delay response
+          respond resp
+
+postgrestResponse
+  :: AppState.AppState
+  -> AppConfig
+  -> Maybe SchemaCache
+  -> AuthResult
+  -> Wai.Request
+  -> Handler IO Wai.Response
+postgrestResponse appState conf@AppConfig{..} maybeSchemaCache authResult@AuthResult{..} req = do
+  let observer = AppState.getObserver appState
+
+  sCache <-
+    case maybeSchemaCache of
+      Just sCache ->
+        return sCache
+      Nothing -> do
+        lift $ observer SchemaCacheEmptyObs
+        throwError Error.NoSchemaCacheError
+
+  body <- lift $ Wai.strictRequestBody req
+
+  let jwtTime = if configServerTimingEnabled then Auth.getJwtDur req else Nothing
+      timezones = dbTimezones sCache
+      prefs = ApiRequest.userPreferences conf req timezones
+
+  (parseTime, apiReq@ApiRequest{..}) <- withTiming $ liftEither . mapLeft Error.ApiRequestErr $ ApiRequest.userApiRequest conf prefs req body
+  (planTime, plan)                   <- withTiming $ liftEither $ Plan.actionPlan iAction conf apiReq sCache
+
+  let mainQ = Query.mainQuery plan conf apiReq authResult configDbPreRequest
+      tx = MainTx.mainTx mainQ conf authResult apiReq plan sCache
+      obsQuery s = when configLogQuery $ observer $ QueryObs mainQ s
+
+  (txTime, txResult) <- withTiming $ do
+    case tx of
+      MainTx.NoDbTx r -> pure r
+      MainTx.DbTx{..} -> do
+        dbRes <- lift $ AppState.usePool appState (dqTransaction dqIsoLevel dqTxMode $ runExceptT dqDbHandler)
+        let eitherResp = join $ mapLeft (Error.PgErr . Error.PgError (Just authRole /= configDbAnonRole)) dbRes
+
+        -- TODO: we use obsQuery twice, one here and one below because in case of an error with the usePool above, the request will finish here and return an error message.
+        -- This is because of a combination of ExceptT + our Error module which has Wai.responseLBS.
+        -- This needs refactoring so only the below obsQuery is used.
+        lift $ whenLeft eitherResp $ obsQuery . Error.status
+        liftEither eitherResp
+
+  (respTime, resp) <- withTiming $ do
+    let response = Response.actionResponse txResult apiReq (T.decodeUtf8 prettyVersion, docsVersion) conf sCache
+        status' = either Error.status Response.pgrstStatus response
+
+    -- TODO: see above obsQuery, only this obsQuery should remain after refactoring (because the QueryObs depends on the status)
+    lift $ obsQuery status'
+    liftEither response
+
+  return $ toWaiResponse (ServerTiming jwtTime parseTime planTime txTime respTime) resp
+
+  where
+    toWaiResponse :: ServerTiming -> Response.PgrstResponse -> Wai.Response
+    toWaiResponse timing (Response.PgrstResponse st hdrs bod) =
+      Wai.responseLBS st (hdrs ++ serverTimingHeaders timing ++ [varyHeader | not $ varyHeaderPresent hdrs]) bod
+
+    serverTimingHeaders :: ServerTiming -> [HTTP.Header]
+    serverTimingHeaders timing = [serverTimingHeader timing | configServerTimingEnabled]
+
+    varyHeader :: HTTP.Header
+    varyHeader = (HTTP.hVary, "Accept, Prefer, Range")
+
+    varyHeaderPresent :: [HTTP.Header] -> Bool
+    varyHeaderPresent = any (\(h, _v) -> h == HTTP.hVary)
+
+    withTiming :: Handler IO a -> Handler IO (Maybe Double, a)
+    withTiming f = if configServerTimingEnabled
+        then do
+          (t, r) <- timeItT f
+          pure (Just t, r)
+        else do
+          r <- f
+          pure (Nothing, r)
+
+traceHeaderMiddleware :: AppState -> Wai.Middleware
+traceHeaderMiddleware appState app req respond = do
+  conf <- AppState.getConfig appState
+
+  case configServerTraceHeader conf of
+    Nothing -> app req respond
+    Just hdr ->
+      let hdrVal = L.lookup hdr $ Wai.requestHeaders req in
+      app req (respond . Wai.mapResponseHeaders ([(hdr, fromMaybe mempty hdrVal)] ++))
+
+addRetryHint :: Int -> Wai.Response -> Wai.Response
+addRetryHint delay response = do
+  let h = ("Retry-After", BS.pack $ show delay)
+  Wai.mapResponseHeaders (\hs -> if isServiceUnavailable response then h:hs else hs) response
+
+isServiceUnavailable :: Wai.Response -> Bool
+isServiceUnavailable response = Wai.responseStatus response == HTTP.status503
+
+type AppSockets = (NS.Socket, Maybe NS.Socket)
+
+initSockets :: AppConfig -> IO AppSockets
+initSockets AppConfig{..} = do
+  let
+    cfg'usp = configServerUnixSocket
+    cfg'uspm = configServerUnixSocketMode
+    cfg'host = configServerHost
+    cfg'port = configServerPort
+    cfg'adminHost = configAdminServerHost
+    cfg'adminPort = configAdminServerPort
+
+  sock <- case cfg'usp of
+    -- I'm not using `streaming-commons`' bindPath function here because it's not defined for Windows,
+    -- but we need to have runtime error if we try to use it in Windows, not compile time error
+    Just path -> createAndBindDomainSocket path cfg'uspm
+    Nothing -> do
+      (_, sock) <-
+        if cfg'port /= 0
+          then do
+            sock <- bindPortTCP cfg'port (fromString $ T.unpack cfg'host)
+            pure (cfg'port, sock)
+          else do
+            -- explicitly bind to a random port, returning bound port number
+            (num, sock) <- bindRandomPortTCP (fromString $ T.unpack cfg'host)
+            pure (num, sock)
+      pure sock
+
+  adminSock <- case cfg'adminPort of
+    Just adminPort -> do
+      adminSock <- bindPortTCP adminPort (fromString $ T.unpack cfg'adminHost)
+      pure $ Just adminSock
+    Nothing -> pure Nothing
+
+  pure (sock, adminSock)

src/PostgREST/AppState.hs

@@ -0,0 +1,436 @@
+{-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE NamedFieldPuns  #-}
+{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE RecursiveDo     #-}
+
+module PostgREST.AppState
+  ( AppState
+  , destroy
+  , getConfig
+  , getSchemaCache
+  , getMainThreadId
+  , getPgVersion
+  , getNextDelay
+  , getNextListenerDelay
+  , getTime
+  , getJwtCacheState
+  , init
+  , initWithPool
+  , putConfig -- For tests TODO refactoring
+  , putNextListenerDelay
+  , putSchemaCache
+  , putPgVersion
+  , putIsListenerOn
+  , usePool
+  , readInDbConfig
+  , schemaCacheLoader
+  , getObserver
+  , isLoaded
+  , isPending
+  ) where
+
+import qualified Data.ByteString.Char8      as BS
+import           Data.Either.Combinators    (whenLeft)
+import qualified Hasql.Pool                 as SQL
+import qualified Hasql.Pool.Config          as SQL
+import qualified Hasql.Session              as SQL
+import qualified Hasql.Transaction.Sessions as SQL
+import qualified Network.HTTP.Types.Status  as HTTP
+import qualified PostgREST.Auth.JwtCache    as JwtCache
+import qualified PostgREST.Error            as Error
+import qualified PostgREST.Logger           as Logger
+import qualified PostgREST.Metrics          as Metrics
+import           PostgREST.Observation
+import           PostgREST.TimeIt           (timeItT)
+import           PostgREST.Version          (prettyVersion)
+
+import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
+                           updateAction)
+import Control.Retry      (RetryPolicy, RetryStatus (..), capDelay,
+                           exponentialBackoff, retrying,
+                           rsPreviousDelay)
+import Data.IORef         (IORef, atomicWriteIORef, newIORef,
+                           readIORef)
+import Data.Time.Clock    (UTCTime, getCurrentTime)
+
+import PostgREST.Auth.JwtCache           (JwtCacheState, update)
+import PostgREST.Config                  (AppConfig (..),
+                                          readAppConfig,
+                                          toConnectionSettings)
+import PostgREST.Config.Database         (queryDbSettings,
+                                          queryPgVersion,
+                                          queryRoleSettings)
+import PostgREST.Config.PgVersion        (PgVersion (..),
+                                          minimumPgVersion)
+import PostgREST.Debounce                (makeDebouncer)
+import PostgREST.SchemaCache             (SchemaCache (..),
+                                          querySchemaCache,
+                                          showSummary)
+import PostgREST.SchemaCache.Identifiers (quoteQi)
+
+import Protolude
+
+data AppState = AppState
+  -- | Database connection pool
+  { statePool              :: SQL.Pool
+  -- | Database server version
+  , statePgVersion         :: IORef PgVersion
+  -- | Schema cache
+  , stateSchemaCache       :: IORef (Maybe SchemaCache)
+  -- | The schema cache status
+  , stateSCacheStatus      :: SchemaCacheStatus
+  -- | State of the LISTEN channel
+  , stateIsListenerOn      :: IORef Bool
+  -- | starts the connection worker with a debounce
+  , debouncedSCacheLoader  :: IO ()
+  -- | Config that can change at runtime
+  , stateConf              :: IORef AppConfig
+  -- | Time used for verifying JWT expiration
+  , stateGetTime           :: IO UTCTime
+  -- | Used for killing the main thread in case a subthread fails
+  , stateMainThreadId      :: ThreadId
+  -- | Keeps track of the next delay for db connection retry
+  , stateNextDelay         :: IORef Int
+  -- | Keeps track of the next delay for the listener
+  , stateNextListenerDelay :: IORef Int
+  -- | Observation handler
+  , stateObserver          :: ObservationHandler
+  -- | JWT Cache
+  , stateJwtCache          :: JwtCache.JwtCacheState
+  , stateLogger            :: Logger.LoggerState
+  , stateMetrics           :: Metrics.MetricsState
+  }
+
+-- | Schema cache status.
+-- Empty means pending and full means loaded.
+newtype SchemaCacheStatus = SchemaCacheStatus
+  { getSCStatusMVar :: MVar ()
+  }
+
+init :: AppConfig -> IO AppState
+init conf@AppConfig{configLogLevel, configDbPoolSize} = do
+  loggerState  <- Logger.init
+  metricsState <- Metrics.init configDbPoolSize
+  let observer = liftA2 (>>) (Logger.observationLogger loggerState configLogLevel) (Metrics.observationMetrics metricsState)
+
+  observer $ AppStartObs prettyVersion
+
+  pool <- initPool conf observer
+  initWithPool pool conf loggerState metricsState observer
+
+initWithPool :: SQL.Pool -> AppConfig -> Logger.LoggerState -> Metrics.MetricsState -> ObservationHandler -> IO AppState
+initWithPool pool conf loggerState metricsState observer = mdo
+
+  appState <- AppState pool
+    <$> newIORef minimumPgVersion -- assume we're in a supported version when starting, this will be corrected on a later step
+    <*> newIORef Nothing
+    <*> newSchemaCacheStatus
+    <*> newIORef False
+    <*> makeDebouncer (retryingSchemaCacheLoad appState *> threadDelay 100000)  -- 100ms cooldown
+    <*> newIORef conf
+    <*> mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime }
+    <*> myThreadId
+    <*> newIORef 0
+    <*> newIORef 1
+    <*> pure observer
+    <*> JwtCache.init conf observer
+    <*> pure loggerState
+    <*> pure metricsState
+
+  return appState
+
+destroy :: AppState -> IO ()
+destroy = destroyPool
+
+initPool :: AppConfig -> ObservationHandler -> IO SQL.Pool
+initPool cfg@AppConfig{..} observer = do
+  SQL.acquire $ SQL.settings
+    [ SQL.size configDbPoolSize
+    , SQL.acquisitionTimeout $ fromIntegral configDbPoolAcquisitionTimeout
+    , SQL.agingTimeout $ fromIntegral configDbPoolMaxLifetime
+    , SQL.idlenessTimeout $ fromIntegral configDbPoolMaxIdletime
+    , SQL.staticConnectionSettings $ toConnectionSettings identity cfg
+    , SQL.observationHandler $ observer . HasqlPoolObs
+    ]
+
+-- | Run an action with a database connection.
+usePool :: AppState -> SQL.Session a -> IO (Either SQL.UsageError a)
+usePool AppState{stateObserver=observer, stateMainThreadId=mainThreadId, ..} sess = do
+    observer PoolRequest
+
+    res <- SQL.use statePool sess
+
+    observer PoolRequestFullfilled
+
+    whenLeft res (\case
+      SQL.AcquisitionTimeoutUsageError ->
+        observer PoolAcqTimeoutObs
+      err@(SQL.ConnectionUsageError e) ->
+        let failureMessage = BS.unpack $ fromMaybe mempty e in
+        when (("FATAL:  password authentication failed" `isInfixOf` failureMessage) || ("no password supplied" `isInfixOf` failureMessage)) $ do
+          observer $ ExitDBFatalError ServerAuthError err
+          killThread mainThreadId
+      err@(SQL.SessionUsageError (SQL.QueryError tpl _ (SQL.ResultError resultErr))) ->
+        handleResultError err tpl resultErr
+      err@(SQL.SessionUsageError (SQL.PipelineError (SQL.ResultError resultErr))) ->
+        -- Passing the empty template will not work for schema cache queries, see TODO further below.
+        handleResultError err mempty resultErr
+      err@(SQL.SessionUsageError (SQL.QueryError _ _ (SQL.ClientError _))) ->
+        -- An error on the client-side, usually indicates problems with connection
+        observer $ QueryErrorCodeHighObs err
+      SQL.SessionUsageError (SQL.PipelineError (SQL.ClientError _))  -> pure ()
+      )
+
+    return res
+  where
+    handleResultError err tpl resultErr = do
+      case resultErr of
+        SQL.UnexpectedResult{} -> do
+          observer $ ExitDBFatalError ServerPgrstBug err
+          killThread mainThreadId
+        SQL.RowError{} -> do
+          observer $ ExitDBFatalError ServerPgrstBug err
+          killThread mainThreadId
+        SQL.UnexpectedAmountOfRows{} -> do
+          observer $ ExitDBFatalError ServerPgrstBug err
+          killThread mainThreadId
+        -- Check for a syntax error (42601 is the pg code) only for queries that don't have `WITH pgrst_source` as prefix.
+        -- This would mean the error is on our schema cache queries, so we treat it as fatal.
+        -- TODO have a better way to mark this as a schema cache query
+        SQL.ServerError "42601" _ _ _ _ ->
+          unless ("WITH pgrst_source" `BS.isPrefixOf` tpl) $ do
+            observer $ ExitDBFatalError ServerPgrstBug err
+            killThread mainThreadId
+        -- Check for a "prepared statement <name> already exists" error (Code 42P05: duplicate_prepared_statement).
+        -- This would mean that a connection pooler in transaction mode is being used
+        -- while prepared statements are enabled in the PostgREST configuration,
+        -- both of which are incompatible with each other.
+        SQL.ServerError "42P05" _ _ _ _ -> do
+          observer $ ExitDBFatalError ServerError42P05 err
+          killThread mainThreadId
+        -- Check for a "transaction blocks not allowed in statement pooling mode" error (Code 08P01: protocol_violation).
+        -- This would mean that a connection pooler in statement mode is being used which is not supported in PostgREST.
+        SQL.ServerError "08P01" "transaction blocks not allowed in statement pooling mode" _ _ _ -> do
+          observer $ ExitDBFatalError ServerError08P01 err
+          killThread mainThreadId
+        SQL.ServerError{} ->
+          when (Error.status (Error.PgError False err) >= HTTP.status500) $
+            observer $ QueryErrorCodeHighObs err
+
+-- | Flush the connection pool so that any future use of the pool will
+-- use connections freshly established after this call.
+-- | Emits PoolFlushed observation
+flushPool :: AppState -> IO ()
+flushPool AppState{..} = do
+  SQL.release statePool
+  stateObserver PoolFlushed
+
+-- | Destroy the pool on shutdown.
+-- | Differs from flushPool in not emiting PoolFlushed observation.
+destroyPool :: AppState -> IO ()
+destroyPool AppState{..} = SQL.release statePool
+
+getPgVersion :: AppState -> IO PgVersion
+getPgVersion = readIORef . statePgVersion
+
+putPgVersion :: AppState -> PgVersion -> IO ()
+putPgVersion = atomicWriteIORef . statePgVersion
+
+getSchemaCache :: AppState -> IO (Maybe SchemaCache)
+getSchemaCache = readIORef . stateSchemaCache
+
+putSchemaCache :: AppState -> Maybe SchemaCache -> IO ()
+putSchemaCache appState = atomicWriteIORef (stateSchemaCache appState)
+
+schemaCacheLoader :: AppState -> IO ()
+schemaCacheLoader = debouncedSCacheLoader
+
+getNextDelay :: AppState -> IO Int
+getNextDelay = readIORef . stateNextDelay
+
+getNextListenerDelay :: AppState -> IO Int
+getNextListenerDelay = readIORef . stateNextListenerDelay
+
+putNextListenerDelay :: AppState -> Int -> IO ()
+putNextListenerDelay = atomicWriteIORef . stateNextListenerDelay
+
+getConfig :: AppState -> IO AppConfig
+getConfig = readIORef . stateConf
+
+putConfig :: AppState -> AppConfig -> IO ()
+putConfig = atomicWriteIORef . stateConf
+
+getTime :: AppState -> IO UTCTime
+getTime = stateGetTime
+
+getJwtCacheState :: AppState -> JwtCacheState
+getJwtCacheState = stateJwtCache
+
+getMainThreadId :: AppState -> ThreadId
+getMainThreadId = stateMainThreadId
+
+isConnEstablished :: AppState -> IO Bool
+isConnEstablished appState = do
+  AppConfig{..} <- getConfig appState
+  if configDbChannelEnabled then -- if the listener is enabled, we can be sure the connection is up
+    readIORef $ stateIsListenerOn appState
+  else -- otherwise the only way to check the connection is to make a query
+    isRight <$> usePool appState (SQL.sql "SELECT 1")
+
+putIsListenerOn :: AppState -> Bool -> IO ()
+putIsListenerOn = atomicWriteIORef . stateIsListenerOn
+
+isLoaded :: AppState -> IO Bool
+isLoaded x = do
+  scacheLoaded <- isSchemaCacheLoaded x
+  connEstablished <- isConnEstablished x
+  return $ scacheLoaded && connEstablished
+
+isPending :: AppState -> IO Bool
+isPending x = do
+  scacheLoaded <- isSchemaCacheLoaded x
+  connEstablished <- isConnEstablished x
+  return $ not scacheLoaded || not connEstablished
+
+getObserver :: AppState -> ObservationHandler
+getObserver = stateObserver
+
+-- | Try to load the schema cache and retry if it fails.
+--
+-- This is done by repeatedly: 1) flushing the pool, 2) querying the version and validating that the postgres version is supported by us, and 3) loading the schema cache.
+-- It's necessary to flush the pool:
+--
+-- + Because connections cache the pg catalog(see #2620)
+-- + For rapid recovery. Otherwise, the pool idle or lifetime timeout would have to be reached for new healthy connections to be acquired.
+retryingSchemaCacheLoad :: AppState -> IO ()
+retryingSchemaCacheLoad appState@AppState{stateObserver=observer, stateMainThreadId=mainThreadId} =
+  void $ retrying retryPolicy shouldRetry (\RetryStatus{rsIterNumber, rsPreviousDelay} -> do
+    when (rsIterNumber > 0) $ do
+      let delay = fromMaybe 0 rsPreviousDelay `div` oneSecondInUs
+      observer $ ConnectionRetryObs delay
+
+    (,) <$> qPgVersion <*> (qInDbConfig *> qSchemaCache)
+  )
+  where
+    qPgVersion :: IO (Maybe PgVersion)
+    qPgVersion = do
+      AppConfig{..} <- getConfig appState
+      pgVersion <- usePool appState queryPgVersion
+      case pgVersion of
+        Left e -> do
+          observer $ QueryPgVersionError e
+          unless configDbPoolAutomaticRecovery $ do
+            observer ExitDBNoRecoveryObs
+            killThread mainThreadId
+          return Nothing
+        Right actualPgVersion ->
+          if actualPgVersion < minimumPgVersion then do
+            observer $ ExitUnsupportedPgVersion actualPgVersion minimumPgVersion
+            killThread mainThreadId
+            return Nothing
+          else do
+            observer $ DBConnectedObs $ pgvFullName actualPgVersion
+            observer $ PoolInit configDbPoolSize
+            putPgVersion appState actualPgVersion
+            return $ Just actualPgVersion
+
+    qInDbConfig :: IO ()
+    qInDbConfig = do
+      AppConfig{..} <- getConfig appState
+      when configDbConfig $ readInDbConfig False appState
+
+    qSchemaCache :: IO (Maybe SchemaCache)
+    qSchemaCache = do
+      conf@AppConfig{..} <- getConfig appState
+      (resultTime, result) <-
+        timeItT $ usePool appState (SQL.transactionNoRetry SQL.ReadCommitted SQL.Read $ querySchemaCache conf)
+      case result of
+        Left e -> do
+          markSchemaCachePending appState
+          putSchemaCache appState Nothing
+          observer $ SchemaCacheErrorObs configDbSchemas configDbExtraSearchPath e
+          return Nothing
+
+        Right sCache -> do
+          -- IMPORTANT: While the pending schema cache state starts from running the above querySchemaCache, only at this stage we block API requests due to the usage of an
+          -- IORef on putSchemaCache. This is why schema cache status is marked as pending here to signal the Admin server (using isPending) that we're on a recovery state.
+          markSchemaCachePending appState
+          putSchemaCache appState $ Just sCache
+          (loadTime, summary) <- timeItT (evaluate $ showSummary sCache)
+          -- Flush the pool after loading the schema cache to reset any stale session cache entries
+          -- We do it after successfully querying the schema cache (because this can fail and during retries we would flush the pool repeatedly unnecessarily)
+          -- and after marking sCacheStatus as pending,
+          flushPool appState
+          observer $ SchemaCacheQueriedObs resultTime $ dbQueryTimings sCache
+          observer $ SchemaCacheLoadedObs loadTime summary
+          markSchemaCacheLoaded appState
+          return $ Just sCache
+
+    shouldRetry :: RetryStatus -> (Maybe PgVersion, Maybe SchemaCache) -> IO Bool
+    shouldRetry _ (pgVer, sCache) = do
+      AppConfig{..} <- getConfig appState
+      let itShould = configDbPoolAutomaticRecovery && (isNothing pgVer || isNothing sCache)
+      return itShould
+
+    retryPolicy :: RetryPolicy
+    retryPolicy =
+      let delayMicroseconds = 32*oneSecondInUs {-32 seconds-} in
+      capDelay delayMicroseconds $ exponentialBackoff oneSecondInUs
+
+    oneSecondInUs = 1000000 -- one second in microseconds
+
+newSchemaCacheStatus :: IO SchemaCacheStatus
+newSchemaCacheStatus = SchemaCacheStatus <$> newEmptyMVar
+
+markSchemaCachePending :: AppState -> IO ()
+markSchemaCachePending = void . tryTakeMVar . getSCStatusMVar . stateSCacheStatus
+
+markSchemaCacheLoaded :: AppState -> IO ()
+markSchemaCacheLoaded = void . (`tryPutMVar` ()) . getSCStatusMVar . stateSCacheStatus
+
+isSchemaCacheLoaded :: AppState -> IO Bool
+isSchemaCacheLoaded = fmap not . isEmptyMVar . getSCStatusMVar . stateSCacheStatus
+
+-- | Reads the in-db config and reads the config file again
+-- | We don't retry reading the in-db config after it fails immediately, because it could have user errors. We just report the error and continue.
+readInDbConfig :: Bool -> AppState -> IO ()
+readInDbConfig startingUp appState@AppState{stateObserver=observer} = do
+  conf <- getConfig appState
+  pgVer <- getPgVersion appState
+  dbSettings <-
+    if configDbConfig conf then do
+      qDbSettings <- usePool appState (queryDbSettings (quoteQi <$> configDbPreConfig conf))
+      case qDbSettings of
+        Left e -> do
+          observer $ ConfigReadErrorObs e
+          pure mempty
+        Right x -> pure x
+    else
+      pure mempty
+  (roleSettings, roleIsolationLvl) <-
+    if configDbConfig conf then do
+      rSettings <- usePool appState (queryRoleSettings pgVer)
+      case rSettings of
+        Left e -> do
+          observer $ QueryRoleSettingsErrorObs e
+          pure (mempty, mempty)
+        Right x -> pure x
+    else
+      pure mempty
+  readAppConfig dbSettings (configFilePath conf) (Just $ configDbUri conf) roleSettings roleIsolationLvl >>= \case
+    Left err   ->
+      if startingUp then
+        panic err -- die on invalid config if the program is starting up
+      else
+        observer $ ConfigInvalidObs err
+    Right newConf -> do
+      putConfig appState newConf
+      -- After the config has reloaded, jwt-secret might have changed, so
+      -- if it has changed, it is important to invalidate the jwt cache
+      -- entries, because they were cached using the old secret
+      update (getJwtCacheState appState) newConf
+
+      if startingUp then
+        pass
+      else
+        observer ConfigSucceededObs

src/PostgREST/Auth.hs

@@ -0,0 +1,77 @@
+{-# LANGUAGE RecordWildCards #-}
+{-|
+Module      : PostgREST.Auth
+Description : PostgREST authentication functions.
+
+This module provides functions to deal with the JWT authentication (http://jwt.io).
+It also can be used to define other authentication functions,
+in the future Oauth, LDAP and similar integrations can be coded here.
+
+Authentication should always be implemented in an external service.
+In the test suite there is an example of simple login function that can be used for a
+very simple authentication system inside the PostgreSQL database.
+-}
+module PostgREST.Auth
+  ( getResult
+  , getJwtDur
+  , getRole
+  , middleware
+  ) where
+
+import qualified Data.ByteString                 as BS
+import qualified Data.Vault.Lazy                 as Vault
+import qualified Network.HTTP.Types.Header       as HTTP
+import qualified Network.Wai                     as Wai
+import qualified Network.Wai.Middleware.HttpAuth as Wai
+
+import Data.List        (lookup)
+import PostgREST.TimeIt (timeItT)
+import System.IO.Unsafe (unsafePerformIO)
+
+import PostgREST.AppState      (AppState, getConfig, getJwtCacheState,
+                                getTime)
+import PostgREST.Auth.Jwt      (parseClaims)
+import PostgREST.Auth.JwtCache (lookupJwtCache)
+import PostgREST.Auth.Types    (AuthResult (..))
+import PostgREST.Config        (AppConfig (..))
+import PostgREST.Error         (Error (..))
+
+import Protolude
+
+-- | Validate authorization header
+--   Parse and store JWT claims for future use in the request.
+middleware :: AppState -> Wai.Middleware
+middleware appState app req respond = do
+  conf@AppConfig{..} <- getConfig appState
+  time <- getTime appState
+
+  let token  = Wai.extractBearerAuth =<< lookup HTTP.hAuthorization (Wai.requestHeaders req)
+      parseJwt = runExceptT $ lookupJwtCache jwtCacheState token >>= parseClaims conf time
+      jwtCacheState = getJwtCacheState appState
+
+  -- If ServerTimingEnabled -> calculate JWT validation time
+  req' <- if configServerTimingEnabled then do
+      (dur, authResult) <- timeItT parseJwt
+      pure $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult & Vault.insert jwtDurKey dur }
+    else do
+      authResult <- parseJwt
+      pure $ req { Wai.vault = Wai.vault req & Vault.insert authResultKey authResult }
+
+  app req' respond
+
+authResultKey :: Vault.Key (Either Error AuthResult)
+authResultKey = unsafePerformIO Vault.newKey
+{-# NOINLINE authResultKey #-}
+
+getResult :: Wai.Request -> Maybe (Either Error AuthResult)
+getResult = Vault.lookup authResultKey . Wai.vault
+
+jwtDurKey :: Vault.Key Double
+jwtDurKey = unsafePerformIO Vault.newKey
+{-# NOINLINE jwtDurKey #-}
+
+getJwtDur :: Wai.Request -> Maybe Double
+getJwtDur =  Vault.lookup jwtDurKey . Wai.vault
+
+getRole :: Wai.Request -> Maybe BS.ByteString
+getRole req = authRole <$> (rightToMaybe =<< getResult req)

src/PostgREST/Auth/Jwt.hs

@@ -4,6 +4,8 @@ Description : PostgREST JWT support functions.
 
 This module provides functions to deal with JWT parsing and validation (http://jwt.io).
 -}
+{-# LANGUAGE DeriveGeneric         #-}
+{-# LANGUAGE FlexibleContexts      #-}
 {-# LANGUAGE ImpredicativeTypes    #-}
 {-# LANGUAGE LambdaCase            #-}
 {-# LANGUAGE NamedFieldPuns        #-}
@@ -29,7 +31,7 @@ import Data.Time.Clock.POSIX   (utcTimeToPOSIXSeconds)
 
 import PostgREST.Auth.Types    (AuthResult (..))
 import PostgREST.Config        (AppConfig (..), audMatchesCfg)
-import PostgREST.Config.JSPath (evaluateJSPath)
+import PostgREST.Config.JSPath (walkJSPath)
 import PostgREST.Error         (Error (..), JwtClaimsError (..),
                                 JwtDecodeError (..), JwtError (..))
 
@@ -112,7 +114,7 @@ parseClaims cfg@AppConfig{configJwtRoleClaimKey, configDbAnonRole} time mclaims
   validateClaims time (audMatchesCfg cfg) mclaims
   -- role defaults to anon if not specified in jwt
   role <- liftEither . maybeToRight (JwtErr JwtTokenRequired) $
-    unquoted <$> evaluateJSPath (Just $ JSON.Object mclaims) configJwtRoleClaimKey <|> configDbAnonRole
+    unquoted <$> walkJSPath (Just $ JSON.Object mclaims) configJwtRoleClaimKey <|> configDbAnonRole
   pure AuthResult
            { authClaims = mclaims
            , authRole = role

src/PostgREST/Auth/JwtCache.hs

@@ -4,9 +4,12 @@ Description : PostgREST JWT validation results Cache.
 
 This module provides functions to deal with the JWT cache.
 -}
-{-# LANGUAGE LambdaCase     #-}
-{-# LANGUAGE NamedFieldPuns #-}
-{-# LANGUAGE StrictData     #-}
+{-# LANGUAGE ExistentialQuantification #-}
+{-# LANGUAGE FlexibleInstances         #-}
+{-# LANGUAGE LambdaCase                #-}
+{-# LANGUAGE MultiParamTypeClasses     #-}
+{-# LANGUAGE NamedFieldPuns            #-}
+{-# LANGUAGE StrictData                #-}
 
 module PostgREST.Auth.JwtCache
   ( init
@@ -20,12 +23,13 @@ import qualified Data.Aeson.KeyMap as KM
 
 import PostgREST.Error (Error (..), JwtError (JwtSecretMissing))
 
-import           Control.Concurrent.STM      (newTVarIO, readTVar, writeTVar)
+import           Control.Concurrent.STM      (newTVarIO, readTVar,
+                                              writeTVar)
 import           Control.Concurrent.STM.TVar (TVar)
 import           Control.Monad.Error.Class   (liftEither)
 import           Data.ByteString             hiding (all, init)
-import           Data.IORef                  (IORef, newIORef, readIORef,
-                                              writeIORef)
+import           Data.IORef                  (IORef, newIORef,
+                                              readIORef, writeIORef)
 import           Jose.Jwk                    (JwkSet)
 import           PostgREST.Auth.Jwt          (parseAndDecodeClaims)
 import           PostgREST.Cache.Sieve       (alwaysValid)
@@ -38,7 +42,7 @@ import           Protolude
 data JwtCacheState = JwtCacheState ObservationHandler (IORef JwtCache)
 
 class CacheVariant m v where
-  cached :: (MonadError Error n, MonadIO n) => SC.Cache m ByteString v -> ByteString -> n JSON.Object
+  cached :: SC.Cache m ByteString v -> ByteString -> ExceptT Error IO JSON.Object
 
 {-|
 Jwt caching can have three different configurations:
@@ -56,12 +60,12 @@ data JwtCache =
   forall m v. CacheVariant m v => JwtCache JwkSet (TVar Int) (SC.Cache m ByteString v)
 
 instance CacheVariant IO (Either Error JSON.Object) where
-  cached c = liftIO . SC.cached c >=> liftEither
+  cached c = lift . SC.cached c >=> liftEither
 
 instance CacheVariant (ExceptT Error IO) JSON.Object where
-  cached c = liftIO . runExceptT . SC.cached c >=> liftEither
+  cached = SC.cached
 
-decode :: (MonadError Error m, MonadIO m) => JwtCache -> ByteString -> m JSON.Object
+decode :: JwtCache -> ByteString -> ExceptT Error IO JSON.Object
 decode JwtNoJwks        = const $ throwError (JwtErr JwtSecretMissing)
 decode (JwtNoCache key) = parseAndDecodeClaims key
 decode (JwtCache _ _ c) = cached c
@@ -106,5 +110,5 @@ newJwtCache AppConfig{configJWKS, configJwtCacheMaxEntries} observationHandler =
             (const . const $ lift $ observationHandler JwtCacheEviction) -- evictions metrics
             alwaysValid) -- no invalidation for now
 
-lookupJwtCache :: (MonadError Error m, MonadIO m) => JwtCacheState -> Maybe ByteString -> m JSON.Object
+lookupJwtCache :: JwtCacheState -> Maybe ByteString -> ExceptT Error IO JSON.Object
 lookupJwtCache (JwtCacheState _ cacheState) k = liftIO (readIORef cacheState) >>= flip (maybe (pure KM.empty)) k . decode

src/PostgREST/CLI.hs

@@ -42,13 +42,11 @@ runClientCommand conf CmdReady = Client.ready conf
 -- | Run postgrest with command
 runAppCommand :: AppConfig -> RunCommand -> IO ()
 runAppCommand conf@AppConfig{..} runCmd = do
-  mainThreadId <- myThreadId
-  mainThreadIdRef <- mkWeakThreadId mainThreadId
   -- Per https://github.com/PostgREST/postgrest/issues/268, we want to
   -- explicitly close the connections to PostgreSQL on shutdown.
   -- 'AppState.destroy' takes care of that.
   bracket
-    (AppState.init conf (killThread mainThreadId))
+    (AppState.init conf)
     AppState.destroy
     (\appState -> case runCmd of
       CmdDumpConfig -> do
@@ -57,21 +55,20 @@ runAppCommand conf@AppConfig{..} runCmd = do
       CmdDumpSchema -> do
         when configDbConfig $ AppState.readInDbConfig True appState
         putStrLn =<< dumpSchema appState
-      CmdRun -> App.run appState mainThreadIdRef)
+      CmdRun -> App.run appState)
 
 -- | Dump SchemaCache schema to JSON
 dumpSchema :: AppState -> IO LBS.ByteString
 dumpSchema appState = do
   conf@AppConfig{..} <- AppState.getConfig appState
-  pgVer <- AppState.getPgVersion appState
   result <-
-    AppState.usePool appState (SQL.transactionNoRetry SQL.ReadCommitted SQL.Read $ querySchemaCache pgVer conf)
+    AppState.usePool appState (SQL.transactionNoRetry SQL.ReadCommitted SQL.Read $ querySchemaCache conf)
   case result of
     Left e -> do
       let observer = AppState.getObserver appState
       observer $ SchemaCacheErrorObs configDbSchemas configDbExtraSearchPath e
       exitFailure
-    Right (sCache, _) -> return $ JSON.encode sCache
+    Right sCache -> return $ JSON.encode sCache
 
 -- | Command line interface options
 data CLI = CLI

src/PostgREST/Cache/Sieve.hs

@@ -8,9 +8,11 @@ This module provides implementation of a mutable cache on Sieve algorithm.
 {-# LANGUAGE GADTs           #-}
 {-# LANGUAGE LambdaCase      #-}
 {-# LANGUAGE NamedFieldPuns  #-}
+{-# LANGUAGE PolyKinds       #-}
 {-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE RecursiveDo     #-}
 {-# LANGUAGE StrictData      #-}
+{-# LANGUAGE TupleSections   #-}
 
 module PostgREST.Cache.Sieve (
       Cache

src/PostgREST/Config.hs

@@ -3,16 +3,20 @@ Module      : PostgREST.Config
 Description : Manages PostgREST configuration type and parser.
 
 -}
-{-# LANGUAGE LambdaCase      #-}
-{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE FlexibleContexts      #-}
+{-# LANGUAGE FlexibleInstances     #-}
+{-# LANGUAGE LambdaCase            #-}
+{-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE RecordWildCards       #-}
 {-# OPTIONS_GHC -fno-warn-type-defaults #-}
-{-# LANGUAGE NamedFieldPuns  #-}
+{-# LANGUAGE NamedFieldPuns        #-}
 
 module PostgREST.Config
   ( AppConfig (..)
   , Environment
   , JSPath
-  , defaultRoleJSPathKey
+  , JSPathExp(..)
+  , FilterExp(..)
   , LogLevel(..)
   , OpenAPIMode(..)
   , Proxy(..)
@@ -57,12 +61,15 @@ import Numeric                 (readOct, showOct)
 import System.Environment      (getEnvironment)
 import System.Posix.Types      (FileMode)
 
-import PostgREST.Config.Database         (RoleIsolationLvl, RoleSettings)
-import PostgREST.Config.JSPath           (JSPath (..), defaultRoleJSPathKey,
-                                          dumpJSPath, pRoleClaimKey)
-import PostgREST.Config.Proxy            (Proxy (..), isMalformedProxyUri,
-                                          toURI)
-import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..), toQi)
+import PostgREST.Config.Database         (RoleIsolationLvl,
+                                          RoleSettings)
+import PostgREST.Config.JSPath           (FilterExp (..), JSPath,
+                                          JSPathExp (..), dumpJSPath,
+                                          pRoleClaimKey)
+import PostgREST.Config.Proxy            (Proxy (..),
+                                          isMalformedProxyUri, toURI)
+import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
+                                          toQi)
 
 import PostgREST.Version (prettyVersion)
 import Protolude         hiding (Proxy, toList)
@@ -71,59 +78,57 @@ audMatchesCfg :: AppConfig -> Text -> Bool
 audMatchesCfg =  maybe (const True) (==) . configJwtAudience
 
 data AppConfig = AppConfig
-  { configAppSettings               :: [(Text, Text)]
-  , configClientErrorVerbosity      :: Verbosity
-  , configDbAggregates              :: Bool
-  , configDbAnonRole                :: Maybe BS.ByteString
-  , configDbChannel                 :: Text
-  , configDbChannelEnabled          :: Bool
-  , configDbExtraSearchPath         :: [Text]
-  , configDbHoistedTxSettings       :: [Text]
-  , configDbMaxRows                 :: Maybe Integer
-  , configDbPlanEnabled             :: Bool
-  , configDbPoolSize                :: Int
-  , configDbPoolAcquisitionTimeout  :: Int
-  , configDbPoolMaxLifetime         :: Int
-  , configDbPoolMaxIdletime         :: Int
-  , configDbPoolAutomaticRecovery   :: Bool
-  , configDbPreRequest              :: Maybe QualifiedIdentifier
-  , configDbPreparedStatements      :: Bool
-  , configDbRootSpec                :: Maybe QualifiedIdentifier
-  , configDbSchemas                 :: NonEmpty Text
-  , configDbConfig                  :: Bool
-  , configDbPreConfig               :: Maybe QualifiedIdentifier
-  , configDbTxAllowOverride         :: Bool
-  , configDbTxRollbackAll           :: Bool
-  , configDbUri                     :: Text
-  , configFilePath                  :: Maybe FilePath
-  , configJWKS                      :: Maybe JwkSet
-  , configJwtAudience               :: Maybe Text
-  , configJwtRoleClaimKey           :: JSPath
-  , configJwtSecret                 :: Maybe BS.ByteString
-  , configJwtSecretIsBase64         :: Bool
-  , configJwtCacheMaxEntries        :: Int
-  , configLogLevel                  :: LogLevel
-  , configLogQuery                  :: Bool
-  , configOpenApiMode               :: OpenAPIMode
-  , configOpenApiSecurityActive     :: Bool
-  , configOpenApiServerProxyUri     :: Maybe Text
-  , configServerCorsAllowedOrigins  :: [Text]
-  , configServerHost                :: Text
-  , configServerPort                :: Int
-  , configServerReusePort           :: Bool
-  , configServerTraceHeader         :: Maybe (CI.CI BS.ByteString)
-  , configServerTimingEnabled       :: Bool
-  , configServerUnixSocket          :: Maybe FilePath
-  , configServerUnixSocketMode      :: FileMode
-  , configUrlUseLegacyTargetNames   :: Bool
-  , configAdminServerHost           :: Text
-  , configAdminServerPort           :: Maybe Int
-  , configAdminServerUnixSocket     :: Maybe FilePath
-  , configAdminServerUnixSocketMode :: FileMode
-  , configRoleSettings              :: RoleSettings
-  , configRoleIsoLvl                :: RoleIsolationLvl
-  , configInternalSCQuerySleepFst   :: Maybe Int32
-  , configInternalSCQuerySleepSnd   :: Maybe Int32
+  { configAppSettings              :: [(Text, Text)]
+  , configClientErrorVerbosity     :: Verbosity
+  , configDbAggregates             :: Bool
+  , configDbAnonRole               :: Maybe BS.ByteString
+  , configDbChannel                :: Text
+  , configDbChannelEnabled         :: Bool
+  , configDbExtraSearchPath        :: [Text]
+  , configDbHoistedTxSettings      :: [Text]
+  , configDbMaxRows                :: Maybe Integer
+  , configDbPlanEnabled            :: Bool
+  , configDbPoolSize               :: Int
+  , configDbPoolAcquisitionTimeout :: Int
+  , configDbPoolMaxLifetime        :: Int
+  , configDbPoolMaxIdletime        :: Int
+  , configDbPoolAutomaticRecovery  :: Bool
+  , configDbPreRequest             :: Maybe QualifiedIdentifier
+  , configDbPreparedStatements     :: Bool
+  , configDbRootSpec               :: Maybe QualifiedIdentifier
+  , configDbSchemas                :: NonEmpty Text
+  , configDbConfig                 :: Bool
+  , configDbPreConfig              :: Maybe QualifiedIdentifier
+  , configDbTimezoneEnabled        :: Bool
+  , configDbTxAllowOverride        :: Bool
+  , configDbTxRollbackAll          :: Bool
+  , configDbUri                    :: Text
+  , configFilePath                 :: Maybe FilePath
+  , configJWKS                     :: Maybe JwkSet
+  , configJwtAudience              :: Maybe Text
+  , configJwtRoleClaimKey          :: JSPath
+  , configJwtSecret                :: Maybe BS.ByteString
+  , configJwtSecretIsBase64        :: Bool
+  , configJwtCacheMaxEntries       :: Int
+  , configLogLevel                 :: LogLevel
+  , configLogQuery                 :: Bool
+  , configOpenApiMode              :: OpenAPIMode
+  , configOpenApiSecurityActive    :: Bool
+  , configOpenApiServerProxyUri    :: Maybe Text
+  , configServerCorsAllowedOrigins :: Maybe [Text]
+  , configServerHost               :: Text
+  , configServerPort               :: Int
+  , configServerTraceHeader        :: Maybe (CI.CI BS.ByteString)
+  , configServerTimingEnabled      :: Bool
+  , configServerUnixSocket         :: Maybe FilePath
+  , configServerUnixSocketMode     :: FileMode
+  , configAdminServerHost          :: Text
+  , configAdminServerPort          :: Maybe Int
+  , configRoleSettings             :: RoleSettings
+  , configRoleIsoLvl               :: RoleIsolationLvl
+  , configInternalSCQuerySleep     :: Maybe Int32
+  , configInternalSCLoadSleep      :: Maybe Int32
+  , configInternalSCRelLoadSleep   :: Maybe Int32
   }
 
 data LogLevel = LogCrit | LogError | LogWarn | LogInfo | LogDebug
@@ -158,7 +163,7 @@ dumpOpenApiMode = \case
 -- | Dump the config
 toText :: AppConfig -> Text
 toText conf =
-  unlines $ sort $ (\(k, v) -> k <> " = " <> v) <$> pgrstSettings ++ appSettings
+  unlines $ (\(k, v) -> k <> " = " <> v) <$> pgrstSettings ++ appSettings
   where
     -- apply conf to all pgrst settings
     pgrstSettings = (\(k, v) -> (k, v conf)) <$>
@@ -182,10 +187,11 @@ toText conf =
       ,("db-schemas",                q . T.intercalate "," . toList . configDbSchemas)
       ,("db-config",                     T.toLower . show . configDbConfig)
       ,("db-pre-config",             q . maybe mempty dumpQi . configDbPreConfig)
+      ,("db-timezone-enabled",           T.toLower . show . configDbTimezoneEnabled)
       ,("db-tx-end",                 q . showTxEnd)
       ,("db-uri",                    q . configDbUri)
       ,("jwt-aud",                   q . fromMaybe mempty . configJwtAudience)
-      ,("jwt-role-claim-key",        q . dumpJSPath . configJwtRoleClaimKey)
+      ,("jwt-role-claim-key",        q . T.intercalate mempty . fmap dumpJSPath . configJwtRoleClaimKey)
       ,("jwt-secret",                q . T.decodeUtf8 . showJwtSecret)
       ,("jwt-secret-is-base64",          T.toLower . show . configJwtSecretIsBase64)
       ,("jwt-cache-max-entries",         show . configJwtCacheMaxEntries)
@@ -194,19 +200,15 @@ toText conf =
       ,("openapi-mode",              q . dumpOpenApiMode . configOpenApiMode)
       ,("openapi-security-active",       T.toLower . show . configOpenApiSecurityActive)
       ,("openapi-server-proxy-uri",  q . fromMaybe mempty . configOpenApiServerProxyUri)
-      ,("server-cors-allowed-origins", q . T.intercalate "," . configServerCorsAllowedOrigins)
+      ,("server-cors-allowed-origins",      q . maybe "" (T.intercalate ",") . configServerCorsAllowedOrigins)
       ,("server-host",               q . configServerHost)
       ,("server-port",                   show . configServerPort)
-      ,("server-reuseport",              T.toLower . show . configServerReusePort)
       ,("server-trace-header",       q . T.decodeUtf8 . maybe mempty CI.original . configServerTraceHeader)
       ,("server-timing-enabled",         T.toLower . show . configServerTimingEnabled)
       ,("server-unix-socket",        q . maybe mempty T.pack . configServerUnixSocket)
       ,("server-unix-socket-mode",   q . T.pack . showSocketMode)
-      ,("url-use-legacy-target-names",   T.toLower . show . configUrlUseLegacyTargetNames)
       ,("admin-server-host",         q . configAdminServerHost)
       ,("admin-server-port",             maybe "\"\"" show . configAdminServerPort)
-      ,("admin-server-unix-socket",  q . maybe mempty T.pack . configAdminServerUnixSocket)
-      ,("admin-server-unix-socket-mode", q . T.pack . showAdminSocketMode)
       ]
 
     -- quote all app.settings
@@ -230,7 +232,6 @@ toText conf =
       where
         secret = fromMaybe mempty $ configJwtSecret c
     showSocketMode c = showOct (configServerUnixSocketMode c) mempty
-    showAdminSocketMode c = showOct (configAdminServerUnixSocketMode c) mempty
 
 -- This class is needed for the polymorphism of overrideFromDbOrEnvironment
 -- because C.required and C.optional have different signatures
@@ -296,6 +297,7 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
     <*> parseDbSchemas "db-schemas" "db-schema"
     <*> (fromMaybe True <$> optBool "db-config")
     <*> (fmap toQi <$> optString "db-pre-config")
+    <*> (fromMaybe True <$> optBool "db-timezone-enabled")
     <*> parseTxEnd "db-tx-end" snd
     <*> parseTxEnd "db-tx-end" fst
     <*> (fromMaybe "postgresql://" <$> optString "db-uri")
@@ -316,21 +318,18 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
     <*> parseCORSAllowedOrigins "server-cors-allowed-origins"
     <*> (defaultServerHost <$> optString "server-host")
     <*> parseServerPort "server-port"
-    <*> (fromMaybe False <$> optBool "server-reuseport")
     <*> (fmap (CI.mk . encodeUtf8) <$> optString "server-trace-header")
     <*> (fromMaybe False <$> optBool "server-timing-enabled")
     <*> (fmap T.unpack <$> optString "server-unix-socket")
     <*> parseSocketFileMode "server-unix-socket-mode"
-    <*> (fromMaybe True <$> optBool "url-use-legacy-target-names")
     <*> (defaultServerHost <$> optWithAlias (optString "admin-server-host")
                                             (optString "server-host"))
     <*> parseAdminServerPort "admin-server-port"
-    <*> (fmap T.unpack <$> optString "admin-server-unix-socket")
-    <*> parseSocketFileMode "admin-server-unix-socket-mode"
     <*> pure roleSettings
     <*> pure roleIsolationLvl
-    <*> optInt "internal-schema-cache-query-sleep-before-queries"
     <*> optInt "internal-schema-cache-query-sleep"
+    <*> optInt "internal-schema-cache-load-sleep"
+    <*> optInt "internal-schema-cache-relationship-load-sleep"
   where
     parseErrorVerbosity :: C.Key -> C.Parser C.Config Verbosity
     parseErrorVerbosity k =
@@ -377,10 +376,10 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
         Just fileModeText ->
           case readOct $ T.unpack fileModeText of
             []              ->
-              fail $ "Invalid " <> T.unpack k <> ": not an octal"
+              fail "Invalid server-unix-socket-mode: not an octal"
             (fileMode, _):_ ->
               if fileMode < 384 || fileMode > 511
-                then fail $ "Invalid " <> T.unpack k <> ": needs to be between 600 and 777"
+                then fail "Invalid server-unix-socket-mode: needs to be between 600 and 777"
                 else pure fileMode
 
     parseOpenAPIMode :: C.Key -> C.Parser C.Config OpenAPIMode
@@ -424,13 +423,13 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
     parseRoleClaimKey :: C.Key -> C.Key -> C.Parser C.Config JSPath
     parseRoleClaimKey k al =
       optWithAlias (optString k) (optString al) >>= \case
-        Nothing  -> pure defaultRoleJSPathKey -- $.role
+        Nothing  -> pure [JSPKey "role"]
         Just rck -> either (fail . show) pure $ pRoleClaimKey rck
 
     parseCORSAllowedOrigins k =
       optString k >>= \case
-        Nothing   -> pure []
-        Just orig -> pure (T.strip <$> T.splitOn "," orig)
+        Nothing   -> pure Nothing
+        Just orig -> pure $ Just (T.strip <$> T.splitOn "," orig)
 
     optWithAlias :: C.Parser C.Config (Maybe a) -> C.Parser C.Config (Maybe a) -> C.Parser C.Config (Maybe a)
     optWithAlias orig alias =
@@ -591,7 +590,6 @@ pgConnString conn | uriDesignator `T.isPrefixOf` conn || shortUriDesignator `T.i
 
 -- | Adds a `fallback_application_name` value to the connection string. This allows querying the PostgREST version on pg_stat_activity.
 --
--- >>> import Protolude
 -- >>> let ver = "11.1.0 (5a04ec7)"::ByteString
 -- >>> let strangeVer = "11'1&0@#$%,.:\"[]{}?+^()=asdfqwer"::ByteString
 --
@@ -731,6 +729,9 @@ exampleConfigFile = S.unlines
   , "## The name of which database schema to expose to REST clients"
   , "db-schemas = \"public\""
   , ""
+  , "## Enable quering pg_timezone_names from db"
+  , "# db-timezone-enabled = true"
+  , ""
   , "## How to terminate database transactions"
   , "## Possible values are:"
   , "## commit (default)"
@@ -778,7 +779,6 @@ exampleConfigFile = S.unlines
   , ""
   , "server-host = \"!4\""
   , "server-port = 3000"
-  , "server-reuseport = false"
   , ""
   , "## Allow getting the request-response timing information through the `Server-Timing` header"
   , "server-timing-enabled = false"
@@ -790,9 +790,4 @@ exampleConfigFile = S.unlines
   , "## Unix socket file mode"
   , "## When none is provided, 660 is applied by default"
   , "# server-unix-socket-mode = \"660\""
-  , ""
-  , "## Use legacy target names in relationship filters"
-  , "## If active, allows using the target name of the relationship in filters even if it has an alias."
-  , "## Otherwise it only allows the alias in filters"
-  , "url-use-legacy-target-names = true"
   ]

src/PostgREST/Config/Database.hs

@@ -7,6 +7,7 @@ module PostgREST.Config.Database
   , queryRoleSettings
   , RoleSettings
   , RoleIsolationLvl
+  , TimezoneNames
   , toIsolationLevel
   ) where
 
@@ -15,7 +16,6 @@ import Control.Arrow ((***))
 import PostgREST.Config.PgVersion (PgVersion (..), pgVersion150)
 
 import qualified Data.HashMap.Strict as HM
-import qualified Data.Text           as T
 
 import qualified Hasql.Decoders             as HD
 import qualified Hasql.Encoders             as HE
@@ -30,9 +30,10 @@ import Protolude
 
 type RoleSettings     = (HM.HashMap ByteString (HM.HashMap ByteString ByteString))
 type RoleIsolationLvl = HM.HashMap ByteString SQL.IsolationLevel
+type TimezoneNames    = Set Text -- cache timezone names for prefer timezone=
 
-toIsolationLevel :: Text -> SQL.IsolationLevel
-toIsolationLevel a = case T.toLower a of
+toIsolationLevel :: (Eq a, IsString a) => a -> SQL.IsolationLevel
+toIsolationLevel a = case a of
   "repeatable read" -> SQL.RepeatableRead
   "serializable"    -> SQL.Serializable
   _                 -> SQL.ReadCommitted
@@ -55,6 +56,7 @@ dbSettingsNames =
   ,"db_prepared_statements"
   ,"db_root_spec"
   ,"db_schemas"
+  ,"db_timezone_enabled"
   ,"db_tx_end"
   ,"db_hoisted_tx_settings"
   ,"jwt_aud"
@@ -68,7 +70,6 @@ dbSettingsNames =
   ,"server_cors_allowed_origins"
   ,"server_trace_header"
   ,"server_timing_enabled"
-  ,"url_use_legacy_target_names"
   ]
 
 queryPgVersion :: Session PgVersion
@@ -147,7 +148,7 @@ queryRoleSettings pgVer =
         SELECT
           rolname,
           substr(setting, 1, strpos(setting, '=') - 1) as key,
-          substr(setting, strpos(setting, '=') + 1) as value
+          lower(substr(setting, strpos(setting, '=') + 1)) as value
         FROM role_setting
       ),
       iso_setting AS (

src/PostgREST/Config/JSPath.hs

@@ -0,0 +1,150 @@
+{-# OPTIONS_GHC -Wno-unused-do-bind #-}
+{-# LANGUAGE LambdaCase #-}
+module PostgREST.Config.JSPath
+  ( JSPath
+  , JSPathExp(..)
+  , FilterExp(..)
+  , dumpJSPath
+  , pRoleClaimKey
+  , walkJSPath
+  ) where
+
+import qualified Data.Aeson                    as JSON
+import qualified Data.Aeson.Key                as K
+import qualified Data.Aeson.KeyMap             as KM
+import qualified Data.Text                     as T
+import qualified Data.Vector                   as V
+import qualified Text.ParserCombinators.Parsec as P
+
+import Data.Either.Combinators       (mapLeft)
+import Text.ParserCombinators.Parsec ((<?>))
+import Text.Read                     (read)
+
+import Protolude
+
+
+-- | full jspath, e.g. .property[0].attr.detail[?(@ == "role1")]
+type JSPath = [JSPathExp]
+
+-- NOTE: We only accept one JSPFilter expr (at the end of input)
+-- | jspath expression
+data JSPathExp
+  = JSPKey Text                      -- .property or ."property-dash"
+  | JSPIdx Int                       -- [0]
+  | JSPSlice (Maybe Int) (Maybe Int) -- [0:5] or [0:] or [:5] or [:]
+  | JSPFilter FilterExp              -- [?(@ == "match")]
+
+data FilterExp
+  = EqualsCond Text
+  | NotEqualsCond Text
+  | StartsWithCond Text
+  | EndsWithCond Text
+  | ContainsCond Text
+
+dumpJSPath :: JSPathExp -> Text
+-- TODO: this needs to be quoted properly for special chars
+dumpJSPath (JSPKey k) = "." <> show k
+dumpJSPath (JSPIdx i) = "[" <> show i <> "]"
+dumpJSPath (JSPSlice s e) = "[" <> maybe "" show s <> ":" <> maybe "" show e <> "]"
+dumpJSPath (JSPFilter cond) = "[?(@" <> expr <> ")]"
+  where
+    expr =
+      case cond of
+        EqualsCond text     -> " == " <> show text
+        NotEqualsCond text  -> " != " <> show text
+        StartsWithCond text -> " ^== " <> show text
+        EndsWithCond text   -> " ==^ " <> show text
+        ContainsCond text   -> " *== " <> show text
+
+-- | Evaluate JSPath on a JSON
+walkJSPath :: Maybe JSON.Value -> JSPath -> Maybe JSON.Value
+walkJSPath x                      []                = x
+walkJSPath (Just (JSON.Object o)) (JSPKey key:rest) = walkJSPath (KM.lookup (K.fromText key) o) rest
+walkJSPath (Just (JSON.Array ar)) (JSPIdx idx:rest) = walkJSPath (ar V.!? idx) rest
+walkJSPath (Just (JSON.String str)) (JSPSlice start end:rest) =
+  let
+    len = T.length str
+
+    norm :: Maybe Int -> Maybe Int -- Normalize negative indices to positive
+    norm = fmap (\i -> max 0 $ min len $ if i < 0 then len + i else i)
+
+    s = fromMaybe 0 $ norm start -- normalized start index
+    e = fromMaybe len $ norm end -- normalized end index
+    slicedString = if s >= e then T.empty else T.take (e-s) $ T.drop s str
+  in
+    walkJSPath (Just $ JSON.String slicedString) rest
+
+walkJSPath (Just (JSON.Array ar)) (JSPFilter jspFilter:rest) = case jspFilter of
+    EqualsCond txt     -> walkJSPath (findFirstMatch (==) txt ar) rest
+    NotEqualsCond txt  -> walkJSPath (findFirstMatch (/=) txt ar) rest
+    StartsWithCond txt -> walkJSPath (findFirstMatch T.isPrefixOf txt ar) rest
+    EndsWithCond txt   -> walkJSPath (findFirstMatch T.isSuffixOf txt ar) rest
+    ContainsCond txt   -> walkJSPath (findFirstMatch T.isInfixOf txt ar) rest
+  where
+    findFirstMatch matchWith pattern = find (\case
+      JSON.String txt -> pattern `matchWith` txt
+      _               -> False)
+walkJSPath _                      _                 = Nothing
+
+-- Used for the config value "role-claim-key"
+pRoleClaimKey :: Text -> Either Text JSPath
+pRoleClaimKey selStr =
+  mapLeft show $ P.parse pJSPath ("failed to parse role-claim-key value (" <> toS selStr <> ")") (toS selStr)
+
+pJSPath :: P.Parser JSPath
+pJSPath = P.many1 pJSPathExp <* P.eof
+
+pJSPathExp :: P.Parser JSPathExp
+pJSPathExp = P.try pJSPKey <|> P.try pJSPFilter <|> P.try pJSPIdx <|> pJSPSlice
+
+pJSPKey :: P.Parser JSPathExp
+pJSPKey = do
+  P.char '.'
+  val <- toS <$> P.many1 (P.alphaNum <|> P.oneOf "_$@") <|> pQuotedValue
+  return (JSPKey val) <?> "pJSPKey: JSPath attribute key"
+
+pJSPIdx :: P.Parser JSPathExp
+pJSPIdx = do
+  P.char '['
+  num <- read <$> P.many1 P.digit
+  P.char ']'
+  return (JSPIdx num) <?> "pJSPIdx: JSPath array index"
+
+pJSPSlice :: P.Parser JSPathExp
+pJSPSlice = do
+  P.char '['
+  startSign <- P.optionMaybe $ P.char '-'
+  startIndex <- P.optionMaybe (read <$> P.many1 P.digit)
+  P.char ':'
+  endSign <- P.optionMaybe $ P.char '-'
+  endIndex <- P.optionMaybe (read <$> P.many1 P.digit)
+  P.char ']'
+  let start' = if isJust startSign then ((-1) *) <$> startIndex else startIndex
+      end'   = if isJust endSign   then ((-1) *) <$> endIndex   else endIndex
+  return (JSPSlice start' end') <?> "pJSPSlice: JSPath string slice"
+
+pJSPFilter :: P.Parser JSPathExp
+pJSPFilter = do
+  P.try $ P.string "[?("
+  condition <- pFilterConditionParser
+  P.char ')'
+  P.char ']'
+  return (JSPFilter condition) <?> "pJSPFilter: JSPath filter exp"
+
+pFilterConditionParser :: P.Parser FilterExp
+pFilterConditionParser = do
+  P.char '@'
+  P.spaces
+  filt <- matchOperator
+  P.spaces
+  filt <$> pQuotedValue
+    where
+      matchOperator =
+        P.try (P.string "==^" $> EndsWithCond)
+        <|> P.try (P.string "==" $> EqualsCond)
+        <|> P.try (P.string "!=" $> NotEqualsCond)
+        <|> P.try (P.string "^==" $> StartsWithCond)
+        <|> P.try (P.string "*==" $> ContainsCond)
+
+pQuotedValue :: P.Parser Text
+pQuotedValue = toS <$> (P.char '"' *> P.many (P.noneOf "\"") <* P.char '"')

src/PostgREST/Config/PgVersion.hs

@@ -1,11 +1,11 @@
 {-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric  #-}
 module PostgREST.Config.PgVersion
   ( PgVersion(..)
   , minimumPgVersion
   , pgVersion150
   , pgVersion170
   , pgVersion180
-  , pgVersion190
   ) where
 
 import qualified Data.Aeson as JSON
@@ -38,6 +38,3 @@ pgVersion170 = PgVersion 170000 "17.0" "17.0"
 
 pgVersion180 :: PgVersion
 pgVersion180 = PgVersion 180000 "18.0" "18.0"
-
-pgVersion190 :: PgVersion
-pgVersion190 = PgVersion 190000 "19.0" "19.0"

src/PostgREST/Cors.hs

@@ -2,6 +2,9 @@
 Module      : PostgREST.Cors
 Description : Wai Middleware to set cors policy.
 -}
+
+{-# LANGUAGE TupleSections #-}
+
 module PostgREST.Cors (middleware) where
 
 import qualified Data.ByteString.Char8       as BS
@@ -23,13 +26,11 @@ middleware appState app req res = do
   Wai.cors (corsPolicy $ configServerCorsAllowedOrigins conf) app req res
 
 -- | CORS policy to be used in by Wai Cors middleware
-corsPolicy :: [Text] -> Wai.Request -> Maybe Wai.CorsResourcePolicy
+corsPolicy :: Maybe [Text] -> Wai.Request -> Maybe Wai.CorsResourcePolicy
 corsPolicy corsAllowedOrigins req = case lookup "origin" headers of
   Just _ ->
     Just Wai.CorsResourcePolicy
-    { Wai.corsOrigins = case corsAllowedOrigins of
-        []      -> Nothing
-        origins -> Just (map T.encodeUtf8 origins, True)
+    { Wai.corsOrigins = (, True) . map T.encodeUtf8 <$> corsAllowedOrigins
     , Wai.corsMethods = ["GET", "POST", "PATCH", "PUT", "DELETE", "OPTIONS"]
     , Wai.corsRequestHeaders = "Authorization" : accHeaders
     , Wai.corsExposedHeaders = Just

src/PostgREST/Error.hs

@@ -19,8 +19,6 @@ module PostgREST.Error
   , JwtClaimsError(..)
   , errorPayload
   , status
-  , noRelBetweenHint
-  , noRpcHint
   ) where
 
 import qualified Data.Aeson                as JSON
@@ -46,21 +44,19 @@ import qualified PostgREST.MediaType as MediaType
 
 import PostgREST.Config                   (Verbosity (..))
 import PostgREST.SchemaCache              (SchemaCache (SchemaCache, dbTablesFuzzyIndex))
-import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..), Schema)
-import PostgREST.SchemaCache.Relationship (Cardinality (..), Junction (..),
-                                           Relationship (..), RelationshipsMap)
-import PostgREST.SchemaCache.Routine      (Routine (..), RoutineParam (..))
+import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..),
+                                           Schema)
+import PostgREST.SchemaCache.Relationship (Cardinality (..),
+                                           Junction (..),
+                                           Relationship (..),
+                                           RelationshipsMap)
+import PostgREST.SchemaCache.Routine      (Routine (..),
+                                           RoutineParam (..))
 
 import PostgREST.Error.Types
 
 import Protolude
 
--- $setup
--- >>> import qualified Data.HashMap.Strict as HM
--- >>> import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..))
--- >>> import PostgREST.SchemaCache.Relationship (Relationship (..))
--- >>> import PostgREST.SchemaCache.Routine (Routine (..), RoutineParam (..))
-
 -- | Encode Error to ByteString
 errorPayload :: (ErrorBody a, ErrorHeaders a) => Verbosity -> a -> LByteString
 errorPayload verb = JSON.encode . toJsonPgrstError verb
@@ -179,7 +175,7 @@ instance ErrorBody ApiRequestError where
   message InvalidFilters               = "Filters must include all and only primary key columns with 'eq' operators"
   message (UnacceptableSchema sch _)   = "Invalid schema: " <> sch
   message (MediaTypeError cts)         = "None of these media types are available: " <> T.intercalate ", " (map T.decodeUtf8 cts)
-  message (NotEmbedded resource _)     = "'" <> resource <> "' is not an embedded resource in this request"
+  message (NotEmbedded resource)       = "'" <> resource <> "' is not an embedded resource in this request"
   message GucHeadersError              = "response.headers guc must be a JSON array composed of objects with a single key and a string value"
   message GucStatusError               = "response.status guc must be a valid status code"
   message PutLimitNotAllowedError      = "limit/offset querystring parameters are not allowed for PUT"
@@ -211,13 +207,11 @@ instance ErrorBody ApiRequestError where
   details (InvalidPreferences prefs) = Just $ JSON.String $ T.decodeUtf8 ("Invalid preferences: " <> BS.intercalate ", " prefs)
   details (MaxAffectedViolationError n) = Just $ JSON.String $ T.unwords ["The query affects", show n, "rows"]
   details (NotImplemented details') = Just $ JSON.String details'
-  details (NotEmbedded _ (Just _)) = Just $ JSON.String "Target names are not allowed in filters if they have an alias"
 
   details _ = Nothing
 
   -- HINT: Maybe JSON.Value
-  hint (NotEmbedded resource Nothing) = Just $ JSON.String $ "Verify that '" <> resource <> "' is included in the 'select' query parameter."
-  hint (NotEmbedded _ (Just (name, alias))) = Just $ JSON.String $ "Change '" <> name <> "' to '" <> alias <> "' in filters, orders or limits."
+  hint (NotEmbedded resource) = Just $ JSON.String $ "Verify that '" <> resource <> "' is included in the 'select' query parameter."
   hint (PGRSTParseError raiseErr) = Just $ JSON.String $ pgrstParseErrorHint raiseErr
   hint (UnacceptableSchema _ schemas) = Just $ JSON.String $ "Only the following schemas are exposed: "  <> T.intercalate ", " schemas
 
@@ -481,7 +475,7 @@ instance ErrorBody SQL.UsageError where
   code    (SQL.SessionUsageError (SQL.QueryError _ _ e)) = code e
   code    SQL.AcquisitionTimeoutUsageError               = "PGRST003"
 
-  message (SQL.ConnectionUsageError _) = "Database connection error."
+  message (SQL.ConnectionUsageError _) = "Database connection error. Retrying the connection."
   message (SQL.SessionUsageError (SQL.PipelineError e))  = message e
   message (SQL.SessionUsageError (SQL.QueryError _ _ e)) = message e
   message SQL.AcquisitionTimeoutUsageError = "Timed out acquiring connection from connection pool."

src/PostgREST/Error/Types.hs

@@ -22,7 +22,8 @@ import qualified Hasql.Pool as SQL
 import PostgREST.MediaType                (MediaType (..))
 import PostgREST.SchemaCache              (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers  (QualifiedIdentifier (..))
-import PostgREST.SchemaCache.Relationship (Relationship (..), RelationshipsMap)
+import PostgREST.SchemaCache.Relationship (Relationship (..),
+                                           RelationshipsMap)
 import PostgREST.SchemaCache.Routine      (Routine (..))
 import Protolude
 
@@ -43,7 +44,7 @@ data ApiRequestError
   | InvalidPreferences [ByteString]
   | InvalidRange RangeError
   | InvalidRpcMethod ByteString
-  | NotEmbedded Text (Maybe (Text, Text))
+  | NotEmbedded Text
   | NotImplemented Text
   | PutLimitNotAllowedError
   | QueryParamError QPError

src/PostgREST/Listener.hs

@@ -0,0 +1,110 @@
+{-# LANGUAGE LambdaCase      #-}
+{-# LANGUAGE MultiWayIf      #-}
+{-# LANGUAGE RecordWildCards #-}
+
+module PostgREST.Listener (runListener) where
+
+import qualified Data.ByteString.Char8 as BS
+
+import qualified Hasql.Connection      as SQL
+import qualified Hasql.Notifications   as SQL
+import           PostgREST.AppState    (AppState, getConfig)
+import           PostgREST.Config      (AppConfig (..))
+import           PostgREST.Observation (Observation (..))
+
+import qualified PostgREST.AppState as AppState
+import qualified PostgREST.Config   as Config
+
+import           Control.Arrow              ((&&&))
+import           Data.Bitraversable         (bisequence)
+import           Data.Either.Combinators    (whenRight)
+import qualified Data.Text                  as T
+import qualified Database.PostgreSQL.LibPQ  as LibPQ
+import qualified Hasql.Session              as SQL
+import           PostgREST.Config.Database  (queryPgVersion)
+import           PostgREST.Config.PgVersion (pgvFullName)
+import           Protolude
+
+-- | Starts the Listener in a thread
+runListener :: AppState -> IO ()
+runListener appState = do
+  AppConfig{..} <- getConfig appState
+  when configDbChannelEnabled $
+    void . forkIO . void $ retryingListen appState
+
+-- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff with a cap of 32 seconds, if the LISTEN connection is lost.
+-- | This function never returns (but can throw) and return type enforces that.
+retryingListen :: AppState -> IO Void
+retryingListen appState = do
+  cfg@AppConfig{..} <- AppState.getConfig appState
+  let
+    dbChannel = toS configDbChannel
+    onError err = do
+      AppState.putIsListenerOn appState False
+      observer $ DBListenFail dbChannel (Right err)
+      when (isDbListenerBug err) $
+        observer DBListenBugHint
+      unless configDbPoolAutomaticRecovery $
+        killThread mainThreadId
+
+      -- retry the listener
+      delay <- AppState.getNextListenerDelay appState
+      observer $ DBListenRetry delay
+      threadDelay (delay * oneSecondInMicro)
+      unless (delay == maxDelay) $
+        AppState.putNextListenerDelay appState (delay * 2)
+      -- loop running the listener
+      retryingListen appState
+
+  -- Execute the listener with with error handling
+  handle onError $ do
+    -- Make sure we don't leak connections on errors
+    bracket
+      -- acquire connection
+      (SQL.acquire $
+        Config.toConnectionSettings Config.addTargetSessionAttrs cfg)
+      -- release connection
+      (`whenRight` releaseConnection) $
+      -- use connection
+      \case
+        Right db -> do
+          SQL.listen db $ SQL.toPgIdentifier dbChannel
+          (pqHost, pqPort) <- SQL.withLibPQConnection db $ bisequence . (LibPQ.host &&& LibPQ.port)
+          pgFullName <- SQL.run queryPgVersion db >>= either throwIO (pure . pgvFullName)
+
+          AppState.putIsListenerOn appState True
+
+          delay <- AppState.getNextListenerDelay appState
+          when (delay > 1) $ do -- if we did a retry
+            -- assume we lost notifications, refresh the schema cache
+            AppState.schemaCacheLoader appState
+            -- reset the delay
+            AppState.putNextListenerDelay appState 1
+
+          observer $ DBListenStart pqHost pqPort pgFullName dbChannel
+
+          -- wait for notifications
+          -- this will never return, in case of an error it will throw and be caught by onError
+          forever $ SQL.waitForNotifications handleNotification db
+
+        Left err -> do
+          observer $ DBListenFail dbChannel (Left err)
+          exitFailure
+  where
+    observer = AppState.getObserver appState
+    mainThreadId = AppState.getMainThreadId appState
+    oneSecondInMicro = 1000000
+    maxDelay = 32
+
+    handleNotification channel msg =
+      if | BS.null msg            -> observer (DBListenerGotSCacheMsg channel) >> cacheReloader
+         | msg == "reload schema" -> observer (DBListenerGotSCacheMsg channel) >> cacheReloader
+         | msg == "reload config" -> observer (DBListenerGotConfigMsg channel) >> AppState.readInDbConfig False appState
+         | otherwise              -> pure () -- Do nothing if anything else than an empty message is sent
+
+    cacheReloader =
+      AppState.schemaCacheLoader appState
+
+    releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release
+
+    isDbListenerBug e = "could not access status of transaction" `T.isInfixOf` show e

src/PostgREST/Logger.hs

@@ -7,13 +7,15 @@ Description : Logging based on the Observation.hs module. Access logs get sent t
 -}
 -- TODO log with buffering enabled to not lose throughput on logging levels higher than LogError
 module PostgREST.Logger
-  (observationLogger
+  ( middleware
+  , observationLogger
   , init
   , LoggerState
   ) where
 
 import           Control.AutoUpdate                (defaultUpdateSettings,
-                                                    mkAutoUpdate, updateAction)
+                                                    mkAutoUpdate,
+                                                    updateAction)
 import qualified Data.ByteString.Char8             as BS
 import qualified Data.Text.Encoding                as T
 import qualified Hasql.Decoders                    as HD
@@ -21,16 +23,20 @@ import qualified Hasql.DynamicStatements.Snippet   as SQL hiding (sql)
 import qualified Hasql.DynamicStatements.Statement as SQL
 import qualified Hasql.Statement                   as SQL
 
-import Data.Time (ZonedTime, defaultTimeLocale, formatTime, getZonedTime)
+import Data.Time (ZonedTime, defaultTimeLocale, formatTime,
+                  getZonedTime)
+
+import qualified Network.Wai                          as Wai
+import qualified Network.Wai.Middleware.RequestLogger as Wai
 
 import Network.HTTP.Types.Status (Status, status400, status500)
+import System.IO.Unsafe          (unsafePerformIO)
 
-import PostgREST.Config        (LogLevel (..), Verbosity (..))
-import PostgREST.Debounce      (makeDebouncer)
-import PostgREST.Logger.Apache (apacheFormat)
+import PostgREST.Config      (LogLevel (..), Verbosity (..))
+import PostgREST.Debounce    (makeDebouncer)
 import PostgREST.Observation
-import PostgREST.Query         (MainQuery (..))
-import PostgREST.SchemaCache   (queryTimingsWLabels)
+import PostgREST.Query       (MainQuery (..))
+import PostgREST.SchemaCache (queryTimingsWLabels)
 
 import qualified Data.ByteString.Lazy       as LBS
 import qualified Data.Text                  as T
@@ -45,19 +51,32 @@ import           Protolude
 data LoggerState = LoggerState
   { stateGetZTime               :: IO ZonedTime  -- ^ Time with time zone used for logs
   , stateLogDebouncePoolTimeout :: IO ()         -- ^ Logs with a debounce
-  , getLogLevel                 :: IO LogLevel   -- ^ Get LogLevel from Config
   }
 
-init :: IO LogLevel -> IO LoggerState
-init getLogLvl = mdo
+init :: IO LoggerState
+init = mdo
   let
-    oneSecond = 1_000_000
-    loggerState = LoggerState zTime debouncePoolTimeout getLogLvl
+    oneSecond = 1000000
+    loggerState = LoggerState zTime debouncePoolTimeout
   zTime <- mkAutoUpdate defaultUpdateSettings { updateAction = getZonedTime }
   debouncePoolTimeout <- makeDebouncer $
     logWithZTime loggerState (observationMessages PoolAcqTimeoutObs) *> threadDelay (5 * oneSecond)
   pure loggerState
 
+-- TODO stop using this middleware to reuse the same "observer" pattern for all our logs
+middleware :: LogLevel -> (Wai.Request -> Maybe BS.ByteString) -> Wai.Middleware
+middleware logLevel getAuthRole =
+    unsafePerformIO $
+      Wai.mkRequestLogger Wai.defaultRequestLoggerSettings
+      { Wai.outputFormat =
+         Wai.ApacheWithSettings $
+           Wai.defaultApacheSettings &
+           Wai.setApacheRequestFilter (\_ res -> shouldLogResponse logLevel $ Wai.responseStatus res) &
+           Wai.setApacheUserGetter getAuthRole
+      , Wai.autoFlush = True
+      , Wai.destination = Wai.Handle stdout
+      }
+
 shouldLogResponse :: LogLevel -> Status -> Bool
 shouldLogResponse logLevel = case logLevel of
   LogCrit  -> const False
@@ -67,58 +86,48 @@ shouldLogResponse logLevel = case logLevel of
   LogDebug -> const True
 
 -- All observations are logged except some that depend on the log-level
-observationLogger :: LoggerState -> ObservationHandler
-observationLogger loggerState obs = do
-  logLevel <- getLogLevel loggerState -- We need to do the IO action to read the "log-level" config value because it can be reloaded
-  case obs of
-    PoolAcqTimeoutObs -> do
-      when (logLevel >= LogError) $
-        stateLogDebouncePoolTimeout loggerState
-    o@(QueryErrorCodeHighObs _) -> do
-      when (logLevel >= LogError) $ do
-        logWithZTime loggerState $ observationMessages o
-    o@SchemaCacheEmptyObs ->
-      when (logLevel >= LogError) $ do
+observationLogger :: LoggerState -> LogLevel -> ObservationHandler
+observationLogger loggerState logLevel obs = case obs of
+  PoolAcqTimeoutObs -> do
+    when (logLevel >= LogError) $
+      stateLogDebouncePoolTimeout loggerState
+  o@(QueryErrorCodeHighObs _) -> do
+    when (logLevel >= LogError) $ do
       logWithZTime loggerState $ observationMessages o
-    o@(HasqlPoolObs _) -> do
-      when (logLevel >= LogDebug) $ do
-        logWithZTime loggerState $ observationMessages o
-    o@(QueryObs _ status) -> do
-      when (shouldLogResponse logLevel status) $
-        logWithZTime loggerState $ observationMessages o
-    o@PoolRequest ->
-      when (logLevel >= LogDebug) $ do
-        logWithZTime loggerState $ observationMessages o
-    o@PoolRequestFullfilled ->
-      when (logLevel >= LogDebug) $ do
-        logWithZTime loggerState $ observationMessages o
-    ResponseObs maybeRole req status contentLen ->
-      when (shouldLogResponse logLevel status) $ do
-        zTime <- stateGetZTime loggerState
-        putStr $ apacheFormat maybeRole (BS.pack $ formatZonedTime zTime) req status contentLen -- putStr prints to stdout
-    o@PoolFlushed ->
-      when (logLevel >= LogDebug) $ do
-        logWithZTime loggerState $ observationMessages o
-    o@JwtCacheEviction ->
-      when (logLevel >= LogDebug) $ do
-        logWithZTime loggerState $ observationMessages o
-    o@(JwtCacheLookup _) ->
-      when (logLevel >= LogDebug) $ do
-        logWithZTime loggerState $ observationMessages o
-    o@(WarpServerObs _) ->
-      when (logLevel >= LogDebug) $ do
-        logWithZTime loggerState $ observationMessages o
-    o ->
+  o@SchemaCacheEmptyObs ->
+    when (logLevel >= LogError) $ do
+    logWithZTime loggerState $ observationMessages o
+  o@(HasqlPoolObs _) -> do
+    when (logLevel >= LogDebug) $ do
       logWithZTime loggerState $ observationMessages o
+  o@(QueryObs _ status) -> do
+    when (shouldLogResponse logLevel status) $
+      logWithZTime loggerState $ observationMessages o
+  o@PoolRequest ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessages o
+  o@PoolRequestFullfilled ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessages o
+  o@PoolFlushed ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessages o
+  o@JwtCacheEviction ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessages o
+  o@(JwtCacheLookup _) ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessages o
+  o@(WarpServerObs _) ->
+    when (logLevel >= LogDebug) $ do
+      logWithZTime loggerState $ observationMessages o
+  o ->
+    logWithZTime loggerState $ observationMessages o
 
 logWithZTime :: LoggerState -> [Text] -> IO ()
 logWithZTime loggerState txts = do
   zTime <- stateGetZTime loggerState
-  let prefix = toS (formatZonedTime zTime) <> ": "
-  traverse_ (hPutStrLn stderr . (prefix <>)) txts
-
-formatZonedTime :: ZonedTime -> [Char]
-formatZonedTime = formatTime defaultTimeLocale "%d/%b/%Y:%T %z"
+  traverse_ (hPutStrLn stderr . (toS (formatTime defaultTimeLocale "%d/%b/%Y:%T %z: " zTime) <>)) txts
 
 -- TODO: maybe patch upstream hasql-dynamic-statements so we have a less hackish way to convert
 -- the SQL.Snippet or maybe don't use hasql-dynamic-statements and resort to plain strings for the queries and use regular hasql
@@ -133,8 +142,6 @@ observationMessages :: Observation -> [Text]
 observationMessages = \case
   AdminStartObs address ->
     pure $ "Admin server listening on " <> address
-  AdminServerCrashedObs ex ->
-    pure $ "Admin server crashed unexpectedly: " <> (showOnSingleLine '\t' . show) ex
   AppStartObs ver ->
     pure $ "Starting PostgREST " <> T.decodeUtf8 ver <> "..."
   AppServerAddressObs address ->
@@ -181,8 +188,8 @@ observationMessages = \case
       either showListenerConnError showListenerException listenErr
   DBListenRetry delay ->
     pure $ "Retrying listening for database notifications in " <> (show delay::Text) <> " seconds..."
-  DBListenBugCallQueryFix ->
-    pure "This is likely a PostgreSQL bug in the notification queue, executing the following to try to solve it: SELECT pg_notification_queue_usage();"
+  DBListenBugHint ->
+    pure "HINT:  This is likely a bug in the notification queue, try executing the following to solve it: select pg_notification_queue_usage();"
   DBListenerGotSCacheMsg channel ->
     pure $ "Received a schema cache reload message on the " <> show channel <> " channel"
   DBListenerGotConfigMsg channel ->
@@ -193,10 +200,6 @@ observationMessages = \case
       let snipts  = renderSnippet <$> [mqTxVars, fromMaybe mempty mqPreReq, mqMain, x, y, z, fromMaybe mempty mqExplain]
       in
         showOnSingleLine '\n' . T.decodeUtf8 <$> filter (/= mempty) snipts
-  LegacyTargetNameWarningObs (warningMsg, warningHints) requestMethod requestTarget ->
-    [ "WARNING: " <> warningMsg
-    , "Update filters, orders or limits that use " <> warningHints <> " in " <> "`" <> T.decodeUtf8 (requestMethod <> " " <> requestTarget) <> "`"
-    ]
   ConfigReadErrorObs usageErr ->
     pure $ "Failed to query database settings for the config parameters." <> jsonMessage usageErr
   QueryRoleSettingsErrorObs usageErr ->
@@ -240,8 +243,6 @@ observationMessages = \case
     pure $ "Received termination unix signal " <> signal
   WarpServerObs txt ->
     pure $ "Warp server: " <> txt
-  ResponseObs {} ->
-    mempty -- Control flow never reaches here, the observation message is returned in observationLogger function
   where
     showMillis :: Double -> Text
     showMillis x = toS $ showFFloat (Just 1) x ""

src/PostgREST/MainTx.hs

@@ -16,7 +16,8 @@ module PostgREST.MainTx
 import           Control.Lens                      ((^?))
 import           Control.Monad.Extra               (whenJust)
 import qualified Data.Aeson.Lens                   as L
-import qualified Data.ByteString                   as BS hiding (break)
+import qualified Data.ByteString                   as BS hiding
+                                                         (break)
 import qualified Data.ByteString.Char8             as BS
 import qualified Data.HashMap.Strict               as HM
 import qualified Data.Set                          as S
@@ -31,17 +32,21 @@ import qualified PostgREST.SchemaCache as SchemaCache
 
 
 import PostgREST.ApiRequest              (ApiRequest (..))
-import PostgREST.ApiRequest.Preferences  (PreferCount (..), PreferHandling (..),
+import PostgREST.ApiRequest.Preferences  (PreferCount (..),
+                                          PreferHandling (..),
                                           PreferMaxAffected (..),
                                           PreferTransaction (..),
                                           Preferences (..))
 import PostgREST.ApiRequest.Types        (Mutation (..))
 import PostgREST.Auth.Types              (AuthResult (..))
-import PostgREST.Config                  (AppConfig (..), OpenAPIMode (..))
+import PostgREST.Config                  (AppConfig (..),
+                                          OpenAPIMode (..))
 import PostgREST.Error                   (Error)
 import PostgREST.MediaType               (MediaType (..))
-import PostgREST.Plan                    (ActionPlan (..), CrudPlan (..),
-                                          DbActionPlan (..), InfoPlan (..),
+import PostgREST.Plan                    (ActionPlan (..),
+                                          CrudPlan (..),
+                                          DbActionPlan (..),
+                                          InfoPlan (..),
                                           InspectPlan (..))
 import PostgREST.Query                   (MainQuery (..))
 import PostgREST.SchemaCache             (SchemaCache (..))
@@ -54,7 +59,12 @@ import Protolude hiding (Handler)
 type DbHandler = ExceptT Error SQL.Transaction
 
 data MainTx
-  = DbTx (SQL.Session (Either Error DbResult))
+  = DbTx {
+      dqIsoLevel    :: SQL.IsolationLevel
+    , dqTxMode      :: SQL.Mode
+    , dqDbHandler   :: DbHandler DbResult
+    , dqTransaction :: SQL.IsolationLevel -> SQL.Mode -> SQL.Transaction (Either Error DbResult) -> SQL.Session (Either Error DbResult)
+    }
   | NoDbTx DbResult
 
 data DbResult
@@ -86,7 +96,7 @@ data ResultSet
 mainTx :: MainQuery -> AppConfig -> AuthResult -> ApiRequest -> ActionPlan -> SchemaCache -> MainTx
 mainTx _ _ _ _ (NoDb x) _ = NoDbTx $ NoDbResult x
 mainTx genQ@MainQuery{..} conf@AppConfig{..} AuthResult{..} apiReq (Db plan) sCache =
-  DbTx $ SQL.transactionNoRetry isoLvl txMode $ runExceptT dbHandler
+  DbTx isoLvl txMode dbHandler SQL.transactionNoRetry
   where
     isoLvl = planIsoLvl conf authRole plan
     txMode = planTxMode plan
@@ -105,7 +115,7 @@ planTxMode (MayUseDb x) = ipTxmode x
 planIsoLvl :: AppConfig -> ByteString -> DbActionPlan -> SQL.IsolationLevel
 planIsoLvl AppConfig{configRoleIsoLvl} role actPlan = case actPlan of
   DbCrud _ CallReadPlan{crProc} -> fromMaybe roleIsoLvl $ pdIsoLvl crProc
-  _                             -> roleIsoLvl
+  _                           -> roleIsoLvl
   where
     roleIsoLvl = HM.findWithDefault SQL.ReadCommitted role configRoleIsoLvl
 

src/PostgREST/MediaType.hs

@@ -1,4 +1,5 @@
 {-# LANGUAGE DeriveAnyClass        #-}
+{-# LANGUAGE DeriveGeneric         #-}
 {-# LANGUAGE DuplicateRecordFields #-}
 {-# OPTIONS_GHC -Wno-unused-do-bind #-}
 module PostgREST.MediaType
@@ -8,7 +9,6 @@ module PostgREST.MediaType
   , toContentType
   , toMime
   , decodeMediaType
-  , tokenizeMediaType
   ) where
 
 import qualified Data.Aeson                    as JSON
@@ -22,9 +22,6 @@ import Network.HTTP.Types.Header (Header, hContentType)
 
 import Protolude
 
--- $setup
--- >>> import qualified Text.ParserCombinators.Parsec as P
-
 -- | Enumeration of currently supported media types
 data MediaType
   = MTApplicationJSON

src/PostgREST/Metrics.hs

@@ -0,0 +1,79 @@
+{-# LANGUAGE RecordWildCards #-}
+{-|
+Module      : PostgREST.Logger
+Description : Metrics based on the Observation module. See Observation.hs.
+-}
+module PostgREST.Metrics
+  ( init
+  , MetricsState (..)
+  , observationMetrics
+  , metricsToText
+  ) where
+
+import qualified Data.ByteString.Lazy   as LBS
+import qualified Hasql.Pool.Observation as SQL
+
+import Prometheus
+
+import PostgREST.Observation
+
+import Protolude
+
+data MetricsState =
+  MetricsState {
+    poolTimeouts         :: Counter,
+    poolAvailable        :: Gauge,
+    poolWaiting          :: Gauge,
+    poolMaxSize          :: Gauge,
+    schemaCacheLoads     :: Vector Label1 Counter,
+    schemaCacheQueryTime :: Gauge,
+    jwtCacheRequests     :: Counter,
+    jwtCacheHits         :: Counter,
+    jwtCacheEvictions    :: Counter
+  }
+
+init :: Int -> IO MetricsState
+init configDbPoolSize = do
+  metricState <- MetricsState <$>
+    register (counter (Info "pgrst_db_pool_timeouts_total" "The total number of pool connection timeouts")) <*>
+    register (gauge (Info "pgrst_db_pool_available" "Available connections in the pool")) <*>
+    register (gauge (Info "pgrst_db_pool_waiting" "Requests waiting to acquire a pool connection")) <*>
+    register (gauge (Info "pgrst_db_pool_max" "Max pool connections")) <*>
+    register (vector "status" $ counter (Info "pgrst_schema_cache_loads_total" "The total number of times the schema cache was loaded")) <*>
+    register (gauge (Info "pgrst_schema_cache_query_time_seconds" "The query time in seconds of the last schema cache load")) <*>
+    register (counter (Info "pgrst_jwt_cache_requests_total" "The total number of JWT cache lookups")) <*>
+    register (counter (Info "pgrst_jwt_cache_hits_total" "The total number of JWT cache hits")) <*>
+    register (counter (Info "pgrst_jwt_cache_evictions_total" "The total number of JWT cache evictions"))
+  setGauge (poolMaxSize metricState) (fromIntegral configDbPoolSize)
+  pure metricState
+
+-- Only some observations are used as metrics
+observationMetrics :: MetricsState -> ObservationHandler
+observationMetrics MetricsState{..} obs = case obs of
+  PoolAcqTimeoutObs -> do
+    incCounter poolTimeouts
+  (HasqlPoolObs (SQL.ConnectionObservation _ status)) -> case status of
+     SQL.ReadyForUseConnectionStatus _ -> do
+      incGauge poolAvailable
+     SQL.InUseConnectionStatus        -> do
+      decGauge poolAvailable
+     SQL.TerminatedConnectionStatus  _ -> do
+      decGauge poolAvailable
+     SQL.ConnectingConnectionStatus -> pure ()
+  PoolRequest ->
+    incGauge poolWaiting
+  PoolRequestFullfilled ->
+    decGauge poolWaiting
+  SchemaCacheLoadedObs resTime _ -> do
+    withLabel schemaCacheLoads "SUCCESS" incCounter
+    setGauge schemaCacheQueryTime resTime
+  SchemaCacheErrorObs{} -> do
+    withLabel schemaCacheLoads "FAIL" incCounter
+  JwtCacheLookup True -> incCounter jwtCacheRequests *> incCounter jwtCacheHits
+  JwtCacheLookup False -> incCounter jwtCacheRequests
+  JwtCacheEviction -> incCounter jwtCacheEvictions
+  _ ->
+    pure ()
+
+metricsToText :: IO LBS.ByteString
+metricsToText = exportMetricsAsText

src/PostgREST/Network.hs

@@ -20,7 +20,23 @@ import Protolude
 resolveSocketToAddress :: NS.Socket -> IO Text
 resolveSocketToAddress sock = do
   sn <- NS.getSocketName sock
-  return $ fromString $ show sn
+  return $ showSocketAddr sn
+
+-- |
+-- >>> let addr_ipv4 = NS.SockAddrInet 80 (NS.tupleToHostAddress (127,0,0,1))
+-- >>> let addr_ipv6 = NS.SockAddrInet6 80 0 (0,0,0,1) 0
+-- >>> let addr_unix = NS.SockAddrUnix "/tmp/pgrst.sock"
+--
+-- >>> showSocketAddr addr_ipv4
+-- "127.0.0.1:80"
+
+-- >>> showSocketAddr addr_ipv6
+-- "[::1]:80"
+--
+-- >>> showSocketAddr addr_unix
+-- "/tmp/pgrst.sock"
+showSocketAddr :: NS.SockAddr -> Text
+showSocketAddr = fromString . show
 
 -- | When printing special addresses like !4 or *6, we use the following mapping.
 --   These special addresses come from:

src/PostgREST/Observation.hs

@@ -1,3 +1,4 @@
+{-# LANGUAGE DeriveGeneric #-}
 {-|
 Module      : PostgREST.Observation
 Description : This module holds an Observation type which is the core of Observability for PostgREST.
@@ -15,7 +16,6 @@ import qualified Hasql.Connection           as SQL
 import qualified Hasql.Pool                 as SQL
 import qualified Hasql.Pool.Observation     as SQL
 import           Network.HTTP.Types.Status  (Status)
-import qualified Network.Wai                as Wai
 import           PostgREST.Config.PgVersion
 import           PostgREST.Query            (MainQuery)
 import           PostgREST.SchemaCache      (QueryTimings)
@@ -24,7 +24,6 @@ import Protolude hiding (toList)
 
 data Observation
   = AdminStartObs Text
-  | AdminServerCrashedObs SomeException
   | AppStartObs ByteString
   | AppServerAddressObs Text
   | ExitUnsupportedPgVersion PgVersion PgVersion
@@ -39,12 +38,11 @@ data Observation
   | DBListenStart (Maybe ByteString) (Maybe ByteString) Text Text -- host, port, version string, channel
   | DBListenFail Text (Either SQL.ConnectionError SomeException)
   | DBListenRetry Int
-  | DBListenBugCallQueryFix
+  | DBListenBugHint -- https://github.com/PostgREST/postgrest/issues/3147
   | DBListenerGotSCacheMsg ByteString
   | DBListenerGotConfigMsg ByteString
   | DBListenerConnectionCleanupFail SomeException
   | QueryObs MainQuery Status
-  | LegacyTargetNameWarningObs (Text, Text) ByteString ByteString
   | ConfigReadErrorObs SQL.UsageError
   | ConfigInvalidObs Text
   | ConfigSucceededObs
@@ -54,7 +52,6 @@ data Observation
   | PoolInit Int
   | PoolAcqTimeoutObs
   | HasqlPoolObs SQL.Observation
-  | ResponseObs (Maybe ByteString) Wai.Request Status (Maybe Integer)
   | PoolRequest
   | PoolRequestFullfilled
   | PoolFlushed

src/PostgREST/Plan.hs

@@ -23,8 +23,6 @@ module PostgREST.Plan
   , InspectPlan(..)
   , InfoPlan(..)
   , CrudPlan(..)
-  , legacyWarnings
-  , addNullEmbedFilters
   ) where
 
 import qualified Data.HashMap.Strict           as HM
@@ -41,7 +39,8 @@ import Data.Tree               (Tree (..))
 
 import PostgREST.ApiRequest                  (ApiRequest (..))
 import PostgREST.Config                      (AppConfig (..))
-import PostgREST.Error                       (ApiRequestError (..), Error (..),
+import PostgREST.Error                       (ApiRequestError (..),
+                                              Error (..),
                                               SchemaCacheError (..))
 import PostgREST.MediaType                   (MediaType (..))
 import PostgREST.Plan.Negotiate              (negotiateContent)
@@ -51,19 +50,25 @@ import PostgREST.RangeQuery                  (NonnegRange, allRange,
                                               restrictRange)
 import PostgREST.SchemaCache                 (SchemaCache (..))
 import PostgREST.SchemaCache.Identifiers     (FieldName,
-                                              QualifiedIdentifier (..), Schema)
-import PostgREST.SchemaCache.Relationship    (Cardinality (..), Junction (..),
+                                              QualifiedIdentifier (..),
+                                              Schema)
+import PostgREST.SchemaCache.Relationship    (Cardinality (..),
+                                              Junction (..),
                                               Relationship (..),
-                                              RelationshipsMap, relIsToOne)
+                                              RelationshipsMap,
+                                              relIsToOne)
 import PostgREST.SchemaCache.Representations (DataRepresentation (..),
                                               RepresentationsMap)
-import PostgREST.SchemaCache.Routine         (MediaHandler (..), Routine (..),
-                                              RoutineMap, RoutineParam (..),
+import PostgREST.SchemaCache.Routine         (MediaHandler (..),
+                                              Routine (..),
+                                              RoutineMap,
+                                              RoutineParam (..),
                                               funcReturnsScalar,
                                               funcReturnsSetOfScalar,
                                               funcReturnsSingle)
 import PostgREST.SchemaCache.Table           (Column (..), Table (..),
-                                              TablesMap, tableColumnsList,
+                                              TablesMap,
+                                              tableColumnsList,
                                               tablePKCols)
 
 import PostgREST.ApiRequest.Preferences
@@ -81,16 +86,7 @@ import Protolude hiding (from)
 
 -- $setup
 -- Setup for doctests
--- >>> :set -XDuplicateRecordFields
 -- >>> import Data.Ranged.Ranges (fullRange)
--- >>> import Data.Tree (Tree (..))
--- >>> import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..))
--- >>> import PostgREST.ApiRequest.Types
--- >>> import PostgREST.Plan.CallPlan
--- >>> import PostgREST.Plan.MutatePlan
--- >>> import PostgREST.Plan.ReadPlan as ReadPlan
--- >>> import PostgREST.Plan.Types
--- >>> import Protolude
 
 -- Plan for reading or writing to the db
 data CrudPlan
@@ -130,7 +126,7 @@ data InspectPlan = InspectPlan {
   , ipSchema   :: Schema
   }
 
--- A Plan may use the database or not
+-- A Plan may use the the database or not
 data ActionPlan
   = Db DbActionPlan
   | NoDb InfoPlan
@@ -148,25 +144,6 @@ data InfoPlan
   | RoutineInfoPlan Routine -- info about function
   | SchemaInfoPlan -- info about schema cache
 
-legacyWarnings :: ActionPlan -> [(Text, Text)]
-legacyWarnings (NoDb _) = []
-legacyWarnings (Db dbPlan) =
-  case dbPlan of
-    DbCrud _ crudPlan ->
-      readPlanWarnings $ case crudPlan of
-        WrappedReadPlan{wrReadPlan} -> wrReadPlan
-        MutateReadPlan{mrReadPlan}  -> mrReadPlan
-        CallReadPlan{crReadPlan}    -> crReadPlan
-    MayUseDb _ -> []
-  where
-    readPlanWarnings :: ReadPlanTree -> [(Text, Text)]
-    readPlanWarnings (Node rp forest) =
-      maybeToList (readPlanWarning rp) <> foldMap readPlanWarnings forest
-
-readPlanWarning :: ReadPlan -> Maybe (Text, Text)
-readPlanWarning ReadPlan{relName, relAlias = Just alias, relIsLegacyTargetNameMatch = True} = Just (relName, alias)
-readPlanWarning _ = Nothing
-
 actionPlan :: Action -> AppConfig -> ApiRequest -> SchemaCache -> Either Error ActionPlan
 actionPlan act conf apiReq sCache = case act of
   ActDb dbAct              -> Db <$> dbActionPlan dbAct conf apiReq sCache
@@ -377,7 +354,7 @@ resolveQueryInputField ctx field opExpr = withTextParse ctx $ resolveTypeOrUnkno
 -- | Adds filters, order, limits on its respective nodes.
 -- | Adds joins conditions obtained from resource embedding.
 readPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Either Error ReadPlanTree
-readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows, configDbAggregates, configUrlUseLegacyTargetNames} SchemaCache{dbTables, dbRelationships, dbRepresentations} apiRequest  =
+readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows, configDbAggregates} SchemaCache{dbTables, dbRelationships, dbRepresentations} apiRequest  =
   let
     -- JSON output format hardcoded for now. In the future we might want to support other output mappings such as CSV.
     ctx = ResolverContext dbTables dbRepresentations qi "json"
@@ -392,10 +369,10 @@ readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows, configDbAggregate
     addAliases =<<
     expandStars ctx =<<
     addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
-    addLogicTrees ctx apiRequest configUrlUseLegacyTargetNames =<<
-    addRanges apiRequest configUrlUseLegacyTargetNames =<<
-    addOrders ctx apiRequest configUrlUseLegacyTargetNames =<<
-    addFilters ctx apiRequest configUrlUseLegacyTargetNames (initReadRequest ctx $ QueryParams.qsSelect $ iQueryParams apiRequest)
+    addLogicTrees ctx apiRequest =<<
+    addRanges apiRequest =<<
+    addOrders ctx apiRequest =<<
+    addFilters ctx apiRequest (initReadRequest ctx $ QueryParams.qsSelect $ iQueryParams apiRequest)
 
 -- Build the initial read plan tree
 initReadRequest :: ResolverContext -> [Tree SelectItem] -> ReadPlanTree
@@ -403,7 +380,7 @@ initReadRequest ctx@ResolverContext{qi=QualifiedIdentifier{..}} =
   foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi ctx, relName=qiName, depth=rootDepth} []
   where
     rootDepth = 0
-    defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing Nothing [] rootDepth False
+    defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing Nothing [] rootDepth
     treeEntry :: Depth -> Tree SelectItem -> ReadPlanTree -> ReadPlanTree
     treeEntry depth (Node si fldForest) (Node q rForest) =
       let nxtDepth = succ depth in
@@ -456,7 +433,7 @@ addAliases = Right . fmap addAliasToPlan
     isJsonKeyPath _                                 = False
 
     isTransformPath CoercibleField{cfTransform=(Just _), cfName=_} = True
-    isTransformPath _                                              = False
+    isTransformPath _ = False
 
     lastJsonKey CoercibleField{cfName=fieldName, cfJsonPath=jsonPath} =
       case jOp <$> lastMay jsonPath of
@@ -563,7 +540,7 @@ addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,re
           -- we use the table name as an alias so findRel can find the right relationship.
           ActDb (ActRelationMut _ _) -> rPlan{from=newFrom, fromAlias=newAlias}
           ActDb (ActRoutine _ _)     -> rPlan{from=newFrom, fromAlias=newAlias}
-          _                          -> rPlan
+          _                  -> rPlan
       in
       Node newReadPlan <$> updateForest (Just $ Node newReadPlan forest)
   where
@@ -837,10 +814,10 @@ findTable :: QualifiedIdentifier -> SchemaCache -> Either Error QualifiedIdentif
 findTable qi@QualifiedIdentifier{..} sc@SchemaCache{dbTables} =
   case HM.lookup qi dbTables of
     Nothing -> Left $ SchemaCacheErr $ TableNotFound qiSchema qiName sc
-    Just _  -> Right qi
+    Just _ -> Right qi
 
-addFilters :: ResolverContext -> ApiRequest -> Bool -> ReadPlanTree -> Either Error ReadPlanTree
-addFilters ctx ApiRequest{..} useTargetNames rReq =
+addFilters :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
+addFilters ctx ApiRequest{..} rReq =
   foldr addFilterToNode (Right rReq) flts
   where
     QueryParams.QueryParams{..} = iQueryParams
@@ -852,15 +829,15 @@ addFilters ctx ApiRequest{..} useTargetNames rReq =
 
     addFilterToNode :: (EmbedPath, Filter) -> Either Error ReadPlanTree ->  Either Error ReadPlanTree
     addFilterToNode =
-      updateNode useTargetNames (\flt (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest (resolveFilter ctx{qi=fromTable} flt) lf}  f)
+      updateNode (\flt (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=addFilterToLogicForest (resolveFilter ctx{qi=fromTable} flt) lf}  f)
 
-addOrders :: ResolverContext -> ApiRequest -> Bool -> ReadPlanTree -> Either Error ReadPlanTree
-addOrders ctx ApiRequest{..} useTargetNames rReq = foldr addOrderToNode (Right rReq) qsOrder
+addOrders :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
+addOrders ctx ApiRequest{..} rReq = foldr addOrderToNode (Right rReq) qsOrder
   where
     QueryParams.QueryParams{..} = iQueryParams
 
     addOrderToNode :: (EmbedPath, [OrderTerm]) -> Either Error ReadPlanTree -> Either Error ReadPlanTree
-    addOrderToNode = updateNode useTargetNames (\o (Node q f) -> Node q{order=resolveOrder ctx <$> o} f)
+    addOrderToNode = updateNode (\o (Node q f) -> Node q{order=resolveOrder ctx <$> o} f)
 
 resolveOrder :: ResolverContext -> OrderTerm -> CoercibleOrderTerm
 resolveOrder _ (OrderRelationTerm a b c d) = CoercibleOrderRelationTerm a b c d
@@ -885,7 +862,7 @@ addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
             then Right $ cot{coRelation=relAggAlias}
             else Left $ ApiRequestErr $ RelatedOrderNotToOne (qiName from) name
         Nothing ->
-          Left $ ApiRequestErr $ NotEmbedded coRelation Nothing
+          Left $ ApiRequestErr $ NotEmbedded coRelation
 
 -- | Searches for null filters on embeds, e.g. `projects=not.is.null` on `GET /clients?select=*,projects(*)&projects=not.is.null`
 --
@@ -910,8 +887,7 @@ addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
 --         relToParent = Nothing,
 --         relJoinConds = [],
 --         relAlias = Nothing, relAggAlias = "clients_projects_1", relHint = Nothing, relJoinType = Nothing, relSpread = Nothing, depth = 1,
---         relSelect = [],
---         relIsLegacyTargetNameMatch = False
+--         relSelect = []
 --       },
 --       subForest = []
 --     }
@@ -937,8 +913,7 @@ addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
 --       ],
 --       order = [], range_ = fullRange, relName = "clients", relToParent = Nothing, relJoinConds = [], relAlias = Nothing, relAggAlias = "", relHint = Nothing,
 --       relJoinType = Nothing, relSpread = Nothing, depth = 0,
---       relSelect = [],
---       relIsLegacyTargetNameMatch = False
+--       relSelect = []
 --     },
 --     subForest = subForst
 --   }
@@ -974,8 +949,8 @@ addNullEmbedFilters (Node rp@ReadPlan{where_=curLogic} forest) = do
       flt@(CoercibleStmnt _) ->
         Right flt
 
-addRanges :: ApiRequest -> Bool -> ReadPlanTree -> Either Error ReadPlanTree
-addRanges ApiRequest{..} useTargetNames rReq =
+addRanges :: ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
+addRanges ApiRequest{..} rReq =
   case iAction of
     ActDb (ActRelationMut _ _) -> Right rReq
     _                          -> foldr addRangeToNode (Right rReq) =<< ranges
@@ -984,10 +959,10 @@ addRanges ApiRequest{..} useTargetNames rReq =
     ranges = first (ApiRequestErr . QueryParamError) $ QueryParams.pRequestRange `traverse` HM.toList iRange
 
     addRangeToNode :: (EmbedPath, NonnegRange) -> Either Error ReadPlanTree -> Either Error ReadPlanTree
-    addRangeToNode = updateNode useTargetNames (\r (Node q f) -> Node q{range_=r} f)
+    addRangeToNode = updateNode (\r (Node q f) -> Node q{range_=r} f)
 
-addLogicTrees :: ResolverContext -> ApiRequest -> Bool -> ReadPlanTree -> Either Error ReadPlanTree
-addLogicTrees ctx ApiRequest{..} useTargetNames rReq =
+addLogicTrees :: ResolverContext -> ApiRequest -> ReadPlanTree -> Either Error ReadPlanTree
+addLogicTrees ctx ApiRequest{..} rReq =
   foldr addLogicTreeToNode (Right rReq) logic
   where
     QueryParams.QueryParams{..} = iQueryParams
@@ -1000,7 +975,7 @@ addLogicTrees ctx ApiRequest{..} useTargetNames rReq =
         _                            -> filter (not . null . fst) qsLogic
 
     addLogicTreeToNode :: (EmbedPath, LogicTree) -> Either Error ReadPlanTree -> Either Error ReadPlanTree
-    addLogicTreeToNode = updateNode useTargetNames (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)
+    addLogicTreeToNode = updateNode (\t (Node q@ReadPlan{from=fromTable, where_=lf} f) -> Node q{ReadPlan.where_=resolveLogicTree ctx{qi=fromTable} t:lf} f)
 
 resolveLogicTree :: ResolverContext -> LogicTree -> CoercibleLogicTree
 resolveLogicTree ctx (Stmnt flt) = CoercibleStmnt $ resolveFilter ctx flt
@@ -1010,35 +985,18 @@ resolveFilter :: ResolverContext -> Filter -> CoercibleFilter
 resolveFilter ctx (Filter fld opExpr) = CoercibleFilter{field=resolveQueryInputField ctx fld opExpr, opExpr=opExpr}
 
 -- Find a Node of the Tree and apply a function to it
-updateNode :: Bool -> (a -> ReadPlanTree -> ReadPlanTree) -> (EmbedPath, a) -> Either Error ReadPlanTree -> Either Error ReadPlanTree
-updateNode _ f ([], a) rr = f a <$> rr
-updateNode _ _ _ (Left e) = Left e
-updateNode useTargetNames f (targetNodeName:remainingPath, a) (Right (Node rootNode forest)) =
-  case findNode useTargetNames of
-    Nothing ->
-      Left $ ApiRequestErr $ NotEmbedded targetNodeName findLegacyUsage
+updateNode :: (a -> ReadPlanTree -> ReadPlanTree) -> (EmbedPath, a) -> Either Error ReadPlanTree -> Either Error ReadPlanTree
+updateNode f ([], a) rr = f a <$> rr
+updateNode _ _ (Left e) = Left e
+updateNode f (targetNodeName:remainingPath, a) (Right (Node rootNode forest)) =
+  case findNode of
+    Nothing -> Left $ ApiRequestErr $ NotEmbedded targetNodeName
     Just target ->
       (\node -> Node rootNode $ node : delete target forest) <$>
-      updateNode useTargetNames f (remainingPath, a) (Right $ updateLegacyAttrs target)
+      updateNode f (remainingPath, a) (Right target)
   where
-    findNode :: Bool -> Maybe ReadPlanTree
-    findNode isLegacy = find (matchTarget isLegacy) forest
-
-    matchTarget :: Bool -> ReadPlanTree -> Bool
-    matchTarget isLegacy (Node ReadPlan{relName, relAlias} _)
-      | isLegacy = relName == targetNodeName || relAlias == Just targetNodeName
-      | otherwise = fromMaybe relName relAlias == targetNodeName
-
-    updateLegacyAttrs :: ReadPlanTree -> ReadPlanTree
-    updateLegacyAttrs node@(Node rPlan@ReadPlan{relName, relAlias} children)
-      | relName == targetNodeName && isJust relAlias && relAlias /= Just targetNodeName =
-          Node rPlan{relIsLegacyTargetNameMatch=True} children
-      | otherwise = node
-
-    findLegacyUsage :: Maybe (Text, Text)
-    findLegacyUsage
-      | useTargetNames = Nothing
-      | otherwise = (\(Node rp _) -> fromMaybe mempty $ readPlanWarning rp) . updateLegacyAttrs <$> findNode True
+    findNode :: Maybe ReadPlanTree
+    findNode = find (\(Node ReadPlan{relName, relAlias} _) -> fromMaybe relName relAlias == targetNodeName) forest
 
 mutatePlan :: Mutation -> QualifiedIdentifier -> ApiRequest -> SchemaCache -> ReadPlanTree -> Either Error MutatePlan
 mutatePlan mutation qi ApiRequest{iPreferences=Preferences{..}, ..} SchemaCache{dbTables, dbRepresentations} readReq =

src/PostgREST/Plan/MutatePlan.hs

@@ -6,8 +6,10 @@ where
 import qualified Data.ByteString.Lazy as LBS
 
 import PostgREST.ApiRequest.Preferences  (PreferResolution)
-import PostgREST.Plan.Types              (CoercibleField, CoercibleLogicTree)
-import PostgREST.SchemaCache.Identifiers (FieldName, QualifiedIdentifier)
+import PostgREST.Plan.Types              (CoercibleField,
+                                          CoercibleLogicTree)
+import PostgREST.SchemaCache.Identifiers (FieldName,
+                                          QualifiedIdentifier)
 
 
 import Protolude

src/PostgREST/Plan/Negotiate.hs

@@ -18,7 +18,8 @@ import PostgREST.Error                   (ApiRequestError (..))
 import PostgREST.MediaType               (MediaType (..))
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
                                           RelIdentifier (..))
-import PostgREST.SchemaCache.Routine     (MediaHandler (..), MediaHandlerMap,
+import PostgREST.SchemaCache.Routine     (MediaHandler (..),
+                                          MediaHandlerMap,
                                           ResolvedHandler)
 
 import           PostgREST.ApiRequest.Preferences

src/PostgREST/Plan/ReadPlan.hs

@@ -0,0 +1,53 @@
+module PostgREST.Plan.ReadPlan
+  ( ReadPlanTree
+  , ReadPlan(..)
+  , JoinCondition(..)
+  , SpreadType(..)
+  ) where
+
+import Data.Tree (Tree (..))
+
+import PostgREST.ApiRequest.Types         (Alias, Depth, Hint,
+                                           JoinType, NodeName)
+import PostgREST.Plan.Types               (CoercibleLogicTree,
+                                           CoercibleOrderTerm,
+                                           CoercibleSelectField (..),
+                                           RelSelectField (..),
+                                           SpreadType (..))
+import PostgREST.RangeQuery               (NonnegRange)
+import PostgREST.SchemaCache.Identifiers  (FieldName,
+                                           QualifiedIdentifier)
+import PostgREST.SchemaCache.Relationship (Relationship)
+
+
+import Protolude
+
+type ReadPlanTree = Tree ReadPlan
+
+data JoinCondition =
+  JoinCondition
+    (QualifiedIdentifier, FieldName)
+    (QualifiedIdentifier, FieldName)
+  deriving (Eq, Show)
+
+-- TODO: Enforce uniqueness of columns by changing to a Set instead of a List where applicable
+data ReadPlan = ReadPlan
+  { select       :: [CoercibleSelectField]
+  , from         :: QualifiedIdentifier
+  , fromAlias    :: Maybe Alias
+  , where_       :: [CoercibleLogicTree]
+  , order        :: [CoercibleOrderTerm]
+  , range_       :: NonnegRange
+  , relName      :: NodeName
+  , relToParent  :: Maybe Relationship
+  , relJoinConds :: [JoinCondition]
+  , relAlias     :: Maybe Alias
+  , relAggAlias  :: Alias
+  , relHint      :: Maybe Hint
+  , relJoinType  :: Maybe JoinType
+  , relSpread    :: Maybe SpreadType
+  , relSelect    :: [RelSelectField]
+  , depth        :: Depth
+  -- ^ used for aliasing
+  }
+  deriving (Eq, Show)