bump version to 12.0.1

fix: any handler sets a default application/json
Now it sets application/octet-stream as the generic type.
2023-12-12 19:20:47 -05:00 · 2023-12-12 18:46:40 -05:00 · 2023-12-12 18:46:40 -05:00 · 2023-12-12 18:46:40 -05:00 · 2023-12-12 18:46:40 -05:00 · 2023-12-12 18:46:40 -05:00
25 changed files with 203 additions and 109 deletions
@@ -20,11 +20,10 @@ runs:
    - name: Restore and cache Nix store
      uses: nix-community/cache-nix-action@v4.0.3
      with:
-        key: cache-nix-${{ runner.os }}-id-${{ inputs.cache-id }}-${{ hashFiles('nix/**/*.nix') }}
+        key: cache-nix-${{ runner.os }}-id-${{ inputs.cache-id }}-${{ hashFiles('nix/**/*.nix', '.github/actions/setup-nix/*') }}
        restore-keys: |
          cache-nix-${{ runner.os }}-common-
-        restore-key-hit: true
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
      with:
        name: postgrest
        authToken: ${{ inputs.authToken }}
@@ -11,18 +11,39 @@ on:
    branches:
      - main
      - rel-*
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  
+  # Terminate all previous runs of the same workflow and branch/tag, except for main and release branches/tags
+  cancel-in-progress: "${{ !(github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/rel-')) }}"

 jobs:
-  Lint-Style:
-    name: Lint & check code style
+  Prepopulate-Nix-Cache-Linux:
+    name: Prepopulate Nix cache for Linux runners
    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+        with:
+          cache-id: common
+      - name: Put all tools to store to be cached afterwards
+        run: |
+          # shellcheck disable=SC2046
+          nix-store -v --realize $( nix-instantiate default.nix )
+        shell: bash
+    
+  Lint-Style:
+    name: Lint & check code style
+    runs-on: ubuntu-latest
+    needs: [Prepopulate-Nix-Cache-Linux]
    steps:
      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: style
-          cache-id: style
+          cache-id: common
      - name: Run linter (check locally with `nix-shell --run postgrest-lint`)
        run: postgrest-lint
      - name: Run style check (auto-format with `nix-shell --run postgrest-style`)
@@ -32,6 +53,7 @@ jobs:
  Test-Nix:
    name: Test (Nix)
    runs-on: ubuntu-latest
+    needs: [Prepopulate-Nix-Cache-Linux]
    defaults:
      run:
        # Hack for enabling color output, see:
@@ -43,7 +65,7 @@ jobs:
        uses: ./.github/actions/setup-nix
        with:
          tools: tests
-          cache-id: test-pg
+          cache-id: common

      - name: Run coverage (IO tests and Spec tests against PostgreSQL 15)
        run: postgrest-coverage
@@ -68,6 +90,7 @@ jobs:
        pgVersion: [9.6, 10, 11, 12, 13, 14, 15, 16]
    name: Test PG ${{ matrix.pgVersion }} (Nix)
    runs-on: ubuntu-latest
+    needs: [Prepopulate-Nix-Cache-Linux]
    defaults:
      run:
        # Hack for enabling color output, see:
@@ -81,7 +104,7 @@ jobs:
          tools: tests withTools
          # It seems like they are installing the same set of derivations, so we can assign them the same cache id. 
          # This would decrease the amount of caches dowloaded on merge cache step and will prevent disk space issues.
-          cache-id: test-pg 
+          cache-id: common 

      - name: Run spec tests
        if: always()
@@ -95,13 +118,14 @@ jobs:
  Test-Memory-Nix:
    name: Test memory (Nix)
    runs-on: ubuntu-latest
+    needs: [Prepopulate-Nix-Cache-Linux]
    steps:
      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: memory
-          cache-id: test-memory
+          cache-id: common
      - name: Run memory tests
        run: postgrest-test-memory

@@ -109,13 +133,14 @@ jobs:
  Build-Static-Nix:
    name: Build Linux static (Nix)
    runs-on: ubuntu-latest
+    needs: [Prepopulate-Nix-Cache-Linux]
    steps:
      - uses: actions/checkout@v4
      - name: Setup Nix Environment
        uses: ./.github/actions/setup-nix
        with:
          tools: tests
-          cache-id: static-nix
+          cache-id: common

      - name: Build static executable
        run: nix-build -A postgrestStatic
@@ -137,33 +162,6 @@ jobs:
          path: postgrest-docker.tar.gz
          if-no-files-found: error

-  # TODO: Enable this again in a PR by PostgREST admins, because regular users don't have permission to delete cache entries, which this job does.
-  #
-  # merge-nix-caches-linux:
-  #   name: "Merge Nix caches (Linux)"
-  #   needs: [Test-Nix, Test-Pg-Nix, Test-Memory-Nix, Build-Static-Nix, Lint-Style]
-  #   runs-on: ubuntu-latest
-  #   strategy:
-  #     max-parallel: 1
-  #     matrix:
-  #       cache-id: ['static-nix', 'test-pg', 'style', 'test-memory']
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: nixbuild/nix-quick-install-action@v26
-  #       with:
-  #         nix_version: '2.13.6'
-  #     - name: Restore and cache Nix store
-  #       uses: nix-community/cache-nix-action@v4
-  #       with:
-  #         key: cache-nix-${{ runner.os }}-common-${{ hashFiles('nix/**/*.nix') }}
-  #         extra-restore-keys: |
-  #           cache-nix-${{ runner.os }}-cid-
-  #         purge: true
-  #         purge-keys: |
-  #           cache-nix-${{ runner.os }}-cid-
-  #           cache-nix-${{ runner.os }}-common-
-  #         purge-created-max-age: 0
-
  Build-Macos-Nix:
    name: Build MacOS (Nix)
    runs-on: macos-latest
@@ -15,6 +15,9 @@ jobs:
    name: Loadtest PR (Nix)
    if: ${{ github.event_name == 'pull_request' }}
    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v4
        with:
@@ -23,7 +26,7 @@ jobs:
        uses: ./.github/actions/setup-nix
        with:
          tools: loadtest
-          cache-id: test-pg
+          cache-id: test-loadtest
      - uses: actions-ecosystem/action-get-latest-tag@v1
        id: get-latest-tag
        with:
@@ -55,7 +58,7 @@ jobs:
        uses: ./.github/actions/setup-nix
        with:
          tools: loadtest
-          cache-id: test-pg
+          cache-id: test-loadtest
      - name: Run loadtest
        run: |
          postgrest-loadtest-against ${{ steps.get-latest-tag.outputs.tag }}
@@ -17,20 +17,26 @@ PostgREST ongoing development is only possible thanks to our Sponsors and Backer
          <img width="296px" src="static/gnuhost.png">
        </a>
      </td>
-    </tr>
-    <tr></tr>
-    <tr>
-      <td align="center" valign="middle">
-        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
-        </a>
-      </td>
      <td align="center" valign="middle">
        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
          <img width="296px" src="static/neon.jpg">
        </a>
      </td>
    </tr>
+    <tr></tr>
+    <tr>
+      <td align="center" valign="middle">
+        <a href="https://code.build/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/code-build.png">
+        </a>
+      </td>
+      <td align="center" valign="middle">
+        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
+        </a>
+      </td>
+      <td></td>
+    </tr>
  </tbody>
 </table>

@@ -5,6 +5,17 @@ This project adheres to [Semantic Versioning](http://semver.org/).

 ## Unreleased

+## [12.0.1] - 2023-12-12
+
+### Fixed
+
+ - #3054, Fix not allowing special characters in JSON keys - @laurenceisla
+ - #2344, Replace JSON parser error with a clearer generic message - @develop7
+ - #3100, Add missing in-database configuration option for `jwt-cache-max-lifetime` - @laurenceisla
+ - #3089, The any media type handler now sets `Content-Type: application/octet-stream` by default instead of `Content-Type: application/json` - @steve-chavez
+
+## [12.0.0] - 2023-12-01
+
 ### Added

 - #1614, Add `db-pool-automatic-recovery` configuration to disable connection retrying - @taimoorzaeem
@@ -28,20 +28,26 @@ API than you are likely to write from scratch.
          <img width="296px" src="static/gnuhost.png">
        </a>
      </td>
-    </tr>
-    <tr></tr>
-    <tr>
-      <td align="center" valign="middle">
-        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
-        </a>
-      </td>
      <td align="center" valign="middle">
        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
          <img width="296px" src="static/neon.jpg">
        </a>
      </td>
    </tr>
+    <tr></tr>
+    <tr>
+      <td align="center" valign="middle">
+        <a href="https://code.build/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="static/code-build.png">
+        </a>
+      </td>
+      <td align="center" valign="middle">
+        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
+        </a>
+      </td>
+      <td></td>
+    </tr>
  </tbody>
 </table>

@@ -25,20 +25,26 @@ write from scratch.
          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/gnuhost.png">
        </a>
      </td>
-    </tr>
-    <tr></tr>
-    <tr>
-      <td align="center" valign="middle">
-        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
-          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
-        </a>
-      </td>
      <td align="center" valign="middle">
        <a href="https://neon.tech/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/neon.jpg">
        </a>
      </td>
    </tr>
+    <tr></tr>
+    <tr>
+      <td align="center" valign="middle">
+        <a href="https://code.build/?utm_source=sponsor&utm_campaign=postgrest" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/code-build.png">
+        </a>
+      </td>
+      <td align="center" valign="middle">
+        <a href="https://supabase.io?utm_source=postgrest%20backers&utm_medium=open%20source%20partner&utm_campaign=postgrest%20backers%20github&utm_term=homepage" target="_blank">
+          <img width="296px" src="https://raw.githubusercontent.com/PostgREST/postgrest/main/static/supabase.png">
+        </a>
+      </td>
+      <td></td>
+    </tr>
  </tbody>
 </table>

@@ -1,5 +1,5 @@
 name:               postgrest
-version:            11.2.0
+version:            12.0.1
 synopsis:           REST API for any Postgres database
 description:        Reads the schema of a PostgreSQL database and creates RESTful routes
                    for tables, views, and functions, supporting all HTTP methods that security
@@ -259,7 +259,9 @@ getPayload reqBody contentMediaType QueryParams{qsColumns} action PathInfo{pathI
          else note "All object keys must match" . payloadAttributes reqBody
                 =<< if LBS.null reqBody && pathIsProc
                       then Right emptyObject
-                       else first BS.pack $ JSON.eitherDecode reqBody
+                       else first BS.pack $
+                          -- Drop parsing error message in favor of generic one (https://github.com/PostgREST/postgrest/issues/2344)
+                          maybe (Left "Empty or invalid json") Right $ JSON.decode reqBody
      (MTTextCSV, _) -> do
        json <- csvToJson <$> first BS.pack (CSV.decodeByName reqBody)
        note "All lines must have same number of fields" $ payloadAttributes (JSON.encode json) json
@@ -350,8 +350,12 @@ pFieldForest = pFieldTree `sepBy` lexeme (char ',')
 pFieldName :: Parser Text
 pFieldName =
  pQuotedValue <|>
-  T.intercalate "-" . map toS <$> (pIdentifier `sepBy1` dash) <?>
+  sepByDash pIdentifier <?>
  "field name (* or [a..z0..9_$])"
+
+sepByDash :: Parser Text -> Parser Text
+sepByDash fieldIdent =
+  T.intercalate "-" . map toS <$> (fieldIdent `sepBy1` dash)
  where
    isDash :: GenParser Char st ()
    isDash = try ( char '-' >> notFollowedBy (char '>') )
@@ -364,12 +368,18 @@ pFieldName =
 -- >>> P.parse pJsonPath "" "->text"
 -- Right [JArrow {jOp = JKey {jVal = "text"}}]
 --
+-- >>> P.parse pJsonPath "" "->!@#$%^&*_a"
+-- Right [JArrow {jOp = JKey {jVal = "!@#$%^&*_a"}}]
+--
 -- >>> P.parse pJsonPath "" "->1"
 -- Right [JArrow {jOp = JIdx {jVal = "+1"}}]
 --
 -- >>> P.parse pJsonPath "" "->>text"
 -- Right [J2Arrow {jOp = JKey {jVal = "text"}}]
 --
+-- >>> P.parse pJsonPath "" "->>!@#$%^&*_a"
+-- Right [J2Arrow {jOp = JKey {jVal = "!@#$%^&*_a"}}]
+--
 -- >>> P.parse pJsonPath "" "->>1"
 -- Right [J2Arrow {jOp = JIdx {jVal = "+1"}}]
 --
@@ -406,7 +416,7 @@ pJsonPath = many pJsonOperation
      try (string "->" $> JArrow)

    pJsonOperand =
-      let pJKey = JKey . toS <$> pFieldName
+      let pJKey = JKey . toS <$> pJsonKeyName
          pJIdx = JIdx . toS <$> ((:) <$> P.option '+' (char '-') <*> many1 digit) <* pEnd
          pEnd = try (void $ lookAhead (string "->")) <|>
                 try (void $ lookAhead (string "::")) <|>
@@ -415,6 +425,15 @@ pJsonPath = many pJsonOperation
                 try eof in
      try pJIdx <|> try pJKey

+pJsonKeyName :: Parser Text
+pJsonKeyName =
+  pQuotedValue <|>
+  sepByDash pJsonKeyIdentifier <?>
+  "any non reserved character different from: .,>()"
+
+pJsonKeyIdentifier :: Parser Text
+pJsonKeyIdentifier = T.strip . toS <$> many1 (noneOf "(-:.,>)")
+
 pField :: Parser Field
 pField = lexeme $ (,) <$> pFieldName <*> P.option [] pJsonPath

@@ -477,6 +496,9 @@ pRelationSelect = lexeme $ do
 -- >>> P.parse pFieldSelect "" "alias:name->jsonpath::cast"
 -- Right (SelectField {selField = ("name",[JArrow {jOp = JKey {jVal = "jsonpath"}}]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Just "cast", selAlias = Just "alias"})
 --
+-- >>> P.parse pFieldSelect "" "alias:name->!@#$%^&*_a::cast"
+-- Right (SelectField {selField = ("name",[JArrow {jOp = JKey {jVal = "!@#$%^&*_a"}}]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Just "cast", selAlias = Just "alias"})
+--
 -- >>> P.parse pFieldSelect "" "*"
 -- Right (SelectField {selField = ("*",[]), selAggregateFunction = Nothing, selAggregateCast = Nothing, selCast = Nothing, selAlias = Nothing})
 --
@@ -678,9 +700,15 @@ pDelimiter = char '.' <?> "delimiter (.)"
 -- >>> P.parse pOrder "" "json_col->key.asc.nullslast"
 -- Right [OrderTerm {otTerm = ("json_col",[JArrow {jOp = JKey {jVal = "key"}}]), otDirection = Just OrderAsc, otNullOrder = Just OrderNullsLast}]
 --
+-- >>> P.parse pOrder "" "json_col->!@#$%^&*_a.asc.nullslast"
+-- Right [OrderTerm {otTerm = ("json_col",[JArrow {jOp = JKey {jVal = "!@#$%^&*_a"}}]), otDirection = Just OrderAsc, otNullOrder = Just OrderNullsLast}]
+--
 -- >>> P.parse pOrder "" "clients(json_col->key).desc.nullsfirst"
 -- Right [OrderRelationTerm {otRelation = "clients", otRelTerm = ("json_col",[JArrow {jOp = JKey {jVal = "key"}}]), otDirection = Just OrderDesc, otNullOrder = Just OrderNullsFirst}]
 --
+-- >>> P.parse pOrder "" "clients(json_col->!@#$%^&*_a).desc.nullsfirst"
+-- Right [OrderRelationTerm {otRelation = "clients", otRelTerm = ("json_col",[JArrow {jOp = JKey {jVal = "!@#$%^&*_a"}}]), otDirection = Just OrderDesc, otNullOrder = Just OrderNullsFirst}]
+--
 -- >>> P.parse pOrder "" "clients(name,id)"
 -- Left (line 1, column 8):
 -- unexpected '('
@@ -60,6 +60,7 @@ dbSettingsNames =
  ,"jwt_role_claim_key"
  ,"jwt_secret"
  ,"jwt_secret_is_base64"
+  ,"jwt_cache_max_lifetime"
  ,"openapi_mode"
  ,"openapi_security_active"
  ,"openapi_server_proxy_uri"
@@ -63,6 +63,7 @@ import PostgREST.SchemaCache.Representations (DataRepresentation (..),
                                              RepresentationsMap)
 import PostgREST.SchemaCache.Routine         (MediaHandler (..),
                                              MediaHandlerMap,
+                                              ResolvedHandler,
                                              Routine (..),
                                              RoutineMap,
                                              RoutineParam (..),
@@ -992,9 +993,9 @@ addFilterToLogicForest :: CoercibleFilter -> [CoercibleLogicTree] -> [CoercibleL
 addFilterToLogicForest flt lf = CoercibleStmnt flt : lf

 -- | Do content negotiation. i.e. choose a media type based on the intersection of accepted/produced media types.
-negotiateContent :: AppConfig -> ApiRequest -> QualifiedIdentifier -> [MediaType] -> MediaHandlerMap -> Either ApiRequestError (MediaHandler, MediaType)
+negotiateContent :: AppConfig -> ApiRequest -> QualifiedIdentifier -> [MediaType] -> MediaHandlerMap -> Either ApiRequestError ResolvedHandler
 negotiateContent conf ApiRequest{iAction=act, iPreferences=Preferences{preferRepresentation=rep}} identifier accepts produces =
-  defaultMTAnyToMTJSON $ case (act, firstAcceptedPick) of
+  case (act, firstAcceptedPick) of
    (_, Nothing)                         -> Left . MediaTypeError $ map MediaType.toMime accepts
    (ActionMutate _, Just (x, mt))       -> Right (if rep == Just Full then x else NoAgg, mt)
    -- no need for an aggregate on HEAD https://github.com/PostgREST/postgrest/issues/2849
@@ -1003,11 +1004,6 @@ negotiateContent conf ApiRequest{iAction=act, iPreferences=Preferences{preferRep
    (ActionInvoke InvHead, Just (_, mt)) -> Right (NoAgg, mt)
    (_, Just (x, mt))                    -> Right (x, mt)
  where
-    -- the initial handler in the schema cache has a */* to BuiltinAggJson but it doesn't preserve the media type (application/json)
-    -- we just convert the default */* to application/json here
-    -- TODO resolving to "application/json" for "*/*" is not correct when using a "*/*" custom handler media type.
-    -- We should return "application/octet-stream" as the generic type instead.
-    defaultMTAnyToMTJSON = mapRight (\(x, y) -> (x, if y == MTAny then MTApplicationJSON else y))
    firstAcceptedPick = listToMaybe $ mapMaybe matchMT accepts -- If there are multiple accepted media types, pick the first. This is usual in content negotiation.
    matchMT mt = case mt of
      -- all the vendored media types have special handling as they have media type parameters, they cannot be overridden
@@ -1015,11 +1011,12 @@ negotiateContent conf ApiRequest{iAction=act, iPreferences=Preferences{preferRep
      m@MTVndArrayJSONStrip                       -> Just (BuiltinAggArrayJsonStrip, m)
      m@(MTVndPlan (MTVndSingularJSON strip) _ _) -> mtPlanToNothing $ Just (BuiltinAggSingleJson strip, m)
      m@(MTVndPlan MTVndArrayJSONStrip _ _)       -> mtPlanToNothing $ Just (BuiltinAggArrayJsonStrip, m)
+      -- TODO the plan should have its own MediaHandler instead of relying on MediaType
+      m@(MTVndPlan mType _ _)                     -> mtPlanToNothing $ (,) <$> (fst <$> lookupHandler mType) <*> pure m
      -- all the other media types can be overridden
-      m@(MTVndPlan mType _ _)                     -> mtPlanToNothing $ (,) <$> lookupHandler mType <*> pure m
-      x                                           -> (,) <$> lookupHandler x <*> pure x
+      x                                           -> lookupHandler x
    mtPlanToNothing x = if configDbPlanEnabled conf then x else Nothing -- don't find anything if the plan media type is not allowed
    lookupHandler mt =
-      HM.lookup (RelId identifier, MTAny) produces <|> -- lookup handler that applies to `*/*` and identifier
-      HM.lookup (RelId identifier, mt) produces <|>    -- lookup handler that applies to a particular media type and identifier
-      HM.lookup (RelAnyElement, mt) produces           -- lookup handler that applies to a particular media type and anyelement
+      HM.lookup (RelId identifier, MTAny) produces <|> -- lookup for identifier and `*/*`
+      HM.lookup (RelId identifier, mt) produces <|>    -- lookup for identifier and a particular media type
+      HM.lookup (RelAnyElement, mt) produces           -- lookup for anyelement and a particular media type
@@ -1115,10 +1115,10 @@ allViewsKeyDependencies =

 initialMediaHandlers :: MediaHandlerMap
 initialMediaHandlers =
-  HM.insert (RelAnyElement, MediaType.MTAny            ) BuiltinOvAggJson $
-  HM.insert (RelAnyElement, MediaType.MTApplicationJSON) BuiltinOvAggJson $
-  HM.insert (RelAnyElement, MediaType.MTTextCSV        ) BuiltinOvAggCsv $
-  HM.insert (RelAnyElement, MediaType.MTGeoJSON        ) BuiltinOvAggGeoJson
+  HM.insert (RelAnyElement, MediaType.MTAny            ) (BuiltinOvAggJson,    MediaType.MTApplicationJSON) $
+  HM.insert (RelAnyElement, MediaType.MTApplicationJSON) (BuiltinOvAggJson,    MediaType.MTApplicationJSON) $
+  HM.insert (RelAnyElement, MediaType.MTTextCSV        ) (BuiltinOvAggCsv,     MediaType.MTTextCSV) $
+  HM.insert (RelAnyElement, MediaType.MTGeoJSON        ) (BuiltinOvAggGeoJson, MediaType.MTGeoJSON)
  HM.empty

 mediaHandlers :: PgVersion -> Bool -> SQL.Statement [Schema] MediaHandlerMap
@@ -1142,7 +1142,11 @@ mediaHandlers pgVer =
            lower(t.typname) as typname,
            b.oid as base_oid,
            b.typname AS basetypname,
-            t.typnamespace
+            t.typnamespace,
+            case t.typname
+              when '*/*' then 'application/octet-stream'
+              else t.typname
+            end as resolved_media_type
          FROM pg_type t
          JOIN pg_type b ON t.typbasetype = b.oid
          WHERE
@@ -1154,7 +1158,8 @@ mediaHandlers pgVer =
        proc.proname                  as handler_name,
        arg_schema.nspname::text      as target_schema,
        arg_name.typname::text        as target_name,
-        media_types.typname           as media_type
+        media_types.typname           as media_type,
+        media_types.resolved_media_type
      from media_types
        join pg_proc      proc         on proc.prorettype = media_types.oid
        join pg_namespace proc_schema  on proc_schema.oid = proc.pronamespace
@@ -1162,7 +1167,7 @@ mediaHandlers pgVer =
        join pg_type      arg_name     on arg_name.oid = proc.proargtypes[0]
        join pg_namespace arg_schema   on arg_schema.oid = arg_name.typnamespace
      where
-        proc_schema.nspname = ANY($1) and
+        proc_schema.nspname = ANY('{test}') and
        proc.pronargs = 1 and
        arg_name.oid in (select reltype from all_relations)
      union
@@ -1171,7 +1176,8 @@ mediaHandlers pgVer =
          mtype.typname   as handler_name,
          pro_sch.nspname as target_schema,
          proname         as target_name,
-          mtype.typname   as media_type
+          mtype.typname   as media_type,
+          mtype.resolved_media_type
      from pg_proc proc
        join pg_namespace pro_sch on pro_sch.oid = proc.pronamespace
        join media_types mtype on proc.prorettype = mtype.oid
@@ -1181,12 +1187,13 @@ mediaHandlers pgVer =

 decodeMediaHandlers :: HD.Result MediaHandlerMap
 decodeMediaHandlers =
-  HM.fromList . fmap (\(x, y, z) -> ((if isAnyElement y then RelAnyElement else RelId y, z), CustomFunc x) ) <$> HD.rowList caggRow
+  HM.fromList . fmap (\(x, y, z, w) -> ((if isAnyElement y then RelAnyElement else RelId y, z), (CustomFunc x, w)) ) <$> HD.rowList caggRow
  where
-    caggRow = (,,)
+    caggRow = (,,,)
              <$> (QualifiedIdentifier <$> column HD.text <*> column HD.text)
              <*> (QualifiedIdentifier <$> column HD.text <*> column HD.text)
              <*> (MediaType.decodeMediaType . encodeUtf8 <$> column HD.text)
+              <*> (MediaType.decodeMediaType . encodeUtf8 <$> column HD.text)

 timezones :: Bool -> SQL.Statement () TimezoneNames
 timezones = SQL.Statement sql HE.noParams decodeTimezones
@@ -16,6 +16,7 @@ module PostgREST.SchemaCache.Routine
  , funcReturnsCompositeAlias
  , funcReturnsSingle
  , MediaHandlerMap
+  , ResolvedHandler
  , MediaHandler(..)
  ) where

@@ -145,4 +146,6 @@ funcTableName proc = case pdReturnType proc of
  Single (Composite qi _) -> Just $ qiName qi
  _                       -> Nothing

-type MediaHandlerMap = HM.HashMap (RelIdentifier, MediaType.MediaType) MediaHandler
+-- the resolved handler also carries the media type because MTAny (*/*) is resolved to a different media type
+type ResolvedHandler = (MediaHandler, MediaType.MediaType)
+type MediaHandlerMap = HM.HashMap (RelIdentifier, MediaType.MediaType) ResolvedHandler
@@ -22,7 +22,7 @@ jwt-aud = "https://otherexample.org"
 jwt-role-claim-key = ".\"other\".\"pre_config_role\""
 jwt-secret = "ODERREALLYREALLYREALLYREALLYVERYSAFE"
 jwt-secret-is-base64 = true
-jwt-cache-max-lifetime = 86400
+jwt-cache-max-lifetime = 7200
 log-level = "info"
 openapi-mode = "disabled"
 openapi-security-active = false
@@ -22,7 +22,7 @@ jwt-aud = "https://example.org"
 jwt-role-claim-key = ".\"a\".\"role\""
 jwt-secret = "OVERRIDE=REALLY=REALLY=REALLY=REALLY=VERY=SAFE"
 jwt-secret-is-base64 = false
-jwt-cache-max-lifetime = 86400
+jwt-cache-max-lifetime = 3600
 log-level = "info"
 openapi-mode = "ignore-privileges"
 openapi-security-active = true
@@ -6,6 +6,7 @@ ALTER ROLE db_config_authenticator SET pgrst.openapi_server_proxy_uri = 'https:/
 ALTER ROLE db_config_authenticator SET pgrst.jwt_secret = 'REALLY=REALLY=REALLY=REALLY=VERY=SAFE';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_secret_is_base64 = 'false';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_role_claim_key = '."a"."role"';
+ALTER ROLE db_config_authenticator SET pgrst.jwt_cache_max_lifetime = '3600';
 ALTER ROLE db_config_authenticator SET pgrst.db_aggregates_enabled = 'false';
 ALTER ROLE db_config_authenticator SET pgrst.db_anon_role = 'anonymous';
 ALTER ROLE db_config_authenticator SET pgrst.db_tx_end = 'commit-allow-override';
@@ -54,6 +55,7 @@ ALTER ROLE other_authenticator SET pgrst.jwt_aud = 'https://otherexample.org';
 ALTER ROLE other_authenticator SET pgrst.openapi_server_proxy_uri = 'https://otherexample.org/api';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret = 'ODERREALLYREALLYREALLYREALLYVERYSAFE';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret_is_base64 = 'true';
+ALTER ROLE other_authenticator SET pgrst.jwt_cache_max_lifetime = '7200';
 ALTER ROLE other_authenticator SET pgrst.db_aggregates_enabled = 'false';
 ALTER ROLE other_authenticator SET pgrst.db_schemas = 'test, other_tenant1, other_tenant2';
 ALTER ROLE other_authenticator SET pgrst.db_root_spec = 'other_root';
@@ -102,7 +102,7 @@ postJsonArrayTest(){

 echo "Running memory usage tests.."

-jsonKeyTest "1M" "POST" "/rpc/leak?columns=blob" "24M"
+jsonKeyTest "1M" "POST" "/rpc/leak?columns=blob" "26M"
 jsonKeyTest "1M" "POST" "/leak?columns=blob" "16M"
 jsonKeyTest "1M" "PATCH" "/leak?id=eq.1&columns=blob" "16M"

@@ -225,31 +225,30 @@ spec = describe "custom media types" $ do

  context "any media type" $ do
    context "on functions" $ do
-      -- TODO not correct, it should return the generic "application/octet-stream"
      it "returns application/json for */* if not explicitly set" $ do
        request methodGet "/rpc/ret_any_mt" (acceptHdrs "*/*") ""
          `shouldRespondWith` "any"
          { matchStatus  = 200
-          , matchHeaders = [matchContentTypeJson]
+          , matchHeaders = ["Content-Type" <:> "application/octet-stream"]
          }

-      it "accepts any media type and sets it as a header" $ do
+      it "accepts any media type and sets the generic octet-stream as content type" $ do
        request methodGet "/rpc/ret_any_mt" (acceptHdrs "app/bingo") ""
          `shouldRespondWith` "any"
          { matchStatus  = 200
-          , matchHeaders = ["Content-Type" <:> "app/bingo"]
+          , matchHeaders = ["Content-Type" <:> "application/octet-stream"]
          }

        request methodGet "/rpc/ret_any_mt" (acceptHdrs "text/bango") ""
          `shouldRespondWith` "any"
          { matchStatus  = 200
-          , matchHeaders = ["Content-Type" <:> "text/bango"]
+          , matchHeaders = ["Content-Type" <:> "application/octet-stream"]
          }

        request methodGet "/rpc/ret_any_mt" (acceptHdrs "image/boingo") ""
          `shouldRespondWith` "any"
          { matchStatus  = 200
-          , matchHeaders = ["Content-Type" <:> "image/boingo"]
+          , matchHeaders = ["Content-Type" <:> "application/octet-stream"]
          }

      it "returns custom media type for */* if explicitly set" $ do
@@ -276,12 +275,11 @@ spec = describe "custom media types" $ do
          `shouldRespondWith` 415

    context "on tables" $ do
-      -- TODO not correct, it should return the generic "application/octet-stream"
      it "returns application/json for */* if not explicitly set" $ do
        request methodGet "/some_numbers?val=eq.1" (acceptHdrs "*/*") ""
          `shouldRespondWith` "anything\n1"
          { matchStatus  = 200
-          , matchHeaders = [matchContentTypeJson]
+          , matchHeaders = ["Content-Type" <:> "application/octet-stream"]
          }

      it "accepts any media type and sets it as a header" $ do
@@ -298,5 +296,5 @@ spec = describe "custom media types" $ do
        request methodGet "/some_numbers?val=eq.4" (acceptHdrs "unknown/unknown") ""
          `shouldRespondWith` "anything\n4"
          { matchStatus  = 200
-          , matchHeaders = ["Content-Type" <:> "unknown/unknown"]
+          , matchHeaders = ["Content-Type" <:> "application/octet-stream"]
          }
@@ -287,7 +287,7 @@ spec actualPgVersion = do
      it "fails with 400 and error" $
        post "/simple_pk" "}{ x = 2"
        `shouldRespondWith`
-        [json|{"message":"Error in $: Failed reading: not a valid json value at '}{x=2'","code":"PGRST102","details":null,"hint":null}|]
+        [json|{"message":"Empty or invalid json","code":"PGRST102","details":null,"hint":null}|]
        { matchStatus  = 400
        , matchHeaders = [matchContentTypeJson]
        }
@@ -296,7 +296,7 @@ spec actualPgVersion = do
      it "fails with 400 and error" $
        post "/simple_pk" ""
        `shouldRespondWith`
-        [json|{"message":"Error in $: not enough input","code":"PGRST102","details":null,"hint":null}|]
+        [json|{"message":"Empty or invalid json","code":"PGRST102","details":null,"hint":null}|]
        { matchStatus  = 400
        , matchHeaders = [matchContentTypeJson]
        }
@@ -56,6 +56,21 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
        [json| [{"myInt":1}] |] -- the value in the db is an int, but here we expect a string for now
        { matchHeaders = [matchContentTypeJson] }

+    it "accepts non reserved special characters in the key's name" $
+      get "/json_arr?id=eq.10&select=data->!@#$%^%26*_d->>!@#$%^%26*_e::integer" `shouldRespondWith`
+        [json| [{"!@#$%^&*_e":3}] |]
+        { matchHeaders = [matchContentTypeJson] }
+
+    it "fails when there is a reserved special character in the key's name" $
+      get "/json_arr?id=eq.10&select=data->(!@#$%^%26*_d->>!@#$%^%26*_e::integer" `shouldRespondWith`
+        [json| {
+          "code":"PGRST100",
+          "details":"unexpected \"(\" expecting \"-\", digit or any non reserved character different from: .,>()",
+          "hint":null,
+          "message":"\"failed to parse select parameter (data->(!@#$%^&*_d->>!@#$%^&*_e::integer)\" (line 1, column 7)"}
+        |]
+        { matchStatus  = 400 , matchHeaders = [] }
+
    -- TODO the status code for the error is 404, this is because 42883 represents undefined function
    -- this works fine for /rpc/unexistent requests, but for this case a 500 seems more appropriate
    it "fails when a double arrow ->> is followed with a single arrow ->" $ do
@@ -178,6 +193,11 @@ spec actualPgVersion = describe "json and jsonb operators" $ do
      get "/grandchild_entities?or=(jsonb_col->a->>b.eq.foo, jsonb_col->>b.eq.bar)&select=id" `shouldRespondWith`
        [json|[{id: 4}, {id: 5}]|] { matchStatus = 200, matchHeaders = [matchContentTypeJson] }

+    it "can filter when the key's name has non reserved special characters" $
+      get "/json_arr?select=data->!@#$%^%26*_d&data->!@#$%^%26*_d->>!@#$%^%26*_e=eq.3" `shouldRespondWith`
+        [json| [{"!@#$%^&*_d": {"!@#$%^&*_e": 3}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+
    it "can filter by array indexes" $ do
      get "/json_arr?select=data&data->>0=eq.1" `shouldRespondWith`
        [json| [{"data":[1, 2, 3]}] |]
@@ -44,7 +44,7 @@ spec actualPgVersion = do
      it "fails with 400 and error" $
        request methodPatch "/simple_pk" [] "}{ x = 2"
          `shouldRespondWith`
-          [json|{"message":"Error in $: Failed reading: not a valid json value at '}{x=2'","code":"PGRST102","details":null,"hint":null}|]
+          [json|{"message":"Empty or invalid json","code":"PGRST102","details":null,"hint":null}|]
          { matchStatus  = 400,
            matchHeaders = [matchContentTypeJson]
          }
@@ -53,7 +53,7 @@ spec actualPgVersion = do
      it "fails with 400 and error" $
        request methodPatch "/items" [] ""
          `shouldRespondWith`
-          [json|{"message":"Error in $: not enough input","code":"PGRST102","details":null,"hint":null}|]
+          [json|{"message":"Empty or invalid json","code":"PGRST102","details":null,"hint":null}|]
          { matchStatus  = 400,
            matchHeaders = [matchContentTypeJson]
          }
@@ -512,6 +512,7 @@ INSERT INTO json_arr VALUES (6, '[{"a": [1,2,3]}, {"b": [4,5]}]');
 INSERT INTO json_arr VALUES (7, '{"c": [1,2,3], "d": [4,5]}');
 INSERT INTO json_arr VALUES (8, '{"c": [{"d": [4,5,6,7,8]}]}');
 INSERT INTO json_arr VALUES (9, '[{"0xy1": [1,{"23-xy-45": [2, {"xy-6": [3]}]}]}]');
+INSERT INTO json_arr VALUES (10, '{"!@#$%^&*_a": [{"!@#$%^&*_b": 1}, {"!@#$%^&*_c": [2]}], "!@#$%^&*_d": {"!@#$%^&*_e": 3}}');

 TRUNCATE TABLE jsonb_test CASCADE;
 INSERT INTO jsonb_test VALUES (1, '{ "a": {"b": 2} }');
@@ -3663,10 +3663,14 @@ declare
  resp bytea;
 begin
  case req_accept
-    when 'app/chico'   then resp := 'chico';
-    when 'app/harpo'   then resp := 'harpo';
+    when 'app/chico'   then
+      perform set_config('response.headers', json_build_array(json_build_object('Content-Type', req_accept))::text, true);
+      resp := 'chico';
+    when 'app/harpo'   then
+      perform set_config('response.headers', json_build_array(json_build_object('Content-Type', req_accept))::text, true);
+      resp := 'harpo';
    when '*/*'         then
-      perform set_config('response.headers', '[{"Content-Type": "app/groucho"}]', true);
+      perform set_config('response.headers', json_build_array(json_build_object('Content-Type', 'app/groucho'))::text, true);
      resp := 'groucho';
    else
      raise sqlstate 'PT415' using message = 'Unsupported Media Type';
@@ -3689,8 +3693,10 @@ declare
 begin
  case req_accept
    when 'magic/number' then
+      perform set_config('response.headers', json_build_array(json_build_object('Content-Type', req_accept))::text, true);
      prefix := 'magic';
    when 'crazy/bingo'  then
+      perform set_config('response.headers', json_build_array(json_build_object('Content-Type', req_accept))::text, true);
      prefix := 'crazy';
    else
      prefix := 'anything';
Author	SHA1	Message	Date
Laurence Isla	cd38da56d5	bump version to 12.0.1	2023-12-12 19:20:47 -05:00
steve-chavezandLaurence Isla	229a4e4cd6	fix: any handler sets a default application/json Now it sets application/octet-stream as the generic type.	2023-12-12 18:46:40 -05:00
Laurence Isla	c3f7440e33	fix: add `jwt_cache_max_lifetime` as an in-database configuration option (#3102 )	2023-12-12 18:46:40 -05:00
Laurence Isla	4e03ee252b	chore: add new sponsor	2023-12-12 18:46:40 -05:00
Andrei DziahelandLaurence Isla	0ac4d0d0a9	fix: replace json parser error with generic msg (#3090 )	2023-12-12 18:46:40 -05:00
Andrei DziahelandLaurence Isla	ceda77c5de	ci: cancel in-progress loadtests for PRs as well	2023-12-12 18:46:40 -05:00
Andrei DziahelandLaurence Isla	a6e4a0078b	ci: test only head of branch	2023-12-12 18:46:40 -05:00
Laurence Isla	15f4157cc5	changelog: move missplaced unreleased fix (#3094 )	2023-12-12 18:46:40 -05:00
Laurence Isla	50ea1999b2	fix: allow using special characters in json keys (#3081 ) * increase memory size test	2023-12-12 18:46:40 -05:00
Andrei DziahelandLaurence Isla	c92f16a2cb	ci: similar jobs use a single cache for Nix stores Jobs that use the setup-nix action will share a single cache, which is created by a previous warm up Job.	2023-12-12 18:46:40 -05:00
dependabot[bot]andLaurence Isla	1cebc03313	build(deps): bump cachix/cachix-action in /.github/actions/setup-nix (#3085 )	2023-12-12 18:46:40 -05:00
steve-chavez	00580bc8cb	bump version to 12.0.0	2023-12-01 19:18:07 -05:00