feat: data representations allow custom parsing and formatting of API fields.

See PR #2523. Most notable code changes: - Load data representation casts into schema cache. - Data representations for reads, filters, inserts, updates, views, over joins. - `CoercibleField` represents name references in queries where coercion may be needed. - `ResolverContext` help facilitate field resolution during planning. - Planner 'resolves' names in the API query and pairs them with any implicit conversions to be used in the query builder stage. - Tests for all of the above. - More consistent naming (TypedX -> CoercibleX). New: unit tests for more data representation use cases; helpful as examples as well. New: update CHANGELOG with data representations feature description. Fixed failing idempotence test. New: replace date formatter test with one that does something. Fixup: inadvertent CHANGELOG change after rebase. Cleanup: `tfName` -> `cfName` and related. Document what IRType means. Formatting. New: use a subquery to interpret `IN` literals requiring data rep transformation. - With the previous method, very long queries such as `ANY (ARRAY[test.color('000100'), test.color('CAFE12'), test.color('01E240'), ...` could be generated. Consider the case where the parser function name is 45 characters and there's a hundred literals. That's 4.5kB of SQL just for the function name alone! - New version uses `unnest`: `ANY (SELECT test.color(unnest('{000100,CAFE12,01E240,...}'::text[]))` to produce a much shorter query. - This is likely to be more performant and either way much more readable and debuggable in the logs.
2023-06-29 15:01:58 -05:00
parent 078c6ec08c
commit 0a1564ba5a
17 changed files with 1045 additions and 163 deletions
@@ -1328,3 +1328,122 @@ spec actualPgVersion = do
      get "/articles?body=imatch(any).{stop,thing}&select=id" `shouldRespondWith`
        [json|[{"id":1}, {"id":2}]|]
        { matchHeaders = [matchContentTypeJson] }
+
+  describe "Data representations for customisable value formatting and parsing" $ do
+    it "formats a single column" $
+      get "/datarep_todos?select=id,label_color&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"label_color":"#000000"},{"id":2,"label_color":"#000100"},{"id":3,"label_color":"#01E240"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats two columns with different formatters" $
+      get "/datarep_todos?select=id,label_color,due_at&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"},{"id":2,"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"},{"id":3,"label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "fails in some reasonable way when selecting fields that don't exist" $
+      get "/datarep_todos?select=id,label_color,banana" `shouldRespondWith`
+        [json| {"code":"42703","details":null,"hint":null,"message":"column datarep_todos.banana does not exist"} |]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+    it "formats columns in views including computed columns" $
+      get "/datarep_todos_computed?select=id,label_color,dark_color" `shouldRespondWith`
+        [json| [
+          {"id":1, "label_color":"#000000", "dark_color":"#000000"},
+          {"id":2, "label_color":"#000100", "dark_color":"#000080"},
+          {"id":3, "label_color":"#01E240", "dark_color":"#00F120"},
+          {"id":4, "label_color":"", "dark_color":""}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats and allows rename" $
+      get "/datarep_todos?select=id,clr:label_color&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"clr":"#000000"},{"id":2,"clr":"#000100"},{"id":3,"clr":"#01E240"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats, renames and allows manual casting on top" $
+      get "/datarep_todos?select=id,clr:label_color::text&id=lt.4" `shouldRespondWith`
+        [json| [{"id":1,"clr":"\"#000000\""},{"id":2,"clr":"\"#000100\""},{"id":3,"clr":"\"#01E240\""}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats nulls" $
+      -- due_at is formatted as NULL but label_color NULLs become empty strings-- it's up to the formatting function.
+      get "/datarep_todos?select=id,label_color,due_at&id=gt.2&id=lt.5" `shouldRespondWith`
+        [json| [{"id":3,"label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z"},{"id":4,"label_color":"","due_at":null}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats star select" $
+      get "/datarep_todos?select=*&id=lt.4" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+           {"id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+           {"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats implicit star select" $
+      get "/datarep_todos?id=lt.4" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+ {"id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+ {"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats star and explicit mix" $
+      get "/datarep_todos?select=due_at,*&id=lt.4" `shouldRespondWith`
+        [json| [
+          {"due_at":"2018-01-02T00:00:00Z","id":1,"name":"Report","label_color":"#000000","due_at":"2018-01-02T00:00:00Z","icon_image":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAAABBJREFUeJxiYAEAAAAA//8DAAAABgAFBXv6vUAAAAAASUVORK5CYII=","created_at":1513213350,"budget":"12.50"},
+           {"due_at":"2018-01-03T00:00:00Z","id":2,"name":"Essay","label_color":"#000100","due_at":"2018-01-03T00:00:00Z","icon_image":null,"created_at":1513213350,"budget":"100000000000000.13"},
+           {"due_at":"2018-01-01T14:12:34.123456Z","id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats through join" $
+      get "/datarep_next_two_todos?select=id,name,first_item:datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)" `shouldRespondWith`
+        [json| [{"id":1,"name":"school related","first_item":{"label_color":"#000100","due_at":"2018-01-03T00:00:00Z"}},{"id":2,"name":"do these first","first_item":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "formats through join with star select" $
+      get "/datarep_next_two_todos?select=id,name,second_item:datarep_todos!datarep_next_two_todos_second_item_id_fkey(*)" `shouldRespondWith`
+        [json| [
+          {"id":1,"name":"school related","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}},
+          {"id":2,"name":"do these first","second_item":{"id":3,"name":"Algebra","label_color":"#01E240","due_at":"2018-01-01T14:12:34.123456Z","icon_image":null,"created_at":1513213350,"budget":"0.00"}}
+        ] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "uses text parser on value for filter given through query parameters" $
+      get "/datarep_todos?select=id,due_at&label_color=eq.000100" `shouldRespondWith`
+        [json| [{"id":2,"due_at":"2018-01-03T00:00:00Z"}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    it "in the absense of text parser, does not try to use the JSON parser for query parameters" $
+      get "/datarep_todos?select=id,due_at&due_at=eq.Z" `shouldRespondWith`
+        -- we prove the parser is not used because it'd replace the Z with `+00:00` and a different error message.
+        [json| {"code":"22007","details":null,"hint":null,"message":"invalid input syntax for type timestamp with time zone: \"Z\""} |]
+        { matchStatus  = 400
+        , matchHeaders = [matchContentTypeJson]
+        }
+    -- Before PG 11, this will fail because we need arrays of domain type values. The docs should explain data reps are
+    -- not supported in this case.
+    when (actualPgVersion >= pgVersion110) $ do
+      it "uses text parser for filter with 'IN' predicates" $
+        get "/datarep_todos?select=id,due_at&label_color=in.(000100,01E240)" `shouldRespondWith`
+          [json| [
+            {"id":2, "due_at": "2018-01-03T00:00:00Z"},
+            {"id":3, "due_at": "2018-01-01T14:12:34.123456Z"}
+          ] |]
+          { matchHeaders = [matchContentTypeJson] }
+      it "uses text parser for filter with 'NOT IN' predicates" $
+        get "/datarep_todos?select=id,due_at&label_color=not.in.(000000,01E240)" `shouldRespondWith`
+          [json| [
+            {"id":2, "due_at": "2018-01-03T00:00:00Z"}
+          ] |]
+          { matchHeaders = [matchContentTypeJson] }
+    it "uses text parser on value for filter across relations" $
+      get "/datarep_next_two_todos?select=id,name,datarep_todos!datarep_next_two_todos_first_item_id_fkey(label_color,due_at)&datarep_todos.label_color=neq.000100" `shouldRespondWith`
+        [json| [{"id":1,"name":"school related","datarep_todos":null},{"id":2,"name":"do these first","datarep_todos":{"label_color":"#000000","due_at":"2018-01-02T00:00:00Z"}}] |]
+        { matchHeaders = [matchContentTypeJson] }
+    -- This is not supported by data reps (would be hard to make it work with high performance). So the test just
+    -- verifies we don't panic or add inappropriate SQL to the filters.
+    it "fails safely on user trying to use ilike operator on data reps column" $
+      get "/datarep_todos?select=id,name&label_color=ilike.#*100" `shouldRespondWith` (
+        if actualPgVersion >= pgVersion110 then
+        [json|
+          {"code":"42883","details":null,"hint":"No operator matches the given name and argument types. You might need to add explicit type casts.","message":"operator does not exist: public.color ~~* unknown"}
+        |]
+        else
+        [json|
+          {"code":"42883","details":null,"hint":"No operator matches the given name and argument type(s). You might need to add explicit type casts.","message":"operator does not exist: public.color ~~* unknown"}
+        |])
+        { matchStatus  = 404
+        , matchHeaders = [matchContentTypeJson]
+        }