Merge pull request #524 from begriffs/unicode-inserts

Preserve unicode in requests and responses

CHANGELOG.md

@@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ## Unreleased
 
 ### Fixed
+- Preserve unicode values in insert,update,rpc (regression) - @begriffs
 - Prevent duplicate call to stored procs (regression) - @begriffs
 - Allow SQL functions to generate registered JWT claims - @begriffs
 - Terminate gracefully on SIGTERM (for use in Docker) - @recmo

src/PostgREST/ApiRequest.hs

@@ -41,8 +41,8 @@ data PreferRepresentation = Full | HeadersOnly | None deriving Eq
 -- route responses and upload payloads
 data ContentType = ApplicationJSON | TextCSV deriving Eq
 instance Show ContentType where
-  show ApplicationJSON = "application/json"
-  show TextCSV         = "text/csv"
+  show ApplicationJSON = "application/json; charset=utf-8"
+  show TextCSV         = "text/csv; charset=utf-8"
 
 {-|
   Describes what the user wants to do. This data type is a

src/PostgREST/App.hs

@@ -249,7 +249,7 @@ contentRangeH frm to total =
       fromInRange   = frm <= to
 
 jsonH :: Header
-jsonH = (hContentType, "application/json")
+jsonH = (hContentType, "application/json; charset=utf-8")
 
 formatRelationError :: Text -> Text
 formatRelationError = formatGeneralError

src/PostgREST/QueryBuilder.hs

@@ -43,6 +43,7 @@ import           Data.List               (find, (\\))
 import           Data.Monoid             ((<>))
 import           Data.Text               (Text, intercalate, unwords, replace, isInfixOf, toLower, split)
 import qualified Data.Text as T          (map, takeWhile)
+import qualified Data.Text.Encoding as T
 import           Data.String.Conversions (cs)
 import           Control.Applicative     ((<|>))
 import           Control.Monad           (join)
@@ -93,7 +94,7 @@ encodeUniformObjs =
 createReadStatement :: SqlQuery -> SqlQuery -> NonnegRange -> Bool -> Bool -> Bool ->
                        H.Query () ResultsWithCount
 createReadStatement selectQuery countQuery range isSingle countTotal asCsv =
-  H.statement sql HE.unit decodeStandard True
+  unicodeStatement sql HE.unit decodeStandard True
  where
   sql = [qc|
       WITH {sourceCTEName} AS ({selectQuery}) SELECT {cols}
@@ -116,7 +117,7 @@ createWriteStatement :: QualifiedIdentifier -> SqlQuery -> SqlQuery -> Bool ->
 createWriteStatement _ _ _ _ _ _ _ (PayloadParseError _) = undefined
 createWriteStatement _ _ mutateQuery _ None
                      _ _ (PayloadJSON (UniformObjects _)) =
-  H.statement sql encodeUniformObjs decodeStandardMay True
+  unicodeStatement sql encodeUniformObjs decodeStandardMay True
  where
   sql = [qc|
       WITH {sourceCTEName} AS ({mutateQuery})
@@ -124,7 +125,7 @@ createWriteStatement _ _ mutateQuery _ None
 
 createWriteStatement qi _ mutateQuery isSingle HeadersOnly
                      pKeys _ (PayloadJSON (UniformObjects _)) =
-  H.statement sql encodeUniformObjs decodeStandardMay True
+  unicodeStatement sql encodeUniformObjs decodeStandardMay True
  where
   sql = [qc|
       WITH {sourceCTEName} AS ({mutateQuery} RETURNING {fromQi qi}.*)
@@ -139,7 +140,7 @@ createWriteStatement qi _ mutateQuery isSingle HeadersOnly
 
 createWriteStatement qi selectQuery mutateQuery isSingle Full
                      pKeys asCsv (PayloadJSON (UniformObjects _)) =
-  H.statement sql encodeUniformObjs decodeStandardMay True
+  unicodeStatement sql encodeUniformObjs decodeStandardMay True
  where
   sql = [qc|
       WITH {sourceCTEName} AS ({mutateQuery} RETURNING {fromQi qi}.*)
@@ -206,7 +207,7 @@ addJoinConditions schema (Node (query, (n, r)) forest) =
 type ProcResults = (Maybe Int64, Int64, JSON.Value)
 callProc :: QualifiedIdentifier -> JSON.Object -> NonnegRange -> Bool -> H.Query () (Maybe ProcResults)
 callProc qi params range countTotal =
-  H.statement sql HE.unit decodeProc True
+  unicodeStatement sql HE.unit decodeProc True
   where
     sql = [qc|
             WITH t AS (select * {_callSql})
@@ -220,10 +221,10 @@ callProc qi params range countTotal =
           |]
     _args = intercalate "," $ map _assignment (HM.toList params)
     _assignment (n,v) = pgFmtIdent n <> ":=" <> insertableValue v
-    _callSql = [qc| from {fromQi qi}({_args}) |] :: BS.ByteString
+    _callSql = [qc| from {fromQi qi}({_args}) |] :: Text
     _countExpr = if countTotal
                    then "(select pg_catalog.count(1) from t)"
-                   else "null::bigint" :: BS.ByteString
+                   else "null::bigint" :: Text
     decodeProc = HD.maybeRow procRow
     procRow = (,,) <$> HD.nullableValue HD.int8 <*> HD.value HD.int8
                    <*> HD.value HD.json
@@ -439,6 +440,9 @@ getJoinConditions (Relation t cols ft fcs typ lt lc1 lc2) =
     toFilter :: Text -> Text -> Column -> Column -> Filter
     toFilter tb ftb c fc = Filter (colName c, Nothing) "=" (VForeignKey (QualifiedIdentifier s tb) (ForeignKey fc{colTable=(colTable fc){tableName=ftb}}))
 
+unicodeStatement :: Text -> HE.Params a -> HD.Result b -> Bool -> H.Query a b
+unicodeStatement = H.statement . T.encodeUtf8
+
 emptyOnNull :: Text -> [a] -> Text
 emptyOnNull val x = if null x then "" else val
 

test/Feature/AuthSpec.hs

@@ -21,7 +21,7 @@ spec = describe "authorization" $ do
       `shouldRespondWith` ResponseMatcher {
           matchBody = Just [json| {"token":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.y4vZuu1dDdwAl0-S00MCRWRYMlJ5YAMSir6Es6WtWx0"} |]
         , matchStatus = 200
-        , matchHeaders = ["Content-Type" <:> "application/json"]
+        , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
         }
 
   it "sql functions can encode custom and standard claims" $
@@ -29,7 +29,7 @@ spec = describe "authorization" $ do
       `shouldRespondWith` ResponseMatcher {
           matchBody = Just [json| {"token":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJmdW4iLCJqdGkiOiJmb28iLCJuYmYiOjEzMDA4MTkzODAsImV4cCI6MTMwMDgxOTM4MCwiaHR0cDovL3Bvc3RncmVzdC5jb20vZm9vIjp0cnVlLCJpc3MiOiJqb2UiLCJyb2xlIjoicG9zdGdyZXN0X3Rlc3QiLCJpYXQiOjEzMDA4MTkzODAsImF1ZCI6ImV2ZXJ5b25lIn0._tQCF79-ZZGMlLktd3csM_bVaiMg7A8YvIb6K2hcu5w"} |]
         , matchStatus = 200
-        , matchHeaders = ["Content-Type" <:> "application/json"]
+        , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
         }
 
   it "sql functions can read custom and standard claims variables" $ do

test/Feature/InsertSpec.hs

@@ -9,10 +9,11 @@ import SpecHelper
 
 import qualified Data.Aeson as JSON
 import Data.Maybe (fromJust)
+import Data.Monoid ((<>))
 import Text.Heredoc
 import Network.HTTP.Types.Header
 import Network.HTTP.Types
-import Control.Monad (replicateM_)
+import Control.Monad (replicateM_, void)
 
 import TestTypes(IncPK(..), CompoundPK(..))
 import Network.Wai (Application)
@@ -41,7 +42,7 @@ spec = do
           } |] `shouldRespondWith` ResponseMatcher {
             matchBody    = Just [str|{"integer":14,"varchar":"testing!"}|]
           , matchStatus  = 201
-          , matchHeaders = ["Content-Type" <:> "application/json"]
+          , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"]
           }
 
       it "includes related data after insert" $
@@ -49,7 +50,7 @@ spec = do
           [str|{"id":6,"name":"New Project","client_id":2}|] `shouldRespondWith` ResponseMatcher {
             matchBody    = Just [str|{"id":6,"name":"New Project","clients":{"id":2,"name":"Apple"}}|]
           , matchStatus  = 201
-          , matchHeaders = ["Content-Type" <:> "application/json", "Location" <:> "/projects?id=eq.6"]
+          , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8", "Location" <:> "/projects?id=eq.6"]
           }
 
 
@@ -145,13 +146,6 @@ spec = do
           , matchHeaders = ["Location" <:> [str|/json?data=eq.{"foo":"bar"}|]]
           }
 
-        -- TODO! the test above seems right, why was the one below working before and not now
-        -- p <- request methodPost "/json" [("Prefer", "return=representation")] inserted
-        -- liftIO $ do
-        --   simpleBody p `shouldBe` inserted
-        --   simpleHeaders p `shouldSatisfy` matchHeader hLocation "/json\\?data=eq\\.%7B%22foo%22%3A%22bar%22%7D"
-        --   simpleStatus p `shouldBe` created201
-
       it "serializes nested array" $ do
         let inserted = [json| { "data": [1,2,3] } |]
         request methodPost "/json"
@@ -162,12 +156,6 @@ spec = do
           , matchStatus  = 201
           , matchHeaders = ["Location" <:> [str|/json?data=eq.[1,2,3]|]]
           }
-        -- TODO! the test above seems right, why was the one below working before and not now
-        -- p <- request methodPost "/json" [("Prefer", "return=representation")] inserted
-        -- liftIO $ do
-        --   simpleBody p `shouldBe` inserted
-        --   simpleHeaders p `shouldSatisfy` matchHeader hLocation "/json\\?data=eq\\.%5B1%2C2%2C3%5D"
-        --   simpleStatus p `shouldBe` created201
 
   describe "CSV insert" $ do
 
@@ -183,16 +171,8 @@ spec = do
            `shouldRespondWith` ResponseMatcher {
              matchBody    = Just inserted
            , matchStatus  = 201
-           , matchHeaders = ["Content-Type" <:> "text/csv"]
+           , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8"]
            }
-        -- p <- request methodPost "/menagerie" [("Content-Type", "text/csv")]
-        --        [str|integer,double,varchar,boolean,date,money,enum
-        --            |13,3.14159,testing!,false,1900-01-01,$3.99,foo
-        --            |12,0.1,a string,true,1929-10-01,12,bar
-        --            |]
-        -- liftIO $ do
-        --   simpleBody p `shouldBe` "Content-Type: application/json\nLocation: /menagerie?integer=eq.13\n\n\n--postgrest_boundary\nContent-Type: application/json\nLocation: /menagerie?integer=eq.12\n\n"
-        --   simpleStatus p `shouldBe` created201
 
     context "requesting full representation" $ do
       it "returns full details of inserted record" $
@@ -202,21 +182,10 @@ spec = do
           `shouldRespondWith` ResponseMatcher {
             matchBody    = Just "a,b\nbar,baz"
           , matchStatus  = 201
-          , matchHeaders = ["Content-Type" <:> "text/csv",
+          , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8",
                             "Location" <:> "/no_pk?a=eq.bar&b=eq.baz"]
           }
 
-      -- it "can post nulls (old way)" $ do
-      --   pendingWith "changed the response when in csv mode"
-      --   request methodPost "/no_pk"
-      --                [("Content-Type", "text/csv"), ("Prefer", "return=representation")]
-      --                "a,b\nNULL,foo"
-      --     `shouldRespondWith` ResponseMatcher {
-      --       matchBody    = Just [json| { "a":null, "b":"foo" } |]
-      --     , matchStatus  = 201
-      --     , matchHeaders = ["Content-Type" <:> "application/json",
-      --                       "Location" <:> "/no_pk?a=is.null&b=eq.foo"]
-      --     }
       it "can post nulls" $
         request methodPost "/no_pk"
                      [("Content-Type", "text/csv"), ("Accept", "text/csv"), ("Prefer", "return=representation")]
@@ -224,7 +193,7 @@ spec = do
           `shouldRespondWith` ResponseMatcher {
             matchBody    = Just "a,b\n,foo"
           , matchStatus  = 201
-          , matchHeaders = ["Content-Type" <:> "text/csv",
+          , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8",
                             "Location" <:> "/no_pk?a=is.null&b=eq.foo"]
           }
 
@@ -233,10 +202,21 @@ spec = do
       it "fails for too few" $ do
         p <- request methodPost "/no_pk" [("Content-Type", "text/csv")] "a,b\nfoo,bar\nbaz"
         liftIO $ simpleStatus p `shouldBe` badRequest400
-      -- it does not fail because the extra columns are ignored
-      -- it "fails for too many" $ do
-      --   p <- request methodPost "/no_pk" [("Content-Type", "text/csv")] "a,b\nfoo,bar\nbaz,bat,bad"
-      --   liftIO $ simpleStatus p `shouldBe` badRequest400
+
+    context "with unicode values" $
+      it "succeeds and returns usable location header" $ do
+        let payload = [json| { "a":"圍棋", "b":"￥" } |]
+        p <- request methodPost "/no_pk"
+                     [("Prefer", "return=representation")]
+                     payload
+        liftIO $ do
+          simpleBody p `shouldBe` payload
+          simpleStatus p `shouldBe` created201
+
+        let Just location = lookup hLocation $ simpleHeaders p
+        r <- get location
+        liftIO $ simpleBody r `shouldBe` "["<>payload<>"]"
+
 
   describe "Putting record" $ do
 
@@ -387,6 +367,17 @@ spec = do
           , matchHeaders = []
           }
 
+    context "with unicode values" $
+      it "succeeds and returns values intact" $ do
+        void $ request methodPost "/no_pk" []
+          [json| { "a":"patchme", "b":"patchme" } |]
+        let payload = [json| { "a":"圍棋", "b":"￥" } |]
+        p <- request methodPatch "/no_pk?a=eq.patchme&b=eq.patchme"
+          [("Prefer", "return=representation")] payload
+        liftIO $ do
+          simpleBody p `shouldBe` "["<>payload<>"]"
+          simpleStatus p `shouldBe` ok200
+
   describe "Row level permission" $
     it "set user_id when inserting rows" $ do
       let auth = authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.y4vZuu1dDdwAl0-S00MCRWRYMlJ5YAMSir6Es6WtWx0"

test/Feature/QuerySpec.hs

@@ -337,7 +337,7 @@ spec = do
         `shouldRespondWith` ResponseMatcher {
           matchBody    = Just "k,extra\nxyyx,u\nxYYx,v"
         , matchStatus  = 200
-        , matchHeaders = ["Content-Type" <:> "text/csv"]
+        , matchHeaders = ["Content-Type" <:> "text/csv; charset=utf-8"]
         }
 
   describe "Canonical location" $ do
@@ -390,11 +390,15 @@ spec = do
         post "/rpc/test_empty_rowset" [json| {} |] `shouldRespondWith`
           [json| [] |]
 
-    context "a proc that returns plain text" $
+    context "a proc that returns plain text" $ do
       it "returns proper json" $
         post "/rpc/sayhello" [json| { "name": "world" } |] `shouldRespondWith`
           [json| [{"sayhello":"Hello, world"}] |]
 
+      it "can handle unicode" $
+        post "/rpc/sayhello" [json| { "name": "￥" } |] `shouldRespondWith`
+          [json| [{"sayhello":"Hello, ￥"}] |]
+
     context "improper input" $ do
       it "rejects unknown content type even if payload is good" $
         request methodPost "/rpc/sayhello"