Merge pull request #964 from steve-chavez/tsquery-options

Allow specifying dictionary and plain/phrase in full text search

.circleci/config.yml

@@ -53,7 +53,14 @@ build-distro-bin: &build-distro-bin
 
 jobs:
   build-test:
-    machine: true
+    docker:
+      - image: circleci/buildpack-deps:trusty
+        environment:
+          - PGHOST=localhost
+      - image: circleci/postgres:9.6.2
+        environment:
+          - POSTGRES_USER=circleci
+          - POSTGRES_DB=circleci
     steps:
       - checkout
       - restore_cache:
@@ -65,8 +72,9 @@ jobs:
             curl -L https://github.com/commercialhaskell/stack/releases/download/v1.1.2/stack-1.1.2-linux-x86_64.tar.gz | tar zx -C /tmp
             sudo mv /tmp/stack-1.1.2-linux-x86_64/stack /usr/bin
             sudo apt-get update
-            sudo apt-get install libgmp-dev
-            sudo apt-get install --only-upgrade binutils
+            sudo apt-get install -y libgmp-dev
+            sudo apt-get install -y --only-upgrade binutils
+            sudo apt-get install -y postgresql-client
             stack setup
             rm -rf $(stack path --dist-dir) $(stack path --local-install-root)
             stack install hlint packdeps cabal-install
@@ -78,7 +86,6 @@ jobs:
       - run:
           name: run tests
           command: |
-            sudo service postgresql start
             POSTGREST_TEST_CONNECTION=$(test/create_test_db "postgres://circleci@localhost" postgrest_test) stack test
             test/io-tests.sh
       - run:

CHANGELOG.md

@@ -7,6 +7,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- #887, #601, Allow specifying dictionary and plain/phrase tsquery in full text search - @steve-chavez
+
 ### Fixed
 
 ## [0.4.3.0] - 2017-09-06

src/PostgREST/Parsers.hs

@@ -1,6 +1,6 @@
 module PostgREST.Parsers where
 
-import           Protolude                     hiding (try, intercalate, replace)
+import           Protolude                     hiding (try, intercalate, replace, option)
 import           Control.Monad                 ((>>))
 import           Data.Foldable                 (foldl1)
 import qualified Data.HashMap.Strict           as M
@@ -21,7 +21,7 @@ pRequestFilter :: (Text, Text) -> Either ApiRequestError (EmbedPath, Filter)
 pRequestFilter (k, v) = mapError $ (,) <$> path <*> (Filter <$> fld <*> oper)
   where
     treePath = parse pTreePath ("failed to parser tree path (" ++ toS k ++ ")") $ toS k
-    oper = parse (pOperation pVText pVTextL) ("failed to parse filter (" ++ toS v ++ ")") $ toS v
+    oper = parse (pOpExpr pSingleVal pListVal) ("failed to parse filter (" ++ toS v ++ ")") $ toS v
     path = fst <$> treePath
     fld = snd <$> treePath
 
@@ -78,7 +78,7 @@ pFieldForest :: Parser [Tree SelectItem]
 pFieldForest = pFieldTree `sepBy1` lexeme (char ',')
 
 pFieldTree :: Parser (Tree SelectItem)
-pFieldTree =  try (Node <$> pRelationSelect <*> between (char '{') (char '}') pFieldForest)
+pFieldTree =  try (Node <$> pRelationSelect <*> between (char '{') (char '}') pFieldForest) -- TODO: "{}" deprecated
           <|> try (Node <$> pRelationSelect <*> between (char '(') (char ')') pFieldForest)
           <|> Node <$> pFieldSelect <*> pure []
 
@@ -96,7 +96,6 @@ pFieldName = do
     dash :: Parser Char
     dash = isDash *> pure '-'
 
-
 pJsonPathStep :: Parser Text
 pJsonPathStep = toS <$> try (string "->" *> pFieldName)
 
@@ -131,26 +130,35 @@ pFieldSelect = lexeme $
     s <- pStar
     return ((s, Nothing), Nothing, Nothing, Nothing)
 
-pOperation :: Parser Operand -> Parser Operand -> Parser Operation
-pOperation parserVText parserVTextL = try ( string "not" *> pDelimiter *> (Operation True <$> pExpr)) <|> Operation False <$> pExpr
+pOpExpr :: Parser SingleVal -> Parser ListVal -> Parser OpExpr
+pOpExpr pSVal pLVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)) <|> OpExpr False <$> pOperation
   where
-    pExpr :: Parser (Operator, Operand)
-    pExpr =
-          ((,) <$> (toS <$> foldl1 (<|>) (try . ((<* pDelimiter) . string) . toS <$> M.keys notInOps)) <*> parserVText)
-      <|> ((,) <$> (toS <$> foldl1 (<|>) (try . ((<* pDelimiter) . string) . toS <$> M.keys inOps)) <*> parserVTextL)
+    pOperation :: Parser Operation
+    pOperation =
+          Op . toS <$> foldl1 (<|>) (try . ((<* pDelimiter) . string) . toS <$> M.keys ops) <*> pSVal
+      <|> In <$> (string "in" *> pDelimiter *> pLVal)
+      <|> pFts
       <?> "operator (eq, gt, ...)"
-    inOps = M.filterWithKey (const . flip elem ["in", "notin"]) operators
-    notInOps = M.difference operators inOps
+    pFts = do
+      mode <- option Normal $
+              try (string "phrase" *> pDelimiter *> pure Phrase)
+          <|> try (string "plain" *> pDelimiter *> pure Plain)
 
-pVText :: Parser Operand
-pVText = VText . toS <$> many anyChar
+      lang <- try (Just <$> manyTill (letter <|> digit <|> oneOf "_") (try (string ".fts") <|> try (string ".@@")) <* pDelimiter) -- TODO: '@@' deprecated
+          <|> try (string "fts" *> pDelimiter) *> pure Nothing
+          <|> try (string "@@" *> pDelimiter)  *> pure Nothing -- TODO: '@@' deprecated
+      Fts mode (toS <$> lang) <$> pSVal
+    ops = M.filterWithKey (const . flip notElem ["in", "fts", "@@"]) operators -- TODO: '@@' deprecated
 
-pVTextL :: Parser Operand
-pVTextL =     VTextL <$> try (lexeme (char '(') *> pVTextLElement `sepBy1` char ',' <* lexeme (char ')'))
-          <|> VTextL <$> lexeme pVTextLElement `sepBy1` char ','
+pSingleVal :: Parser SingleVal
+pSingleVal = toS <$> many anyChar
 
-pVTextLElement :: Parser Text
-pVTextLElement = try pQuotedValue <|> (toS <$> many (noneOf ",)"))
+pListVal :: Parser ListVal
+pListVal =    try (lexeme (char '(') *> pListElement `sepBy1` char ',' <* lexeme (char ')'))
+          <|> lexeme pListElement `sepBy1` char ',' -- TODO: "in.3,4,5" deprecated, parens e.g. "in.(3,4,5)" should be used
+
+pListElement :: Parser Text
+pListElement = try pQuotedValue <|> (toS <$> many (noneOf ",)"))
 
 pQuotedValue :: Parser Text
 pQuotedValue = toS <$> (char '"' *> many (noneOf "\"") <* char '"' <* notFollowedBy (noneOf ",)"))
@@ -182,7 +190,7 @@ pLogicTree = Stmnt <$> try pLogicFilter
              <|> Expr <$> pNot <*> pLogicOp <*> (lexeme (char '(') *> pLogicTree `sepBy1` lexeme (char ',') <* lexeme (char ')'))
   where
     pLogicFilter :: Parser Filter
-    pLogicFilter = Filter <$> pField <* pDelimiter <*> pOperation pLogicVText pLogicVTextL
+    pLogicFilter = Filter <$> pField <* pDelimiter <*> pOpExpr pLogicSingleVal pLogicListVal
     pNot :: Parser Bool
     pNot = try (string "not" *> pDelimiter *> pure True)
            <|> pure False
@@ -192,9 +200,10 @@ pLogicTree = Stmnt <$> try pLogicFilter
                <|> string "or" *> pure Or
                <?> "logic operator (and, or)"
 
-pLogicVText :: Parser Operand
-pLogicVText = VText <$> (try pQuotedValue <|> try pPgArray <|> (toS <$> many (noneOf ",)")))
+pLogicSingleVal :: Parser SingleVal
+pLogicSingleVal = try pQuotedValue <|> try pPgArray <|> (toS <$> many (noneOf ",)"))
   where
+    -- TODO: "{}" deprecated, after removal pPgArray can be removed
     pPgArray :: Parser Text
     pPgArray =  do
       a <- string "{"
@@ -202,8 +211,8 @@ pLogicVText = VText <$> (try pQuotedValue <|> try pPgArray <|> (toS <$> many (no
       c <- string "}"
       toS <$> pure (a ++ b ++ c)
 
-pLogicVTextL :: Parser Operand
-pLogicVTextL = VTextL <$> (lexeme (char '(') *> pVTextLElement `sepBy1` char ',' <* lexeme (char ')'))
+pLogicListVal :: Parser ListVal
+pLogicListVal = lexeme (char '(') *> pListElement `sepBy1` char ',' <* lexeme (char ')')
 
 pLogicPath :: Parser (EmbedPath, Text)
 pLogicPath = do

src/PostgREST/QueryBuilder.hs

@@ -208,7 +208,7 @@ requestToCountQuery schema (DbRead (Node (Select _ _ logicForest _ _, (mainTbl,
    qi = removeSourceCTESchema schema mainTbl
    -- all foreing key filters are root nodes(see addFilterToLogicForest), only those are filtered
    nonFKRoot :: LogicTree -> Bool
-   nonFKRoot (Stmnt (Filter _ Operation{expr=(_, VForeignKey _ _)})) = False
+   nonFKRoot (Stmnt (Filter _ (OpExpr _ (Join _ _)))) = False
    nonFKRoot (Stmnt _) = True
    nonFKRoot Expr{} = True
    filteredLogic = filter nonFKRoot logicForest
@@ -253,7 +253,7 @@ requestToQuery schema isParent (DbRead (Node (Select colSelects tbls logicForest
       where
         node_name = fromMaybe name alias
         local_table_name = table <> "_" <> node_name
-        replaceTableName localTableName (Filter a (Operation b (c, VForeignKey (QualifiedIdentifier "" _) d))) = Filter a (Operation b (c, VForeignKey (QualifiedIdentifier "" localTableName) d))
+        replaceTableName localTableName (Filter a (OpExpr b (Join (QualifiedIdentifier "" _) c))) = Filter a (OpExpr b (Join (QualifiedIdentifier "" localTableName) c))
         replaceTableName _ x = x
         sel = "row_to_json(" <> pgFmtIdent local_table_name <> ".*) AS " <> pgFmtIdent node_name
         joi = " LEFT OUTER JOIN ( " <> subquery <> " ) AS " <> pgFmtIdent local_table_name  <>
@@ -385,7 +385,7 @@ getJoinFilters (Relation t cols ft fcs typ lt lc1 lc2) =
     ftN = tableName ft
     ltN = fromMaybe "" (tableName <$> lt)
     toFilter :: Text -> Text -> Column -> Column -> Filter
-    toFilter tb ftb c fc = Filter (colName c, Nothing) (Operation False ("=", VForeignKey (QualifiedIdentifier s tb) (ForeignKey fc{colTable=(colTable fc){tableName=ftb}})))
+    toFilter tb ftb c fc = Filter (colName c, Nothing) (OpExpr False (Join (QualifiedIdentifier s tb) (ForeignKey fc{colTable=(colTable fc){tableName=ftb}})))
 
 unicodeStatement :: Text -> HE.Params a -> HD.Result b -> Bool -> H.Query a b
 unicodeStatement = H.statement . T.encodeUtf8
@@ -413,43 +413,43 @@ pgFmtSelectItem table (f@(_, jp), Nothing, alias, _) = pgFmtField table f <> pgF
 pgFmtSelectItem table (f@(_, jp), Just cast, alias, _) = "CAST (" <> pgFmtField table f <> " AS " <> cast <> " )" <> pgFmtAs jp alias
 
 pgFmtFilter :: QualifiedIdentifier -> Filter -> SqlFragment
-pgFmtFilter table (Filter fld (Operation hasNot_ ex)) = notOp <> " " <> case ex of
-   (op, VText val) -> pgFmtFieldOp op <> " " <> case op of
-     "like"  -> unknownLiteral (T.map star val)
-     "ilike" -> unknownLiteral (T.map star val)
-     -- TODO: The '@@' was deprecated, remove in v0.5.0.0
-     "@@"    -> "to_tsquery(" <> unknownLiteral val <> ") "
-     "fts"   -> "to_tsquery(" <> unknownLiteral val <> ") "
-     "is"    -> whiteList val
-     "isnot" -> whiteList val
-     _       -> unknownLiteral val
-   (op, VTextL vals) -> pgFmtIn op vals -- in and notin
-   (op, VForeignKey fQi (ForeignKey Column{colTable=Table{tableName=fTableName}, colName=fColName})) ->
-     pgFmtField fQi fld <> " " <> sqlOperator op <> " " <> pgFmtColumn (removeSourceCTESchema (qiSchema fQi) fTableName) fColName
+pgFmtFilter table (Filter fld (OpExpr hasNot oper)) = notOp <> " " <> case oper of
+   Op op val  -> pgFmtFieldOp op <> " " <> case op of
+     "like"   -> unknownLiteral (T.map star val)
+     "ilike"  -> unknownLiteral (T.map star val)
+     "is"     -> whiteList val
+     _        -> unknownLiteral val
+
+   In vals -> pgFmtField table fld <> " " <>
+    let emptyValForIn = "= any('{}') " in -- Workaround because for postgresql "col IN ()" is invalid syntax, we instead do "col = any('{}')"
+    case ((&&) (length vals == 1) . T.null) <$> headMay vals of
+      Just False -> sqlOperator "in" <> "(" <> intercalate ", " (map unknownLiteral vals) <> ") "
+      Just True  -> emptyValForIn
+      Nothing    -> emptyValForIn
+
+   Fts mode lang val ->
+     pgFmtFieldOp "fts" <> " " <> case mode of
+       Normal -> "to_tsquery("
+       Plain  -> "plainto_tsquery("
+       Phrase -> "phraseto_tsquery("
+     <> maybe "" (flip (<>) ", " . pgFmtLit) lang <> unknownLiteral val <> ") "
+
+   Join fQi (ForeignKey Column{colTable=Table{tableName=fTableName}, colName=fColName}) ->
+     pgFmtField fQi fld <> " = " <> pgFmtColumn (removeSourceCTESchema (qiSchema fQi) fTableName) fColName
  where
    pgFmtFieldOp op = pgFmtField table fld <> " " <> sqlOperator op
    sqlOperator o = HM.lookupDefault "=" o operators
-   notOp = if hasNot_ then "NOT" else ""
+   notOp = if hasNot then "NOT" else ""
    star c = if c == '*' then '%' else c
    unknownLiteral = (<> "::unknown ") . pgFmtLit
    whiteList :: Text -> SqlFragment
    whiteList v = fromMaybe
      (toS (pgFmtLit v) <> "::unknown ")
      (find ((==) . toLower $ v) ["null","true","false"])
-   pgFmtIn :: Operator -> [Text] -> SqlFragment
-   pgFmtIn op vals =
-    -- Workaround because for postgresql "col IN ()" is invalid syntax, we instead do "col = any('{}')"
-    let emptyValForIn o = (if "not" `isInfixOf` o then "NOT " else "") -- handle case of "notin" operator
-                          <> pgFmtField table fld <> " = any('{}') " in
-    case T.null <$> headMay vals of
-      Just isNull -> if isNull && length vals == 1
-                        then emptyValForIn op
-                        else pgFmtFieldOp op <> "(" <> intercalate ", " (map unknownLiteral vals) <> ") "
-      Nothing -> emptyValForIn op
 
 pgFmtLogicTree :: QualifiedIdentifier -> LogicTree -> SqlFragment
-pgFmtLogicTree qi (Expr hasNot_ op forest) = notOp <> " (" <> intercalate (" " <> show op <> " ") (pgFmtLogicTree qi <$> forest) <> ")"
-  where notOp =  if hasNot_ then "NOT" else ""
+pgFmtLogicTree qi (Expr hasNot op forest) = notOp <> " (" <> intercalate (" " <> show op <> " ") (pgFmtLogicTree qi <$> forest) <> ")"
+  where notOp =  if hasNot then "NOT" else ""
 pgFmtLogicTree qi (Stmnt flt) = pgFmtFilter qi flt
 
 pgFmtJsonPath :: Maybe JsonPath -> SqlFragment

src/PostgREST/Types.hs

@@ -158,8 +158,6 @@ operators = M.fromList [
   ("like", "LIKE"),
   ("ilike", "ILIKE"),
   ("in", "IN"),
-  ("notin", "NOT IN"),
-  ("isnot", "IS NOT"),
   ("is", "IS"),
   ("fts", "@@"),
   ("cs", "@>"),
@@ -174,8 +172,19 @@ operators = M.fromList [
   ("@@", "@@"),
   ("@>", "@>"),
   ("<@", "<@")]
-data Operation = Operation{ hasNot::Bool, expr::(Operator, Operand) } deriving (Eq, Show)
-data Operand = VText Text | VTextL [Text] | VForeignKey QualifiedIdentifier ForeignKey deriving (Show, Eq)
+
+data OpExpr = OpExpr Bool Operation deriving (Eq, Show)
+data Operation = Op Operator SingleVal |
+                 In ListVal |
+                 Fts FtsMode (Maybe Language) SingleVal |
+                 Join QualifiedIdentifier ForeignKey deriving (Eq, Show)
+
+data FtsMode = Normal | Plain | Phrase deriving (Eq, Show)
+type Language = Text
+-- | Represents a single value in a filter, e.g. id=eq.singleval
+type SingleVal = Text
+-- | Represents a list value in a filter, e.g. id=in.(val1,val2,val3)
+type ListVal = [Text]
 
 data LogicOperator = And | Or deriving Eq
 instance Show LogicOperator where
@@ -207,7 +216,7 @@ type RelationDetail = Text
 type SelectItem = (Field, Maybe Cast, Maybe Alias, Maybe RelationDetail)
 -- | Path of the embedded levels, e.g "clients.projects.name=eq.." gives Path ["clients", "projects"]
 type EmbedPath = [Text]
-data Filter = Filter { field::Field, operation::Operation } deriving (Show, Eq)
+data Filter = Filter { field::Field, opExpr::OpExpr } deriving (Show, Eq)
 
 data ReadQuery = Select { select::[SelectItem], from::[TableName], where_::[LogicTree], order::Maybe [OrderTerm], range_::NonnegRange } deriving (Show, Eq)
 data MutateQuery = Insert { in_::TableName, qPayload::PayloadJSON, returning::[FieldName] }

test/Feature/AndOrParamsSpec.hs

@@ -73,6 +73,12 @@ spec =
         it "can handle fts" $ do
           get "/entities?or=(text_search_vector.fts.bar,text_search_vector.fts.baz)&select=id" `shouldRespondWith`
             [json|[{ "id": 1 }, { "id": 2 }]|] { matchHeaders = [matchContentTypeJson] }
+          get "/tsearch?or=(text_search_vector.phrase.german.fts.Art%20Spass, text_search_vector.plain.french.fts.amusant%20impossible, text_search_vector.english.fts.impossible)" `shouldRespondWith`
+            [json|[
+              {"text_search_vector": "'fun':5 'imposs':9 'kind':3" },
+              {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" },
+              {"text_search_vector": "'art':4 'spass':5 'unmog':7"}
+            ]|] { matchHeaders = [matchContentTypeJson] }
           -- TODO: remove in 0.5.0 as deprecated
           get "/entities?or=(text_search_vector.@@.bar,text_search_vector.@@.baz)&select=id" `shouldRespondWith`
             [json|[{ "id": 1 }, { "id": 2 }]|] { matchHeaders = [matchContentTypeJson] }

test/Feature/QuerySpec.hs

@@ -55,11 +55,6 @@ spec = do
         `shouldRespondWith` [json| [{"id":1},{"id":3},{"id":5}] |]
         { matchHeaders = ["Content-Range" <:> "0-2/*"] }
 
-    it "matches items NOT IN" $
-      get "/items?id=notin.2,4,6,7,8,9,10,11,12,13,14,15"
-        `shouldRespondWith` [json| [{"id":1},{"id":3},{"id":5}] |]
-        { matchHeaders = ["Content-Range" <:> "0-2/*"] }
-
     it "matches items NOT IN using not operator" $
       get "/items?id=not.in.2,4,6,7,8,9,10,11,12,13,14,15"
         `shouldRespondWith` [json| [{"id":1},{"id":3},{"id":5}] |]
@@ -98,29 +93,107 @@ spec = do
     it "matches with ilike using not operator" $
       get "/simple_pk?k=not.ilike.xy*&order=extra.asc" `shouldRespondWith` "[]"
 
-    it "matches with tsearch fts" $ do
-      get "/tsearch?text_search_vector=fts.impossible" `shouldRespondWith`
-        [json| [{"text_search_vector": "'fun':5 'imposs':9 'kind':3" }] |]
-        { matchHeaders = [matchContentTypeJson] }
-      get "/tsearch?text_search_vector=fts.possible" `shouldRespondWith`
-        [json| [{"text_search_vector": "'also':2 'fun':3 'possibl':8" }] |]
-        { matchHeaders = [matchContentTypeJson] }
-      -- TODO: remove in 0.5.0 as deprecated
-      get "/tsearch?text_search_vector=@@.impossible" `shouldRespondWith`
-        [json| [{"text_search_vector": "'fun':5 'imposs':9 'kind':3" }] |]
-        { matchHeaders = [matchContentTypeJson] }
-      get "/tsearch?text_search_vector=@@.possible" `shouldRespondWith`
-        [json| [{"text_search_vector": "'also':2 'fun':3 'possibl':8" }] |]
-        { matchHeaders = [matchContentTypeJson] }
+    describe "Full text search operator" $ do
+      it "finds matches with to_tsquery" $
+        get "/tsearch?text_search_vector=fts.impossible" `shouldRespondWith`
+          [json| [{"text_search_vector": "'fun':5 'imposs':9 'kind':3" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "can use lexeme boolean operators(&=%26, |=%7C, !) in to_tsquery" $ do
+        get "/tsearch?text_search_vector=fts.fun%26possible" `shouldRespondWith`
+          [json| [ {"text_search_vector": "'also':2 'fun':3 'possibl':8"}] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=fts.impossible%7Cpossible" `shouldRespondWith`
+          [json| [
+          {"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
+          {"text_search_vector": "'also':2 'fun':3 'possibl':8"}] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=fts.fun%26!possible" `shouldRespondWith`
+          [json| [ {"text_search_vector": "'fun':5 'imposs':9 'kind':3"}] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "finds matches with plainto_tsquery" $
+        get "/tsearch?text_search_vector=plain.fts.The%20Fat%20Rats" `shouldRespondWith`
+          [json| [ {"text_search_vector": "'ate':3 'cat':2 'fat':1 'rat':4" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "finds matches with phraseto_tsquery" $
+        get "/tsearch?text_search_vector=phrase.fts.The%20Fat%20Cats" `shouldRespondWith`
+          [json| [{"text_search_vector": "'ate':3 'cat':2 'fat':1 'rat':4" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "finds matches with different dictionaries" $ do
+        get "/tsearch?text_search_vector=french.fts.amusant" `shouldRespondWith`
+          [json| [{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=plain.french.fts.amusant%20impossible" `shouldRespondWith`
+          [json| [{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=phrase.german.fts.Art%20Spass" `shouldRespondWith`
+          [json| [{"text_search_vector": "'art':4 'spass':5 'unmog':7" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+
+      it "can be negated with not operator" $ do
+        get "/tsearch?text_search_vector=not.fts.impossible%7Cfat%7Cfun" `shouldRespondWith`
+          [json| [
+            {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
+            {"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=not.english.fts.impossible%7Cfat%7Cfun" `shouldRespondWith`
+          [json| [
+            {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
+            {"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=not.plain.fts.The%20Fat%20Rats" `shouldRespondWith`
+          [json| [
+            {"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
+            {"text_search_vector": "'also':2 'fun':3 'possibl':8"},
+            {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
+            {"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=not.phrase.english.fts.The%20Fat%20Cats" `shouldRespondWith`
+          [json| [
+            {"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
+            {"text_search_vector": "'also':2 'fun':3 'possibl':8"},
+            {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
+            {"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
+          { matchHeaders = [matchContentTypeJson] }
 
-    it "matches with tsearch fts using not operator" $ do
-      get "/tsearch?text_search_vector=not.fts.impossible" `shouldRespondWith`
-        [json| [{"text_search_vector": "'also':2 'fun':3 'possibl':8" }] |]
-        { matchHeaders = [matchContentTypeJson] }
       -- TODO: remove in 0.5.0 as deprecated
-      get "/tsearch?text_search_vector=not.@@.impossible" `shouldRespondWith`
-        [json| [{"text_search_vector": "'also':2 'fun':3 'possibl':8" }] |]
-        { matchHeaders = [matchContentTypeJson] }
+      it "Deprecated @@ operator, pending to remove" $ do
+        get "/tsearch?text_search_vector=@@.impossible" `shouldRespondWith`
+          [json| [{"text_search_vector": "'fun':5 'imposs':9 'kind':3" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=plain.@@.The%20Fat%20Rats" `shouldRespondWith`
+          [json| [ {"text_search_vector": "'ate':3 'cat':2 'fat':1 'rat':4" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=phrase.@@.The%20Fat%20Cats" `shouldRespondWith`
+          [json| [{"text_search_vector": "'ate':3 'cat':2 'fat':1 'rat':4" }] |]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=not.@@.impossible%7Cfat%7Cfun" `shouldRespondWith`
+          [json| [
+            {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
+            {"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=not.english.@@.impossible%7Cfat%7Cfun" `shouldRespondWith`
+          [json| [
+            {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
+            {"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=not.plain.@@.The%20Fat%20Rats" `shouldRespondWith`
+          [json| [
+            {"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
+            {"text_search_vector": "'also':2 'fun':3 'possibl':8"},
+            {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
+            {"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
+          { matchHeaders = [matchContentTypeJson] }
+        get "/tsearch?text_search_vector=not.phrase.english.@@.The%20Fat%20Cats" `shouldRespondWith`
+          [json| [
+            {"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
+            {"text_search_vector": "'also':2 'fun':3 'possibl':8"},
+            {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
+            {"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
+          { matchHeaders = [matchContentTypeJson] }
 
     it "matches with computed column" $
       get "/items?always_true=eq.true&order=id.asc" `shouldRespondWith`
@@ -709,7 +782,7 @@ spec = do
         { matchHeaders = [matchContentTypeJson] }
 
     it "fails if an operator is not given" $
-      get "/ghostBusters?id=0" `shouldRespondWith` [json| {"details":"unexpected \"0\" expecting \"not\" or operator (eq, gt, ...)","message":"\"failed to parse filter (0)\" (line 1, column 1)"} |]
+      get "/ghostBusters?id=0" `shouldRespondWith` [json| {"details":"unexpected end of input expecting operator (eq, gt, ...)","message":"\"failed to parse filter (0)\" (line 1, column 2)"} |]
         { matchStatus  = 400
         , matchHeaders = [matchContentTypeJson]
         }
@@ -814,7 +887,7 @@ spec = do
           , matchHeaders = []
           }
 
-  describe "values with quotes in IN and NOTIN operators" $ do
+  describe "values with quotes in IN and NOT IN" $ do
     it "succeeds when only quoted values are present" $ do
       get "/w_or_wo_comma_names?name=in.\"Hebdon, John\"" `shouldRespondWith`
         [json| [{"name":"Hebdon, John"}] |]
@@ -822,9 +895,6 @@ spec = do
       get "/w_or_wo_comma_names?name=in.\"Hebdon, John\",\"Williams, Mary\",\"Smith, Joseph\"" `shouldRespondWith`
         [json| [{"name":"Hebdon, John"},{"name":"Williams, Mary"},{"name":"Smith, Joseph"}] |]
         { matchHeaders = [matchContentTypeJson] }
-      get "/w_or_wo_comma_names?name=notin.\"Hebdon, John\",\"Williams, Mary\",\"Smith, Joseph\"" `shouldRespondWith`
-        [json| [{"name":"David White"},{"name":"Larry Thompson"}] |]
-        { matchHeaders = [matchContentTypeJson] }
       get "/w_or_wo_comma_names?name=not.in.\"Hebdon, John\",\"Williams, Mary\",\"Smith, Joseph\"" `shouldRespondWith`
         [json| [{"name":"David White"},{"name":"Larry Thompson"}] |]
         { matchHeaders = [matchContentTypeJson] }
@@ -836,9 +906,6 @@ spec = do
       get "/w_or_wo_comma_names?name=not.in.\"Hebdon, John\",Larry Thompson,\"Smith, Joseph\"" `shouldRespondWith`
         [json| [{"name":"Williams, Mary"},{"name":"David White"}] |]
         { matchHeaders = [matchContentTypeJson] }
-      get "/w_or_wo_comma_names?name=notin.\"Hebdon, John\",David White,\"Williams, Mary\",Larry Thompson" `shouldRespondWith`
-        [json| [{"name":"Smith, Joseph"}] |]
-        { matchHeaders = [matchContentTypeJson] }
 
     it "checks well formed quoted values" $ do
       get "/w_or_wo_comma_names?name=in.\"\"Hebdon, John\"" `shouldRespondWith`
@@ -875,10 +942,6 @@ spec = do
         get "/items_with_different_col_types?time_data=in." `shouldRespondWith`
           [json| [] |] { matchHeaders = [matchContentTypeJson] }
 
-    it "returns all results for notin when no value is present" $
-      get "/items_with_different_col_types?int_data=notin.&select=int_data" `shouldRespondWith`
-        [json| [{int_data: 1}] |] { matchHeaders = [matchContentTypeJson] }
-
     it "returns all results for not.in when no value is present" $
       get "/items_with_different_col_types?int_data=not.in.&select=int_data" `shouldRespondWith`
         [json| [{int_data: 1}] |] { matchHeaders = [matchContentTypeJson] }
@@ -894,9 +957,7 @@ spec = do
       get "/items_with_different_col_types?int_data=in.()" `shouldRespondWith`
         [json| [] |] { matchHeaders = [matchContentTypeJson] }
 
-    it "returns all results when the notin value is empty between parentheses" $ do
-      get "/items_with_different_col_types?int_data=notin.()&select=int_data" `shouldRespondWith`
-        [json| [{int_data: 1}] |] { matchHeaders = [matchContentTypeJson] }
+    it "returns all results when the not.in value is empty between parentheses" $
       get "/items_with_different_col_types?int_data=not.in.()&select=int_data" `shouldRespondWith`
         [json| [{int_data: 1}] |] { matchHeaders = [matchContentTypeJson] }
 

test/fixtures/data.sql

@@ -248,7 +248,9 @@ INSERT INTO nullable_integer VALUES (NULL);
 TRUNCATE TABLE tsearch CASCADE;
 INSERT INTO tsearch VALUES (to_tsvector('It''s kind of fun to do the impossible'));
 INSERT INTO tsearch VALUES (to_tsvector('But also fun to do what is possible'));
-
+INSERT INTO tsearch VALUES (to_tsvector('Fat cats ate rats'));
+INSERT INTO tsearch VALUES (to_tsvector('french', 'C''est un peu amusant de faire l''impossible'));
+INSERT INTO tsearch VALUES (to_tsvector('german', 'Es ist eine Art Spaß, das Unmögliche zu machen')); 
 
 --
 -- Data for Name: users_projects; Type: TABLE DATA; Schema: test; Owner: -