Allow dictionary and plain/phrase in fts

This commit is contained in:
steve-chavez
2017-09-08 19:56:00 -05:00
parent d98a05023d
commit dff4d766a8
7 changed files with 175 additions and 65 deletions
+2
View File
@@ -7,6 +7,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Added
- #887, #601, Allow specifying dictionary and plain/phrase tsquery in full text search - @steve-chavez
### Fixed
## [0.4.3.0] - 2017-09-06
+32 -23
View File
@@ -1,6 +1,6 @@
module PostgREST.Parsers where
import Protolude hiding (try, intercalate, replace)
import Protolude hiding (try, intercalate, replace, option)
import Control.Monad ((>>))
import Data.Foldable (foldl1)
import qualified Data.HashMap.Strict as M
@@ -21,7 +21,7 @@ pRequestFilter :: (Text, Text) -> Either ApiRequestError (EmbedPath, Filter)
pRequestFilter (k, v) = mapError $ (,) <$> path <*> (Filter <$> fld <*> oper)
where
treePath = parse pTreePath ("failed to parser tree path (" ++ toS k ++ ")") $ toS k
oper = parse (pOperation pVText pVTextL) ("failed to parse filter (" ++ toS v ++ ")") $ toS v
oper = parse (pOpExpr pSingleVal pListVal) ("failed to parse filter (" ++ toS v ++ ")") $ toS v
path = fst <$> treePath
fld = snd <$> treePath
@@ -96,7 +96,6 @@ pFieldName = do
dash :: Parser Char
dash = isDash *> pure '-'
pJsonPathStep :: Parser Text
pJsonPathStep = toS <$> try (string "->" *> pFieldName)
@@ -131,26 +130,36 @@ pFieldSelect = lexeme $
s <- pStar
return ((s, Nothing), Nothing, Nothing, Nothing)
pOperation :: Parser Operand -> Parser Operand -> Parser Operation
pOperation parserVText parserVTextL = try ( string "not" *> pDelimiter *> (Operation True <$> pExpr)) <|> Operation False <$> pExpr
pOpExpr :: Parser Text -> Parser [Text] -> Parser OpExpr
pOpExpr pSVal pLVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)) <|> OpExpr False <$> pOperation
where
pExpr :: Parser (Operator, Operand)
pExpr =
((,) <$> (toS <$> foldl1 (<|>) (try . ((<* pDelimiter) . string) . toS <$> M.keys notInOps)) <*> parserVText)
<|> ((,) <$> (toS <$> foldl1 (<|>) (try . ((<* pDelimiter) . string) . toS <$> M.keys inOps)) <*> parserVTextL)
pOperation :: Parser Operation
pOperation =
Op . toS <$> foldl1 (<|>) (try . ((<* pDelimiter) . string) . toS <$> M.keys ops) <*> pSVal
<|> In . toS <$> (string "in" <* pDelimiter) <*> pLVal
<|> In . toS <$> (string "notin" <* pDelimiter) <*> pLVal
<|> pFts
<?> "operator (eq, gt, ...)"
inOps = M.filterWithKey (const . flip elem ["in", "notin"]) operators
notInOps = M.difference operators inOps
pFts = do
mode <- option Normal $
try (string "phrase" *> pDelimiter *> pure Phrase)
<|> try (string "plain" *> pDelimiter *> pure Plain)
pVText :: Parser Operand
pVText = VText . toS <$> many anyChar
lang <- try (Just <$> manyTill (letter <|> digit <|> oneOf "_") (try (string ".fts") <|> try (string ".@@")) <* pDelimiter) -- TODO: '@@' deprecated
<|> try (string "fts" *> pDelimiter) *> pure Nothing
<|> try (string "@@" *> pDelimiter) *> pure Nothing -- TODO: '@@' deprecated
Fts mode (toS <$> lang) <$> pSVal
ops = M.filterWithKey (const . flip notElem ["in", "notin", "fts", "@@"]) operators -- TODO: '@@' deprecated
pVTextL :: Parser Operand
pVTextL = VTextL <$> try (lexeme (char '(') *> pVTextLElement `sepBy1` char ',' <* lexeme (char ')'))
<|> VTextL <$> lexeme pVTextLElement `sepBy1` char ','
pSingleVal :: Parser Text
pSingleVal = toS <$> many anyChar
pVTextLElement :: Parser Text
pVTextLElement = try pQuotedValue <|> (toS <$> many (noneOf ",)"))
pListVal :: Parser [Text]
pListVal = try (lexeme (char '(') *> pListElement `sepBy1` char ',' <* lexeme (char ')'))
<|> lexeme pListElement `sepBy1` char ','
pListElement :: Parser Text
pListElement = try pQuotedValue <|> (toS <$> many (noneOf ",)"))
pQuotedValue :: Parser Text
pQuotedValue = toS <$> (char '"' *> many (noneOf "\"") <* char '"' <* notFollowedBy (noneOf ",)"))
@@ -182,7 +191,7 @@ pLogicTree = Stmnt <$> try pLogicFilter
<|> Expr <$> pNot <*> pLogicOp <*> (lexeme (char '(') *> pLogicTree `sepBy1` lexeme (char ',') <* lexeme (char ')'))
where
pLogicFilter :: Parser Filter
pLogicFilter = Filter <$> pField <* pDelimiter <*> pOperation pLogicVText pLogicVTextL
pLogicFilter = Filter <$> pField <* pDelimiter <*> pOpExpr pLogicSingleVal pLogicListVal
pNot :: Parser Bool
pNot = try (string "not" *> pDelimiter *> pure True)
<|> pure False
@@ -192,8 +201,8 @@ pLogicTree = Stmnt <$> try pLogicFilter
<|> string "or" *> pure Or
<?> "logic operator (and, or)"
pLogicVText :: Parser Operand
pLogicVText = VText <$> (try pQuotedValue <|> try pPgArray <|> (toS <$> many (noneOf ",)")))
pLogicSingleVal :: Parser Text
pLogicSingleVal = try pQuotedValue <|> try pPgArray <|> (toS <$> many (noneOf ",)"))
where
pPgArray :: Parser Text
pPgArray = do
@@ -202,8 +211,8 @@ pLogicVText = VText <$> (try pQuotedValue <|> try pPgArray <|> (toS <$> many (no
c <- string "}"
toS <$> pure (a ++ b ++ c)
pLogicVTextL :: Parser Operand
pLogicVTextL = VTextL <$> (lexeme (char '(') *> pVTextLElement `sepBy1` char ',' <* lexeme (char ')'))
pLogicListVal :: Parser [Text]
pLogicListVal = lexeme (char '(') *> pListElement `sepBy1` char ',' <* lexeme (char ')')
pLogicPath :: Parser (EmbedPath, Text)
pLogicPath = do
+22 -16
View File
@@ -208,7 +208,7 @@ requestToCountQuery schema (DbRead (Node (Select _ _ logicForest _ _, (mainTbl,
qi = removeSourceCTESchema schema mainTbl
-- all foreing key filters are root nodes(see addFilterToLogicForest), only those are filtered
nonFKRoot :: LogicTree -> Bool
nonFKRoot (Stmnt (Filter _ Operation{expr=(_, VForeignKey _ _)})) = False
nonFKRoot (Stmnt (Filter _ (OpExpr _ (Join _ _)))) = False
nonFKRoot (Stmnt _) = True
nonFKRoot Expr{} = True
filteredLogic = filter nonFKRoot logicForest
@@ -253,7 +253,7 @@ requestToQuery schema isParent (DbRead (Node (Select colSelects tbls logicForest
where
node_name = fromMaybe name alias
local_table_name = table <> "_" <> node_name
replaceTableName localTableName (Filter a (Operation b (c, VForeignKey (QualifiedIdentifier "" _) d))) = Filter a (Operation b (c, VForeignKey (QualifiedIdentifier "" localTableName) d))
replaceTableName localTableName (Filter a (OpExpr b (Join (QualifiedIdentifier "" _) c))) = Filter a (OpExpr b (Join (QualifiedIdentifier "" localTableName) c))
replaceTableName _ x = x
sel = "row_to_json(" <> pgFmtIdent local_table_name <> ".*) AS " <> pgFmtIdent node_name
joi = " LEFT OUTER JOIN ( " <> subquery <> " ) AS " <> pgFmtIdent local_table_name <>
@@ -385,7 +385,7 @@ getJoinFilters (Relation t cols ft fcs typ lt lc1 lc2) =
ftN = tableName ft
ltN = fromMaybe "" (tableName <$> lt)
toFilter :: Text -> Text -> Column -> Column -> Filter
toFilter tb ftb c fc = Filter (colName c, Nothing) (Operation False ("=", VForeignKey (QualifiedIdentifier s tb) (ForeignKey fc{colTable=(colTable fc){tableName=ftb}})))
toFilter tb ftb c fc = Filter (colName c, Nothing) (OpExpr False (Join (QualifiedIdentifier s tb) (ForeignKey fc{colTable=(colTable fc){tableName=ftb}})))
unicodeStatement :: Text -> HE.Params a -> HD.Result b -> Bool -> H.Query a b
unicodeStatement = H.statement . T.encodeUtf8
@@ -413,19 +413,25 @@ pgFmtSelectItem table (f@(_, jp), Nothing, alias, _) = pgFmtField table f <> pgF
pgFmtSelectItem table (f@(_, jp), Just cast, alias, _) = "CAST (" <> pgFmtField table f <> " AS " <> cast <> " )" <> pgFmtAs jp alias
pgFmtFilter :: QualifiedIdentifier -> Filter -> SqlFragment
pgFmtFilter table (Filter fld (Operation hasNot_ ex)) = notOp <> " " <> case ex of
(op, VText val) -> pgFmtFieldOp op <> " " <> case op of
"like" -> unknownLiteral (T.map star val)
"ilike" -> unknownLiteral (T.map star val)
-- TODO: The '@@' was deprecated, remove in v0.5.0.0
"@@" -> "to_tsquery(" <> unknownLiteral val <> ") "
"fts" -> "to_tsquery(" <> unknownLiteral val <> ") "
"is" -> whiteList val
"isnot" -> whiteList val
_ -> unknownLiteral val
(op, VTextL vals) -> pgFmtIn op vals -- in and notin
(op, VForeignKey fQi (ForeignKey Column{colTable=Table{tableName=fTableName}, colName=fColName})) ->
pgFmtField fQi fld <> " " <> sqlOperator op <> " " <> pgFmtColumn (removeSourceCTESchema (qiSchema fQi) fTableName) fColName
pgFmtFilter table (Filter fld (OpExpr hasNot_ oper)) = notOp <> " " <> case oper of
Op op val -> pgFmtFieldOp op <> " " <> case op of
"like" -> unknownLiteral (T.map star val)
"ilike" -> unknownLiteral (T.map star val)
"is" -> whiteList val
"isnot" -> whiteList val
_ -> unknownLiteral val
In op vals -> pgFmtIn op vals -- in and notin
Fts mode lang val ->
pgFmtFieldOp "fts" <> " " <> case mode of
Normal -> "to_tsquery("
Plain -> "plainto_tsquery("
Phrase -> "phraseto_tsquery("
<> maybe "" (flip (<>) ", " . pgFmtLit) lang <> unknownLiteral val <> ") "
Join fQi (ForeignKey Column{colTable=Table{tableName=fTableName}, colName=fColName}) ->
pgFmtField fQi fld <> " = " <> pgFmtColumn (removeSourceCTESchema (qiSchema fQi) fTableName) fColName
where
pgFmtFieldOp op = pgFmtField table fld <> " " <> sqlOperator op
sqlOperator o = HM.lookupDefault "=" o operators
+10 -3
View File
@@ -174,8 +174,15 @@ operators = M.fromList [
("@@", "@@"),
("@>", "@>"),
("<@", "<@")]
data Operation = Operation{ hasNot::Bool, expr::(Operator, Operand) } deriving (Eq, Show)
data Operand = VText Text | VTextL [Text] | VForeignKey QualifiedIdentifier ForeignKey deriving (Show, Eq)
data OpExpr = OpExpr Bool Operation deriving (Eq, Show)
data Operation = Op Operator Text |
In Operator [Text] |
Fts FtsMode (Maybe Language) Text |
Join QualifiedIdentifier ForeignKey deriving (Eq, Show)
data FtsMode = Normal | Plain | Phrase deriving (Eq, Show)
type Language = Text
data LogicOperator = And | Or deriving Eq
instance Show LogicOperator where
@@ -207,7 +214,7 @@ type RelationDetail = Text
type SelectItem = (Field, Maybe Cast, Maybe Alias, Maybe RelationDetail)
-- | Path of the embedded levels, e.g "clients.projects.name=eq.." gives Path ["clients", "projects"]
type EmbedPath = [Text]
data Filter = Filter { field::Field, operation::Operation } deriving (Show, Eq)
data Filter = Filter { field::Field, opExpr::OpExpr } deriving (Show, Eq)
data ReadQuery = Select { select::[SelectItem], from::[TableName], where_::[LogicTree], order::Maybe [OrderTerm], range_::NonnegRange } deriving (Show, Eq)
data MutateQuery = Insert { in_::TableName, qPayload::PayloadJSON, returning::[FieldName] }
+6
View File
@@ -73,6 +73,12 @@ spec =
it "can handle fts" $ do
get "/entities?or=(text_search_vector.fts.bar,text_search_vector.fts.baz)&select=id" `shouldRespondWith`
[json|[{ "id": 1 }, { "id": 2 }]|] { matchHeaders = [matchContentTypeJson] }
get "/tsearch?or=(text_search_vector.phrase.german.fts.Art%20Spass, text_search_vector.plain.french.fts.amusant%20impossible, text_search_vector.english.fts.impossible)" `shouldRespondWith`
[json|[
{"text_search_vector": "'fun':5 'imposs':9 'kind':3" },
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" },
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}
]|] { matchHeaders = [matchContentTypeJson] }
-- TODO: remove in 0.5.0 as deprecated
get "/entities?or=(text_search_vector.@@.bar,text_search_vector.@@.baz)&select=id" `shouldRespondWith`
[json|[{ "id": 1 }, { "id": 2 }]|] { matchHeaders = [matchContentTypeJson] }
+100 -22
View File
@@ -98,29 +98,107 @@ spec = do
it "matches with ilike using not operator" $
get "/simple_pk?k=not.ilike.xy*&order=extra.asc" `shouldRespondWith` "[]"
it "matches with tsearch fts" $ do
get "/tsearch?text_search_vector=fts.impossible" `shouldRespondWith`
[json| [{"text_search_vector": "'fun':5 'imposs':9 'kind':3" }] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=fts.possible" `shouldRespondWith`
[json| [{"text_search_vector": "'also':2 'fun':3 'possibl':8" }] |]
{ matchHeaders = [matchContentTypeJson] }
-- TODO: remove in 0.5.0 as deprecated
get "/tsearch?text_search_vector=@@.impossible" `shouldRespondWith`
[json| [{"text_search_vector": "'fun':5 'imposs':9 'kind':3" }] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=@@.possible" `shouldRespondWith`
[json| [{"text_search_vector": "'also':2 'fun':3 'possibl':8" }] |]
{ matchHeaders = [matchContentTypeJson] }
describe "Full text search operator" $ do
it "finds matches with to_tsquery" $
get "/tsearch?text_search_vector=fts.impossible" `shouldRespondWith`
[json| [{"text_search_vector": "'fun':5 'imposs':9 'kind':3" }] |]
{ matchHeaders = [matchContentTypeJson] }
it "can use lexeme boolean operators(&=%26, |=%7C, !) in to_tsquery" $ do
get "/tsearch?text_search_vector=fts.fun%26possible" `shouldRespondWith`
[json| [ {"text_search_vector": "'also':2 'fun':3 'possibl':8"}] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=fts.impossible%7Cpossible" `shouldRespondWith`
[json| [
{"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
{"text_search_vector": "'also':2 'fun':3 'possibl':8"}] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=fts.fun%26!possible" `shouldRespondWith`
[json| [ {"text_search_vector": "'fun':5 'imposs':9 'kind':3"}] |]
{ matchHeaders = [matchContentTypeJson] }
it "finds matches with plainto_tsquery" $
get "/tsearch?text_search_vector=plain.fts.The%20Fat%20Rats" `shouldRespondWith`
[json| [ {"text_search_vector": "'ate':3 'cat':2 'fat':1 'rat':4" }] |]
{ matchHeaders = [matchContentTypeJson] }
it "finds matches with phraseto_tsquery" $
get "/tsearch?text_search_vector=phrase.fts.The%20Fat%20Cats" `shouldRespondWith`
[json| [{"text_search_vector": "'ate':3 'cat':2 'fat':1 'rat':4" }] |]
{ matchHeaders = [matchContentTypeJson] }
it "finds matches with different dictionaries" $ do
get "/tsearch?text_search_vector=french.fts.amusant" `shouldRespondWith`
[json| [{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" }] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=plain.french.fts.amusant%20impossible" `shouldRespondWith`
[json| [{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" }] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=phrase.german.fts.Art%20Spass" `shouldRespondWith`
[json| [{"text_search_vector": "'art':4 'spass':5 'unmog':7" }] |]
{ matchHeaders = [matchContentTypeJson] }
it "can be negated with not operator" $ do
get "/tsearch?text_search_vector=not.fts.impossible%7Cfat%7Cfun" `shouldRespondWith`
[json| [
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=not.english.fts.impossible%7Cfat%7Cfun" `shouldRespondWith`
[json| [
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=not.plain.fts.The%20Fat%20Rats" `shouldRespondWith`
[json| [
{"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
{"text_search_vector": "'also':2 'fun':3 'possibl':8"},
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=not.phrase.english.fts.The%20Fat%20Cats" `shouldRespondWith`
[json| [
{"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
{"text_search_vector": "'also':2 'fun':3 'possibl':8"},
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
{ matchHeaders = [matchContentTypeJson] }
it "matches with tsearch fts using not operator" $ do
get "/tsearch?text_search_vector=not.fts.impossible" `shouldRespondWith`
[json| [{"text_search_vector": "'also':2 'fun':3 'possibl':8" }] |]
{ matchHeaders = [matchContentTypeJson] }
-- TODO: remove in 0.5.0 as deprecated
get "/tsearch?text_search_vector=not.@@.impossible" `shouldRespondWith`
[json| [{"text_search_vector": "'also':2 'fun':3 'possibl':8" }] |]
{ matchHeaders = [matchContentTypeJson] }
it "Deprecated @@ operator, pending to remove" $ do
get "/tsearch?text_search_vector=@@.impossible" `shouldRespondWith`
[json| [{"text_search_vector": "'fun':5 'imposs':9 'kind':3" }] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=plain.@@.The%20Fat%20Rats" `shouldRespondWith`
[json| [ {"text_search_vector": "'ate':3 'cat':2 'fat':1 'rat':4" }] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=phrase.@@.The%20Fat%20Cats" `shouldRespondWith`
[json| [{"text_search_vector": "'ate':3 'cat':2 'fat':1 'rat':4" }] |]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=not.@@.impossible%7Cfat%7Cfun" `shouldRespondWith`
[json| [
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=not.english.@@.impossible%7Cfat%7Cfun" `shouldRespondWith`
[json| [
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=not.plain.@@.The%20Fat%20Rats" `shouldRespondWith`
[json| [
{"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
{"text_search_vector": "'also':2 'fun':3 'possibl':8"},
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
{ matchHeaders = [matchContentTypeJson] }
get "/tsearch?text_search_vector=not.phrase.english.@@.The%20Fat%20Cats" `shouldRespondWith`
[json| [
{"text_search_vector": "'fun':5 'imposs':9 'kind':3"},
{"text_search_vector": "'also':2 'fun':3 'possibl':8"},
{"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4"},
{"text_search_vector": "'art':4 'spass':5 'unmog':7"}]|]
{ matchHeaders = [matchContentTypeJson] }
it "matches with computed column" $
get "/items?always_true=eq.true&order=id.asc" `shouldRespondWith`
@@ -709,7 +787,7 @@ spec = do
{ matchHeaders = [matchContentTypeJson] }
it "fails if an operator is not given" $
get "/ghostBusters?id=0" `shouldRespondWith` [json| {"details":"unexpected \"0\" expecting \"not\" or operator (eq, gt, ...)","message":"\"failed to parse filter (0)\" (line 1, column 1)"} |]
get "/ghostBusters?id=0" `shouldRespondWith` [json| {"details":"unexpected end of input expecting operator (eq, gt, ...)","message":"\"failed to parse filter (0)\" (line 1, column 2)"} |]
{ matchStatus = 400
, matchHeaders = [matchContentTypeJson]
}
+3 -1
View File
@@ -248,7 +248,9 @@ INSERT INTO nullable_integer VALUES (NULL);
TRUNCATE TABLE tsearch CASCADE;
INSERT INTO tsearch VALUES (to_tsvector('It''s kind of fun to do the impossible'));
INSERT INTO tsearch VALUES (to_tsvector('But also fun to do what is possible'));
INSERT INTO tsearch VALUES (to_tsvector('Fat cats ate rats'));
INSERT INTO tsearch VALUES (to_tsvector('french', 'C''est un peu amusant de faire l''impossible'));
INSERT INTO tsearch VALUES (to_tsvector('german', 'Es ist eine Art Spaß, das Unmögliche zu machen'));
--
-- Data for Name: users_projects; Type: TABLE DATA; Schema: test; Owner: -