Added tsvector based fulltext search for DataSync

This allows a simple search fulltext search to implemented like this:

function ProductSearch() {
    const [searchQuery, setSearchQuery] = useState('');
    const onChange = useCallback(event => setSearchQuery(event.target.value), [ setSearchQuery ]);

    const products = useQuery(query('products').whereTextSearchStartsWith('ts', searchQuery));

    return <form>
        <input type="text" className="form-control" value={searchQuery} onChange={onChange}/>

        <div>
            Result: {products?.map(product => <div>{product.name}</div>)}
        </div>
    </form>
}

Author: mpscholten

IHP/DataSync/Controller.hs

@@ -1,7 +1,7 @@
 {-# LANGUAGE UndecidableInstances #-}
 module IHP.DataSync.Controller where
 
-import IHP.ControllerPrelude
+import IHP.ControllerPrelude hiding (OrderByClause)
 import qualified Control.Exception as Exception
 import qualified IHP.Log as Log
 import qualified Data.Aeson as Aeson
@@ -354,13 +354,6 @@ changesToValue changes = object (map changeToPair changes)
     where
         changeToPair ChangeNotifications.Change { col, new } = (columnNameToFieldName col) .= new
 
-queryFieldNamesToColumnNames :: SQLQuery -> SQLQuery
-queryFieldNamesToColumnNames sqlQuery = sqlQuery
-        |> modify #orderByClause (map convertOrderByClause)
-    where
-        convertOrderByClause OrderByClause { orderByColumn, orderByDirection } = OrderByClause { orderByColumn = cs (fieldNameToColumnName (cs orderByColumn)), orderByDirection }
-
-
 runInModelContextWithTransaction :: (?state :: IORef DataSyncController, _) => ((?modelContext :: ModelContext) => IO result) -> Maybe UUID -> IO result
 runInModelContextWithTransaction function (Just transactionId) = do
     let globalModelContext = ?modelContext

IHP/DataSync/DynamicQuery.hs

@@ -1,11 +1,12 @@
+{-# LANGUAGE DeriveAnyClass #-}
 {-|
 Module: IHP.DataSync.DynamicQuery
 Description: The normal IHP query functionality is type-safe. This module provides type-unsafe access to the database.
 Copyright: (c) digitally induced GmbH, 2021
 -}
 module IHP.DataSync.DynamicQuery where
 
-import IHP.ControllerPrelude
+import IHP.ControllerPrelude hiding (OrderByClause)
 import Data.Aeson
 import qualified Data.HashMap.Strict as HashMap
 import qualified Database.PostgreSQL.Simple as PG
@@ -16,6 +17,8 @@ import qualified Database.PostgreSQL.Simple.Types as PG
 import qualified Database.PostgreSQL.Simple.Notification as PG
 import qualified IHP.QueryBuilder as QueryBuilder
 import Data.Aeson.TH
+import qualified GHC.Generics
+import qualified Control.DeepSeq as DeepSeq
 
 data Field = Field { fieldName :: Text, fieldValue :: DynamicValue }
 
@@ -45,14 +48,26 @@ data DynamicSQLQuery = DynamicSQLQuery
     , offset :: !(Maybe Int)
     } deriving (Show, Eq)
 
+data OrderByClause
+    = OrderByClause
+        { orderByColumn :: !ByteString
+        , orderByDirection :: !OrderByDirection }
+    | OrderByTSRank { tsvector :: Text, tsquery :: !Text }
+    deriving (Show, Eq, GHC.Generics.Generic, DeepSeq.NFData)
+
 -- | Represents a WHERE conditions of a 'DynamicSQLQuery'
 data ConditionExpression
     = ColumnExpression { field :: !Text }
     | NullExpression
     | InfixOperatorExpression { left :: !ConditionExpression, op :: !ConditionOperator, right :: !ConditionExpression }
     | LiteralExpression { value :: !DynamicValue }
+    | CallExpression { functionCall :: !FunctionCall }
     deriving (Show, Eq)
 
+data FunctionCall
+    = ToTSQuery { text :: !Text } -- ^ to_tsquery('english', text)
+    deriving (Show, Eq, GHC.Generics.Generic, DeepSeq.NFData)
+
 -- | Operators available in WHERE conditions
 data ConditionOperator
     = OpEqual -- ^ a = b
@@ -65,6 +80,7 @@ data ConditionOperator
     | OpOr -- ^ a OR b
     | OpIs -- ^ a IS b
     | OpIsNot -- ^ a IS NOT b
+    | OpTSMatch -- ^ tsvec_a @@ tsvec_b
     deriving (Show, Eq)
 
 data SelectedColumns
@@ -148,6 +164,7 @@ $(deriveFromJSON defaultOptions 'SelectAll)
 $(deriveFromJSON defaultOptions ''ConditionOperator)
 $(deriveFromJSON defaultOptions ''ConditionExpression)
 $(deriveFromJSON defaultOptions ''DynamicValue)
+$(deriveFromJSON defaultOptions ''FunctionCall)
 
 instance FromJSON DynamicSQLQuery where
     parseJSON = withObject "DynamicSQLQuery" $ \v -> DynamicSQLQuery
@@ -156,4 +173,19 @@ instance FromJSON DynamicSQLQuery where
         <*> v .: "whereCondition"
         <*> v .: "orderByClause"
         <*> v .:? "limit" -- Limit can be absent in older versions of ihp-datasync.js
-        <*> v .:? "offset" -- Offset can be absent in older versions of ihp-datasync.js
+        <*> v .:? "offset" -- Offset can be absent in older versions of ihp-datasync.js
+
+
+instance FromJSON OrderByClause where
+    parseJSON = withObject "OrderByClause" $ \v -> do
+        let oldFormat = OrderByClause
+                <$> v .: "orderByColumn"
+                <*> v .: "orderByDirection"
+        let tagged = do
+                tag <- v .: "tag"
+                case tag of
+                    "OrderByClause" -> oldFormat
+                    "OrderByTSRank" -> OrderByTSRank <$> v .: "tsvector" <*> v .: "tsquery"
+                    otherwise -> error ("Invalid tag: " <> otherwise)
+        tagged <|> oldFormat
+

IHP/DataSync/DynamicQueryCompiler.hs

@@ -32,15 +32,20 @@ compileQuery DynamicSQLQuery { .. } = (sql, args)
         (orderBySql, orderByArgs) = case orderByClause of
                 [] -> ("", [])
                 orderByClauses ->
-                    ( PG.Query $ cs $ " ORDER BY " <> (intercalate ", " (map (const "? ?") orderByClauses))
+                    ( PG.Query $ cs $ " ORDER BY " <> (intercalate ", " (map compileOrderByClause orderByClauses))
                     , orderByClauses
-                        |> map (\QueryBuilder.OrderByClause { orderByColumn, orderByDirection } ->
+                        |> map (\case
+                            OrderByClause { orderByColumn, orderByDirection } ->
                                     [ PG.toField $ PG.Identifier (fieldNameToColumnName $ cs orderByColumn)
                                     , PG.toField $ if orderByDirection == QueryBuilder.Desc
                                         then PG.Plain "DESC"
                                         else PG.Plain ""
                                     ]
-                                )
+                            OrderByTSRank { tsvector, tsquery } ->
+                                    [ PG.toField $ PG.Identifier (fieldNameToColumnName tsvector)
+                                    , PG.toField tsquery
+                                    ]
+                        )
                         |> concat
                     )
 
@@ -56,6 +61,10 @@ compileQuery DynamicSQLQuery { .. } = (sql, args)
                 Just offset -> (" OFFSET ?", [PG.toField offset])
                 Nothing -> ("", [])
 
+compileOrderByClause :: OrderByClause -> Text
+compileOrderByClause OrderByClause {} = "? ?"
+compileOrderByClause OrderByTSRank { tsvector, tsquery } = "ts_rank(?, to_tsquery('english', ?))"
+
 compileSelectedColumns :: SelectedColumns -> PG.Action
 compileSelectedColumns SelectAll = PG.Plain "*"
 compileSelectedColumns (SelectSpecific fields) = PG.Many args
@@ -93,6 +102,7 @@ compileCondition (LiteralExpression literal) = ("?", [toValue literal])
         toValue (DateTimeValue utcTime) = PG.toField utcTime
         toValue (PointValue point) = PG.toField point
         toValue Null = PG.toField PG.Null
+compileCondition (CallExpression { functionCall = ToTSQuery { text } }) = ("to_tsquery('english', ?)", [PG.toField text])
 
 compileOperator :: ConditionOperator -> PG.Query
 compileOperator OpEqual = "="
@@ -104,4 +114,5 @@ compileOperator OpNotEqual = "<>"
 compileOperator OpAnd = "AND"
 compileOperator OpOr = "OR"
 compileOperator OpIs = "IS"
-compileOperator OpIsNot = "IS NOT"
+compileOperator OpIsNot = "IS NOT"
+compileOperator OpTSMatch = "@@"

IHP/DataSync/REST/Controller.hs

@@ -1,7 +1,7 @@
 {-# LANGUAGE UndecidableInstances #-}
 module IHP.DataSync.REST.Controller where
 
-import IHP.ControllerPrelude
+import IHP.ControllerPrelude hiding (OrderByClause)
 import IHP.DataSync.REST.Types
 import Data.Aeson
 import Data.Aeson.TH

Test/DataSync/DynamicQueryCompiler.hs

@@ -7,7 +7,7 @@ import Test.Hspec
 import IHP.Prelude
 import IHP.DataSync.DynamicQueryCompiler
 import IHP.DataSync.DynamicQuery
-import IHP.QueryBuilder
+import IHP.QueryBuilder hiding (OrderByClause)
 import qualified Database.PostgreSQL.Simple.ToField as PG
 
 tests = do
@@ -164,4 +164,19 @@ tests = do
                 compileQuery query `shouldBe`
                         ( "SELECT ? FROM ? WHERE (?) IS NOT NULL"
                         , [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "user_id"]
+                        )
+
+            it "compile queries with TS expressions" do
+                let query = DynamicSQLQuery
+                        { table = "products"
+                        , selectedColumns = SelectAll
+                        , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "ts") OpTSMatch (CallExpression { functionCall = ToTSQuery { text = "test" }})
+                        , orderByClause = [ OrderByTSRank { tsvector = "ts", tsquery = "test" } ]
+                        , limit = Nothing
+                        , offset = Nothing
+                        }
+                
+                compileQuery query `shouldBe`
+                        ( "SELECT ? FROM ? WHERE (?) @@ (to_tsquery('english', ?)) ORDER BY ts_rank(?, to_tsquery('english', ?))"
+                        , [PG.Plain "*", PG.EscapeIdentifier "products", PG.EscapeIdentifier "ts", PG.Escape "test", PG.EscapeIdentifier "ts", PG.Escape "test"]
                         )

lib/IHP/DataSync/ihp-querybuilder.js

@@ -341,6 +341,34 @@ class QueryBuilder {
     filterWhereGreaterThanOrEqual() {
 	return this.whereGreaterThanOrEqual.apply(this, Array.from(arguments))
     }
+    
+    whereTextSearchStartsWith(field, value) {
+        let normalized = value.trim().split(' ').map(s => s.trim()).filter(v => v.length > 0).join('&');
+        if (normalized.length > 0) {
+            normalized +=  ':*';
+        }
+        const expression = {
+                tag: 'InfixOperatorExpression',
+                left: {
+                    tag: 'ColumnExpression',
+                    field,
+                },
+                op: 'OpTSMatch',
+                right: {
+                    tag: 'CallExpression',
+                    functionCall: {
+                        tag: 'ToTSQuery',
+                        text: normalized
+                    }
+                },
+            };
+
+        this.#addWhereCondition('OpAnd', expression);
+
+        this.query.orderByClause.push({ tag: 'OrderByTSRank', tsvector: field, tsquery: normalized });
+
+        return this;
+    }
 
     or(conditionBuilder) {
 	if (this.query.whereCondition === null) {