Merge pull request #1408 from digitallyinduced/s0kil/fix-issue-1407

DataSync Support distinctOn

Author: mpscholten

Guide/realtime-spas.markdown

@@ -689,7 +689,7 @@ const todos = await query('todos')
         .where('id', 'd94173ec-1d91-421e-8fdc-20a3161b7802')
         .or(where('id', '173ecd94-911d-1e42-dc8f-1b780320a316'))
         .fetch()
-	
+
 // SQL:
 // SELECT * FROM todos
 // WHERE id = 'd94173ec-1d91-421e-8fdc-20a3161b7802'
@@ -711,7 +711,7 @@ const todos = await query('todos')
             )
         )
         .fetch()
-	
+
 // SQL:
 // SELECT * FROM todos
 // WHERE id = 'd94173ec-1d91-421e-8fdc-20a3161b7802'
@@ -754,7 +754,7 @@ const todos = await query('todos')
             )
         )
         .fetch()
-	
+
 // SQL:
 // SELECT * FROM todos
 // WHERE user_id = '173ecd94-911d-1e42-dc8f-1b780320a316'
@@ -791,6 +791,17 @@ const oldestTodos = await query('todos')
         .fetchOne();
 ````
 
+
+#### Unique/Distinct
+
+Use `distinctOn`, to get a unique result:
+
+```javascript
+const userTodos = await query('todos')
+        .distinctOn('userId')
+        .fetch();
+````
+
 ### Create Record
 
 To insert a record into the database, call `createRecord` with a plain javascript object:

IHP/DataSync/DynamicQuery.hs

@@ -47,6 +47,7 @@ data DynamicSQLQuery = DynamicSQLQuery
     , selectedColumns :: SelectedColumns
     , whereCondition :: !(Maybe ConditionExpression)
     , orderByClause :: ![OrderByClause]
+    , distinctOnColumn :: !(Maybe ByteString)
     , limit :: !(Maybe Int)
     , offset :: !(Maybe Int)
     } deriving (Show, Eq)
@@ -178,6 +179,7 @@ instance FromJSON DynamicSQLQuery where
         <*> v .: "selectedColumns"
         <*> v .: "whereCondition"
         <*> v .: "orderByClause"
+        <*> v .:? "distinctOnColumn" -- distinctOnColumn can be absent in older versions of ihp-datasync.js
         <*> v .:? "limit" -- Limit can be absent in older versions of ihp-datasync.js
         <*> v .:? "offset" -- Offset can be absent in older versions of ihp-datasync.js
 
@@ -194,4 +196,3 @@ instance FromJSON OrderByClause where
                     "OrderByTSRank" -> OrderByTSRank <$> v .: "tsvector" <*> v .: "tsquery"
                     otherwise -> error ("Invalid tag: " <> otherwise)
         tagged <|> oldFormat
-

IHP/DataSync/DynamicQueryCompiler.hs

@@ -19,16 +19,21 @@ import qualified Data.List as List
 compileQuery :: DynamicSQLQuery -> (PG.Query, [PG.Action])
 compileQuery DynamicSQLQuery { .. } = (sql, args)
     where
-        sql = "SELECT ? FROM ?" <> whereSql <> orderBySql <> limitSql <> offsetSql
-        args = catMaybes
-                [ Just (compileSelectedColumns selectedColumns)
-                , Just (PG.toField (PG.Identifier table))
-                ]
+        sql = "SELECT" <> distinctOnSql <> "? FROM ?" <> whereSql <> orderBySql <> limitSql <> offsetSql
+        args = distinctOnArgs
+                <> catMaybes
+                    [ Just (compileSelectedColumns selectedColumns)
+                    , Just (PG.toField (PG.Identifier table))
+                    ]
                 <> whereArgs
                 <> orderByArgs
                 <> limitArgs
                 <> offsetArgs
 
+        (distinctOnSql, distinctOnArgs) = case distinctOnColumn of
+            Just column -> (" DISTINCT ON (?) ", [PG.toField $ PG.Identifier (fieldNameToColumnName $ cs column)])
+            Nothing     -> (" ", [])
+
         (orderBySql, orderByArgs) = case orderByClause of
                 [] -> ("", [])
                 orderByClauses ->
@@ -56,7 +61,7 @@ compileQuery DynamicSQLQuery { .. } = (sql, args)
         (limitSql, limitArgs) = case limit of
                 Just limit -> (" LIMIT ?", [PG.toField limit])
                 Nothing -> ("", [])
-        
+
         (offsetSql, offsetArgs) = case offset of
                 Just offset -> (" OFFSET ?", [PG.toField offset])
                 Nothing -> ("", [])
@@ -115,4 +120,4 @@ compileOperator OpAnd = "AND"
 compileOperator OpOr = "OR"
 compileOperator OpIs = "IS"
 compileOperator OpIsNot = "IS NOT"
-compileOperator OpTSMatch = "@@"
+compileOperator OpTSMatch = "@@"

IHP/DataSync/REST/Controller.hs

@@ -55,7 +55,7 @@ instance (
                         |> map aesonValueToPostgresValue
 
                 let params = (PG.Identifier table, PG.In (map PG.Identifier columns), PG.In values)
-                
+
                 result :: Either EnhancedSqlError [[Field]] <- Exception.try do
                     sqlQueryWithRLS query params
 
@@ -87,14 +87,14 @@ instance (
                                 |> HashMap.elems
                                 |> map aesonValueToPostgresValue
                             )
-                        
+
 
                 let params = (PG.Identifier table, PG.In (map PG.Identifier columns), PG.Values [] values)
 
                 result :: [[Field]] <- sqlQueryWithRLS query params
                 renderJson result
 
-        
+
 
     action UpdateRecordAction { table, id } = do
         ensureRLSEnabled table
@@ -170,6 +170,7 @@ buildDynamicQueryFromRequest table = DynamicSQLQuery
     , selectedColumns = paramOrDefault SelectAll "fields"
     , whereCondition = Nothing
     , orderByClause = paramList "orderBy"
+    , distinctOnColumn = paramOrNothing "distinctOnColumn"
     , limit = paramOrNothing "limit"
     , offset = paramOrNothing "offset"
     }
@@ -220,18 +221,18 @@ aesonValueToPostgresValue (Number value) = case Scientific.floatingOrInteger val
     Left (floating :: Double) -> PG.toField floating
     Right (integer :: Integer) -> PG.toField integer
 aesonValueToPostgresValue Data.Aeson.Null = PG.toField PG.Null
-aesonValueToPostgresValue object@(Object values) = 
+aesonValueToPostgresValue object@(Object values) =
     let
         tryDecodeAsPoint :: Maybe Point
         tryDecodeAsPoint = do
                 xValue <- HashMap.lookup "x" values
                 yValue <- HashMap.lookup "y" values
                 x <- case xValue of
                         Number number -> pure (Scientific.toRealFloat number)
-                        otherwise -> Nothing 
+                        otherwise -> Nothing
                 y <- case yValue of
                         Number number -> pure (Scientific.toRealFloat number)
-                        otherwise -> Nothing 
+                        otherwise -> Nothing
                 pure Point { x, y }
     in
         -- This is really hacky and is mostly duck typing. We should refactor this in the future to

Test/DataSync/DynamicQueryCompiler.hs

@@ -19,6 +19,7 @@ tests = do
                         , selectedColumns = SelectAll
                         , whereCondition = Nothing
                         , orderByClause = []
+                        , distinctOnColumn = Nothing
                         , limit = Nothing
                         , offset = Nothing
                         }
@@ -27,12 +28,13 @@ tests = do
                         ( "SELECT ? FROM ?"
                         , [PG.Plain "*", PG.EscapeIdentifier "posts"]
                         )
-            
+
             it "compile a select query with order by" do
                 let query = DynamicSQLQuery
                         { table = "posts"
                         , selectedColumns = SelectAll
                         , whereCondition = Nothing
+                        , distinctOnColumn = Nothing
                         , orderByClause = [OrderByClause { orderByColumn = "title", orderByDirection = Desc }]
                         , limit = Nothing
                         , offset = Nothing
@@ -42,7 +44,7 @@ tests = do
                         ( "SELECT ? FROM ? ORDER BY ? ?"
                         , [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "title", PG.Plain "DESC"]
                         )
-            
+
             it "compile a select query with multiple order bys" do
                 let query = DynamicSQLQuery
                         { table = "posts"
@@ -52,6 +54,7 @@ tests = do
                             OrderByClause { orderByColumn = "createdAt", orderByDirection = Desc },
                             OrderByClause { orderByColumn = "title", orderByDirection = Asc }
                         ]
+                        , distinctOnColumn = Nothing
                         , limit = Nothing
                         , offset = Nothing
                         }
@@ -60,13 +63,14 @@ tests = do
                         ( "SELECT ? FROM ? ORDER BY ? ?, ? ?"
                         , [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "created_at", PG.Plain "DESC", PG.EscapeIdentifier "title", PG.Plain ""]
                         )
-            
+
             it "compile a basic select query with a where condition" do
                 let query = DynamicSQLQuery
                         { table = "posts"
                         , selectedColumns = SelectAll
                         , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
                         , orderByClause = []
+                        , distinctOnColumn = Nothing
                         , limit = Nothing
                         , offset = Nothing
                         }
@@ -82,6 +86,7 @@ tests = do
                         , selectedColumns = SelectAll
                         , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
                         , orderByClause = [ OrderByClause { orderByColumn = "createdAt", orderByDirection = Desc } ]
+                        , distinctOnColumn = Nothing
                         , limit = Nothing
                         , offset = Nothing
                         }
@@ -96,6 +101,7 @@ tests = do
                         { table = "posts"
                         , selectedColumns = SelectAll
                         , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
+                        , distinctOnColumn = Nothing
                         , orderByClause = []
                         , limit = Just 50
                         , offset = Nothing
@@ -112,6 +118,7 @@ tests = do
                         , selectedColumns = SelectAll
                         , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
                         , orderByClause = []
+                        , distinctOnColumn = Nothing
                         , limit = Nothing
                         , offset = Just 50
                         }
@@ -127,25 +134,27 @@ tests = do
                         , selectedColumns = SelectAll
                         , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual (LiteralExpression (TextValue "b8553ce9-6a42-4a68-b5fc-259be3e2acdc"))
                         , orderByClause = []
+                        , distinctOnColumn = Nothing
                         , limit = Just 25
                         , offset = Just 50
                         }
-                
+
                 compileQuery query `shouldBe`
                         ( "SELECT ? FROM ? WHERE (?) = (?) LIMIT ? OFFSET ?"
                         , [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "user_id", PG.Escape "b8553ce9-6a42-4a68-b5fc-259be3e2acdc", PG.Plain "25", PG.Plain "50"]
                         )
-            
+
             it "compile 'field = NULL' conditions to 'field IS NULL'" do
                 let query = DynamicSQLQuery
                         { table = "posts"
                         , selectedColumns = SelectAll
                         , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpEqual NullExpression
                         , orderByClause = []
+                        , distinctOnColumn = Nothing
                         , limit = Nothing
                         , offset = Nothing
                         }
-                
+
                 compileQuery query `shouldBe`
                         ( "SELECT ? FROM ? WHERE (?) IS NULL"
                         , [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "user_id"]
@@ -157,10 +166,11 @@ tests = do
                         , selectedColumns = SelectAll
                         , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "userId") OpNotEqual NullExpression
                         , orderByClause = []
+                        , distinctOnColumn = Nothing
                         , limit = Nothing
                         , offset = Nothing
                         }
-                
+
                 compileQuery query `shouldBe`
                         ( "SELECT ? FROM ? WHERE (?) IS NOT NULL"
                         , [PG.Plain "*", PG.EscapeIdentifier "posts", PG.EscapeIdentifier "user_id"]
@@ -172,11 +182,28 @@ tests = do
                         , selectedColumns = SelectAll
                         , whereCondition = Just $ InfixOperatorExpression (ColumnExpression "ts") OpTSMatch (CallExpression { functionCall = ToTSQuery { text = "test" }})
                         , orderByClause = [ OrderByTSRank { tsvector = "ts", tsquery = "test" } ]
+                        , distinctOnColumn = Nothing
                         , limit = Nothing
                         , offset = Nothing
                         }
-                
+
                 compileQuery query `shouldBe`
                         ( "SELECT ? FROM ? WHERE (?) @@ (to_tsquery('english', ?)) ORDER BY ts_rank(?, to_tsquery('english', ?))"
                         , [PG.Plain "*", PG.EscapeIdentifier "products", PG.EscapeIdentifier "ts", PG.Escape "test", PG.EscapeIdentifier "ts", PG.Escape "test"]
-                        )
+                        )
+
+            it "compile a basic select query with distinctOn" do
+                let query = DynamicSQLQuery
+                        { table = "posts"
+                        , selectedColumns = SelectAll
+                        , whereCondition = Nothing
+                        , orderByClause = []
+                        , distinctOnColumn = Just "groupId"
+                        , limit = Nothing
+                        , offset = Nothing
+                        }
+
+                compileQuery query `shouldBe`
+                        ( "SELECT DISTINCT ON (?) ? FROM ?"
+                        , [PG.EscapeIdentifier "group_id", PG.Plain "*", PG.EscapeIdentifier "posts"]
+                        )

lib/IHP/DataSync/ihp-querybuilder.js

@@ -229,6 +229,7 @@ class QueryBuilder extends ConditionBuildable {
             selectedColumns: { tag: 'SelectAll' },
             whereCondition: null,
             orderByClause: [],
+            distinctOnColumn: null,
             limit: null,
             offset: null
         };
@@ -308,6 +309,12 @@ class QueryBuilder extends ConditionBuildable {
         return this;
     }
 
+    distinctOn(column) {
+        this.query.distinctOnColumn = column
+
+        return this
+    }
+
     limit(limit) {
         if (limit !== null && !Number.isInteger(limit)) {
             throw new Error('limit needs to be an integer, or null if no limit should be used');
@@ -339,7 +346,7 @@ class QueryBuilder extends ConditionBuildable {
         const result = await this.limit(1).fetch();
         return result.length > 0 ? result[0] : null;
     }
-    
+
     subscribe(callback) {
         const dataSubscription = new DataSubscription(this.query);
         dataSubscription.createOnServer();