Skip to content

Add “in” filter #64

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
winsmith merged 2 commits into from
Nov 28, 2025
Merged

Add “in” filter #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f06d7fe
Add “in” filter
winsmith Nov 28, 2025
e65b877
Fix linter warnings
winsmith Nov 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Sources/DataTransferObjects/Query/CustomQuery+CompileDown.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ public extension CustomQuery {
return filter
case .null:
return filter
case .in:
return filter
case .and(let filterExpression):
return Filter.and(.init(fields: filterExpression.fields.map { compileRelativeFilterInterval(filter: $0) }))
case .or(let filterExpression):
Expand Down
35 changes: 28 additions & 7 deletions Sources/DataTransferObjects/Query/Filter.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,22 +193,34 @@ extension FilterEquals.MatchValue: Codable {
var container = encoder.singleValueContainer()

switch self {
case .string(let value):
case let .string(value):
try container.encode(value)
case .int(let value):
case let .int(value):
try container.encode(value)
case .double(let value):
case let .double(value):
try container.encode(value)
case .arrayString(let value):
case let .arrayString(value):
try container.encode(value)
case .arrayInt(let value):
case let .arrayInt(value):
try container.encode(value)
case .arrayDouble(let value):
case let .arrayDouble(value):
try container.encode(value)
}
}
}

/// The in filter can match input rows against a set of values, where a match occurs if the value is contained in the set.
public struct FilterIn: Codable, Hashable, Equatable, Sendable {
public init(dimension: String, values: [String]) {
self.dimension = dimension
self.values = values
}

public let dimension: String
public let values: [String]
// extractionFn not supported atm
}

/// The null filter matches rows where a column value is null.
public struct FilterNull: Codable, Hashable, Equatable, Sendable {
public init(column: String) {
Expand All @@ -223,7 +235,7 @@ public struct FilterNull: Codable, Hashable, Equatable, Sendable {
public indirect enum Filter: Codable, Hashable, Equatable, Sendable {
/// The selector filter will match a specific dimension with a specific value.
/// Selector filters can be used as the base filters for more complex Boolean
/// expressions of filters.
/// expressions of filters (deprecated -- use equals instead)
case selector(FilterSelector)

/// The column comparison filter is similar to the selector filter, but instead
Expand Down Expand Up @@ -252,6 +264,10 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable {
/// The null filter matches rows where a column value is null.
case null(FilterNull)

/// The in filter can match input rows against a set of values, where a match occurs
/// if the value is contained in the set.
case `in`(FilterIn)

// logical expression filters
case and(FilterExpression)
case or(FilterExpression)
Expand Down Expand Up @@ -280,6 +296,8 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable {
self = try .equals(FilterEquals(from: decoder))
case "null":
self = try .null(FilterNull(from: decoder))
case "in":
self = try .in(FilterIn(from: decoder))
case "and":
self = try .and(FilterExpression(from: decoder))
case "or":
Expand Down Expand Up @@ -321,6 +339,9 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable {
case let .null(null):
try container.encode("null", forKey: .type)
try null.encode(to: encoder)
case let .in(inFilter):
try container.encode("in", forKey: .type)
try inFilter.self.encode(to: encoder)
case let .and(and):
try container.encode("and", forKey: .type)
try and.encode(to: encoder)
Expand Down
54 changes: 27 additions & 27 deletions Sources/DataTransferObjects/QueryGeneration/CustomQuery+Retention.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,58 +4,58 @@ import DateOperations
extension CustomQuery {
func precompiledRetentionQuery() throws -> CustomQuery {
var query = self

// Get the query intervals - we need at least one interval
guard let queryIntervals = intervals ?? relativeIntervals?.map({ QueryTimeInterval.from(relativeTimeInterval: $0) }),
let firstInterval = queryIntervals.first else {
throw QueryGenerationError.keyMissing(reason: "Missing intervals for retention query")
}

let beginDate = firstInterval.beginningDate
let endDate = firstInterval.endDate

// Use the query's granularity to determine retention period, defaulting to month if not specified
let retentionGranularity = query.granularity ?? .month

// Validate minimum interval based on granularity
try validateMinimumInterval(from: beginDate, to: endDate, granularity: retentionGranularity)

// Split into intervals based on the specified granularity
let retentionIntervals = try splitIntoIntervals(from: beginDate, to: endDate, granularity: retentionGranularity)

// Generate Aggregators
var aggregators = [Aggregator]()
for interval in retentionIntervals {
aggregators.append(aggregator(for: interval))
}

// Generate Post-Aggregators
var postAggregators = [PostAggregator]()
for row in retentionIntervals {
for column in retentionIntervals where column >= row {
postAggregators.append(postAggregatorBetween(interval1: row, interval2: column))
}
}

// Set the query properties
query.queryType = .groupBy
query.granularity = .all
query.aggregations = uniqued(aggregators)
query.postAggregations = uniqued(postAggregators)

return query
}

private func uniqued<T: Hashable>(_ array: [T]) -> [T] {
var set = Set<T>()
return array.filter { set.insert($0).inserted }
}

// MARK: - Helper Methods

private func validateMinimumInterval(from beginDate: Date, to endDate: Date, granularity: QueryGranularity) throws {
let calendar = Calendar.current

switch granularity {
case .day:
let components = calendar.dateComponents([.day], from: beginDate, to: endDate)
Expand Down Expand Up @@ -86,11 +86,11 @@ extension CustomQuery {
throw QueryGenerationError.notImplemented(reason: "Retention queries support day, week, month, quarter, or year granularity")
}
}

private func splitIntoIntervals(from fromDate: Date, to toDate: Date, granularity: QueryGranularity) throws -> [DateInterval] {
let calendar = Calendar.current
var intervals = [DateInterval]()

switch granularity {
case .day:
let numberOfDays = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .day)
Expand All @@ -100,7 +100,7 @@ extension CustomQuery {
let endOfDay = startOfDay.end(of: .day) ?? startOfDay
intervals.append(DateInterval(start: startOfDay, end: endOfDay))
}

case .week:
let numberOfWeeks = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .weekOfYear)
for week in 0...numberOfWeeks {
Expand All @@ -109,7 +109,7 @@ extension CustomQuery {
let endOfWeek = startOfWeek.end(of: .weekOfYear) ?? startOfWeek
intervals.append(DateInterval(start: startOfWeek, end: endOfWeek))
}

case .month:
let numberOfMonths = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .month)
for month in 0...numberOfMonths {
Expand All @@ -118,7 +118,7 @@ extension CustomQuery {
let endOfMonth = startOfMonth.end(of: .month) ?? startOfMonth
intervals.append(DateInterval(start: startOfMonth, end: endOfMonth))
}

case .quarter:
let numberOfQuarters = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .quarter)
for quarter in 0...numberOfQuarters {
Expand All @@ -127,7 +127,7 @@ extension CustomQuery {
let endOfQuarter = startOfQuarter.end(of: .quarter) ?? startOfQuarter
intervals.append(DateInterval(start: startOfQuarter, end: endOfQuarter))
}

case .year:
let numberOfYears = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .year)
for year in 0...numberOfYears {
Expand All @@ -136,18 +136,18 @@ extension CustomQuery {
let endOfYear = startOfYear.end(of: .year) ?? startOfYear
intervals.append(DateInterval(start: startOfYear, end: endOfYear))
}

default:
throw QueryGenerationError.notImplemented(reason: "Retention queries support day, week, month, quarter, or year granularity")
}

return intervals
}

private func numberOfUnitsBetween(beginDate: Date, endDate: Date, component: Calendar.Component) -> Int {
let calendar = Calendar.current
let components = calendar.dateComponents([component], from: beginDate, to: endDate)

switch component {
case .day:
return components.day ?? 0
Expand All @@ -163,13 +163,13 @@ extension CustomQuery {
return 0
}
}

private func title(for interval: DateInterval) -> String {
let formatter = ISO8601DateFormatter()
formatter.formatOptions = [.withFullDate]
return "\(formatter.string(from: interval.start))_\(formatter.string(from: interval.end))"
}

private func aggregator(for interval: DateInterval) -> Aggregator {
.filtered(.init(
filter: .interval(.init(
Expand All @@ -182,7 +182,7 @@ extension CustomQuery {
))
))
}

private func postAggregatorBetween(interval1: DateInterval, interval2: DateInterval) -> PostAggregator {
.thetaSketchEstimate(.init(
name: "retention_\(title(for: interval1))_\(title(for: interval2))",
Expand All @@ -195,4 +195,4 @@ extension CustomQuery {
))
))
}
}
}
36 changes: 18 additions & 18 deletions Tests/QueryGenerationTests/RetentionQueryGenerationTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,57 +129,57 @@ final class RetentionQueryGenerationTests: XCTestCase {
granularity: .month
)
XCTAssertThrowsError(try monthQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))

let monthQuery2 = CustomQuery(
queryType: .retention,
dataSource: "com.telemetrydeck.all",
intervals: [QueryTimeInterval(beginningDate: begin_august, endDate: end_august)],
granularity: .month
)
XCTAssertThrowsError(try monthQuery2.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))

let monthQuery3 = CustomQuery(
queryType: .retention,
dataSource: "com.telemetrydeck.all",
intervals: [QueryTimeInterval(beginningDate: begin_august, endDate: end_september)],
granularity: .month
)
XCTAssertNoThrow(try monthQuery3.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))

// Test daily retention
let startDate = Date(iso8601String: "2022-08-01T00:00:00.000Z")!
let sameDay = Date(iso8601String: "2022-08-01T12:00:00.000Z")!
let nextDay = Date(iso8601String: "2022-08-02T00:00:00.000Z")!

let dayQuery1 = CustomQuery(
queryType: .retention,
dataSource: "com.telemetrydeck.all",
intervals: [QueryTimeInterval(beginningDate: startDate, endDate: sameDay)],
granularity: .day
)
XCTAssertThrowsError(try dayQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))

let dayQuery2 = CustomQuery(
queryType: .retention,
dataSource: "com.telemetrydeck.all",
intervals: [QueryTimeInterval(beginningDate: startDate, endDate: nextDay)],
granularity: .day
)
XCTAssertNoThrow(try dayQuery2.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))

// Test weekly retention
let weekStart = Date(iso8601String: "2022-08-01T00:00:00.000Z")!
let weekMid = Date(iso8601String: "2022-08-05T00:00:00.000Z")!
let weekEnd = Date(iso8601String: "2022-08-08T00:00:00.000Z")!

let weekQuery1 = CustomQuery(
queryType: .retention,
dataSource: "com.telemetrydeck.all",
intervals: [QueryTimeInterval(beginningDate: weekStart, endDate: weekMid)],
granularity: .week
)
XCTAssertThrowsError(try weekQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false))

let weekQuery2 = CustomQuery(
queryType: .retention,
dataSource: "com.telemetrydeck.all",
Expand All @@ -204,22 +204,22 @@ final class RetentionQueryGenerationTests: XCTestCase {
)],
granularity: .month // Explicitly set to month
)

let compiledQuery = try query.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true)

// Verify the compiled query has the expected structure
XCTAssertEqual(compiledQuery.queryType, .groupBy)
XCTAssertEqual(compiledQuery.granularity, .all)
XCTAssertNotNil(compiledQuery.aggregations)
XCTAssertNotNil(compiledQuery.postAggregations)

// The generated query should match the expected structure from tinyQuery
// (though the exact aggregator names might differ due to date formatting)
}

func testRetentionWithDifferentGranularities() throws {
let appID = UUID(uuidString: "79167A27-EBBF-4012-9974-160624E5D07B")!

// Test daily retention - 7 days should generate 8 intervals (0-7 inclusive)
let dailyQuery = CustomQuery(
queryType: .retention,
Expand All @@ -233,12 +233,12 @@ final class RetentionQueryGenerationTests: XCTestCase {
)],
granularity: .day
)

let compiledDailyQuery = try dailyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true)
XCTAssertEqual(compiledDailyQuery.aggregations?.count, 7) // 7 days
// Post-aggregations should be n*(n+1)/2 for n intervals
XCTAssertEqual(compiledDailyQuery.postAggregations?.count, 28) // 7*8/2 = 28

// Test weekly retention - 4 weeks
let weeklyQuery = CustomQuery(
queryType: .retention,
Expand All @@ -252,11 +252,11 @@ final class RetentionQueryGenerationTests: XCTestCase {
)],
granularity: .week
)

let compiledWeeklyQuery = try weeklyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true)
XCTAssertEqual(compiledWeeklyQuery.aggregations?.count, 5) // 5 weeks (spans into 5th week)
XCTAssertEqual(compiledWeeklyQuery.postAggregations?.count, 15) // 5*6/2 = 15

// Test monthly retention - 3 months
let monthlyQuery = CustomQuery(
queryType: .retention,
Expand All @@ -270,7 +270,7 @@ final class RetentionQueryGenerationTests: XCTestCase {
)],
granularity: .month
)

let compiledMonthlyQuery = try monthlyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true)
XCTAssertEqual(compiledMonthlyQuery.aggregations?.count, 3) // 3 months
XCTAssertEqual(compiledMonthlyQuery.postAggregations?.count, 6) // 3*4/2 = 6
Expand Down
Loading
Loading