Skip to content

feat(minor): Support range and relative in static threshold files [3/4] #330

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MahdiBM wants to merge 3 commits into
base: main
Choose a base branch
from
+510 −169
Open

feat(minor): Support range and relative in static threshold files [3/4] #330

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4c5871f
Run swift-format
MahdiBM Jul 14, 2025
f441e9e
Implement relative and range thresholds from static files
MahdiBM Jul 14, 2025
3137c2c
Format
MahdiBM Aug 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ extension BenchmarkBaseline: Equatable {
public func failsAbsoluteThresholdChecks(
benchmarks: [Benchmark],
p90Thresholds: [BenchmarkIdentifier:
[BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]]
[BenchmarkMetric: BenchmarkThreshold]]
) -> BenchmarkResult.ThresholdDeviations {
var allDeviationResults = BenchmarkResult.ThresholdDeviations()

Expand Down
12 changes: 6 additions & 6 deletions Plugins/BenchmarkTool/BenchmarkTool+Export+JMHFormatter.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,23 @@ extension JMHPrimaryMetric {
let factor = result.metric.countable == false ? 1_000 : 1

for p in percentiles {
percentileValues[String(p)] = Statistics.roundToDecimalplaces(
percentileValues[String(p)] = Statistics.roundToDecimalPlaces(
Double(histogram.valueAtPercentile(p)) / Double(factor),
3
)
}

for value in histogram.recordedValues() {
for _ in 0..<value.count {
recordedValues.append(Statistics.roundToDecimalplaces(Double(value.value) / Double(factor), 3))
recordedValues.append(Statistics.roundToDecimalPlaces(Double(value.value) / Double(factor), 3))
}
}

self.score = Statistics.roundToDecimalplaces(score / Double(factor), 3)
scoreError = Statistics.roundToDecimalplaces(error / Double(factor), 3)
self.score = Statistics.roundToDecimalPlaces(score / Double(factor), 3)
scoreError = Statistics.roundToDecimalPlaces(error / Double(factor), 3)
scoreConfidence = [
Statistics.roundToDecimalplaces(score - error) / Double(factor),
Statistics.roundToDecimalplaces(score + error) / Double(factor),
Statistics.roundToDecimalPlaces(score - error) / Double(factor),
Statistics.roundToDecimalPlaces(score + error) / Double(factor),
]
scorePercentiles = percentileValues
if result.metric.countable {
Expand Down
14 changes: 7 additions & 7 deletions Plugins/BenchmarkTool/BenchmarkTool+Operations.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ extension BenchmarkTool {
let benchmarkReply = try read()

switch benchmarkReply {
case let .list(benchmark):
case .list(let benchmark):
benchmark.executablePath = benchmarkPath
benchmark.target = FilePath(benchmarkPath).lastComponent!.description
if metrics.isEmpty == false {
Expand All @@ -38,7 +38,7 @@ extension BenchmarkTool {
benchmarks.append(benchmark)
case .end:
break outerloop
case let .error(description):
case .error(let description):
failBenchmark(description)
break outerloop
default:
Expand All @@ -55,12 +55,12 @@ extension BenchmarkTool {
let benchmarkReply = try read()

switch benchmarkReply {
case let .result(benchmark: benchmark, results: results):
case .result(let benchmark, let results):
let filteredResults = results.filter { benchmark.configuration.metrics.contains($0.metric) }
benchmarkResults[BenchmarkIdentifier(target: target, name: benchmark.name)] = filteredResults
case .end:
break outerloop
case let .error(description):
case .error(let description):
failBenchmark(description, exitCode: .benchmarkJobFailed, "\(target)/\(benchmark.name)")

benchmarkResults[BenchmarkIdentifier(target: target, name: benchmark.name)] = []
Expand Down Expand Up @@ -100,7 +100,7 @@ extension BenchmarkTool {
case .read:
print("Reading thresholds from \"\(thresholdsPath)\"")

var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]] = [:]
var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThreshold]] = [:]
try benchmarks.forEach { benchmark in
if try shouldIncludeBenchmark(benchmark.baseName) {
if let thresholds = BenchmarkTool.makeBenchmarkThresholds(
Expand Down Expand Up @@ -146,7 +146,7 @@ extension BenchmarkTool {
}
}

var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]] = [:]
var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThreshold]] = [:]

if noProgress == false {
print("")
Expand Down Expand Up @@ -299,7 +299,7 @@ extension BenchmarkTool {
}
}

var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]] =
var p90Thresholds: [BenchmarkIdentifier: [BenchmarkMetric: BenchmarkThreshold]] =
[:]

if let benchmarkPath = checkAbsolutePath { // load statically defined thresholds for .p90
Expand Down
143 changes: 85 additions & 58 deletions Plugins/BenchmarkTool/BenchmarkTool+PrettyPrinting.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -547,88 +547,115 @@ extension BenchmarkTool {
printMarkdown("```")

metrics.forEach { metric in
let filteredDeviations =
deviationResults.filter {
$0.name == nameAndTarget.name
&& $0.target == nameAndTarget.target
&& $0.metric == metric
}
.sorted(by: { $0.uxPriority > $1.uxPriority })

let relativeResults = deviationResults.filter {
$0.name == nameAndTarget.name && $0.target == nameAndTarget.target && $0.metric == metric
&& $0.relative == true
}
let absoluteResults = deviationResults.filter {
$0.name == nameAndTarget.name && $0.target == nameAndTarget.target && $0.metric == metric
&& $0.relative == false
}
let width = 40
let percentileWidth = 15
let percentileWidthFor4Columns = 15
let percentileWidthFor3Columns = 20

let table = TextTable<BenchmarkResult.ThresholdDeviation> {
var columns: [Column] = []
columns.reserveCapacity(4)

let sign =
switch $0.deviation {
case .absolute: "Δ"
case .relative: "%"
case .range: "↔"
}
let unitDescription = metric.countable ? $0.units.description : $0.units.timeDescription
columns.append(
Column(
title: "\(metric.description) (\(unitDescription), \(sign))",
value: $0.percentile,
width: width,
align: .left
)
)

// The baseValue is the new baseline that we're using as the comparison base, so...
if absoluteResults.isEmpty == false {
let absoluteTable = TextTable<BenchmarkResult.ThresholdDeviation> {
[
Column(
title:
"\(metric.description) (\(metric.countable ? $0.units.description : $0.units.timeDescription), Δ)",
value: $0.percentile,
width: width,
align: .left
),
let baseValue = $0.baseValue
func baselineColumn(percentileWidth: Int) -> Column {
Column(
title: "\(comparingBaselineName)",
value: baseValue,
width: percentileWidth,
align: .right
)
}

// If absolute or relative add their columns together
var comparisonValue: Int?
var difference: String?
var tolerance: String?
switch $0.deviation {
case .absolute(let compareTo, let diff, let tol):
comparisonValue = compareTo
difference = diff.description
tolerance = tol.description
case .relative(let compareTo, let diff, let tol):
comparisonValue = compareTo
difference = Statistics.roundToDecimalPlaces(diff, 1).description
tolerance = Statistics.roundToDecimalPlaces(tol, 1).description
case .range:
break
}

if let comparisonValue = comparisonValue,
let difference = difference,
let tolerance = tolerance
{
columns.append(contentsOf: [
Column(
title: "\(baselineName)",
value: $0.comparisonValue,
width: percentileWidth,
value: comparisonValue,
width: percentileWidthFor4Columns,
align: .right
),
baselineColumn(percentileWidth: percentileWidthFor4Columns),
Column(
title: "\(comparingBaselineName)",
value: $0.baseValue,
width: percentileWidth,
title: "Difference \(sign)",
value: difference,
width: percentileWidthFor4Columns,
align: .right
),
Column(title: "Difference Δ", value: $0.difference, width: percentileWidth, align: .right),
Column(
title: "Threshold Δ",
value: $0.differenceThreshold,
width: percentileWidth,
title: "Tolerance \(sign)",
value: tolerance,
width: percentileWidthFor4Columns,
align: .right
),
]
])
}

absoluteTable.print(absoluteResults, style: format.tableStyle)
}

if relativeResults.isEmpty == false {
let relativeTable = TextTable<BenchmarkResult.ThresholdDeviation> {
[
Column(
title:
"\(metric.description) (\(metric.countable ? $0.units.description : $0.units.timeDescription), %)",
value: $0.percentile,
width: width,
align: .left
),
// Otherwise if range, then handle it alone
if case .range(let min, let max) = $0.deviation {
columns.append(contentsOf: [
baselineColumn(percentileWidth: percentileWidthFor3Columns),
Column(
title: "\(baselineName)",
value: $0.comparisonValue,
width: percentileWidth,
title: "Minimum",
value: min,
width: percentileWidthFor3Columns,
align: .right
),
Column(
title: "\(comparingBaselineName)",
value: $0.baseValue,
width: percentileWidth,
title: "Maximum",
value: max,
width: percentileWidthFor3Columns,
align: .right
),
Column(title: "Difference %", value: $0.difference, width: percentileWidth, align: .right),
Column(
title: "Threshold %",
value: $0.differenceThreshold,
width: percentileWidth,
align: .right
),
]
])
}

relativeTable.print(relativeResults, style: format.tableStyle)
return columns
}

table.print(filteredDeviations, style: format.tableStyle)
}
}
}
Expand Down
17 changes: 4 additions & 13 deletions Plugins/BenchmarkTool/BenchmarkTool+ReadP90AbsoluteThresholds.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ extension BenchmarkTool {
static func makeBenchmarkThresholds(
path: String,
benchmarkIdentifier: BenchmarkIdentifier
) -> [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]? {
) -> [BenchmarkMetric: BenchmarkThreshold]? {
var path = FilePath(path)
if path.isAbsolute {
path.append("\(benchmarkIdentifier.target).\(benchmarkIdentifier.name).p90.json")
Expand All @@ -44,8 +44,7 @@ extension BenchmarkTool {
path = cwdPath
}

var p90Thresholds: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold] = [:]
var p90ThresholdsRaw: [String: BenchmarkThresholds.AbsoluteThreshold]?
var p90Thresholds: [BenchmarkMetric: BenchmarkThreshold] = [:]

do {
let fileDescriptor = try FileDescriptor.open(path, .readOnly, options: [], permissions: .ownerRead)
Expand All @@ -66,19 +65,11 @@ extension BenchmarkTool {
readBytes.append(contentsOf: nextBytes)
}

p90ThresholdsRaw = try JSONDecoder()
p90Thresholds = try JSONDecoder()
.decode(
[String: BenchmarkThresholds.AbsoluteThreshold].self,
[BenchmarkMetric: BenchmarkThreshold].self,
from: Data(readBytes)
)

if let p90ThresholdsRaw {
p90ThresholdsRaw.forEach { metric, threshold in
if let metric = BenchmarkMetric(argument: metric) {
p90Thresholds[metric] = threshold
}
}
}
} catch {
print(
"Failed to read file at \(path) [\(String(reflecting: error))] \(Errno(rawValue: errno).description)"
Expand Down
Loading
Loading