Skip to content

Commit 30eda26

Browse files
ArthurSensArthurSens
authored
[processor/tailsampling] Add metric for sampled/not sampled spans (#30485)
**Description:** <Describe what has changed.> Add metrics to measure sampled/not sampled spans. **Link to tracking Issue:** Fixes #30482 **Testing:** <Describe what testing was performed and which tests were added.> None **Documentation:** <Describe the documentation added.> None --------- Signed-off-by: Arthur Silva Sens <[email protected]>
1 parent a852c86 commit 30eda26

File tree

6 files changed

+134
-83
lines changed

6 files changed

+134
-83
lines changed

‎.chloggen/sampled_spans_metrics.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Use this changelog template to create an entry for release notes.
2+
3+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
4+
change_type: enhancement
5+
6+
# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
7+
component: processor/tail_sampling
8+
9+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
10+
note: "Add metrics that measure the number of sampled spans and the number of spans that are dropped due to sampling decisions."
11+
12+
# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
13+
issues: [30482]
14+
15+
# (Optional) One or more lines of additional information to render under the primary note.
16+
# These lines will be padded with 2 spaces and then inserted directly into the document.
17+
# Use pipe (|) for multiline entries.
18+
subtext:
19+
20+
# If your change doesn't affect end users or the exported elements of any package,
21+
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
22+
# Optional: The change log or logs in which this entry should be included.
23+
# e.g. '[user]' or '[user, api]'
24+
# Include 'user' if the change is relevant to end users.
25+
# Include 'api' if there is a change to a library API.
26+
# Default: '[user]'
27+
change_logs: [user]

‎processor/tailsamplingprocessor/factory.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,24 @@ import (
1414
"go.opentelemetry.io/collector/component"
1515
"go.opentelemetry.io/collector/config/configtelemetry"
1616
"go.opentelemetry.io/collector/consumer"
17+
"go.opentelemetry.io/collector/featuregate"
1718
"go.opentelemetry.io/collector/processor"
1819

1920
"github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor/internal/metadata"
2021
)
2122

2223
var onceMetrics sync.Once
2324

25+
var metricStatCountSpansSampledFeatureGate = featuregate.GlobalRegistry().MustRegister(
26+
"processor.tailsamplingprocessor.metricstatcountspanssampled",
27+
featuregate.StageAlpha,
28+
featuregate.WithRegisterDescription("When enabled, a new metric stat_count_spans_sampled will be available in the tail sampling processor. Differently from stat_count_traces_sampled, this metric will count the number of spans sampled or not per sampling policy, where the original counts traces."),
29+
)
30+
31+
func isMetricStatCountSpansSampledEnabled() bool {
32+
return metricStatCountSpansSampledFeatureGate.IsEnabled()
33+
}
34+
2435
// NewFactory returns a new factory for the Tail Sampling processor.
2536
func NewFactory() processor.Factory {
2637
onceMetrics.Do(func() {

‎processor/tailsamplingprocessor/go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ require (
1414
go.opentelemetry.io/collector/config/configtelemetry v0.94.1
1515
go.opentelemetry.io/collector/confmap v0.94.1
1616
go.opentelemetry.io/collector/consumer v0.94.1
17+
go.opentelemetry.io/collector/featuregate v1.1.0
1718
go.opentelemetry.io/collector/pdata v1.1.0
1819
go.opentelemetry.io/collector/processor v0.94.1
1920
go.opentelemetry.io/otel/metric v1.23.1
@@ -33,6 +34,7 @@ require (
3334
github.com/gobwas/glob v0.2.3 // indirect
3435
github.com/gogo/protobuf v1.3.2 // indirect
3536
github.com/golang/protobuf v1.5.3 // indirect
37+
github.com/hashicorp/go-version v1.6.0 // indirect
3638
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
3739
github.com/iancoleman/strcase v0.3.0 // indirect
3840
github.com/json-iterator/go v1.1.12 // indirect

‎processor/tailsamplingprocessor/go.sum

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎processor/tailsamplingprocessor/metrics.go

Lines changed: 76 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ var (
2828
statPolicyEvaluationErrorCount = stats.Int64("sampling_policy_evaluation_error", "Count of sampling policy evaluation errors", stats.UnitDimensionless)
2929

3030
statCountTracesSampled = stats.Int64("count_traces_sampled", "Count of traces that were sampled or not per sampling policy", stats.UnitDimensionless)
31+
statCountSpansSampled = stats.Int64("count_spans_sampled", "Count of spans that were sampled or not per sampling policy", stats.UnitDimensionless)
3132
statCountGlobalTracesSampled = stats.Int64("global_count_traces_sampled", "Global count of traces that were sampled or not by at least one policy", stats.UnitDimensionless)
3233

3334
statDroppedTooEarlyCount = stats.Int64("sampling_trace_dropped_too_early", "Count of traces that needed to be dropped before the configured wait time", stats.UnitDimensionless)
@@ -46,90 +47,82 @@ func samplingProcessorMetricViews(level configtelemetry.Level) []*view.View {
4647
latencyDistributionAggregation := view.Distribution(1, 2, 5, 10, 25, 50, 75, 100, 150, 200, 300, 400, 500, 750, 1000, 2000, 3000, 4000, 5000, 10000, 20000, 30000, 50000)
4748
ageDistributionAggregation := view.Distribution(1, 2, 5, 10, 20, 30, 40, 50, 60, 90, 120, 180, 300, 600, 1800, 3600, 7200)
4849

49-
decisionLatencyView := &view.View{
50-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statDecisionLatencyMicroSec.Name()),
51-
Measure: statDecisionLatencyMicroSec,
52-
Description: statDecisionLatencyMicroSec.Description(),
53-
TagKeys: policyTagKeys,
54-
Aggregation: latencyDistributionAggregation,
55-
}
56-
overallDecisionLatencyView := &view.View{
57-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statOverallDecisionLatencyUs.Name()),
58-
Measure: statOverallDecisionLatencyUs,
59-
Description: statOverallDecisionLatencyUs.Description(),
60-
Aggregation: latencyDistributionAggregation,
61-
}
62-
63-
traceRemovalAgeView := &view.View{
64-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statTraceRemovalAgeSec.Name()),
65-
Measure: statTraceRemovalAgeSec,
66-
Description: statTraceRemovalAgeSec.Description(),
67-
Aggregation: ageDistributionAggregation,
68-
}
69-
lateSpanArrivalView := &view.View{
70-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statLateSpanArrivalAfterDecision.Name()),
71-
Measure: statLateSpanArrivalAfterDecision,
72-
Description: statLateSpanArrivalAfterDecision.Description(),
73-
Aggregation: ageDistributionAggregation,
74-
}
75-
76-
countPolicyEvaluationErrorView := &view.View{
77-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statPolicyEvaluationErrorCount.Name()),
78-
Measure: statPolicyEvaluationErrorCount,
79-
Description: statPolicyEvaluationErrorCount.Description(),
80-
Aggregation: view.Sum(),
81-
}
82-
50+
views := make([]*view.View, 0)
8351
sampledTagKeys := []tag.Key{tagPolicyKey, tagSampledKey}
84-
countTracesSampledView := &view.View{
85-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statCountTracesSampled.Name()),
86-
Measure: statCountTracesSampled,
87-
Description: statCountTracesSampled.Description(),
88-
TagKeys: sampledTagKeys,
89-
Aggregation: view.Sum(),
90-
}
91-
92-
countGlobalTracesSampledView := &view.View{
93-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statCountGlobalTracesSampled.Name()),
94-
Measure: statCountGlobalTracesSampled,
95-
Description: statCountGlobalTracesSampled.Description(),
96-
TagKeys: []tag.Key{tagSampledKey},
97-
Aggregation: view.Sum(),
52+
views = append(views,
53+
&view.View{
54+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statDecisionLatencyMicroSec.Name()),
55+
Measure: statDecisionLatencyMicroSec,
56+
Description: statDecisionLatencyMicroSec.Description(),
57+
TagKeys: policyTagKeys,
58+
Aggregation: latencyDistributionAggregation,
59+
},
60+
&view.View{
61+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statOverallDecisionLatencyUs.Name()),
62+
Measure: statOverallDecisionLatencyUs,
63+
Description: statOverallDecisionLatencyUs.Description(),
64+
Aggregation: latencyDistributionAggregation,
65+
},
66+
&view.View{
67+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statTraceRemovalAgeSec.Name()),
68+
Measure: statTraceRemovalAgeSec,
69+
Description: statTraceRemovalAgeSec.Description(),
70+
Aggregation: ageDistributionAggregation,
71+
},
72+
&view.View{
73+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statLateSpanArrivalAfterDecision.Name()),
74+
Measure: statLateSpanArrivalAfterDecision,
75+
Description: statLateSpanArrivalAfterDecision.Description(),
76+
Aggregation: ageDistributionAggregation,
77+
},
78+
&view.View{
79+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statPolicyEvaluationErrorCount.Name()),
80+
Measure: statPolicyEvaluationErrorCount,
81+
Description: statPolicyEvaluationErrorCount.Description(),
82+
Aggregation: view.Sum(),
83+
},
84+
&view.View{
85+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statCountTracesSampled.Name()),
86+
Measure: statCountTracesSampled,
87+
Description: statCountTracesSampled.Description(),
88+
TagKeys: sampledTagKeys,
89+
Aggregation: view.Sum(),
90+
},
91+
&view.View{
92+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statCountGlobalTracesSampled.Name()),
93+
Measure: statCountGlobalTracesSampled,
94+
Description: statCountGlobalTracesSampled.Description(),
95+
TagKeys: []tag.Key{tagSampledKey},
96+
Aggregation: view.Sum(),
97+
},
98+
&view.View{
99+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statDroppedTooEarlyCount.Name()),
100+
Measure: statDroppedTooEarlyCount,
101+
Description: statDroppedTooEarlyCount.Description(),
102+
Aggregation: view.Sum(),
103+
},
104+
&view.View{
105+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statNewTraceIDReceivedCount.Name()),
106+
Measure: statNewTraceIDReceivedCount,
107+
Description: statNewTraceIDReceivedCount.Description(),
108+
Aggregation: view.Sum(),
109+
},
110+
&view.View{
111+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statTracesOnMemoryGauge.Name()),
112+
Measure: statTracesOnMemoryGauge,
113+
Description: statTracesOnMemoryGauge.Description(),
114+
Aggregation: view.LastValue(),
115+
})
116+
117+
if isMetricStatCountSpansSampledEnabled() {
118+
views = append(views, &view.View{
119+
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statCountSpansSampled.Name()),
120+
Measure: statCountSpansSampled,
121+
Description: statCountSpansSampled.Description(),
122+
TagKeys: sampledTagKeys,
123+
Aggregation: view.Sum(),
124+
})
98125
}
99126

100-
countTraceDroppedTooEarlyView := &view.View{
101-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statDroppedTooEarlyCount.Name()),
102-
Measure: statDroppedTooEarlyCount,
103-
Description: statDroppedTooEarlyCount.Description(),
104-
Aggregation: view.Sum(),
105-
}
106-
countTraceIDArrivalView := &view.View{
107-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statNewTraceIDReceivedCount.Name()),
108-
Measure: statNewTraceIDReceivedCount,
109-
Description: statNewTraceIDReceivedCount.Description(),
110-
Aggregation: view.Sum(),
111-
}
112-
trackTracesOnMemorylView := &view.View{
113-
Name: processorhelper.BuildCustomMetricName(metadata.Type.String(), statTracesOnMemoryGauge.Name()),
114-
Measure: statTracesOnMemoryGauge,
115-
Description: statTracesOnMemoryGauge.Description(),
116-
Aggregation: view.LastValue(),
117-
}
118-
119-
return []*view.View{
120-
decisionLatencyView,
121-
overallDecisionLatencyView,
122-
123-
traceRemovalAgeView,
124-
lateSpanArrivalView,
125-
126-
countPolicyEvaluationErrorView,
127-
128-
countTracesSampledView,
129-
countGlobalTracesSampledView,
130-
131-
countTraceDroppedTooEarlyView,
132-
countTraceIDArrivalView,
133-
trackTracesOnMemorylView,
134-
}
127+
return views
135128
}

‎processor/tailsamplingprocessor/processor.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,13 @@ func (tsp *tailSamplingSpanProcessor) makeDecision(id pcommon.TraceID, trace *sa
308308
mutators,
309309
statCountTracesSampled.M(int64(1)),
310310
)
311+
if isMetricStatCountSpansSampledEnabled() {
312+
_ = stats.RecordWithTags(
313+
p.ctx,
314+
mutators,
315+
statCountSpansSampled.M(trace.SpanCount.Load()),
316+
)
317+
}
311318
metrics.decisionSampled++
312319

313320
case sampling.NotSampled:
@@ -317,6 +324,13 @@ func (tsp *tailSamplingSpanProcessor) makeDecision(id pcommon.TraceID, trace *sa
317324
mutators,
318325
statCountTracesSampled.M(int64(1)),
319326
)
327+
if isMetricStatCountSpansSampledEnabled() {
328+
_ = stats.RecordWithTags(
329+
p.ctx,
330+
mutators,
331+
statCountSpansSampled.M(trace.SpanCount.Load()),
332+
)
333+
}
320334
metrics.decisionNotSampled++
321335
}
322336
}

0 commit comments

Comments
 (0)