[datadogexporter] Add support for summary datatype (#3660)

* Add support for summary datatype

* Apply suggestions from code review

Co-authored-by: Kylian Serrania <[email protected]>

Co-authored-by: Kylian Serrania <[email protected]>

Author: mx-psi

exporter/datadogexporter/README.md

@@ -86,3 +86,4 @@ There are a number of optional settings for configuring how to send your metrics
 |-|-|-|
 | `send_monotonic_counter` | Cumulative monotonic metrics are sent as deltas between successive measurements. Disable this flag to send get the raw, monotonically increasing value. | `true` |
 | `delta_ttl` | Maximum number of seconds values from cumulative monotonic metrics are kept in memory. | 3600 |
+| `report_quantiles` | Whether to report quantile values for summary type metrics. | `true` |

exporter/datadogexporter/config/config.go

@@ -63,6 +63,10 @@ type MetricsConfig struct {
 	// Buckets states whether to report buckets from distribution metrics
 	Buckets bool `mapstructure:"report_buckets"`
 
+	// Quantiles states whether to report quantiles from summary metrics.
+	// By default, the minimum, maximum and average are reported.
+	Quantiles bool `mapstructure:"report_quantiles"`
+
 	// SendMonotonic states whether to report cumulative monotonic metrics as counters
 	// or gauges
 	SendMonotonic bool `mapstructure:"send_monotonic_counter"`

exporter/datadogexporter/factory.go

@@ -66,6 +66,7 @@ func createDefaultConfig() config.Exporter {
 			},
 			SendMonotonic: true,
 			DeltaTTL:      3600,
+			Quantiles:     true,
 			ExporterConfig: ddconfig.MetricsExporterConfig{
 				ResourceAttributesAsTags: false,
 			},

exporter/datadogexporter/factory_test.go

@@ -57,6 +57,7 @@ func TestCreateDefaultConfig(t *testing.T) {
 			},
 			DeltaTTL:      3600,
 			SendMonotonic: true,
+			Quantiles:     true,
 		},
 
 		Traces: ddconfig.TracesConfig{
@@ -121,6 +122,7 @@ func TestLoadConfig(t *testing.T) {
 			},
 			DeltaTTL:      3600,
 			SendMonotonic: true,
+			Quantiles:     true,
 		},
 
 		Traces: ddconfig.TracesConfig{
@@ -160,6 +162,7 @@ func TestLoadConfig(t *testing.T) {
 			},
 			SendMonotonic: true,
 			DeltaTTL:      3600,
+			Quantiles:     true,
 		},
 
 		Traces: ddconfig.TracesConfig{
@@ -241,6 +244,7 @@ func TestLoadConfigEnvVariables(t *testing.T) {
 				Endpoint: "https://api.datadoghq.test",
 			},
 			SendMonotonic: true,
+			Quantiles:     false,
 			DeltaTTL:      3600,
 		},
 
@@ -285,6 +289,7 @@ func TestLoadConfigEnvVariables(t *testing.T) {
 			},
 			SendMonotonic: true,
 			DeltaTTL:      3600,
+			Quantiles:     true,
 		},
 
 		Traces: ddconfig.TracesConfig{

exporter/datadogexporter/metrics_translator.go

@@ -17,6 +17,7 @@ package datadogexporter
 import (
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -251,6 +252,52 @@ func mapHistogramMetrics(name string, slice pdata.HistogramDataPointSlice, bucke
 	return ms
 }
 
+// getQuantileTag returns the quantile tag for summary types.
+// Since the summary type is provided as a compatibility feature, we try to format the float number as close as possible to what
+// we do on the Datadog Agent Python OpenMetrics check, which, in turn, tries to
+// follow https://github.com/OpenObservability/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#considerations-canonical-numbers
+func getQuantileTag(quantile float64) string {
+	// We handle 0 and 1 separately since they are special
+	if quantile == 0 {
+		// we do this differently on our check
+		return "quantile:0"
+	} else if quantile == 1.0 {
+		// it needs to have a '.0' added at the end according to the spec
+		return "quantile:1.0"
+	}
+	return fmt.Sprintf("quantile:%s", strconv.FormatFloat(quantile, 'g', -1, 64))
+}
+
+// mapSummaryMetrics maps summary datapoints into Datadog metrics
+func mapSummaryMetrics(name string, slice pdata.SummaryDataPointSlice, quantiles bool, attrTags []string) []datadog.Metric {
+	// Allocate assuming none are nil and no quantiles
+	ms := make([]datadog.Metric, 0, 2*slice.Len())
+	for i := 0; i < slice.Len(); i++ {
+		p := slice.At(i)
+		ts := uint64(p.Timestamp())
+		tags := getTags(p.LabelsMap())
+		tags = append(tags, attrTags...)
+
+		ms = append(ms,
+			metrics.NewGauge(fmt.Sprintf("%s.count", name), ts, float64(p.Count()), tags),
+			metrics.NewGauge(fmt.Sprintf("%s.sum", name), ts, p.Sum(), tags),
+		)
+
+		if quantiles {
+			fullName := fmt.Sprintf("%s.quantile", name)
+			quantiles := p.QuantileValues()
+			for i := 0; i < quantiles.Len(); i++ {
+				q := quantiles.At(i)
+				quantileTags := append(tags, getQuantileTag(q.Quantile()))
+				ms = append(ms,
+					metrics.NewGauge(fullName, ts, q.Value(), quantileTags),
+				)
+			}
+		}
+	}
+	return ms
+}
+
 // mapMetrics maps OTLP metrics into the DataDog format
 func mapMetrics(logger *zap.Logger, cfg config.MetricsConfig, prevPts *ttlmap.TTLMap, fallbackHost string, md pdata.Metrics, buildInfo component.BuildInfo) (series []datadog.Metric, droppedTimeSeries int) {
 	pushTime := uint64(time.Now().UTC().UnixNano())
@@ -301,6 +348,8 @@ func mapMetrics(logger *zap.Logger, cfg config.MetricsConfig, prevPts *ttlmap.TT
 					datapoints = mapIntHistogramMetrics(md.Name(), md.IntHistogram().DataPoints(), cfg.Buckets, attributeTags)
 				case pdata.MetricDataTypeHistogram:
 					datapoints = mapHistogramMetrics(md.Name(), md.Histogram().DataPoints(), cfg.Buckets, attributeTags)
+				case pdata.MetricDataTypeSummary:
+					datapoints = mapSummaryMetrics(md.Name(), md.Summary().DataPoints(), cfg.Quantiles, attributeTags)
 				default: // pdata.MetricDataTypeNone or any other not supported type
 					logger.Debug("Unknown or unsupported metric type", zap.String("metric name", md.Name()), zap.Any("data type", md.DataType()))
 					continue

exporter/datadogexporter/metrics_translator_test.go

@@ -517,6 +517,95 @@ func TestMapHistogramMetrics(t *testing.T) {
 	)
 }
 
+func TestQuantileTag(t *testing.T) {
+	tests := []struct {
+		quantile float64
+		tag      string
+	}{
+		{quantile: 0, tag: "quantile:0"},
+		{quantile: 0.001, tag: "quantile:0.001"},
+		{quantile: 0.9, tag: "quantile:0.9"},
+		{quantile: 0.95, tag: "quantile:0.95"},
+		{quantile: 0.99, tag: "quantile:0.99"},
+		{quantile: 0.999, tag: "quantile:0.999"},
+		{quantile: 1, tag: "quantile:1.0"},
+		{quantile: 1e-10, tag: "quantile:1e-10"},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.tag, getQuantileTag(test.quantile))
+	}
+}
+
+func exampleSummaryDataPointSlice(ts pdata.Timestamp) pdata.SummaryDataPointSlice {
+	slice := pdata.NewSummaryDataPointSlice()
+	point := slice.AppendEmpty()
+	point.SetCount(100)
+	point.SetSum(10_000)
+	qSlice := point.QuantileValues()
+
+	qMin := qSlice.AppendEmpty()
+	qMin.SetQuantile(0.0)
+	qMin.SetValue(0)
+
+	qMedian := qSlice.AppendEmpty()
+	qMedian.SetQuantile(0.5)
+	qMedian.SetValue(100)
+
+	q999 := qSlice.AppendEmpty()
+	q999.SetQuantile(0.999)
+	q999.SetValue(500)
+
+	qMax := qSlice.AppendEmpty()
+	qMax.SetQuantile(1)
+	qMax.SetValue(600)
+	point.SetTimestamp(ts)
+	return slice
+}
+
+func TestMapSummaryMetrics(t *testing.T) {
+	ts := pdata.TimestampFromTime(time.Now())
+	slice := exampleSummaryDataPointSlice(ts)
+
+	noQuantiles := []datadog.Metric{
+		metrics.NewGauge("summary.example.count", uint64(ts), 100, []string{}),
+		metrics.NewGauge("summary.example.sum", uint64(ts), 10_000, []string{}),
+	}
+	quantiles := []datadog.Metric{
+		metrics.NewGauge("summary.example.quantile", uint64(ts), 0, []string{"quantile:0"}),
+		metrics.NewGauge("summary.example.quantile", uint64(ts), 100, []string{"quantile:0.5"}),
+		metrics.NewGauge("summary.example.quantile", uint64(ts), 500, []string{"quantile:0.999"}),
+		metrics.NewGauge("summary.example.quantile", uint64(ts), 600, []string{"quantile:1.0"}),
+	}
+	assert.ElementsMatch(t,
+		mapSummaryMetrics("summary.example", slice, false, []string{}),
+		noQuantiles,
+	)
+	assert.ElementsMatch(t,
+		mapSummaryMetrics("summary.example", slice, true, []string{}),
+		append(noQuantiles, quantiles...),
+	)
+
+	noQuantilesAttr := []datadog.Metric{
+		metrics.NewGauge("summary.example.count", uint64(ts), 100, []string{"attribute_tag:attribute_value"}),
+		metrics.NewGauge("summary.example.sum", uint64(ts), 10_000, []string{"attribute_tag:attribute_value"}),
+	}
+	quantilesAttr := []datadog.Metric{
+		metrics.NewGauge("summary.example.quantile", uint64(ts), 0, []string{"attribute_tag:attribute_value", "quantile:0"}),
+		metrics.NewGauge("summary.example.quantile", uint64(ts), 100, []string{"attribute_tag:attribute_value", "quantile:0.5"}),
+		metrics.NewGauge("summary.example.quantile", uint64(ts), 500, []string{"attribute_tag:attribute_value", "quantile:0.999"}),
+		metrics.NewGauge("summary.example.quantile", uint64(ts), 600, []string{"attribute_tag:attribute_value", "quantile:1.0"}),
+	}
+	assert.ElementsMatch(t,
+		mapSummaryMetrics("summary.example", slice, false, []string{"attribute_tag:attribute_value"}),
+		noQuantilesAttr,
+	)
+	assert.ElementsMatch(t,
+		mapSummaryMetrics("summary.example", slice, true, []string{"attribute_tag:attribute_value"}),
+		append(noQuantilesAttr, quantilesAttr...),
+	)
+}
+
 func TestRunningMetrics(t *testing.T) {
 	ms := pdata.NewMetrics()
 	rms := ms.ResourceMetrics()
@@ -665,6 +754,12 @@ func createTestMetrics() pdata.Metrics {
 	dpDouble.SetTimestamp(seconds(2))
 	dpDouble.SetValue(4 + math.Pi)
 
+	// Summary
+	met = metricsArray.AppendEmpty()
+	met.SetName("summary")
+	met.SetDataType(pdata.MetricDataTypeSummary)
+	slice := exampleSummaryDataPointSlice(seconds(0))
+	slice.CopyTo(met.Summary().DataPoints())
 	return md
 }
 
@@ -712,6 +807,8 @@ func TestMapMetrics(t *testing.T) {
 		testGauge("int.histogram.count", 20),
 		testGauge("double.histogram.sum", math.Phi),
 		testGauge("double.histogram.count", 20),
+		testGauge("summary.sum", 10_000),
+		testGauge("summary.count", 100),
 		testCount("int.cumulative.sum", 3),
 		testCount("double.cumulative.sum", math.Pi),
 	})

exporter/datadogexporter/testdata/config.yaml

@@ -36,6 +36,7 @@ exporters:
 
     metrics:
       endpoint: https://api.datadoghq.test
+      report_quantiles: false
 
     traces:
       sample_rate: 1