From a9c23135e97aad5c455a24622236ce683aad2abe Mon Sep 17 00:00:00 2001 From: Yang Song Date: Thu, 8 Aug 2024 18:10:28 -0400 Subject: [PATCH 1/5] [connector/datadog] Add a config bucket_interval --- .chloggen/dd-conn-bkt-size.yaml | 27 +++++++++++++++++++ connector/datadogconnector/README.md | 6 +++++ connector/datadogconnector/config.go | 10 +++++++ connector/datadogconnector/config_test.go | 14 ++++++++++ connector/datadogconnector/connector.go | 3 +++ .../datadogconnector/connector_native_test.go | 1 + connector/datadogconnector/connector_test.go | 1 + .../datadogconnector/examples/config.yaml | 5 ++++ 8 files changed, 67 insertions(+) create mode 100644 .chloggen/dd-conn-bkt-size.yaml diff --git a/.chloggen/dd-conn-bkt-size.yaml b/.chloggen/dd-conn-bkt-size.yaml new file mode 100644 index 0000000000000..c33c4d9656314 --- /dev/null +++ b/.chloggen/dd-conn-bkt-size.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: datadogconnector + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Add a config `bucket_interval`" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: "`bucket_interval` specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. Default is 10s if unset." + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/connector/datadogconnector/README.md b/connector/datadogconnector/README.md index 550986d40d840..8f61351599c64 100644 --- a/connector/datadogconnector/README.md +++ b/connector/datadogconnector/README.md @@ -173,6 +173,12 @@ connectors: ## A list of resource attributes that should be used as container tags. # # resource_attributes_as_container_tags: ["could.availability_zone", "could.region"] + + ## @param bucket_interval specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. + ## It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. + ## Default is 10s if unset. + # + # bucket_interval: 30s ``` **NOTE**: `compute_stats_by_span_kind` and `peer_tags_aggregation` only work when the feature gate `connector.datadogconnector.performance` is enabled. See below for details on this feature gate. diff --git a/connector/datadogconnector/config.go b/connector/datadogconnector/config.go index bebece6aeb8c8..e8029e1775f16 100644 --- a/connector/datadogconnector/config.go +++ b/connector/datadogconnector/config.go @@ -6,6 +6,7 @@ package datadogconnector // import "github.com/open-telemetry/opentelemetry-coll import ( "fmt" "regexp" + "time" "go.opentelemetry.io/collector/component" ) @@ -75,6 +76,11 @@ type TracesConfig struct { // ResourceAttributesAsContainerTags specifies the list of resource attributes to be used as container tags. ResourceAttributesAsContainerTags []string `mapstructure:"resource_attributes_as_container_tags"` + + // BucketInterval specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. + // It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. + // Default is 10s if unset. + BucketInterval time.Duration `mapstructure:"bucket_interval"` } // Validate the configuration for errors. This is required by component.Config. @@ -103,5 +109,9 @@ func (c *Config) Validate() error { return fmt.Errorf("Trace buffer must be non-negative") } + if c.Traces.BucketInterval < 0 { + return fmt.Errorf("bucket interval must be non-negative") + } + return nil } diff --git a/connector/datadogconnector/config_test.go b/connector/datadogconnector/config_test.go index 307f9e920dcd6..55bf3481eb8d1 100644 --- a/connector/datadogconnector/config_test.go +++ b/connector/datadogconnector/config_test.go @@ -5,6 +5,7 @@ package datadogconnector import ( "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -70,6 +71,19 @@ func TestValidate(t *testing.T) { Traces: TracesConfig{PeerTags: []string{"tag1", "tag2"}}, }, }, + { + name: "With bucket_interval", + cfg: &Config{ + Traces: TracesConfig{BucketInterval: 30 * time.Second}, + }, + }, + { + name: "neg bucket_interval", + cfg: &Config{ + Traces: TracesConfig{BucketInterval: -30 * time.Second}, + }, + err: "bucket interval must be non-negative", + }, } for _, testInstance := range tests { t.Run(testInstance.name, func(t *testing.T) { diff --git a/connector/datadogconnector/connector.go b/connector/datadogconnector/connector.go index 711cbcffc5593..33aed6d20e1f4 100644 --- a/connector/datadogconnector/connector.go +++ b/connector/datadogconnector/connector.go @@ -117,6 +117,9 @@ func getTraceAgentCfg(logger *zap.Logger, cfg TracesConfig, attributesTranslator logger.Info("traces::compute_top_level_by_span_kind needs to be enabled in both the Datadog connector and Datadog exporter configs if both components are being used") acfg.Features["enable_otlp_compute_top_level_by_span_kind"] = struct{}{} } + if v := cfg.BucketInterval; v > 0 { + acfg.BucketInterval = v + } return acfg } diff --git a/connector/datadogconnector/connector_native_test.go b/connector/datadogconnector/connector_native_test.go index 326a7a5dcc15e..eb14051888599 100644 --- a/connector/datadogconnector/connector_native_test.go +++ b/connector/datadogconnector/connector_native_test.go @@ -84,6 +84,7 @@ func creteConnectorNativeWithCfg(t *testing.T, cfg *Config) (*traceToMetricConne creationParams := connectortest.NewNopSettings() metricsSink := &consumertest.MetricsSink{} + cfg.Traces.BucketInterval = 1 * time.Second tconn, err := factory.CreateTracesToMetrics(context.Background(), creationParams, cfg, metricsSink) assert.NoError(t, err) diff --git a/connector/datadogconnector/connector_test.go b/connector/datadogconnector/connector_test.go index 9f88083110104..f5ef6641e8475 100644 --- a/connector/datadogconnector/connector_test.go +++ b/connector/datadogconnector/connector_test.go @@ -106,6 +106,7 @@ func creteConnector(t *testing.T) (*traceToMetricConnector, *consumertest.Metric creationParams := connectortest.NewNopSettings() cfg := factory.CreateDefaultConfig().(*Config) cfg.Traces.ResourceAttributesAsContainerTags = []string{semconv.AttributeCloudAvailabilityZone, semconv.AttributeCloudRegion, "az"} + cfg.Traces.BucketInterval = 1 * time.Second metricsSink := &consumertest.MetricsSink{} diff --git a/connector/datadogconnector/examples/config.yaml b/connector/datadogconnector/examples/config.yaml index d01c9b65ce252..5896bad3832b8 100644 --- a/connector/datadogconnector/examples/config.yaml +++ b/connector/datadogconnector/examples/config.yaml @@ -78,6 +78,11 @@ connectors: ## A list of resource attributes that should be used as container tags. # resource_attributes_as_container_tags: ["could.availability_zone", "could.region"] + ## @param bucket_interval specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. + ## It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. + ## Default is 10s if unset. + # + bucket_interval: 30s exporters: debug: verbosity: detailed From 92ccd0d5444804503acdb314f845dc3e518ea06c Mon Sep 17 00:00:00 2001 From: Yang Song Date: Thu, 8 Aug 2024 18:11:46 -0400 Subject: [PATCH 2/5] Update .chloggen/dd-conn-bkt-size.yaml --- .chloggen/dd-conn-bkt-size.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/dd-conn-bkt-size.yaml b/.chloggen/dd-conn-bkt-size.yaml index c33c4d9656314..d2179aada9fcf 100644 --- a/.chloggen/dd-conn-bkt-size.yaml +++ b/.chloggen/dd-conn-bkt-size.yaml @@ -10,7 +10,7 @@ component: datadogconnector note: "Add a config `bucket_interval`" # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. -issues: [] +issues: [34554] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document. From e3e27d13b69bd5c47ddc3a4c449399b6cfb25f74 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Fri, 9 Aug 2024 08:50:06 -0400 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Pablo Baeyens --- .chloggen/dd-conn-bkt-size.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.chloggen/dd-conn-bkt-size.yaml b/.chloggen/dd-conn-bkt-size.yaml index d2179aada9fcf..f2f206aa66e10 100644 --- a/.chloggen/dd-conn-bkt-size.yaml +++ b/.chloggen/dd-conn-bkt-size.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: datadogconnector # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: "Add a config `bucket_interval`" +note: "Add a config `traces::bucket_interval`" # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [34554] @@ -15,7 +15,7 @@ issues: [34554] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document. # Use pipe (|) for multiline entries. -subtext: "`bucket_interval` specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. Default is 10s if unset." +subtext: "`traces::bucket_interval` specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. Default is 10s if unset." # If your change doesn't affect end users or the exported elements of any package, # you should instead start your pull request title with [chore] or use the "Skip Changelog" label. From 2aa76f3d986956ebfc23857a219c6349434f967e Mon Sep 17 00:00:00 2001 From: Yang Song Date: Wed, 14 Aug 2024 11:25:42 -0400 Subject: [PATCH 4/5] Set the 10s default explicitly --- connector/datadogconnector/factory.go | 2 ++ connector/datadogconnector/factory_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/connector/datadogconnector/factory.go b/connector/datadogconnector/factory.go index a823bf287ecda..c3d3b18ab1265 100644 --- a/connector/datadogconnector/factory.go +++ b/connector/datadogconnector/factory.go @@ -7,6 +7,7 @@ package datadogconnector // import "github.com/open-telemetry/opentelemetry-coll import ( "context" + "time" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient" "github.com/DataDog/datadog-agent/pkg/trace/timing" @@ -44,6 +45,7 @@ func createDefaultConfig() component.Config { Traces: TracesConfig{ IgnoreResources: []string{}, TraceBuffer: 1000, + BucketInterval: 10 * time.Second, }, } } diff --git a/connector/datadogconnector/factory_test.go b/connector/datadogconnector/factory_test.go index 8dc505d731bd8..ed081d6d1d6dc 100644 --- a/connector/datadogconnector/factory_test.go +++ b/connector/datadogconnector/factory_test.go @@ -5,6 +5,7 @@ package datadogconnector import ( "testing" + "time" "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/component/componenttest" @@ -19,6 +20,7 @@ func TestCreateDefaultConfig(t *testing.T) { Traces: TracesConfig{ IgnoreResources: []string{}, TraceBuffer: 1000, + BucketInterval: 10 * time.Second, }, }, cfg, "failed to create default config") From c7f23648e846193f46a59a01fe4a65d980ec19d6 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Wed, 14 Aug 2024 11:32:18 -0400 Subject: [PATCH 5/5] Add a note on implication --- connector/datadogconnector/README.md | 1 + connector/datadogconnector/config.go | 1 + connector/datadogconnector/examples/config.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/connector/datadogconnector/README.md b/connector/datadogconnector/README.md index 8f61351599c64..345814f9dfbbf 100644 --- a/connector/datadogconnector/README.md +++ b/connector/datadogconnector/README.md @@ -176,6 +176,7 @@ connectors: ## @param bucket_interval specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. ## It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. + ## If you are concerned about the metric volume generated by the Datadog connector and the resulting networking egress, try increasing bucket_interval. ## Default is 10s if unset. # # bucket_interval: 30s diff --git a/connector/datadogconnector/config.go b/connector/datadogconnector/config.go index e8029e1775f16..57a8b61825303 100644 --- a/connector/datadogconnector/config.go +++ b/connector/datadogconnector/config.go @@ -79,6 +79,7 @@ type TracesConfig struct { // BucketInterval specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. // It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. + // If you are concerned about the metric volume generated by the Datadog connector and the resulting networking egress, try increasing bucket_interval. // Default is 10s if unset. BucketInterval time.Duration `mapstructure:"bucket_interval"` } diff --git a/connector/datadogconnector/examples/config.yaml b/connector/datadogconnector/examples/config.yaml index 5896bad3832b8..f958baff9a801 100644 --- a/connector/datadogconnector/examples/config.yaml +++ b/connector/datadogconnector/examples/config.yaml @@ -80,6 +80,7 @@ connectors: resource_attributes_as_container_tags: ["could.availability_zone", "could.region"] ## @param bucket_interval specifies the time interval size of aggregation buckets that aggregate the Datadog trace metrics. ## It is also the time interval that Datadog trace metrics payloads are flushed to the pipeline. + ## If you are concerned about the metric volume generated by the Datadog connector and the resulting networking egress, try increasing bucket_interval. ## Default is 10s if unset. # bucket_interval: 30s