[exporter/datadog] Respect confighttp configs and use better defaults (#31733)

**Description:** <Describe what has changed.>
Datadog exporter respects a subset of settings in confighttp client
configs and uses a consistent default HTTP transport settings as Datadog
Agent
(https://github.com/DataDog/datadog-agent/blob/f9ae7f4b842f83b23b2dfe3f15d31f9e6b12e857/pkg/util/http/transport.go#L91-L106).

Author: songy23

.chloggen/ddog-config-change.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Remove config structs `LimitedClientConfig` and `LimitedTLSClientSettings`"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31733]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: This should have no impact to end users as long as they do not import Datadog exporter config structs in their source code.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

.chloggen/ddog-http-defaults.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Datadog exporter uses the same default HTTP settings as Datadog Agent HTTP transport
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31733]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.chloggen/ddog-respect-confighttp.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Datadog exporter respects a subset of settings in confighttp client configs
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31733]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: "Currently the following configs are respected: `read_buffer_size`, `write_buffer_size`, `timeout`, `max_idle_conns`, `max_idle_conns_per_host`, `max_conns_per_host`, `idle_conn_timeout`, `disable_keep_alives` and `tls.insecure_skip_verify`."
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/datadogexporter/config.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/confignet"
 	"go.opentelemetry.io/collector/config/configopaque"
 	"go.opentelemetry.io/collector/config/configretry"
@@ -390,24 +391,11 @@ type HostMetadataConfig struct {
 	Tags []string `mapstructure:"tags"`
 }
 
-// LimitedTLSClientSetting is a subset of TLSClientSetting, see LimitedClientConfig for more details
-type LimitedTLSClientSettings struct {
-	// InsecureSkipVerify controls whether a client verifies the server's
-	// certificate chain and host name.
-	InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"`
-}
-
-type LimitedClientConfig struct {
-	TLSSetting LimitedTLSClientSettings `mapstructure:"tls,omitempty"`
-}
-
 // Config defines configuration for the Datadog exporter.
 type Config struct {
-	exporterhelper.TimeoutSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
-	exporterhelper.QueueSettings   `mapstructure:"sending_queue"`
-	configretry.BackOffConfig      `mapstructure:"retry_on_failure"`
-
-	LimitedClientConfig `mapstructure:",squash"`
+	confighttp.ClientConfig      `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	exporterhelper.QueueSettings `mapstructure:"sending_queue"`
+	configretry.BackOffConfig    `mapstructure:"retry_on_failure"`
 
 	TagsConfig `mapstructure:",squash"`
 
@@ -450,6 +438,10 @@ var _ component.Config = (*Config)(nil)
 
 // Validate the configuration for errors. This is required by component.Config.
 func (c *Config) Validate() error {
+	if err := validateClientConfig(c.ClientConfig); err != nil {
+		return err
+	}
+
 	if c.OnlyMetadata && (!c.HostMetadata.Enabled || c.HostMetadata.HostnameSource != HostnameSourceFirstResource) {
 		return errNoMetadata
 	}
@@ -490,6 +482,36 @@ func (c *Config) Validate() error {
 	return nil
 }
 
+func validateClientConfig(cfg confighttp.ClientConfig) error {
+	var unsupported []string
+	if cfg.Auth != nil {
+		unsupported = append(unsupported, "auth")
+	}
+	if cfg.Endpoint != "" {
+		unsupported = append(unsupported, "endpoint")
+	}
+	if cfg.Compression != "" {
+		unsupported = append(unsupported, "compression")
+	}
+	if cfg.ProxyURL != "" {
+		unsupported = append(unsupported, "proxy_url")
+	}
+	if cfg.Headers != nil {
+		unsupported = append(unsupported, "headers")
+	}
+	if cfg.HTTP2ReadIdleTimeout != 0 {
+		unsupported = append(unsupported, "http2_read_idle_timeout")
+	}
+	if cfg.HTTP2PingTimeout != 0 {
+		unsupported = append(unsupported, "http2_ping_timeout")
+	}
+
+	if len(unsupported) > 0 {
+		return fmt.Errorf("these confighttp client configs are currently not respected by Datadog exporter: %s", strings.Join(unsupported, ", "))
+	}
+	return nil
+}
+
 var _ error = (*renameError)(nil)
 
 // renameError is an error related to a renamed setting.

exporter/datadogexporter/config_test.go

@@ -5,12 +5,25 @@ package datadogexporter
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/configauth"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/config/configopaque"
+	"go.opentelemetry.io/collector/config/configtls"
 	"go.opentelemetry.io/collector/confmap"
 )
 
 func TestValidate(t *testing.T) {
+	idleConnTimeout := 30 * time.Second
+	maxIdleConn := 300
+	maxIdleConnPerHost := 150
+	maxConnPerHost := 250
+	ty, err := component.NewType("ty")
+	assert.NoError(t, err)
+	auth := configauth.Authentication{AuthenticatorID: component.NewID(ty)}
 
 	tests := []struct {
 		name string
@@ -94,8 +107,8 @@ func TestValidate(t *testing.T) {
 			name: "TLS settings are valid",
 			cfg: &Config{
 				API: APIConfig{Key: "notnull"},
-				LimitedClientConfig: LimitedClientConfig{
-					TLSSetting: LimitedTLSClientSettings{
+				ClientConfig: confighttp.ClientConfig{
+					TLSSetting: configtls.ClientConfig{
 						InsecureSkipVerify: true,
 					},
 				},
@@ -115,6 +128,40 @@ func TestValidate(t *testing.T) {
 				Traces: TracesConfig{PeerTags: []string{"tag1", "tag2"}},
 			},
 		},
+		{
+			name: "With confighttp client configs",
+			cfg: &Config{
+				API: APIConfig{Key: "notnull"},
+				ClientConfig: confighttp.ClientConfig{
+					ReadBufferSize:      100,
+					WriteBufferSize:     200,
+					Timeout:             10 * time.Second,
+					IdleConnTimeout:     &idleConnTimeout,
+					MaxIdleConns:        &maxIdleConn,
+					MaxIdleConnsPerHost: &maxIdleConnPerHost,
+					MaxConnsPerHost:     &maxConnPerHost,
+					DisableKeepAlives:   true,
+					TLSSetting:          configtls.ClientConfig{InsecureSkipVerify: true},
+				},
+			},
+		},
+
+		{
+			name: "unsupported confighttp client configs",
+			cfg: &Config{
+				API: APIConfig{Key: "notnull"},
+				ClientConfig: confighttp.ClientConfig{
+					Endpoint:             "endpoint",
+					Compression:          "gzip",
+					ProxyURL:             "proxy",
+					Auth:                 &auth,
+					Headers:              map[string]configopaque.String{"key": "val"},
+					HTTP2ReadIdleTimeout: 250,
+					HTTP2PingTimeout:     200,
+				},
+			},
+			err: "these confighttp client configs are currently not respected by Datadog exporter: auth, endpoint, compression, proxy_url, headers, http2_read_idle_timeout, http2_ping_timeout",
+		},
 	}
 	for _, testInstance := range tests {
 		t.Run(testInstance.name, func(t *testing.T) {
@@ -129,10 +176,26 @@ func TestValidate(t *testing.T) {
 }
 
 func TestUnmarshal(t *testing.T) {
+	cfgWithHTTPConfigs := NewFactory().CreateDefaultConfig().(*Config)
+	idleConnTimeout := 30 * time.Second
+	maxIdleConn := 300
+	maxIdleConnPerHost := 150
+	maxConnPerHost := 250
+	cfgWithHTTPConfigs.ReadBufferSize = 100
+	cfgWithHTTPConfigs.WriteBufferSize = 200
+	cfgWithHTTPConfigs.Timeout = 10 * time.Second
+	cfgWithHTTPConfigs.MaxIdleConns = &maxIdleConn
+	cfgWithHTTPConfigs.MaxIdleConnsPerHost = &maxIdleConnPerHost
+	cfgWithHTTPConfigs.MaxConnsPerHost = &maxConnPerHost
+	cfgWithHTTPConfigs.IdleConnTimeout = &idleConnTimeout
+	cfgWithHTTPConfigs.DisableKeepAlives = true
+	cfgWithHTTPConfigs.TLSSetting.InsecureSkipVerify = true
+	cfgWithHTTPConfigs.warnings = nil
+
 	tests := []struct {
 		name      string
 		configMap *confmap.Conf
-		cfg       Config
+		cfg       *Config
 		err       string
 	}{
 		{
@@ -264,6 +327,21 @@ func TestUnmarshal(t *testing.T) {
 			}),
 			err: "\"metrics::sums::initial_cumulative_monotonic_value\" can only be configured when \"metrics::sums::cumulative_monotonic_mode\" is set to \"to_delta\"",
 		},
+		{
+			name: "unmarshall confighttp client configs",
+			configMap: confmap.NewFromStringMap(map[string]any{
+				"read_buffer_size":        100,
+				"write_buffer_size":       200,
+				"timeout":                 "10s",
+				"max_idle_conns":          300,
+				"max_idle_conns_per_host": 150,
+				"max_conns_per_host":      250,
+				"disable_keep_alives":     true,
+				"idle_conn_timeout":       "30s",
+				"tls":                     map[string]any{"insecure_skip_verify": true},
+			}),
+			cfg: cfgWithHTTPConfigs,
+		},
 	}
 
 	f := NewFactory()

exporter/datadogexporter/examples/collector.yaml

@@ -214,6 +214,27 @@ exporters:
       ## @param tls - boolean - optional - default: false
       # insecure_skip_verify: false
 
+    ## @param read_buffer_size - integer - optional
+    ## @param write_buffer_size - integer - optional
+    ## @param timeout - duration - optional
+    ## @param max_idle_conns - integer - optional
+    ## @param max_idle_conns_per_host - integer - optional
+    ## @param max_conns_per_host - integer - optional
+    ## @param idle_conn_timeout - duration - optional
+    ## @param disable_keep_alives - boolean - optional
+    ##
+    ## Below are a subset of configs in confighttp that are respected by Datadog exporter.
+    ## See https://pkg.go.dev/go.opentelemetry.io/collector/config/confighttp for details on these configs.
+    ##
+    # read_buffer_size: 0
+    # write_buffer_size: 0
+    # timeout: 15s
+    # max_idle_conns: 100
+    # max_idle_conns_per_host: 0
+    # max_conns_per_host: 0
+    # idle_conn_timeout: 0s
+    # disable_keep_alives: false
+
     ## @param metrics - custom object - optional
     ## Metric exporter specific configuration.
     #

exporter/datadogexporter/examples/logs.yaml

@@ -23,6 +23,7 @@ processors:
 
 exporters:
   datadog:
+    idle_conn_timeout: 10s
     api:
       site: ${env:DD_SITE}
       key: ${env:DD_API_KEY}

exporter/datadogexporter/factory.go

@@ -19,6 +19,7 @@ import (
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/confignet"
 	"go.opentelemetry.io/collector/config/configretry"
 	"go.opentelemetry.io/collector/consumer"
@@ -170,18 +171,19 @@ func NewFactory() exporter.Factory {
 	return newFactoryWithRegistry(featuregate.GlobalRegistry())
 }
 
-func defaulttimeoutSettings() exporterhelper.TimeoutSettings {
-	return exporterhelper.TimeoutSettings{
+func defaultClientConfig() confighttp.ClientConfig {
+	// do not use NewDefaultClientConfig for backwards-compatibility
+	return confighttp.ClientConfig{
 		Timeout: 15 * time.Second,
 	}
 }
 
 // createDefaultConfig creates the default exporter configuration
 func (f *factory) createDefaultConfig() component.Config {
 	return &Config{
-		TimeoutSettings: defaulttimeoutSettings(),
-		BackOffConfig:   configretry.NewDefaultBackOffConfig(),
-		QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
+		ClientConfig:  defaultClientConfig(),
+		BackOffConfig: configretry.NewDefaultBackOffConfig(),
+		QueueSettings: exporterhelper.NewDefaultQueueSettings(),
 
 		API: APIConfig{
 			Site: "datadoghq.com",