[pkg/datadog] add hostname_detection_timeout and set default (#40377)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description
Adds `hostname_detection_timeout` config option and sets a reasonable
default (25 seconds)
<!-- Issue number (e.g. #1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes #40373 

<!--Describe what testing was performed and which tests were added.-->
#### Testing
manually run `make kind-build` and `make kind-install-daemonset` and
verify collector does not crash after 30 seconds
<!--Describe the documentation added.-->
#### Documentation
.chloggen and definition in datadogexporter examples folder
<!--Please delete paragraphs that you did not use before submitting.-->

Author: jackgopack4

.chloggen/jackgopack4-pkg-datadog-add-source_timeout.yaml

@@ -0,0 +1,29 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'bug_fix'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: 'exporter/datadog'
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Adds `hostname_detection_timeout` configuration option for Datadog Exporter and sets default to 25 seconds.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [40373, 40377]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  - This fixes a bug on Kubernetes daemonset deployments where collector hangs/reboots on startup.
+  - This configuration option already existed but was not available for customer configuration.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/datadogexporter/README.md

@@ -85,3 +85,9 @@ processors:
         statements:
           - set(attributes["datadog.log.source"], "otel")
 ```
+
+### My Collector K8s pod is getting rebooted on startup when I don't manually set a hostname under `exporters::datadog::hostname`
+
+This is due to a bug with underlying hostname detection blocking the `health_check` extension from responding to liveness/readiness probes on startup. To fix, either set `hostname_detection_timeout` to be less than the pod/daemonset `livenessProbe: failureThreshold * periodSeconds` so that the timeout for hostname detection on startup takes less time than the control plane waits before restarting the pod, or leave `hostname_detection_timeout` at the default `25s` value and double-check the `livenessProbe` and `readinessProbe` settings and ensure that the control plane will in fact wait long enough for startup to complete before restarting the pod.
+
+Hostname detection is currently required to initialize the Datadog Exporter, unless a hostname is specified manually under `hostname`.

exporter/datadogexporter/examples/collector.yaml

@@ -451,6 +451,12 @@ exporters:
       ## The default is 30 minutes.
       # reporter_period: 30m
 
+    ## @param hostname_detection_timeout - duration - optional - default: 25s
+    ## The amount of time in which the exporter will wait for hostname detection to be available.
+    ## Datadog Exporter requires a hostname to send traces to Datadog backend.
+    #
+    # hostname_detection_timeout: 25s
+
     ## @param logs - custom object - optional
     ## Logs exporter specific configuration.
     #

exporter/datadogexporter/factory.go

@@ -268,7 +268,7 @@ func (f *factory) createMetricsExporter(
 	c component.Config,
 ) (exporter.Metrics, error) {
 	cfg := checkAndCastConfig(c, set.Logger)
-	hostProvider, err := f.SourceProvider(set.TelemetrySettings, cfg.Hostname, cfg.HostMetadata.GetSourceTimeout())
+	hostProvider, err := f.SourceProvider(set.TelemetrySettings, cfg.Hostname, cfg.HostnameDetectionTimeout)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build hostname provider: %w", err)
 	}
@@ -453,7 +453,7 @@ func (f *factory) createTracesExporter(
 		wg     sync.WaitGroup // waits for agent to exit
 	)
 
-	hostProvider, err := f.SourceProvider(set.TelemetrySettings, cfg.Hostname, cfg.HostMetadata.GetSourceTimeout())
+	hostProvider, err := f.SourceProvider(set.TelemetrySettings, cfg.Hostname, cfg.HostnameDetectionTimeout)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build hostname provider: %w", err)
 	}
@@ -557,7 +557,7 @@ func (f *factory) createLogsExporter(
 
 	var pusher consumer.ConsumeLogsFunc
 	var logsAgent logsagentpipeline.LogsAgent
-	hostProvider, err := f.SourceProvider(set.TelemetrySettings, cfg.Hostname, cfg.HostMetadata.GetSourceTimeout())
+	hostProvider, err := f.SourceProvider(set.TelemetrySettings, cfg.Hostname, cfg.HostnameDetectionTimeout)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build hostname provider: %w", err)
 	}

exporter/datadogexporter/factory_test.go

@@ -303,8 +303,8 @@ func TestOnlyMetadata(t *testing.T) {
 			Enabled:        true,
 			ReporterPeriod: 30 * time.Minute,
 		},
+		HostnameDetectionTimeout: 50 * time.Millisecond,
 	}
-	cfg.HostMetadata.SetSourceTimeout(50 * time.Millisecond)
 
 	expTraces, err := factory.CreateTraces(
 		ctx,

exporter/datadogexporter/metrics_exporter_test.go

@@ -65,8 +65,8 @@ func TestNewExporter(t *testing.T) {
 			Enabled:        true,
 			ReporterPeriod: 30 * time.Minute,
 		},
+		HostnameDetectionTimeout: 50 * time.Millisecond,
 	}
-	cfg.HostMetadata.SetSourceTimeout(50 * time.Millisecond)
 
 	params := exportertest.NewNopSettings(metadata.Type)
 	f := NewFactory()
@@ -114,8 +114,8 @@ func TestNewExporter_Serializer(t *testing.T) {
 			Enabled:        true,
 			ReporterPeriod: 30 * time.Minute,
 		},
+		HostnameDetectionTimeout: 50 * time.Millisecond,
 	}
-	cfg.HostMetadata.SetSourceTimeout(50 * time.Millisecond)
 
 	params := exportertest.NewNopSettings(metadata.Type)
 	var err error

pkg/datadog/config/config.go

@@ -91,6 +91,14 @@ type Config struct {
 	// HostMetadata defines the host metadata specific configuration
 	HostMetadata HostMetadataConfig `mapstructure:"host_metadata"`
 
+	// HostnameDetectionTimeout defines the timeout for hostname detection.
+	// This is necessary for initializing datadog exporter
+	// On K8s, it must be set to less than `failureThreshold * periodSeconds` due to
+	// initialization blocking health_check liveness probes on startup.
+	// If set to zero duration, there will be no timeout applied.
+	// Default is 25 seconds.
+	HostnameDetectionTimeout time.Duration `mapstructure:"hostname_detection_timeout"`
+
 	// OnlyMetadata defines whether to only send metadata
 	// This is useful for agent-collector setups, so that
 	// metadata about a host is sent to the backend even
@@ -373,5 +381,7 @@ func CreateDefaultConfig() component.Config {
 			HostnameSource: HostnameSourceConfigOrSystem,
 			ReporterPeriod: 30 * time.Minute,
 		},
+
+		HostnameDetectionTimeout: 25 * time.Second, // set to 25 to prevent 30-second pod restart on K8s as reported in issue #40372 and #40373
 	}
 }

pkg/datadog/config/config_test.go

@@ -467,7 +467,8 @@ func TestCreateDefaultConfig(t *testing.T) {
 			HostnameSource: HostnameSourceConfigOrSystem,
 			ReporterPeriod: 30 * time.Minute,
 		},
-		OnlyMetadata: false,
+		HostnameDetectionTimeout: 25 * time.Second,
+		OnlyMetadata:             false,
 	}, cfg, "failed to create default config")
 
 	assert.NoError(t, componenttest.CheckConfigStruct(cfg))
@@ -542,7 +543,8 @@ func TestLoadConfig(t *testing.T) {
 					HostnameSource: HostnameSourceConfigOrSystem,
 					ReporterPeriod: 30 * time.Minute,
 				},
-				OnlyMetadata: false,
+				HostnameDetectionTimeout: 25 * time.Second,
+				OnlyMetadata:             false,
 			},
 		},
 		{
@@ -610,6 +612,7 @@ func TestLoadConfig(t *testing.T) {
 					HostnameSource: HostnameSourceConfigOrSystem,
 					ReporterPeriod: 30 * time.Minute,
 				},
+				HostnameDetectionTimeout: 25 * time.Second,
 			},
 		},
 		{
@@ -675,6 +678,7 @@ func TestLoadConfig(t *testing.T) {
 					Tags:           []string{"example:tag"},
 					ReporterPeriod: 30 * time.Minute,
 				},
+				HostnameDetectionTimeout: 25 * time.Second,
 			},
 		},
 		{
@@ -734,7 +738,8 @@ func TestLoadConfig(t *testing.T) {
 					HostnameSource: HostnameSourceConfigOrSystem,
 					ReporterPeriod: 10 * time.Minute,
 				},
-				OnlyMetadata: false,
+				HostnameDetectionTimeout: 25 * time.Second,
+				OnlyMetadata:             false,
 			},
 		},
 	}

pkg/datadog/config/host.go

@@ -70,21 +70,6 @@ type HostMetadataConfig struct {
 	// To attach tags to telemetry signals regardless of the host, use a processor instead.
 	Tags []string `mapstructure:"tags"`
 
-	// sourceTimeout is the timeout to fetch from each provider - for example AWS IMDS.
-	// If unset, or set to zero duration, there will be no timeout applied.
-	// Default is no timeout.
-	sourceTimeout time.Duration
-
 	// ReporterPeriod is the period at which the host metadata reporter will run.
 	ReporterPeriod time.Duration `mapstructure:"reporter_period"`
 }
-
-// SetSourceTimeout sets the timeout to fetch from each provider - for example AWS IMDS.
-func (h *HostMetadataConfig) SetSourceTimeout(t time.Duration) {
-	h.sourceTimeout = t
-}
-
-// GetSourceTimeout returns the timeout to fetch from each provider - for example AWS IMDS.
-func (h *HostMetadataConfig) GetSourceTimeout() time.Duration {
-	return h.sourceTimeout
-}