Replace the SAPM exporter with OTLP HTTP in default configs

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### 🛑 Breaking changes 🛑
+
+- (Splunk) Change default traces exporter to otlphttp exporter. ([#5588](https://github.com/signalfx/splunk-otel-collector/pull/5588)).
+
 ## v0.114.0
 
 ### 💡 Enhancements 💡

README.md

@@ -176,6 +176,26 @@ manually before the backward compatibility is dropped. For every configuration u
 [the default agent config](https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml)
 as a reference.
 
+### From 0.114.0 to 0.115.0
+
+-  The sapm exporter still works as before but has been deprecated. Use the otlphttp exporter instead
+
+1. Replace the `sapm` exporter with `otlphttp` exporter using the following configuration
+    ```yaml
+    otlphttp:
+      traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+      headers:
+        "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
+    ```
+2. Update traces pipeline to use otlphttp exporter instead of sapm:
+    ```
+    service:
+      pipelines:
+        traces:
+          exporters: [otlphttp, signalfx]
+    ```
+
+
 ### From 0.96.1 to 0.97.0
 
 - `memory_ballast` is no longer effective. The garbage collection is now controlled by the soft memory limit set to 90%

cmd/otelcol/config/collector/agent_config.yaml

@@ -126,9 +126,10 @@ processors:
 
 exporters:
   # Traces
-  sapm:
-    access_token: "${SPLUNK_ACCESS_TOKEN}"
-    endpoint: "${SPLUNK_INGEST_URL}/v2/trace"
+  otlphttp:
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
   # Metrics + Events
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
@@ -157,7 +158,7 @@ exporters:
     endpoint: "${SPLUNK_INGEST_URL}/v1/log"
     log_data_enabled: false
   # Send to gateway
-  otlp:
+  otlp/gateway:
     endpoint: "${SPLUNK_GATEWAY_URL}:4317"
     tls:
       insecure: true
@@ -178,15 +179,15 @@ service:
       - batch
       - resourcedetection
       #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp, signalfx]
+      #exporters: [otlp/gateway, signalfx]
     metrics:
       receivers: [hostmetrics, otlp, signalfx]
       processors: [memory_limiter, batch, resourcedetection]
       exporters: [signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]
     metrics/internal:
       receivers: [prometheus/internal]
       processors: [memory_limiter, batch, resourcedetection]
@@ -211,4 +212,4 @@ service:
       #- resource/add_environment
       exporters: [splunk_hec, splunk_hec/profiling]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]

cmd/otelcol/config/collector/ecs_ec2_config.yaml

@@ -109,9 +109,10 @@ processors:
 
 exporters:
   # Traces
-  sapm:
-    access_token: "${SPLUNK_ACCESS_TOKEN}"
-    endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace"
+  otlphttp:
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
   # Metrics + Events
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
@@ -129,7 +130,7 @@ exporters:
     token: "${SPLUNK_ACCESS_TOKEN}"
     endpoint: "${SPLUNK_INGEST_URL}/v1/log"
     log_data_enabled: false
-
+    
 service:
   extensions: [health_check, http_forwarder, zpages]
   pipelines:
@@ -140,7 +141,7 @@ service:
         - batch
         - resourcedetection
         #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
     metrics:
       receivers: [hostmetrics, otlp, signalfx, awsecscontainermetrics]
       processors: [memory_limiter, batch, filter, resourcedetection]

cmd/otelcol/config/collector/fargate_config.yaml

@@ -92,9 +92,10 @@ processors:
 
 exporters:
   # Traces
-  sapm:
-    access_token: "${SPLUNK_ACCESS_TOKEN}"
-    endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace"
+  otlphttp:
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
   # Metrics + Events
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
@@ -123,7 +124,7 @@ service:
         - batch
         - resourcedetection
         #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
     metrics:
       receivers: [otlp, signalfx, awsecscontainermetrics, prometheus/internal]
       processors: [memory_limiter, batch, resourcedetection]

cmd/otelcol/config/collector/full_config_linux.yaml

@@ -637,7 +637,11 @@ exporters:
   # WARN: https://ingest.${SPLUNK_REALM}.signalfx.com does NOT support gRPC at this time; use otlphttp exporter instead
   otlp:
     endpoint: otelcol2:4317
-
+  sapm:
+    access_token: "${SPLUNK_ACCESS_TOKEN}"
+    endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace"
+    sending_queue:
+      num_consumers: 32
   # Enables the otlp http exporter
   # Full configuration here: https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter
   # NOTE: These settings should be updated to the proper destination

cmd/otelcol/config/collector/gateway_config.yaml

@@ -1,4 +1,4 @@
-# Configuration file that uses the Splunk exporters (SAPM, SignalFx) to push
+# Configuration file that uses the Splunk exporters (OTLP HTTP, SignalFx) to push
 # data to Splunk products.
 
 extensions:
@@ -91,11 +91,10 @@ processors:
 
 exporters:
   # Traces
-  sapm:
-    access_token: "${SPLUNK_ACCESS_TOKEN}"
-    endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace"
-    sending_queue:
-      num_consumers: 32
+  otlphttp:
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
   # Metrics + Events
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
@@ -138,7 +137,7 @@ service:
       - memory_limiter
       - batch
       #- resource/add_environment
-      exporters: [sapm]
+      exporters: [otlphttp]
     metrics:
       receivers: [otlp, signalfx]
       processors: [memory_limiter, batch]

cmd/otelcol/config/collector/upstream_agent_config.yaml

@@ -120,9 +120,10 @@ processors:
 
 exporters:
   # Traces
-  sapm:
-    access_token: "${SPLUNK_ACCESS_TOKEN}"
-    endpoint: "${SPLUNK_INGEST_URL}/v2/trace"
+  otlphttp:
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
   # Metrics + Events
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
@@ -143,7 +144,7 @@ exporters:
     endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v1/log"
     log_data_enabled: false
   # Send to gateway
-  otlp:
+  otlp/gateway:
     endpoint: "${SPLUNK_GATEWAY_URL}:4317"
     tls:
       insecure: true
@@ -162,30 +163,30 @@ service:
       - batch
       - resourcedetection
       #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp, signalfx]
+      #exporters: [otlp/gateway, signalfx]
     # Required for Splunk Infrastructure Monitoring
     metrics:
       receivers: [hostmetrics, otlp, signalfx]
       processors: [memory_limiter, batch, resourcedetection]
       exporters: [signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]
     # Required for Splunk APM and/or Infrastructure Monitoring
     metrics/internal:
       receivers: [prometheus/internal]
       processors: [memory_limiter, batch, resourcedetection]
       exporters: [signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]
     # Required for Splunk Infrastructure Monitoring
     logs/signalfx:
       receivers: [signalfx]
       processors: [memory_limiter, batch]
       exporters: [signalfx]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]
     logs/entities:
       # Receivers are dynamically added if discovery mode is enabled
       receivers: [nop]
@@ -201,4 +202,4 @@ service:
       #- resource/add_environment
       exporters: [splunk_hec/profiling]
       # Use instead when sending to gateway
-      #exporters: [otlp]
+      #exporters: [otlp/gateway]

cmd/otelcol/fips/config/agent_config.yaml

@@ -116,9 +116,10 @@ processors:
 
 exporters:
   # Traces
-  sapm:
-    access_token: "${SPLUNK_ACCESS_TOKEN}"
-    endpoint: "${SPLUNK_INGEST_URL}/v2/trace"
+  otlphttp:
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
   # Metrics + Events
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
@@ -168,7 +169,7 @@ service:
       - batch
       - resourcedetection
       #- resource/add_environment
-      exporters: [sapm, signalfx]
+      exporters: [otlphttp, signalfx]
       # Use instead when sending to gateway
       #exporters: [otlp, signalfx]
     metrics:

deployments/nomad/otel-agent.nomad

@@ -212,6 +212,10 @@ exporters:
     sync_host_metadata: true
   debug:
     verbosity: detailed
+  otlphttp:
+    traces_endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 service:
   extensions:
   - health_check

deployments/nomad/otel-gateway.nomad

@@ -194,6 +194,10 @@ exporters:
     ingest_url: https://ingest.${SPLUNK_REALM}.signalfx.com
   debug:
     verbosity: detailed
+  otlphttp:
+    traces_endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 service:
   extensions:
   - health_check

internal/settings/settings_test.go

@@ -372,12 +372,18 @@ func TestSetDefaultEnvVarsSetsInterfaceFromConfigOption(t *testing.T) {
 func TestSetDefaultFeatureGatesRespectsOverrides(t *testing.T) {
 	t.Cleanup(setRequiredEnvVars(t))
 	for _, args := range [][]string{
-		{"--feature-gates", "some-gate", "--feature-gates", "telemetry.useOtelForInternalMetrics", "--feature-gates",
-			"another-gate"},
-		{"--feature-gates", "some-gate", "--feature-gates", "+telemetry.useOtelForInternalMetrics",
-			"--feature-gates", "another-gate"},
-		{"--feature-gates", "some-gate", "--feature-gates", "-telemetry.useOtelForInternalMetrics",
-			"--feature-gates", "another-gate"},
+		{
+			"--feature-gates", "some-gate", "--feature-gates", "telemetry.useOtelForInternalMetrics", "--feature-gates",
+			"another-gate",
+		},
+		{
+			"--feature-gates", "some-gate", "--feature-gates", "+telemetry.useOtelForInternalMetrics",
+			"--feature-gates", "another-gate",
+		},
+		{
+			"--feature-gates", "some-gate", "--feature-gates", "-telemetry.useOtelForInternalMetrics",
+			"--feature-gates", "another-gate",
+		},
 	} {
 		t.Run(strings.Join(args, " "), func(t *testing.T) {
 			settings, err := New(args)
@@ -388,7 +394,6 @@ func TestSetDefaultFeatureGatesRespectsOverrides(t *testing.T) {
 }
 
 func TestSetSoftMemLimitWithoutGoMemLimitEnvVar(t *testing.T) {
-
 	// if GOLIMIT is not set, we expect soft limit to be 90% of the total memory env var or 90% of default total memory  512 Mib.
 	t.Cleanup(setRequiredEnvVars(t))
 	require.NoError(t, os.Setenv(MemTotalEnvVar, "200"))
@@ -402,7 +407,6 @@ func TestSetSoftMemLimitWithoutGoMemLimitEnvVar(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, settings)
 	require.Equal(t, int64(482344960), debug.SetMemoryLimit(-1))
-
 }
 
 func TestUseConfigPathsFromEnvVar(t *testing.T) {
@@ -591,7 +595,6 @@ func TestConfigArgEnvURIForm(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, []string{"env:SOME_ENV_VAR"}, settings.configPaths.value)
 	require.Equal(t, settings.configPaths.value, settings.ResolverURIs())
-
 }
 
 func TestCheckRuntimeParams_MemTotal(t *testing.T) {

pkg/receiver/smartagentreceiver/README.md

@@ -71,9 +71,11 @@ exporters:
   signalfx:
     access_token: "${SIGNALFX_ACCESS_TOKEN}"
     realm: us1
-  sapm:
-    access_token: "${SIGNALFX_ACCESS_TOKEN}"
-    endpoint: https://ingest.us1.signalfx.com/v2/trace
+  otlphttp:
+    metrics_endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/datapoint/otlp"
+    traces_endpoint: "https://ingest.${SPLUNK_REALM}.signalfx.com/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 
 service:
   pipelines:
@@ -98,5 +100,5 @@ service:
       processors:
         - resourcedetection
       exporters:
-        - sapm
+        - otlphttp
 ```