Fix issue with traces getting agent's k8s.pod.uid (#88)

Fix issue with traces getting agent's k8s.pod.uid

In change 3c312b6 we added some globally set resource attributes like
k8s.pod.uid. The problem is these attributes would get set before k8s_tagger
was run so you would get traces with the k8s.pod.uid of the agent itself
instead of the pod the trace came from.

Instead add a separate metrics pipeline that adds the agent k8s metadata only
to prometheus metrics.

* Remove usage of OTEL_RESOURCE_ATTRIBUTES in favor of processors (user can still set it)
* Use insert for resource attribute updates to prevent overriding upstream values and consistency

Author: jrcamp

helm-charts/splunk-otel-collector/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: splunk-otel-collector
-version: 0.24.2
+version: 0.24.3
 description: Splunk OpenTelemetry Connector for Kubernetes
 type: application
 keywords:

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -15,7 +15,7 @@ extensions:
 receivers:
   {{- include "splunk-otel-collector.otelTraceReceivers" . | nindent 2 }}
   # Prometheus receiver scraping metrics from the pod itself
-  prometheus:
+  prometheus/agent:
     config:
       scrape_configs:
       - job_name: 'otel-agent'
@@ -114,23 +114,43 @@ processors:
     timeout: 200ms
     send_batch_size: 128
 
-  resource/add_cluster_name:
+  resource:
+    # General resource attributes that apply to all telemetry passing through the agent.
     attributes:
-      - action: upsert
-        value: {{ .Values.clusterName }}
+      - action: insert
+        key: host.name
+        value: "${K8S_NODE_NAME}"
+      - action: insert
+        key: k8s.node.name
+        value: "${K8S_NODE_NAME}"
+      - action: insert
         key: k8s.cluster.name
+        value: "{{ .Values.clusterName }}"
       {{- range .Values.extraAttributes.custom }}
-      - action: upsert
-        value: {{ .value }}
-        key: {{ .name }}
+      - action: insert
+        key: "{{ .name }}"
+        value: "{{ .value }}"
       {{- end }}
 
+  # Resource attributes specific to the agent itself.
+  resource/add_agent_k8s:
+    attributes:
+      - action: insert
+        key: k8s.pod.name
+        value: "${K8S_POD_NAME}"
+      - action: insert
+        key: k8s.pod.uid
+        value: "${K8S_POD_UID}"
+      - action: insert
+        key: k8s.namespace.name
+        value: "${K8S_NAMESPACE}"
+
   {{- if .Values.environment }}
   resource/add_environment:
     attributes:
       - action: insert
-        value: {{ .Values.environment }}
         key: deployment.environment
+        value: "{{ .Values.environment }}"
   {{- end }}
 
 # By default only SAPM exporter enabled. It will be pointed to collector deployment if enabled,
@@ -168,9 +188,9 @@ service:
       receivers: [otlp, jaeger, zipkin]
       processors:
         - memory_limiter
+        - resource
         - resourcedetection
         - k8s_tagger
-        - resource/add_cluster_name
         {{- if .Values.environment }}
         - resource/add_environment
         {{- end }}
@@ -185,8 +205,26 @@ service:
 
     # default metrics pipeline
     metrics:
-      receivers: [hostmetrics, prometheus, kubeletstats, receiver_creator]
-      processors: [memory_limiter, resource/add_cluster_name, resourcedetection]
+      receivers: [hostmetrics, kubeletstats, receiver_creator]
+      processors:
+        - memory_limiter
+        - resource
+        - resourcedetection
+      exporters:
+        {{- if .Values.otelCollector.enabled }}
+        - otlp
+        {{- else }}
+        - signalfx
+        {{- end }}
+
+    # Pipeline for metrics collected about the agent pod itself.
+    metrics/agent:
+      receivers: [prometheus/agent]
+      processors:
+        - memory_limiter
+        - resource
+        - resource/add_agent_k8s
+        - resourcedetection
       exporters:
         {{- if .Values.otelCollector.enabled }}
         - otlp

helm-charts/splunk-otel-collector/templates/daemonset.yaml

@@ -152,10 +152,6 @@ spec:
             value: /hostfs/run
           - name: HOST_DEV
             value: /hostfs/dev
-
-          # Host specific resource attributes
-          - name: OTEL_RESOURCE_ATTRIBUTES
-            value: host.name=$(K8S_NODE_NAME),k8s.node.name=$(K8S_NODE_NAME),k8s.pod.name=$(K8S_POD_NAME),k8s.pod.uid=$(K8S_POD_UID),k8s.namespace.name=$(K8S_NAMESPACE)
           {{- end }}
 
         readinessProbe: