Fix metrics sourcetype annotation (#1375)

* Fix sourcetype annotation for metrics

* Update .chloggen, examples and tests

* Update fixmetricssourcetypeannotation.yaml

* Remove metricsSourceType for sourcetype

* Remove metricsSourceType for sourcetype

* Remove metricsSourceType for sourcetype

* Implement code review suggestions

* re-render examples

* Remove unnecessary if clause

Author: mstopa-splunk

.chloggen/fixmetricssourcetypeannotation.yaml

@@ -0,0 +1,12 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+# The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, operator, chart, other)
+component: other
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fixed updating metrics' sourcetype with annotations
+# One or more tracking issues related to the change
+issues: [1375]
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

docs/advanced-configuration.md

@@ -564,7 +564,8 @@ Manage Splunk OTel Collector Logging with these supported annotations.
 * Filter logs using pod and/or namespace annotation
   * If `logsCollection.containers.useSplunkIncludeAnnotation` is `false` (default: false), set `splunk.com/exclude` annotation to `true` on pod and/or namespace to exclude its logs from ingested.
   * If `logsCollection.containers.useSplunkIncludeAnnotation` is `true` (default: false), set `splunk.com/include` annotation to `true` on pod and/or namespace to only include its logs from ingested. All other logs will be ignored.
-* Use `splunk.com/sourcetype` annotation on pod to overwrite `sourcetype` field. If not set, it is dynamically generated to be `kube:container:CONTAINER_NAME`.
+* For logs, use `splunk.com/sourcetype` annotation on pod to overwrite `sourcetype` field. If not set, it is dynamically generated to be `kube:container:CONTAINER_NAME`.
+* For metrics, use the `splunk.com/sourcetype` annotation on the namespace to override the sourcetype field. If not set, it defaults to `httpevent`.
 
 ### Performance of native OpenTelemetry logs collection
 

examples/disable-persistence-queue-traces/rendered_manifests/configmap-agent.yaml

@@ -161,8 +161,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/disable-persistence-queue-traces/rendered_manifests/configmap-cluster-receiver.yaml

@@ -50,8 +50,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/disable-persistence-queue-traces/rendered_manifests/daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: 2ec1c7329487d2d512b428c60b4250ab94617daac606d6743bf55d1d12ba4612
+        checksum/config: e2dd00b071f073e9a34cf91692e448859d2411b6cc37c8aef56e01ec66a7f241
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/disable-persistence-queue-traces/rendered_manifests/deployment-cluster-receiver.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: bc14c64fe9c32c4096cd0e0df15b75d79d168de86101cd6805367a5acc1333c3
+        checksum/config: 28e46c5046b15065a3b498e52faef708337d402885225a0de09f290a9e1da1e9
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

examples/enable-persistence-queue/rendered_manifests/configmap-agent.yaml

@@ -161,8 +161,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/enable-persistence-queue/rendered_manifests/configmap-cluster-receiver.yaml

@@ -50,8 +50,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/enable-persistence-queue/rendered_manifests/daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: f9c882714a6a5086b2f406ad5bd7bc97e6e101d22dc24632acbcbd3985d32e0f
+        checksum/config: 65e1add475e3e56be5f06e7396676f3488a53f49569179de7e8c56f300eaa2a7
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/enable-persistence-queue/rendered_manifests/deployment-cluster-receiver.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: bc14c64fe9c32c4096cd0e0df15b75d79d168de86101cd6805367a5acc1333c3
+        checksum/config: 28e46c5046b15065a3b498e52faef708337d402885225a0de09f290a9e1da1e9
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

examples/multi-metrics/rendered_manifests/configmap-agent.yaml

@@ -133,8 +133,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/multi-metrics/rendered_manifests/configmap-cluster-receiver.yaml

@@ -50,8 +50,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/multi-metrics/rendered_manifests/daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: 30b7626f53600444b8fd92d896123901004a4070e673276e68046fbf33149280
+        checksum/config: 8745d73647576685177b6fbd0d78c980ad9113d44a008f9414cc8ab4adddff3d
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/multi-metrics/rendered_manifests/deployment-cluster-receiver.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: bc14c64fe9c32c4096cd0e0df15b75d79d168de86101cd6805367a5acc1333c3
+        checksum/config: 28e46c5046b15065a3b498e52faef708337d402885225a0de09f290a9e1da1e9
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

examples/only-metrics-platform/rendered_manifests/configmap-agent.yaml

@@ -106,8 +106,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/only-metrics-platform/rendered_manifests/configmap-cluster-receiver.yaml

@@ -50,8 +50,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/only-metrics-platform/rendered_manifests/daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: d54094aac0f8b04d578e2d8ba3b5a5dcda52b09208458de4e9ea58a942dc6355
+        checksum/config: c977b9304cb278d5df51904fe83364224322d0b2a033bf0b718f86ccb45b6d3e
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/only-metrics-platform/rendered_manifests/deployment-cluster-receiver.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: bc14c64fe9c32c4096cd0e0df15b75d79d168de86101cd6805367a5acc1333c3
+        checksum/config: 28e46c5046b15065a3b498e52faef708337d402885225a0de09f290a9e1da1e9
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

helm-charts/splunk-otel-collector/templates/config/_common.tpl

@@ -235,6 +235,10 @@ k8sattributes/clusterReceiver:
     {{- if eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true"}}
     annotations:
       - key: splunk.com/sourcetype
+        tag_name: com.splunk.sourcetype
+        from: namespace
+      - key: splunk.com/sourcetype
+        tag_name: com.splunk.sourcetype
         from: pod
       - key: splunk.com/index
         tag_name: com.splunk.index
@@ -269,6 +273,10 @@ k8sattributes/metrics:
     metadata: []
     annotations:
       - key: splunk.com/sourcetype
+        tag_name: com.splunk.sourcetype
+        from: namespace
+      - key: splunk.com/sourcetype
+        tag_name: com.splunk.sourcetype
         from: pod
       - key: splunk.com/metricsIndex
         tag_name: com.splunk.index
@@ -341,6 +349,18 @@ resource/logs:
     {{- end }}
 {{- end }}
 
+{{/*
+Resource processor for metrics manipulations
+*/}}
+{{- define "splunk-otel-collector.resourceMetricsProcessor" -}}
+resource/metrics:
+  attributes:
+    # Insert the sourcetype value from values.yaml if it has not already been set through annotations.
+    - key: com.splunk.sourcetype
+      value: "{{.Values.splunkPlatform.sourcetype }}"
+      action: insert
+{{- end }}
+
 {{/*
 The transform processor adds service.name attribute to logs the same way as it's done by istio for the generated traces
 https://github.com/istio/istio/blob/6237cb4e63cf9a332327cc0a815d6b46257e6f8a/pkg/config/analysis/analyzers/testdata/common/sidecar-injector-configmap.yaml#L110-L115
@@ -436,9 +456,6 @@ splunk_hec/platform_metrics:
   token: "${SPLUNK_PLATFORM_HEC_TOKEN}"
   index: {{ .Values.splunkPlatform.metricsIndex | quote }}
   source: {{ .Values.splunkPlatform.source | quote }}
-  {{- if .Values.splunkPlatform.sourcetype }}
-  sourcetype: {{ .Values.splunkPlatform.sourcetype | quote }}
-  {{- end }}
   max_idle_conns: {{ .Values.splunkPlatform.maxConnections }}
   max_idle_conns_per_host: {{ .Values.splunkPlatform.maxConnections }}
   disable_compression: {{ .Values.splunkPlatform.disableCompression }}

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -675,6 +675,9 @@ processors:
   {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }}
     filter:
       node_from_env_var: K8S_NODE_NAME
+  {{- if .Values.splunkPlatform.sourcetype }}
+  {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }}
+  {{- end }}
   {{- end }}
 
   {{- if eq .Values.logsEngine "fluentd" }}
@@ -1065,6 +1068,9 @@ service:
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if $gatewayEnabled }}
@@ -1092,6 +1098,9 @@ service:
         - resource/add_mode
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }}

helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl

@@ -33,6 +33,9 @@ processors:
   {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }}
     filter:
       node_from_env_var: K8S_NODE_NAME
+  {{- if .Values.splunkPlatform.sourcetype }}
+  {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }}
+  {{- end }}
   {{- end }}
 
   {{- include "splunk-otel-collector.resourceLogsProcessor" . | nindent 2 }}
@@ -220,6 +223,9 @@ service:
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }}
@@ -275,6 +281,9 @@ service:
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }}

helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl

@@ -79,6 +79,9 @@ processors:
   {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }}
     filter:
       node_from_env_var: K8S_NODE_NAME
+  {{- if .Values.splunkPlatform.sourcetype }}
+  {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }}
+  {{- end }}
   {{- end }}
 
   batch:
@@ -270,6 +273,9 @@ service:
         - resource
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
         - resource/k8s_cluster
       exporters:
@@ -292,6 +298,9 @@ service:
         - resource
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }}
@@ -314,6 +323,9 @@ service:
         - resource/add_mode
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }}

test/k8s_logging_tests/test_config_logging.py

@@ -81,3 +81,24 @@ def test_metric_index_from_annotations(setup, index, expected):
     logger.info("Splunk received %s events in the last minute",
                 len(events))
     assert len(events) >= expected
+
+@pytest.mark.parametrize("index,sourcetype,expected", [
+    ("test_metrics", "sourcetype-anno", 1)
+])
+def test_metric_sourcetype_from_annotations(setup, index, sourcetype, expected):
+
+    '''
+    Test that metrics are being assigned the "sourcetype-anno" sourcetype, as defined by splunk.com/sourcetype annotation added during setup
+    '''
+    logger.info("testing for metrics index={0} sourcetype={1} expected={2} event(s)".format(index, sourcetype, expected))
+    search_query = "index={0} filter=\"sourcetype={1}\"".format(index, sourcetype)
+
+    events = check_events_from_splunk(start_time="-1h@h",
+                                      url=setup["splunkd_url"],
+                                      user=setup["splunk_user"],
+                                      query=["mpreview {0}".format(
+                                          search_query)],
+                                      password=setup["splunk_password"])
+    logger.info("Splunk received %s events in the last minute",
+                len(events))
+    assert len(events) >= expected

test/test_setup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: default
   annotations:
     splunk.com/metricsIndex: test_metrics
+    splunk.com/sourcetype: sourcetype-anno
 ---
 apiVersion: batch/v1
 kind: Job