allow control plane metrics from all distro (#1760)

* allow control plane metrics from all distro

* update issue

Author: jinja2

.chloggen/controlplaneMetrics.yaml

@@ -0,0 +1,23 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+# The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, operator, chart, other)
+component: agent
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow featureGates.useControlPlaneMetricsHistogramData to work with any distribution of k8s cluster.
+# One or more tracking issues related to the change
+issues: [1760]
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Users can enable agent.controlPlaneMetrics for k8s components which run on worker nodes
+  such as kubedns (gke), coredns (aks, eks) and kube-proxy (eks) with the feature gate
+  featureGates.useControlPlaneMetricsHistogramData set to true.
+  To disable collection of metrics from specific control plane components, set the corresponding
+  component to false in the agent.controlPlaneMetrics configuration. For example:
+  agent:
+    controlPlaneMetrics:
+      coredns:
+        enabled: false
+      proxy:
+        enabled: false

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -233,10 +233,21 @@ receivers:
 
       {{- if and (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") .Values.featureGates.useControlPlaneMetricsHistogramData }}
       # Receivers for collecting k8s control plane metrics as native OpenTelemetry metrics, including histogram data.
-      {{- if or (eq .Values.distribution "openshift") (eq .Values.distribution "") }}
       # Below, the TLS certificate verification is often skipped because the k8s default certificate is self signed and
       # will fail the verification.
       {{- if .Values.agent.controlPlaneMetrics.coredns.enabled }}
+      {{- if eq .Values.distribution "gke"}}
+      prometheus/kubedns:
+        rule: type == "pod" && labels["k8s-app"] == "kube-dns"
+        config:
+          config:
+            scrape_configs:
+            - job_name: "kubedns"
+              static_configs:
+                - targets: ['`endpoint`:`"prometheus.io/port" in annotations ? annotations["prometheus.io/port"] : 9153`']
+              tls_config:
+                insecure_skip_verify: true
+      {{- else }}
       prometheus/coredns:
         {{- if eq .Values.distribution "openshift" }}
         rule: type == "pod" && namespace == "openshift-dns" && name contains "dns"
@@ -257,6 +268,7 @@ receivers:
                 - targets: ["`endpoint`:9153"]
               {{- end }}
       {{- end }}
+      {{- end }}
       {{- if .Values.agent.controlPlaneMetrics.etcd.enabled }}
       prometheus/etcd:
         {{- if eq .Values.distribution "openshift" }}
@@ -360,7 +372,6 @@ receivers:
                 credentials_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
                 type: Bearer
       {{- end }}
-      {{- end }}
     {{- end }}
 
   kubeletstats:

helm-charts/splunk-otel-collector/values.yaml

@@ -311,6 +311,8 @@ agent:
       enabled: true
     coredns:
       # Specifies whether to collect coredns metrics.
+      # Note - when using GKE, and this is enabled, the chart will create a
+      # receiver for kubedns instead of coredns.
       enabled: true
     etcd:
       # Specifies whether to collect etcd metrics.
@@ -344,6 +346,8 @@ agent:
       skipVerify: true
     proxy:
       # Specifies whether to collect proxy metrics.
+      # Note - the prometheus receiver introduced for this component might match
+      # kube-proxy deployed in managed k8s cluster like EKS.
       enabled: true
     scheduler:
       # Specifies whether to collect scheduler metrics.