fix: Move cadvisor instrumentation to scraper (#388)

With cadvisor instrumentation in agent we would produce
a significant amount of duplicated data.

This moves cadvisor scraping to the centralized prom scraper,
the same setup we had in previous instrumentation.

Author: obs-gh-jabernet

charts/agent/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: agent
 description: Chart to install K8s collection stack based on Observe Agent
 type: application
-version: 0.64.0
+version: 0.64.1
 appVersion: "2.5.0"
 dependencies:
   - name: opentelemetry-collector

charts/agent/README.md

@@ -1,6 +1,6 @@
 # agent
 
-![Version: 0.64.0](https://img.shields.io/badge/Version-0.64.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.5.0](https://img.shields.io/badge/AppVersion-2.5.0-informational?style=flat-square)
+![Version: 0.64.1](https://img.shields.io/badge/Version-0.64.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.5.0](https://img.shields.io/badge/AppVersion-2.5.0-informational?style=flat-square)
 
 Chart to install K8s collection stack based on Observe Agent
 

charts/agent/templates/_config-receivers.tpl

@@ -112,3 +112,28 @@ prometheus/pod_metrics:
           source_labels:
             - __name__
 {{- end -}}
+
+{{- define "config.receivers.prometheus.cadvisor" -}}
+{{- if .Values.node.metrics.cadvisor.enabled }}
+prometheus/cadvisor:
+  config:
+    scrape_configs:
+      - job_name: 'kubernetes-nodes-cadvisor'
+        scheme: https
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          insecure_skip_verify: true
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+        kubernetes_sd_configs:
+          - role: node
+
+        relabel_configs:
+          - target_label: __address__
+            replacement: kubernetes.default.svc:443
+          - source_labels: [__meta_kubernetes_node_name]
+            regex: (.+)
+            target_label: __metrics_path__
+            replacement: /api/v1/nodes/$$1/proxy/metrics/cadvisor
+{{ end }}
+{{ end }}

charts/agent/templates/_node-logs-metrics-config.tpl

@@ -157,28 +157,7 @@ receivers:
     {{- include "observe.daemonset.logsMetrics.config.filelog.multiline" . | nindent 4 }}
     {{ end }}
   {{ end }}
-  {{- if .Values.node.metrics.cadvisor.enabled }}
-  prometheus/cadvisor:
-    config:
-      scrape_configs:
-        - job_name: 'kubernetes-nodes-cadvisor'
-          scheme: https
-          tls_config:
-            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-            insecure_skip_verify: true
-          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
 
-          kubernetes_sd_configs:
-            - role: node
-
-          relabel_configs:
-            - target_label: __address__
-              replacement: kubernetes.default.svc:443
-            - source_labels: [__meta_kubernetes_node_name]
-              regex: (.+)
-              target_label: __metrics_path__
-              replacement: /api/v1/nodes/$$1/proxy/metrics/cadvisor
-    {{ end }}
 processors:
 {{- include "config.processors.memory_limiter" . | nindent 2 }}
 
@@ -205,23 +184,16 @@ processors:
       - key: debug_source
         action: insert
         value: kubeletstats_metrics
-  attributes/debug_source_cadvisor_metrics:
-    actions:
-      - key: debug_source
-        action: insert
-        value: cadvisor_metrics
 
 # Create intermediate lists for pipeline arrays to then modify based on values.yaml
 {{- $logsExporters := (list "otlphttp/observe/base") -}}
 {{- $hostmetricsExporters := (list "prometheusremotewrite/observe") -}}
 {{- $kubeletstatsExporters := (list "prometheusremotewrite/observe") -}}
-{{- $cadvisorMetricsExporters := (list "prometheusremotewrite/observe") -}}
 
 {{- if eq .Values.agent.config.global.debug.enabled true }}
   {{- $logsExporters = concat $logsExporters ( list "debug/override" ) | uniq }}
   {{- $hostmetricsExporters = concat $hostmetricsExporters ( list "debug/override" ) | uniq }}
   {{- $kubeletstatsExporters = concat $kubeletstatsExporters ( list "debug/override" ) | uniq }}
-  {{- $cadvisorMetricsExporters = concat $cadvisorMetricsExporters ( list "debug/override" ) | uniq }}
 {{- end }}
 
 service:
@@ -244,12 +216,6 @@ service:
         processors: [memory_limiter, k8sattributes, batch, resourcedetection/cloud, resource/observe_common, attributes/debug_source_kubeletstats_metrics]
         exporters: [{{ join ", " $kubeletstatsExporters }}]
       {{- end -}}
-      {{- if .Values.node.metrics.cadvisor.enabled }}
-      metrics/cadvisor:
-        receivers: [prometheus/cadvisor]
-        processors: [memory_limiter, k8sattributes, batch, resourcedetection/cloud, resource/observe_common, attributes/debug_source_cadvisor_metrics]
-        exporters: [{{ join ", " $cadvisorMetricsExporters }}]
-      {{- end -}}
 {{- include "config.service.telemetry" . | nindent 2 }}
 
 {{- end }}

charts/agent/templates/_prometheus-scraper-config.tpl

@@ -8,6 +8,7 @@ exporters:
 receivers:
   nop:
 {{- include "config.receivers.prometheus.pod_metrics" . | nindent 2 }}
+{{- include "config.receivers.prometheus.cadvisor" . | nindent 2 }}
 
 processors:
 {{- include "config.processors.memory_limiter" . | nindent 2 }}
@@ -20,6 +21,12 @@ processors:
 
 {{- include "config.processors.attributes.pod_metrics" . | nindent 2 }}
 
+  attributes/debug_source_cadvisor_metrics:
+    actions:
+      - key: debug_source
+        action: insert
+        value: cadvisor_metrics
+
 # Set up receivers
 {{- $podMetricsReceivers := (list "prometheus/pod_metrics") -}}
 {{- if eq .Values.application.prometheusScrape.enabled false}}
@@ -41,6 +48,13 @@ service:
       receivers: [{{ join ", " $podMetricsReceivers }}]
       processors: [memory_limiter, k8sattributes, batch, resource/observe_common, attributes/debug_source_pod_metrics]
       exporters: [{{ join ", " $podMetricsExporters }}]
+    {{- if .Values.node.metrics.cadvisor.enabled }}
+    metrics/cadvisor:
+      receivers: [prometheus/cadvisor]
+      processors: [memory_limiter, k8sattributes, batch, resource/observe_common, attributes/debug_source_cadvisor_metrics]
+      exporters: [{{ join ", " $podMetricsExporters }}]
+    {{- end -}}
+
 {{- include "config.service.telemetry" . | nindent 2 }}
 
  {{- end }}