Set default send_batch_max_size on k8s cluster receiver (#717)

dmitryax · web-flow · commit c8c11368a19d · 2023-04-03T09:05:27.000-07:00
Big k8s clusters with many pods cause the k8s cluster receiver to fetch metrics batches with many data points which leads to timeouts on signalfx exporter.

This change introduces a maximum limit of data points that one metrics batch can contain. Batches with more than 32k data points will be broken down in several batches.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Update the Kubernetes scheduler monitor to stop using insecure port 10251 and start using secure port 10259 with authentication [#711](https://github.com/signalfx/splunk-otel-collector-chart/pull/711)
 
+### Fixed
+
+- Sending request timeouts in k8s cluster receiver deployment on big k8s clusters [#717](https://github.com/signalfx/splunk-otel-collector-chart/pull/717)
+
 ### Added
 
 - Add experimental support for deploying OpenTelemetry Operator as a subchart [#691](https://github.com/signalfx/splunk-otel-collector-chart/pull/691)
diff --git a/examples/add-receiver-creator/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-receiver-creator/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/add-receiver-creator/rendered_manifests/deployment-cluster-receiver.yaml b/examples/add-receiver-creator/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/add-sampler/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-sampler/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/add-sampler/rendered_manifests/deployment-cluster-receiver.yaml b/examples/add-sampler/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml
@@ -33,7 +33,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 0c43c73ed8b95ee3bf6bdfbe3b60357b0fcb2c8b5306c5a525e3c3a04db55d8b
+        checksum/config: a1be4b2a3f836c3d03dc67eebcefd37e76aee79ee605b02eb53ccaf2cc6eaa21
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/crio-logging/rendered_manifests/configmap-cluster-receiver.yaml b/examples/crio-logging/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/crio-logging/rendered_manifests/deployment-cluster-receiver.yaml b/examples/crio-logging/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/default/rendered_manifests/configmap-cluster-receiver.yaml b/examples/default/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/default/rendered_manifests/deployment-cluster-receiver.yaml b/examples/default/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/distribution-aks/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-aks/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/distribution-aks/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-aks/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: f92818798cea443e8009c342ec0ae2e6cc3418e8301ab666bfef7b45eacea47a
+        checksum/config: 766af110ef6da6a4a474fc7724819d67e831f2eef429dfb0f69d48a127ff23de
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/distribution-eks-fargate/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-eks-fargate/rendered_manifests/configmap-cluster-receiver.yaml
@@ -31,7 +31,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/distribution-eks-fargate/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-eks-fargate/rendered_manifests/deployment-cluster-receiver.yaml
@@ -32,7 +32,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 1b359e355f48e0001516d21cd16e7d9896e4fb03e6b626a988fab488d126bffb
+        checksum/config: b5c9f6008d6ff36f6aea6388f4ba7e5a4dc8653d17f68b8fa1864f255ff129a3
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/distribution-eks/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-eks/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/distribution-eks/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-eks/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 420c3462317cca26689c016cfff600b5ec3529e07e3a758890989d93e976538b
+        checksum/config: 76bfb8d08cf2cb4ff14e34c69e98cc102116a619030a0430ffcb0254cbbb8b9b
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/distribution-gke-autopilot/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-gke-autopilot/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/distribution-gke-autopilot/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-gke-autopilot/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 9f4e518a28726a74ce0d156f944339ad0541a14640151748d28e1757bcd13620
+        checksum/config: 42ac50d7aaac2a656c40808a2591939df84ec3b650e27e384328aa4f9e3f2d71
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/distribution-gke/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-gke/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/distribution-gke/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-gke/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 9f4e518a28726a74ce0d156f944339ad0541a14640151748d28e1757bcd13620
+        checksum/config: 42ac50d7aaac2a656c40808a2591939df84ec3b650e27e384328aa4f9e3f2d71
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/distribution-openshift/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-openshift/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/distribution-openshift/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-openshift/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 707ed21d7355ed365b199c2b04321cca18bd4410add5c21a4265c4516503d6fd
+        checksum/config: 10ac8d5708dd8d894fd97c6ec0b32d7abebd7061ca1deb80acf0116e0d676286
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/enable-network-explorer/rendered_manifests/configmap-cluster-receiver.yaml b/examples/enable-network-explorer/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/enable-network-explorer/rendered_manifests/deployment-cluster-receiver.yaml b/examples/enable-network-explorer/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/enabled-pprof-extension/rendered_manifests/configmap-cluster-receiver.yaml b/examples/enabled-pprof-extension/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/enabled-pprof-extension/rendered_manifests/deployment-cluster-receiver.yaml b/examples/enabled-pprof-extension/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/filter-container-metrics/rendered_manifests/configmap-cluster-receiver.yaml b/examples/filter-container-metrics/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       filter/exclude_containers:
         metrics:
           exclude:
diff --git a/examples/filter-container-metrics/rendered_manifests/deployment-cluster-receiver.yaml b/examples/filter-container-metrics/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 77bff56590873e36c7b0a53d91393ccbd719e2afabdb60bece57ce70f559f09c
+        checksum/config: 00d8b18dea859c37ab2a4b148a753ef21cb670c32fc36b734103502d3da19e11
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/fluentd-multiline-logs-java-stack-traces/rendered_manifests/configmap-cluster-receiver.yaml b/examples/fluentd-multiline-logs-java-stack-traces/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/fluentd-multiline-logs-java-stack-traces/rendered_manifests/deployment-cluster-receiver.yaml b/examples/fluentd-multiline-logs-java-stack-traces/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/kubernetes-windows-nodes/rendered_manifests/configmap-cluster-receiver.yaml b/examples/kubernetes-windows-nodes/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/kubernetes-windows-nodes/rendered_manifests/deployment-cluster-receiver.yaml b/examples/kubernetes-windows-nodes/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/only-metrics/rendered_manifests/configmap-cluster-receiver.yaml b/examples/only-metrics/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/only-metrics/rendered_manifests/deployment-cluster-receiver.yaml b/examples/only-metrics/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/route-data-through-gateway-deployed-separately/rendered_manifests/configmap-cluster-receiver.yaml b/examples/route-data-through-gateway-deployed-separately/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/route-data-through-gateway-deployed-separately/rendered_manifests/deployment-cluster-receiver.yaml b/examples/route-data-through-gateway-deployed-separately/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 5ef6a2963f553a09f4a6f64d586b25bb2564602dfea2b170a4b31a3069797734
+        checksum/config: 2f10fcbfbaa3132a3fdc7ea9eb1e1f43f1cb0425ec6e91ef3fdbaa0f4e26ad97
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/examples/use-proxy/rendered_manifests/configmap-cluster-receiver.yaml b/examples/use-proxy/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
diff --git a/examples/use-proxy/rendered_manifests/deployment-cluster-receiver.yaml b/examples/use-proxy/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:
diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl
@@ -83,6 +83,7 @@ processors:
   {{- include "splunk-otel-collector.otelMemoryLimiterConfig" . | nindent 2 }}
 
   batch:
+    send_batch_max_size: 32768
 
   {{- include "splunk-otel-collector.resourceDetectionProcessor" . | nindent 2 }}
 
