fixing missing logs while pod log file being rolled over

Author: pszkamruk-splunk

.chloggen/dropped-logs-while-logs-rolling-fix.yaml

@@ -0,0 +1,12 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+# The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, operator, chart, other)
+component: agent
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: a fix for a scenario where some logs might be missed due to the pod log file being rolled over during high load, set featureGates.fixMissedLogsDuringLogRotation to true to enable the fix
+# One or more tracking issues related to the change
+issues: [1690]
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -410,14 +410,31 @@ receivers:
   {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (eq .Values.logsEngine "otel") }}
   {{- if .Values.logsCollection.containers.enabled }}
   filelog:
+    {{- if not .Values.featureGates.fixMissedLogsDuringLogRotation }}
     {{- if .Values.isWindows }}
     include: ["C:\\var\\log\\pods\\*\\*\\*.log"]
     {{- else }}
     include: ["/var/log/pods/*/*/*.log"]
     {{- end }}
+    {{- else }}
+    {{- if .Values.isWindows }}
+    include: ["C:\\var\\log\\pods\\*\\*\\*.log*"]
+    {{- else }}
+    include: ["/var/log/pods/*/*/*.log*"]
+    {{- end }}
+    {{- end }}
     # Exclude logs. The file format is
     # /var/log/pods/<namespace_name>_<pod_name>_<pod_uid>/<container_name>/<restart_count>.log
     exclude:
+      {{- if .Values.featureGates.fixMissedLogsDuringLogRotation }}
+      {{- if .Values.isWindows }}
+      - "C:\\var\\log\\pods\\*\\*\\*.log*.gz"
+      - "C:\\var\\log\\pods\\*\\*\\*.log*.tmp"
+      {{- else }}
+      - "/var/log/pods/*/*/*.log*.gz"
+      - "/var/log/pods/*/*/*.log*.tmp"
+      {{- end }}
+      {{- end }}
       {{- if .Values.logsCollection.containers.excludeAgentLogs }}
       {{- if .Values.isWindows }}
       - "C:\\var\\log\\pods\\{{ template "splunk-otel-collector.namespace" . }}_{{ include "splunk-otel-collector.fullname" . }}*_*\\otel-collector\\*.log"
@@ -516,11 +533,19 @@ receivers:
         value: ""
       # Extract metadata from file path
       - type: regex_parser
+        {{- if not .Values.featureGates.fixMissedLogsDuringLogRotation }}
         {{- if .Values.isWindows }}
         regex: '^C:\\var\\log\\pods\\(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\\(?P<container_name>[^\._]+)\\(?P<restart_count>\d+)\.log$'
         {{- else }}
         regex: '^\/var\/log\/pods\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$'
         {{- end }}
+        {{- else }}
+        {{- if .Values.isWindows }}
+        regex: '^C:\\var\\log\\pods\\(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\\(?P<container_name>[^\._]+)\\(?P<restart_count>\d+)\.log'
+        {{- else }}
+        regex: '^\/var\/log\/pods\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log'
+        {{- end }}
+        {{- end }}
         parse_from: attributes["log.file.path"]
       # Move out attributes to Attributes
       - type: move

helm-charts/splunk-otel-collector/values.yaml

@@ -1387,3 +1387,6 @@ featureGates:
   explicitMountServiceAccountToken: false
   # Use a specific metrics pipeline to report control plane metrics as histograms.
   useControlPlaneMetricsHistogramData: false
+  # The feature gate provides a fix for a scenario where some logs might be missed due to the pod log file being
+  # rolled over during high load.
+  fixMissedLogsDuringLogRotation: false