[pkg/ottl] fix handling of nested maps within slices in flatten function (#36204)

Co-authored-by: Evan Bradley <[email protected]>

Author: bacherfl

.chloggen/ottl-fix-flattening-of-nested-slices.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/ottl
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: fix handling of nested maps within slices in the `flatten` function
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [36162]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

pkg/ottl/e2e/e2e_test.go

@@ -71,13 +71,10 @@ func Test_e2e_editors(t *testing.T) {
 				tCtx.GetLogRecord().Attributes().PutStr("foo.nested.test", "pass")
 
 				tCtx.GetLogRecord().Attributes().Remove("things")
-				m1 := tCtx.GetLogRecord().Attributes().PutEmptyMap("things.0")
-				m1.PutStr("name", "foo")
-				m1.PutInt("value", 2)
-
-				m2 := tCtx.GetLogRecord().Attributes().PutEmptyMap("things.1")
-				m2.PutStr("name", "bar")
-				m2.PutInt("value", 5)
+				tCtx.GetLogRecord().Attributes().PutStr("things.0.name", "foo")
+				tCtx.GetLogRecord().Attributes().PutInt("things.0.value", 2)
+				tCtx.GetLogRecord().Attributes().PutStr("things.1.name", "bar")
+				tCtx.GetLogRecord().Attributes().PutInt("things.1.value", 5)
 			},
 		},
 		{
@@ -89,20 +86,18 @@ func Test_e2e_editors(t *testing.T) {
 				m.PutStr("test.http.url", "http://localhost/health")
 				m.PutStr("test.flags", "A|B|C")
 				m.PutStr("test.total.string", "123456789")
+
 				m.PutStr("test.foo.bar", "pass")
 				m.PutStr("test.foo.flags", "pass")
 				m.PutStr("test.foo.bar", "pass")
 				m.PutStr("test.foo.flags", "pass")
 				m.PutStr("test.foo.slice.0", "val")
 				m.PutStr("test.foo.nested.test", "pass")
 
-				m1 := m.PutEmptyMap("test.things.0")
-				m1.PutStr("name", "foo")
-				m1.PutInt("value", 2)
-
-				m2 := m.PutEmptyMap("test.things.1")
-				m2.PutStr("name", "bar")
-				m2.PutInt("value", 5)
+				m.PutStr("test.things.0.name", "foo")
+				m.PutInt("test.things.0.value", 2)
+				m.PutStr("test.things.1.name", "bar")
+				m.PutInt("test.things.1.value", 5)
 				m.CopyTo(tCtx.GetLogRecord().Attributes())
 			},
 		},

pkg/ottl/ottlfuncs/func_flatten.go

@@ -54,28 +54,45 @@ func flatten[K any](target ottl.PMapGetter[K], p ottl.Optional[string], d ottl.O
 		}
 
 		result := pcommon.NewMap()
-		flattenHelper(m, result, prefix, 0, depth)
+		flattenMap(m, result, prefix, 0, depth)
 		result.MoveTo(m)
 
 		return nil, nil
 	}, nil
 }
 
-func flattenHelper(m pcommon.Map, result pcommon.Map, prefix string, currentDepth, maxDepth int64) {
+func flattenMap(m pcommon.Map, result pcommon.Map, prefix string, currentDepth, maxDepth int64) {
 	if len(prefix) > 0 {
 		prefix += "."
 	}
 	m.Range(func(k string, v pcommon.Value) bool {
-		switch {
-		case v.Type() == pcommon.ValueTypeMap && currentDepth < maxDepth:
-			flattenHelper(v.Map(), result, prefix+k, currentDepth+1, maxDepth)
-		case v.Type() == pcommon.ValueTypeSlice && currentDepth < maxDepth:
-			for i := 0; i < v.Slice().Len(); i++ {
+		return flattenValue(k, v, currentDepth, maxDepth, result, prefix)
+	})
+}
+
+func flattenSlice(s pcommon.Slice, result pcommon.Map, prefix string, currentDepth int64, maxDepth int64) {
+	for i := 0; i < s.Len(); i++ {
+		flattenValue(fmt.Sprintf("%d", i), s.At(i), currentDepth+1, maxDepth, result, prefix)
+	}
+}
+
+func flattenValue(k string, v pcommon.Value, currentDepth int64, maxDepth int64, result pcommon.Map, prefix string) bool {
+	switch {
+	case v.Type() == pcommon.ValueTypeMap && currentDepth < maxDepth:
+		flattenMap(v.Map(), result, prefix+k, currentDepth+1, maxDepth)
+	case v.Type() == pcommon.ValueTypeSlice && currentDepth < maxDepth:
+		for i := 0; i < v.Slice().Len(); i++ {
+			switch {
+			case v.Slice().At(i).Type() == pcommon.ValueTypeMap && currentDepth+1 < maxDepth:
+				flattenMap(v.Slice().At(i).Map(), result, fmt.Sprintf("%v.%v", prefix+k, i), currentDepth+2, maxDepth)
+			case v.Slice().At(i).Type() == pcommon.ValueTypeSlice && currentDepth+1 < maxDepth:
+				flattenSlice(v.Slice().At(i).Slice(), result, fmt.Sprintf("%v.%v", prefix+k, i), currentDepth+2, maxDepth)
+			default:
 				v.Slice().At(i).CopyTo(result.PutEmpty(fmt.Sprintf("%v.%v", prefix+k, i)))
 			}
-		default:
-			v.CopyTo(result.PutEmpty(prefix + k))
 		}
-		return true
-	})
+	default:
+		v.CopyTo(result.PutEmpty(prefix + k))
+	}
+	return true
 }

pkg/ottl/ottlfuncs/func_flatten_test.go

@@ -85,6 +85,31 @@ func Test_flatten(t *testing.T) {
 				"occupants.1":    "user 2",
 			},
 		},
+		{
+			name: "combination with mixed nested slices",
+			target: map[string]any{
+				"name": "test",
+				"address": map[string]any{
+					"street": "first",
+					"house":  int64(1234),
+				},
+				"occupants": []any{
+					"user 1",
+					map[string]any{
+						"name": "user 2",
+					},
+				},
+			},
+			prefix: ottl.Optional[string]{},
+			depth:  ottl.Optional[int64]{},
+			expected: map[string]any{
+				"name":             "test",
+				"address.street":   "first",
+				"address.house":    int64(1234),
+				"occupants.0":      "user 1",
+				"occupants.1.name": "user 2",
+			},
+		},
 		{
 			name: "deep nesting",
 			target: map[string]any{