[pkg/ottl] adapt mapGetter to handle nested map items within slices (#37408)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

This PR fixes the limitations described in #37405. The recently
introduced `ValueExpression` had to be adapted, and will now only return
raw types, instead of their `pcommon.Map`/`pcommon.Slice` eqivalents, as
suggested in
#37280 (comment)

<!-- Issue number (e.g. #1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes #37405 

<!--Describe what testing was performed and which tests were added.-->
#### Testing

Adapted and extended the unit and e2e tests

---------

Signed-off-by: Florian Bacher <[email protected]>

Author: bacherfl

.chloggen/ottl-nested-map-literals.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/ottl
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fix limitation of map literals within slice literals not being handled correctly
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [37405]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

pkg/ottl/e2e/e2e_test.go

@@ -226,6 +226,15 @@ func Test_e2e_editors(t *testing.T) {
 				m2.PutStr("test", "pass")
 			},
 		},
+		{
+			statement: `merge_maps(attributes, {"map_literal": {"list": [{"foo":"bar"}, "test"]}}, "upsert")`,
+			want: func(tCtx ottllog.TransformContext) {
+				mapAttr := tCtx.GetLogRecord().Attributes().PutEmptyMap("map_literal")
+				l := mapAttr.PutEmptySlice("list")
+				l.AppendEmpty().SetEmptyMap().PutStr("foo", "bar")
+				l.AppendEmpty().SetStr("test")
+			},
+		},
 		{
 			statement: `replace_all_matches(attributes, "*/*", "test")`,
 			want: func(tCtx ottllog.TransformContext) {
@@ -1125,6 +1134,48 @@ func Test_e2e_converters(t *testing.T) {
 				m.PutInt("bar", 5)
 			},
 		},
+		{
+			statement: `set(attributes["test"], {"list":[{"foo":"bar"}]})`,
+			want: func(tCtx ottllog.TransformContext) {
+				m := tCtx.GetLogRecord().Attributes().PutEmptyMap("test")
+				m2 := m.PutEmptySlice("list").AppendEmpty().SetEmptyMap()
+				m2.PutStr("foo", "bar")
+			},
+		},
+		{
+			statement: `set(attributes, {"list":[{"foo":"bar"}]})`,
+			want: func(tCtx ottllog.TransformContext) {
+				tCtx.GetLogRecord().Attributes().Clear()
+				m2 := tCtx.GetLogRecord().Attributes().PutEmptySlice("list").AppendEmpty().SetEmptyMap()
+				m2.PutStr("foo", "bar")
+			},
+		},
+		{
+			statement: `set(attributes["arr"], [{"list":[{"foo":"bar"}]}, {"bar":"baz"}])`,
+			want: func(tCtx ottllog.TransformContext) {
+				arr := tCtx.GetLogRecord().Attributes().PutEmptySlice("arr")
+				arr.AppendEmpty().SetEmptyMap().PutEmptySlice("list").AppendEmpty().SetEmptyMap().PutStr("foo", "bar")
+				arr.AppendEmpty().SetEmptyMap().PutStr("bar", "baz")
+			},
+		},
+		{
+			statement: `set(attributes["test"], IsList([{"list":[{"foo":"bar"}]}, {"bar":"baz"}]))`,
+			want: func(tCtx ottllog.TransformContext) {
+				tCtx.GetLogRecord().Attributes().PutBool("test", true)
+			},
+		},
+		{
+			statement: `set(attributes["test"], IsMap({"list":[{"foo":"bar"}]}))`,
+			want: func(tCtx ottllog.TransformContext) {
+				tCtx.GetLogRecord().Attributes().PutBool("test", true)
+			},
+		},
+		{
+			statement: `set(attributes["test"], Len([{"list":[{"foo":"bar"}]}, {"bar":"baz"}]))`,
+			want: func(tCtx ottllog.TransformContext) {
+				tCtx.GetLogRecord().Attributes().PutInt("test", 2)
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -1268,41 +1319,176 @@ func Test_e2e_ottl_features(t *testing.T) {
 	}
 }
 
+func Test_e2e_ottl_statement_sequence(t *testing.T) {
+	tests := []struct {
+		name       string
+		statements []string
+		want       func(tCtx ottllog.TransformContext)
+	}{
+		{
+			name: "delete key of map literal",
+			statements: []string{
+				`set(attributes["test"], {"foo":"bar", "list":[{"test":"hello"}]})`,
+				`delete_key(attributes["test"], "foo")`,
+			},
+			want: func(tCtx ottllog.TransformContext) {
+				m := tCtx.GetLogRecord().Attributes().PutEmptyMap("test")
+				m.PutEmptySlice("list").AppendEmpty().SetEmptyMap().PutStr("test", "hello")
+			},
+		},
+		{
+			name: "delete matching keys of map literal",
+			statements: []string{
+				`set(attributes["test"], {"foo":"bar", "list":[{"test":"hello"}]})`,
+				`delete_matching_keys(attributes["test"], ".*oo")`,
+			},
+			want: func(tCtx ottllog.TransformContext) {
+				m := tCtx.GetLogRecord().Attributes().PutEmptyMap("test")
+				m.PutEmptySlice("list").AppendEmpty().SetEmptyMap().PutStr("test", "hello")
+			},
+		},
+		{
+			name: "keep matching keys of map literal",
+			statements: []string{
+				`set(attributes["test"], {"foo":"bar", "list":[{"test":"hello"}]})`,
+				`keep_matching_keys(attributes["test"], ".*ist")`,
+			},
+			want: func(tCtx ottllog.TransformContext) {
+				m := tCtx.GetLogRecord().Attributes().PutEmptyMap("test")
+				m.PutEmptySlice("list").AppendEmpty().SetEmptyMap().PutStr("test", "hello")
+			},
+		},
+		{
+			name: "flatten map literal",
+			statements: []string{
+				`set(attributes["test"], {"foo":"bar", "list":[{"test":"hello"}]})`,
+				`flatten(attributes["test"])`,
+			},
+			want: func(tCtx ottllog.TransformContext) {
+				m := tCtx.GetLogRecord().Attributes().PutEmptyMap("test")
+				m.PutStr("foo", "bar")
+				m.PutStr("list.0.test", "hello")
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tCtx := constructLogTransformContext()
+
+			for _, statement := range tt.statements {
+				logStatements, err := parseStatementWithAndWithoutPathContext(statement)
+				assert.NoError(t, err)
+
+				for _, s := range logStatements {
+					_, _, _ = s.Execute(context.Background(), tCtx)
+				}
+			}
+
+			exTCtx := constructLogTransformContext()
+			tt.want(exTCtx)
+
+			assert.NoError(t, plogtest.CompareResourceLogs(newResourceLogs(exTCtx), newResourceLogs(tCtx)))
+		})
+	}
+}
+
 func Test_e2e_ottl_value_expressions(t *testing.T) {
 	tests := []struct {
 		name      string
 		statement string
-		want      any
+		want      func() any
 	}{
 		{
 			name:      "string literal",
 			statement: `"foo"`,
-			want:      "foo",
+			want: func() any {
+				return "foo"
+			},
 		},
 		{
 			name:      "attribute value",
 			statement: `resource.attributes["host.name"]`,
-			want:      "localhost",
+			want: func() any {
+				return "localhost"
+			},
 		},
 		{
 			name:      "accessing enum",
 			statement: `SEVERITY_NUMBER_TRACE`,
-			want:      int64(1),
+			want: func() any {
+				return int64(1)
+			},
 		},
 		{
 			name:      "Using converter",
 			statement: `TraceID(0x0102030405060708090a0b0c0d0e0f10)`,
-			want:      pcommon.TraceID{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10},
+			want: func() any {
+				return pcommon.TraceID{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10}
+			},
 		},
 		{
 			name:      "Adding results of two converter operations",
 			statement: `Len(attributes) + Len(attributes)`,
-			want:      int64(24),
+			want: func() any {
+				return int64(28)
+			},
 		},
 		{
 			name:      "Nested converter operations",
 			statement: `Hex(Len(attributes) + Len(attributes))`,
-			want:      "0000000000000018",
+			want: func() any {
+				return "000000000000001c"
+			},
+		},
+		{
+			name:      "return map type 1",
+			statement: `attributes["foo"]`,
+			want: func() any {
+				m := pcommon.NewMap()
+				_ = m.FromRaw(map[string]any{
+					"bar": "pass",
+				})
+				return m
+			},
+		},
+		{
+			name:      "return map type 2",
+			statement: `attributes["foo2"]`,
+			want: func() any {
+				m := pcommon.NewMap()
+				_ = m.FromRaw(map[string]any{
+					"slice": []any{
+						"val",
+					},
+				})
+				return m
+			},
+		},
+		{
+			name:      "return map type 3",
+			statement: `attributes["foo3"]`,
+			want: func() any {
+				m := pcommon.NewMap()
+				_ = m.FromRaw(map[string]any{
+					"nested": map[string]any{
+						"test": "pass",
+					},
+				})
+				return m
+			},
+		},
+		{
+			name:      "return list",
+			statement: `attributes["things"]`,
+			want: func() any {
+				s := pcommon.NewSlice()
+				_ = s.FromRaw([]any{
+					map[string]any{"name": "foo"},
+					map[string]any{"name": "bar"},
+				})
+				return s
+			},
 		},
 	}
 
@@ -1314,11 +1500,11 @@ func Test_e2e_ottl_value_expressions(t *testing.T) {
 			valueExpr, err := logParser.ParseValueExpression(tt.statement)
 			assert.NoError(t, err)
 
-			tCtx := constructLogTransformContext()
+			tCtx := constructLogTransformContextValueExpressions()
 			val, err := valueExpr.Eval(context.Background(), tCtx)
 			assert.NoError(t, err)
 
-			assert.Equal(t, tt.want, val)
+			assert.Equal(t, tt.want(), val)
 		})
 	}
 }
@@ -1526,6 +1712,58 @@ func constructLogTransformContextEditors() ottllog.TransformContext {
 	return ottllog.NewTransformContext(logRecord, scope, resource, plog.NewScopeLogs(), plog.NewResourceLogs())
 }
 
+func constructLogTransformContextValueExpressions() ottllog.TransformContext {
+	resource := pcommon.NewResource()
+	resource.Attributes().PutStr("host.name", "localhost")
+	resource.Attributes().PutStr("A|B|C", "newValue")
+
+	scope := pcommon.NewInstrumentationScope()
+	scope.SetName("scope")
+
+	logRecord := plog.NewLogRecord()
+	logRecord.Body().SetStr("operationA")
+	logRecord.SetTimestamp(TestLogTimestamp)
+	logRecord.SetObservedTimestamp(TestObservedTimestamp)
+	logRecord.SetDroppedAttributesCount(1)
+	logRecord.SetFlags(plog.DefaultLogRecordFlags.WithIsSampled(true))
+	logRecord.SetSeverityNumber(1)
+	logRecord.SetTraceID(traceID)
+	logRecord.SetSpanID(spanID)
+	logRecord.Attributes().PutStr("http.method", "get")
+	logRecord.Attributes().PutStr("http.path", "/health")
+	logRecord.Attributes().PutStr("http.url", "http://localhost/health")
+	logRecord.Attributes().PutStr("flags", "A|B|C")
+	logRecord.Attributes().PutStr("total.string", "123456789")
+	logRecord.Attributes().PutStr("A|B|C", "something")
+	logRecord.Attributes().PutStr("foo", "foo")
+	logRecord.Attributes().PutStr("slice", "slice")
+	logRecord.Attributes().PutStr("val", "val2")
+	logRecord.Attributes().PutInt("int_value", 0)
+	arr := logRecord.Attributes().PutEmptySlice("array")
+	arr0 := arr.AppendEmpty()
+	arr0.SetStr("looong")
+	m := logRecord.Attributes().PutEmptyMap("foo")
+	m.PutStr("bar", "pass")
+
+	m2 := logRecord.Attributes().PutEmptyMap("foo2")
+	s := m2.PutEmptySlice("slice")
+	v := s.AppendEmpty()
+	v.SetStr("val")
+
+	m3 := logRecord.Attributes().PutEmptyMap("foo3")
+	m31 := m3.PutEmptyMap("nested")
+	m31.PutStr("test", "pass")
+
+	s2 := logRecord.Attributes().PutEmptySlice("things")
+	thing1 := s2.AppendEmpty().SetEmptyMap()
+	thing1.PutStr("name", "foo")
+
+	thing2 := s2.AppendEmpty().SetEmptyMap()
+	thing2.PutStr("name", "bar")
+
+	return ottllog.NewTransformContext(logRecord, scope, resource, plog.NewScopeLogs(), plog.NewResourceLogs())
+}
+
 func constructSpanTransformContext() ottlspan.TransformContext {
 	resource := pcommon.NewResource()