[processor/transform] Fix basic config cache access (open-telemetry#39290)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

This PR removes the shared cache logic we introduced to support Basic
Config statements, and changes the approach to group all Basic config
statements together into the same sharing-cache
`common.ContextStatements`, embracing the limitation of not being able
to infer the context depending on the user's statements (similar to the
advanced config).
To help with that limitation, the context inferrer validation was
improved, and it now returns specific errors that better describe why it
couldn't infer a valid context.

This PR also introduces a **breaking change**, that consists in forcing
users to use either basic or advanced configuration, for example, mixed
configurations like the following won't be valid anymore, and it will
require users to use only one style:

```yaml
log_statements:
 - set(resource.attributes["foo"], "foo")
 - statements:
   - set(resource.attributes["bar"], "bar")
```

---
Another PR will be open for removing the `WithCache` option from all
OTTL contexts.


<!-- Issue number (e.g. open-telemetry#1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes
open-telemetry#38926

<!--Describe what testing was performed and which tests were added.-->
#### Testing
Unit tests

<!--Describe the documentation added.-->
#### Documentation
Updated README 

<!--Please delete paragraphs that you did not use before submitting.-->

Author: edmocosta

.chloggen/fix-transform-processor-shared-cache.yaml

@@ -0,0 +1,29 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: processor/transform
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fix Basic Config style to properly handle `cache` access.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [38926]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+  The Transform processor now requires only one configuration style per processor's configuration, 
+  which means Advanced Config and Basic Config cannot be used together anymore.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

pkg/ottl/context_inferrer.go

@@ -5,6 +5,7 @@ package ottl // import "github.com/open-telemetry/opentelemetry-collector-contri
 
 import (
 	"cmp"
+	"errors"
 	"fmt"
 	"math"
 	"slices"
@@ -105,7 +106,7 @@ func (s *priorityContextInferrer) infer(ottls []string, hinter hinterFunc) (infe
 		if inferredContext != "" {
 			s.telemetrySettings.Logger.Debug(fmt.Sprintf(`Inferred context: "%s"`, inferredContext))
 		} else {
-			s.telemetrySettings.Logger.Debug("Unable to infer context from statements")
+			s.telemetrySettings.Logger.Debug("Unable to infer context from statements", zap.Error(err))
 		}
 	}()
 
@@ -114,9 +115,9 @@ func (s *priorityContextInferrer) infer(ottls []string, hinter hinterFunc) (infe
 
 	var inferredContextPriority int
 	for _, ottl := range ottls {
-		ottlPaths, ottlFunctions, ottlEnums, err := hinter(ottl)
-		if err != nil {
-			return "", err
+		ottlPaths, ottlFunctions, ottlEnums, hinterErr := hinter(ottl)
+		if hinterErr != nil {
+			return "", hinterErr
 		}
 		for _, p := range ottlPaths {
 			candidate := p.Context
@@ -141,11 +142,13 @@ func (s *priorityContextInferrer) infer(ottls []string, hinter hinterFunc) (infe
 		s.telemetrySettings.Logger.Debug("No context candidate found in the ottls")
 		return inferredContext, nil
 	}
-	ok := s.validateContextCandidate(inferredContext, requiredFunctions, requiredEnums)
-	if ok {
+	if err = s.validateContextCandidate(inferredContext, requiredFunctions, requiredEnums); err == nil {
 		return inferredContext, nil
 	}
-	return s.inferFromLowerContexts(inferredContext, requiredFunctions, requiredEnums), nil
+	if inferredFromLowerContexts, lowerContextErr := s.inferFromLowerContexts(inferredContext, requiredFunctions, requiredEnums); lowerContextErr == nil {
+		return inferredFromLowerContexts, nil
+	}
+	return "", err
 }
 
 // validateContextCandidate checks if the given context candidate has all required functions names
@@ -154,29 +157,26 @@ func (s *priorityContextInferrer) validateContextCandidate(
 	context string,
 	requiredFunctions map[string]struct{},
 	requiredEnums map[enumSymbol]struct{},
-) bool {
+) error {
 	s.telemetrySettings.Logger.Debug(fmt.Sprintf(`Validating selected context candidate: "%s"`, context))
 	candidate, ok := s.contextCandidate[context]
 	if !ok {
-		s.telemetrySettings.Logger.Debug(fmt.Sprintf(`Context "%s" is not a valid candidate`, context))
-		return false
+		return fmt.Errorf(`inferred context "%s" is not a valid candidate`, context)
 	}
 	if len(requiredFunctions) == 0 && len(requiredEnums) == 0 {
-		return true
+		return nil
 	}
 	for function := range requiredFunctions {
 		if !candidate.hasFunctionName(function) {
-			s.telemetrySettings.Logger.Debug(fmt.Sprintf(`Context "%s" does not meet the function requirement: "%s"`, context, function))
-			return false
+			return fmt.Errorf(`inferred context "%s" does not support the function "%s"`, context, function)
 		}
 	}
 	for enum := range requiredEnums {
 		if !candidate.hasEnumSymbol((*EnumSymbol)(&enum)) {
-			s.telemetrySettings.Logger.Debug(fmt.Sprintf(`Context "%s" does not meet the enum requirement: "%s"`, context, string(enum)))
-			return false
+			return fmt.Errorf(`inferred context "%s" does not support the enum symbol "%s"`, context, string(enum))
 		}
 	}
-	return true
+	return nil
 }
 
 // inferFromLowerContexts returns the first lower context that supports all required functions
@@ -187,26 +187,33 @@ func (s *priorityContextInferrer) inferFromLowerContexts(
 	context string,
 	requiredFunctions map[string]struct{},
 	requiredEnums map[enumSymbol]struct{},
-) string {
+) (inferredContext string, err error) {
 	s.telemetrySettings.Logger.Debug(fmt.Sprintf(`Trying to infer context using "%s" lower contexts`, context))
+
+	defer func() {
+		if err != nil {
+			s.telemetrySettings.Logger.Debug("Unable to infer context from lower contexts", zap.Error(err))
+		}
+	}()
+
 	inferredContextCandidate, ok := s.contextCandidate[context]
 	if !ok {
-		return ""
+		return "", fmt.Errorf(`context "%s" is not a valid candidate`, context)
 	}
 
 	lowerContextCandidates := inferredContextCandidate.getLowerContexts(context)
 	if len(lowerContextCandidates) == 0 {
-		return ""
+		return "", fmt.Errorf(`context "%s" has no lower contexts candidates`, context)
 	}
 
 	s.sortContextCandidates(lowerContextCandidates)
 	for _, lowerCandidate := range lowerContextCandidates {
-		ok = s.validateContextCandidate(lowerCandidate, requiredFunctions, requiredEnums)
-		if ok {
-			return lowerCandidate
+		if candidateErr := s.validateContextCandidate(lowerCandidate, requiredFunctions, requiredEnums); candidateErr == nil {
+			return lowerCandidate, nil
 		}
+		s.telemetrySettings.Logger.Debug(fmt.Sprintf(`lower context "%s" is not a valid candidate`, lowerCandidate), zap.Error(err))
 	}
-	return ""
+	return "", errors.New("no valid lower context found")
 }
 
 // sortContextCandidates sorts the slice candidates using the priorityContextInferrer.contextsPriority order.

pkg/ottl/context_inferrer_test.go

@@ -44,6 +44,7 @@ func Test_NewPriorityContextInferrer_InferStatements(t *testing.T) {
 		candidates map[string]*priorityContextInferrerCandidate
 		statements []string
 		expected   string
+		err        string
 	}{
 		{
 			name:     "with priority and statement context",
@@ -109,7 +110,7 @@ func Test_NewPriorityContextInferrer_InferStatements(t *testing.T) {
 				"metric":    newDummyPriorityContextInferrerCandidate(false, false, []string{"datapoint"}),
 				"datapoint": newDummyPriorityContextInferrerCandidate(false, false, []string{}),
 			},
-			expected: "",
+			err: `inferred context "metric" does not support the function "set"`,
 		},
 		{
 			name:       "inferred path context with missing function and no lower context",
@@ -118,7 +119,7 @@ func Test_NewPriorityContextInferrer_InferStatements(t *testing.T) {
 			candidates: map[string]*priorityContextInferrerCandidate{
 				"metric": newDummyPriorityContextInferrerCandidate(false, true, []string{}),
 			},
-			expected: "",
+			err: `inferred context "metric" does not support the function "set"`,
 		},
 		{
 			name:       "inferred path context with missing enum",
@@ -135,7 +136,7 @@ func Test_NewPriorityContextInferrer_InferStatements(t *testing.T) {
 			priority:   []string{"unknown"},
 			statements: []string{`set(unknown.count, 0)`},
 			candidates: map[string]*priorityContextInferrerCandidate{},
-			expected:   "",
+			err:        `inferred context "unknown" is not a valid candidate`,
 		},
 	}
 
@@ -147,6 +148,11 @@ func Test_NewPriorityContextInferrer_InferStatements(t *testing.T) {
 				withContextInferrerPriorities(tt.priority),
 			)
 			inferredContext, err := inferrer.inferFromStatements(tt.statements)
+			if tt.err != "" {
+				require.ErrorContains(t, err, tt.err)
+				return
+			}
+
 			require.NoError(t, err)
 			assert.Equal(t, tt.expected, inferredContext)
 		})

pkg/ottl/parser_collection.go

@@ -333,7 +333,7 @@ func (pc *ParserCollection[R]) ParseStatements(statements StatementsGetter) (R,
 	statementsValues := statements.GetStatements()
 	inferredContext, err := pc.contextInferrer.inferFromStatements(statementsValues)
 	if err != nil {
-		return *new(R), err
+		return *new(R), fmt.Errorf("unable to infer a valid context (%+q) from statements %+q: %w", pc.supportedContextNames(), statementsValues, err)
 	}
 
 	if inferredContext == "" {

pkg/ottl/parser_collection_test.go

@@ -300,18 +300,18 @@ func Test_ParseStatements_NoContextInferredError(t *testing.T) {
 	statements := mockGetter{values: []string{`set(bar.attributes["bar"], "foo")`}}
 	_, err = pc.ParseStatements(statements)
 
-	assert.ErrorContains(t, err, "unable to infer context from statements")
+	assert.ErrorContains(t, err, "unable to infer a valid context")
 }
 
 func Test_ParseStatements_ContextInferenceError(t *testing.T) {
-	pc, err := NewParserCollection[any](componenttest.NewNopTelemetrySettings())
+	pc, err := NewParserCollection[any](componenttest.NewNopTelemetrySettings(), WithParserCollectionContext[any, any]("foo", mockParser(t, WithPathContextNames[any]([]string{"foo"})), WithStatementConverter(newNopParsedStatementsConverter[any]())))
 	require.NoError(t, err)
 	pc.contextInferrer = &mockFailingContextInferrer{err: errors.New("inference error")}
 
 	statements := mockGetter{values: []string{`set(bar.attributes["bar"], "foo")`}}
 	_, err = pc.ParseStatements(statements)
 
-	assert.EqualError(t, err, "inference error")
+	assert.ErrorContains(t, err, "inference error")
 }
 
 func Test_ParseStatements_UnknownContextError(t *testing.T) {

processor/transformprocessor/README.md

@@ -112,7 +112,9 @@ transform:
     - set(log.body, log.attributes["http.route"])
 ```
 
-If you're interested in how OTTL parses these statements, see [Context Inference](#context-inference).
+In some situations a combination of Paths, functions, or enums is not allowed, and the solution 
+might require multiple [Advanced Config](#advanced-config) configuration groups. 
+See [Context Inference](#context-inference) for more details.
 
 ### Advanced Config
 
@@ -171,37 +173,8 @@ transform:
 ```
 
 The Transform Processor will enforce that all the Paths, functions, and enums used in a group's `statements` are parsable.
-In some situations a combination of Paths, functions, or enums is not allowed. For example:
-
-```yaml
-metric_statements:
-  - statements:
-    - convert_sum_to_gauge() where metric.name == "system.processes.count"
-    - limit(datapoint.attributes, 100, ["host.name"])
-```
-
-In this configuration, the `datapoint` Path prefixed is used in the same group of statements as the `convert_sum_to_gauge`
-function. Since `convert_sum_to_gauge` can only be used with the metrics, not datapoints, but the list
-statements contains a reference to the datapoints via the `datapoint` Path prefix, the group of statements cannot
-be parsed.
-
-The solution is to separate the statements into separate groups:
-
-```yaml
-metric_statements:
-  - statements:
-    - limit(datapoint.attributes, 100, ["host.name"])
-  - statements:
-    - convert_sum_to_gauge() where metric.name == "system.processes.count" 
-```
-
-Alternatively, for simplicity, you can use the [basic configuration](#basic-config) style:
-
-```yaml
-metric_statements:
-  - limit(datapoint.attributes, 100, ["host.name"])
-  - convert_sum_to_gauge() where metric.name == "system.processes.count" 
-```
+In some situations a combination of Paths, functions, or enums is not allowed, and it might require multiple configuration groups. 
+See [Context Inference](#context-inference) for more details.
 
 ### Context inference
 
@@ -239,6 +212,29 @@ are both accurate and performant, leveraging the hierarchical structure of conte
 iterations and improve overall processing efficiency.
 All of this happens automatically, leaving you to write OTTL statements without worrying about Context.
 
+In some situations a combination of Paths, functions, or enums is not allowed. For example:
+
+```yaml
+metric_statements:
+  - convert_sum_to_gauge() where metric.name == "system.processes.count"
+  - limit(datapoint.attributes, 100, ["host.name"])
+```
+
+In this configuration, the `datapoint` Path prefixed is used in the same group of statements as the `convert_sum_to_gauge`
+function. Since `convert_sum_to_gauge` can only be used with the metrics, not datapoints, but the list
+statements contains a reference to the datapoints via the `datapoint` Path prefix, the group of statements cannot
+be parsed.
+
+The solution is to separate the statements into separate [Advanced Config](#advanced-config) groups:
+
+```yaml
+metric_statements:
+  - statements:
+    - convert_sum_to_gauge() where metric.name == "system.processes.count"
+  - statements:
+      - limit(datapoint.attributes, 100, ["host.name"])
+```
+
 ## Grammar
 
 You can learn more in-depth details on the capabilities and limitations of the OpenTelemetry Transformation Language used by the Transform Processor by reading about its [grammar](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/LANGUAGE.md).

processor/transformprocessor/config.go

@@ -78,7 +78,6 @@ func (c *Config) Unmarshal(conf *confmap.Conf) error {
 		"log_statements":    &c.LogStatements,
 	}
 
-	flatContextStatements := map[string][]int{}
 	contextStatementsPatch := map[string]any{}
 	for fieldName := range contextStatementsFields {
 		if !conf.IsSet(fieldName) {
@@ -93,17 +92,25 @@ func (c *Config) Unmarshal(conf *confmap.Conf) error {
 			continue
 		}
 
-		stmts := make([]any, 0, len(values))
-		for i, value := range values {
-			// Array of strings means it's a flat configuration style
+		statementsConfigs := make([]any, 0, len(values))
+		var basicStatements []any
+		for _, value := range values {
+			// Array of strings means it's a basic configuration style
 			if reflect.TypeOf(value).Kind() == reflect.String {
-				stmts = append(stmts, map[string]any{"statements": []any{value}})
-				flatContextStatements[fieldName] = append(flatContextStatements[fieldName], i)
+				basicStatements = append(basicStatements, value)
 			} else {
-				stmts = append(stmts, value)
+				if len(basicStatements) > 0 {
+					return errors.New("configuring multiple configuration styles is not supported, please use only Basic configuration or only Advanced configuration")
+				}
+				statementsConfigs = append(statementsConfigs, value)
 			}
 		}
-		contextStatementsPatch[fieldName] = stmts
+
+		if len(basicStatements) > 0 {
+			statementsConfigs = append(statementsConfigs, map[string]any{"statements": basicStatements})
+		}
+
+		contextStatementsPatch[fieldName] = statementsConfigs
 	}
 
 	if len(contextStatementsPatch) > 0 {
@@ -118,12 +125,6 @@ func (c *Config) Unmarshal(conf *confmap.Conf) error {
 		return err
 	}
 
-	for fieldName, indexes := range flatContextStatements {
-		for _, i := range indexes {
-			(*contextStatementsFields[fieldName])[i].SharedCache = true
-		}
-	}
-
 	return err
 }