[confmap] Maintain nil slice values when marshaling and unmarshaling (open-telemetry#11882)

Expands on
open-telemetry#11755 to
show how we could test for a 1:1 mapping between map[string]any <->
`confmap.Conf` <-> Config structs for `[]any(nil)` and `[]any{}` slices.

Author: evan-bradley

.chloggen/define-nil-slice-behavior.yaml

@@ -0,0 +1,28 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: confmap
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Maintain nil values when marshaling or unmarshaling nil slices
+
+# One or more tracking issues or pull requests related to the change
+issues: [11882]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Previously, nil slices were converted to empty lists, which are semantically different
+  than a nil slice. This change makes this conversion more consistent when encoding
+  or decoding config, and these values are now maintained.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

confmap/confmap.go

@@ -141,6 +141,10 @@ func sanitizeExpanded(a any, useOriginal bool) any {
 		return c
 	case []any:
 		var newSlice []any
+		if m == nil {
+			return newSlice
+		}
+		newSlice = []any{}
 		for _, e := range m {
 			newSlice = append(newSlice, sanitizeExpanded(e, useOriginal))
 		}
@@ -236,7 +240,6 @@ func decodeConfig(m *Conf, result any, errorUnused bool, skipTopLevelUnmarshaler
 			// after the main unmarshaler hook is called,
 			// we unmarshal the embedded structs if present to merge with the result:
 			unmarshalerEmbeddedStructsHookFunc(),
-			zeroSliceHookFunc(),
 		),
 	}
 	decoder, err := mapstructure.NewDecoder(dc)
@@ -531,37 +534,6 @@ type Marshaler interface {
 	Marshal(component *Conf) error
 }
 
-// This hook is used to solve the issue: https://github.com/open-telemetry/opentelemetry-collector/issues/4001
-// We adopt the suggestion provided in this issue: https://github.com/mitchellh/mapstructure/issues/74#issuecomment-279886492
-// We should empty every slice before unmarshalling unless user provided slice is nil.
-// Assume that we had a struct with a field of type slice called `keys`, which has default values of ["a", "b"]
-//
-//	type Config struct {
-//	  Keys []string `mapstructure:"keys"`
-//	}
-//
-// The configuration provided by users may have following cases
-// 1. configuration have `keys` field and have a non-nil values for this key, the output should be overridden
-//   - for example, input is {"keys", ["c"]}, then output is Config{ Keys: ["c"]}
-//
-// 2. configuration have `keys` field and have an empty slice for this key, the output should be overridden by empty slices
-//   - for example, input is {"keys", []}, then output is Config{ Keys: []}
-//
-// 3. configuration have `keys` field and have nil value for this key, the output should be default config
-//   - for example, input is {"keys": nil}, then output is Config{ Keys: ["a", "b"]}
-//
-// 4. configuration have no `keys` field specified, the output should be default config
-//   - for example, input is {}, then output is Config{ Keys: ["a", "b"]}
-func zeroSliceHookFunc() mapstructure.DecodeHookFuncValue {
-	return func(from reflect.Value, to reflect.Value) (any, error) {
-		if to.CanSet() && to.Kind() == reflect.Slice && from.Kind() == reflect.Slice {
-			to.Set(reflect.MakeSlice(to.Type(), from.Len(), from.Cap()))
-		}
-
-		return from.Interface(), nil
-	}
-}
-
 type moduleFactory[T any, S any] interface {
 	Create(s S) T
 }

confmap/confmap_test.go

@@ -689,6 +689,50 @@ func TestZeroSliceHookFunc(t *testing.T) {
 	}
 }
 
+func TestNilValuesUnchanged(t *testing.T) {
+	type structWithSlices struct {
+		Strings []string `mapstructure:"strings"`
+	}
+
+	slicesStruct := &structWithSlices{}
+
+	nilCfg := map[string]any{
+		"strings": []any(nil),
+	}
+	nilConf := NewFromStringMap(nilCfg)
+	err := nilConf.Unmarshal(slicesStruct)
+	require.NoError(t, err)
+
+	confFromStruct := New()
+	err = confFromStruct.Marshal(slicesStruct)
+	require.NoError(t, err)
+
+	require.Equal(t, nilCfg, nilConf.ToStringMap())
+	require.EqualValues(t, nilConf.ToStringMap(), confFromStruct.ToStringMap())
+}
+
+func TestEmptySliceUnchanged(t *testing.T) {
+	type structWithSlices struct {
+		Strings []string `mapstructure:"strings"`
+	}
+
+	slicesStruct := &structWithSlices{}
+
+	nilCfg := map[string]any{
+		"strings": []any{},
+	}
+	nilConf := NewFromStringMap(nilCfg)
+	err := nilConf.Unmarshal(slicesStruct)
+	require.NoError(t, err)
+
+	confFromStruct := New()
+	err = confFromStruct.Marshal(slicesStruct)
+	require.NoError(t, err)
+
+	require.Equal(t, nilCfg, nilConf.ToStringMap())
+	require.EqualValues(t, nilConf.ToStringMap(), confFromStruct.ToStringMap())
+}
+
 type C struct {
 	Modifiers []string `mapstructure:"modifiers"`
 }

confmap/confmaptest/configtest_test.go

@@ -30,11 +30,16 @@ func TestLoadConf(t *testing.T) {
 	assert.Equal(t, map[string]any{"floating": 3.14}, cfg.ToStringMap())
 }
 
-func TestToStringMapSanitizeEmptySlice(t *testing.T) {
+func TestToStringMapSanitizeNil(t *testing.T) {
+	cfg, err := LoadConf(filepath.Join("testdata", "nil.yaml"))
+	require.NoError(t, err)
+	assert.Equal(t, map[string]any{"slice": nil}, cfg.ToStringMap())
+}
+
+func TestToStringMapEmptySlice(t *testing.T) {
 	cfg, err := LoadConf(filepath.Join("testdata", "empty-slice.yaml"))
 	require.NoError(t, err)
-	var nilSlice []any
-	assert.Equal(t, map[string]any{"slice": nilSlice}, cfg.ToStringMap())
+	assert.Equal(t, map[string]any{"slice": []any{}}, cfg.ToStringMap())
 }
 
 func TestValidateProviderScheme(t *testing.T) {

confmap/confmaptest/testdata/empty-slice.yaml

@@ -1 +1 @@
-slice: [] # empty slices are sanitized to nil in ToStringMap
+slice: []

confmap/confmaptest/testdata/nil.yaml

@@ -0,0 +1 @@
+slice: 

confmap/internal/mapstructure/encoder.go

@@ -141,6 +141,9 @@ func (e *Encoder) encodeSlice(value reflect.Value) (any, error) {
 			Kind:   value.Kind(),
 		}
 	}
+	if value.IsNil() {
+		return []any(nil), nil
+	}
 	result := make([]any, value.Len())
 	for i := 0; i < value.Len(); i++ {
 		var err error

service/config_test.go

@@ -124,7 +124,7 @@ func TestConfmapMarshalConfig(t *testing.T) {
 									"host": "localhost",
 									"port": 8888,
 									"with_resource_constant_labels": map[string]any{
-										"included": []any(nil),
+										"included": []any{},
 									},
 									"without_scope_info":  true,
 									"without_type_suffix": true,