[exporter/elasticsearch] Support preserving attributes when mapping to ECS (#33670)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
This PR adds support for preserving resource attributes that are valid
ECS fields in addition of mapping them. At the moment the `host.name` is
mapped to the `host.hostname`. Both are valid ECS fields but can differ
in some cases. Since there is no such distinction in SemConv right now,
it does makes sense to preserve both.

refs:
-
https://www.elastic.co/guide/en/ecs/current/ecs-host.html#field-host-name
-
https://www.elastic.co/guide/en/ecs/current/ecs-host.html#field-host-hostname

**Link to tracking Issue:** <Issue number if applicable>

**Testing:** <Describe what testing was performed and which tests were
added.>

Using the testing notes from
#33622.

Stored document:
```json
 {
    "app": {
      "label": {
        "component": "migration-logger"
      }
    },
    "kubernetes": {
      "node": {
        "name": "kind-control-plane"
      },
      "pod": {
        "uid": "0eda57cd-a4ae-4e89-88fa-c771d3bf0c77",
        "name": "daemonset-logs-4sqjq"
      },
      "namespace": "default"
    },
    "agent": {
      "name": "otlp"
    },
    "@timestamp": "2024-06-20T07:27:42.589678923Z",
    "log": {
      "iostream": "stdout",
      "file": {
        "path": "/var/log/pods/default_daemonset-logs-4sqjq_0eda57cd-a4ae-4e89-88fa-c771d3bf0c77/busybox/0.log"
      }
    },
    "service": {
      "name": "migration-logger"
    },
    "k8s": {
      "container": {
        "restart_count": "0",
        "name": "busybox"
      },
      "pod": {
        "start_time": "2024-06-20T07:27:21Z"
      },
      "daemonset": {
        "name": "daemonset-logs"
      }
    },
    "host": {
      "hostname": "daemonset-opentelemetry-collector-agent-l6pzp",
      "os": {
        "type": "linux",
        "platform": "linux"
      },
      "name": "daemonset-opentelemetry-collector-agent-l6pzp"
    },
    "time": "2024-06-20T07:27:42.589678923Z",
    "message": "otel logs at 07:27:42",
    "logtag": "F"
  }
```

**Documentation:** <Describe the documentation added.> ~

/cc @lahsivjar @andrzej-stencel @carsonip

---------

Signed-off-by: ChrsMark <[email protected]>

Author: ChrsMark

.chloggen/es_preserve_host_name.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Preserve `host.name` resource attribute in ECS mode
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33670]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/elasticsearchexporter/model.go

@@ -49,6 +49,13 @@ var resourceAttrsConversionMap = map[string]string{
 	"k8s.pod.uid":                           "kubernetes.pod.uid",
 }
 
+// resourceAttrsToPreserve contains conventions that should be preserved in ECS mode.
+// This can happen when an attribute needs to be mapped to an ECS equivalent but
+// at the same time be preserved to its original form.
+var resourceAttrsToPreserve = map[string]bool{
+	semconv.AttributeHostName: true,
+}
+
 type mappingModel interface {
 	encodeLog(pcommon.Resource, plog.LogRecord, pcommon.InstrumentationScope) ([]byte, error)
 	encodeSpan(pcommon.Resource, ptrace.Span, pcommon.InstrumentationScope) ([]byte, error)
@@ -117,13 +124,13 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo
 	var document objmodel.Document
 
 	// First, try to map resource-level attributes to ECS fields.
-	encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap)
+	encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap, resourceAttrsToPreserve)
 
 	// Then, try to map scope-level attributes to ECS fields.
 	scopeAttrsConversionMap := map[string]string{
 		// None at the moment
 	}
-	encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap)
+	encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap, resourceAttrsToPreserve)
 
 	// Finally, try to map record-level attributes to ECS fields.
 	recordAttrsConversionMap := map[string]string{
@@ -133,7 +140,7 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo
 		semconv.AttributeExceptionType:       "error.type",
 		semconv.AttributeExceptionEscaped:    "event.error.exception.handled",
 	}
-	encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap)
+	encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap, resourceAttrsToPreserve)
 
 	// Handle special cases.
 	encodeLogAgentNameECSMode(&document, resource)
@@ -230,7 +237,7 @@ func scopeToAttributes(scope pcommon.InstrumentationScope) pcommon.Map {
 	return attrs
 }
 
-func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string) {
+func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string, preserveMap map[string]bool) {
 	if len(conversionMap) == 0 {
 		// No conversions to be done; add all attributes at top level of
 		// document.
@@ -247,6 +254,9 @@ func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map,
 			}
 
 			document.AddAttribute(ecsKey, v)
+			if preserve := preserveMap[k]; preserve {
+				document.AddAttribute(k, v)
+			}
 			return true
 		}
 

exporter/elasticsearchexporter/model_test.go

@@ -231,7 +231,7 @@ func TestEncodeLogECSModeDuplication(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}`
+	want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","name":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}`
 	require.NoError(t, err)
 
 	resourceContainerImageTags := resource.Attributes().PutEmptySlice(semconv.AttributeContainerImageTags)
@@ -336,6 +336,7 @@ func TestEncodeLogECSMode(t *testing.T) {
 		"container.image.name":    "my-app",
 		"container.runtime":       "docker",
 		"host.hostname":           "i-103de39e0a.gke.us-west-1b.cloud.google.com",
+		"host.name":               "i-103de39e0a.gke.us-west-1b.cloud.google.com",
 		"host.id":                 "i-103de39e0a",
 		"host.type":               "t2.medium",
 		"host.architecture":       "x86_64",
@@ -671,6 +672,7 @@ func TestMapLogAttributesToECS(t *testing.T) {
 	tests := map[string]struct {
 		attrs         func() pcommon.Map
 		conversionMap map[string]string
+		preserveMap   map[string]bool
 		expectedDoc   func() objmodel.Document
 	}{
 		"no_attrs": {
@@ -775,12 +777,31 @@ func TestMapLogAttributesToECS(t *testing.T) {
 				return d
 			},
 		},
+		"preserve_map": {
+			attrs: func() pcommon.Map {
+				m := pcommon.NewMap()
+				m.PutStr("foo.bar", "baz")
+				return m
+			},
+			conversionMap: map[string]string{
+				"foo.bar": "bar.qux",
+				"qux":     "foo",
+			}, preserveMap: map[string]bool{
+				"foo.bar": true,
+			},
+			expectedDoc: func() objmodel.Document {
+				d := objmodel.Document{}
+				d.AddString("bar.qux", "baz")
+				d.AddString("foo.bar", "baz")
+				return d
+			},
+		},
 	}
 
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
 			var doc objmodel.Document
-			encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap)
+			encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap, test.preserveMap)
 
 			doc.Sort()
 			expectedDoc := test.expectedDoc()