Merge branch 'main' into cache-processor-set-paths

Author: orestisfl

NOTICE.txt

@@ -28749,11 +28749,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/crypto
-Version: v0.44.0
+Version: v0.45.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.44.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.45.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 

changelog/fragments/1763578431-remove-otel.component.id-and-otel.component.kind-from-beat-receiver-events.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: breaking-change
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: remove otel.component.id and otel.component.kind from beat receiver events
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: all
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/beats/pull/47729
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/beats/issues/47600

changelog/fragments/1764007304-bump-elastic-autodiscover.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Not being able to start the add_docker_metadata processor is now consistently a non-fatal error when Docker is not available. 
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: all
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

go.mod

@@ -122,7 +122,7 @@ require (
 	go.etcd.io/bbolt v1.4.0
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.44.0
+	golang.org/x/crypto v0.45.0
 	golang.org/x/mod v0.29.0
 	golang.org/x/net v0.47.0
 	golang.org/x/oauth2 v0.31.0

go.sum

@@ -1260,8 +1260,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
-golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
-golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU=
 golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk=

libbeat/processors/add_docker_metadata/add_docker_metadata.go

@@ -93,7 +93,9 @@ func buildDockerMetadataProcessor(log *logp.Logger, cfg *conf.C, watcherConstruc
 		dockerAvailable = true
 		log.Debugf("%v: docker environment detected", processorName)
 		if err = watcher.Start(); err != nil {
-			return nil, fmt.Errorf("failed to start watcher: %w", err)
+			// mark dockerAvailable as false because watcher creation failed
+			dockerAvailable = false
+			log.Infof("unable to start the docker watcher: %v", err)
 		}
 	}
 

libbeat/processors/add_docker_metadata/add_docker_metadata_test.go

@@ -20,6 +20,7 @@
 package add_docker_metadata
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"runtime"
@@ -34,6 +35,7 @@ import (
 	"github.com/elastic/elastic-agent-autodiscover/docker"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
+	"github.com/elastic/elastic-agent-libs/logp/logptest"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/cgroup"
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/resolve"
@@ -114,7 +116,7 @@ func TestInitializationNoDocker(t *testing.T) {
 func TestInitialization(t *testing.T) {
 	var testConfig = config.NewConfig()
 
-	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil))
+	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{}
@@ -130,7 +132,7 @@ func TestNoMatch(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil))
+	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -148,7 +150,7 @@ func TestMatchNoContainer(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil))
+	p, err := buildDockerMetadataProcessor(logp.L(), testConfig, MockWatcherFactory(nil, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -179,7 +181,7 @@ func TestMatchContainer(t *testing.T) {
 					"b.foo": "3",
 				},
 			},
-		}))
+		}, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -227,7 +229,7 @@ func TestMatchContainerWithDedot(t *testing.T) {
 					"b.foo": "3",
 				},
 			},
-		}))
+		}, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -269,7 +271,7 @@ func TestMatchSource(t *testing.T) {
 					"b": "2",
 				},
 			},
-		}))
+		}, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	var inputSource string
@@ -328,7 +330,7 @@ func TestDisableSource(t *testing.T) {
 					"b": "2",
 				},
 			},
-		}))
+		}, nil))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	input := mapstr.M{
@@ -354,6 +356,7 @@ func TestMatchPIDs(t *testing.T) {
 				},
 			},
 		},
+		nil,
 	))
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
@@ -436,22 +439,45 @@ func TestMatchPIDs(t *testing.T) {
 	})
 }
 
+func TestWatcherError(t *testing.T) {
+	logger, observedLogs := logptest.NewTestingLoggerWithObserver(t, "")
+	testConfig, err := config.NewConfigFrom(map[string]interface{}{
+		"match_fields": []string{"foo"},
+	})
+	assert.NoError(t, err)
+
+	p, err := buildDockerMetadataProcessor(logger, testConfig, MockWatcherFactory(nil, errors.New("mock error")))
+	assert.NoError(t, err, "initializing add_docker_metadata processor")
+	assert.Len(t, observedLogs.FilterMessageSnippet("unable to start the docker watcher").TakeAll(), 1)
+
+	input := mapstr.M{
+		"field": "value",
+	}
+	result, err := p.Run(&beat.Event{Fields: input})
+	assert.NoError(t, err, "processing an event")
+	assert.Equal(t, mapstr.M{"field": "value"}, result.Fields)
+}
+
 // Mock container watcher
 
-func MockWatcherFactory(containers map[string]*docker.Container) docker.WatcherConstructor {
+func MockWatcherFactory(containers map[string]*docker.Container, startErr error) docker.WatcherConstructor {
 	if containers == nil {
 		containers = make(map[string]*docker.Container)
 	}
 	return func(_ *logp.Logger, host string, tls *docker.TLSConfig, shortID bool) (docker.Watcher, error) {
-		return &mockWatcher{containers: containers}, nil
+		return &mockWatcher{containers: containers, startErr: startErr}, nil
 	}
 }
 
 type mockWatcher struct {
 	containers map[string]*docker.Container
+	startErr   error
 }
 
 func (m *mockWatcher) Start() error {
+	if m.startErr != nil {
+		return m.startErr
+	}
 	return nil
 }
 

x-pack/filebeat/fbreceiver/receiver_test.go

@@ -90,8 +90,7 @@ func TestNewReceiver(t *testing.T) {
 		AssertFunc: func(c *assert.CollectT, logs map[string][]mapstr.M, zapLogs *observer.ObservedLogs) {
 			_ = zapLogs
 			require.Lenf(c, logs["r1"], 1, "expected 1 log, got %d", len(logs["r1"]))
-			assert.Equal(c, "filebeatreceiver/r1", logs["r1"][0].Flatten()["agent.otelcol.component.id"], "expected agent.otelcol.component.id field in log record")
-			assert.Equal(c, "receiver", logs["r1"][0].Flatten()["agent.otelcol.component.kind"], "expected agent.otelcol.component.kind field in log record")
+			assert.Equal(c, "test", logs["r1"][0].Flatten()["message"], "expected message field to contain string 'test'")
 			var lastError strings.Builder
 			assert.Conditionf(c, func() bool {
 				return getFromSocket(t, &lastError, monitorSocket, "stats")
@@ -250,10 +249,8 @@ func TestMultipleReceivers(t *testing.T) {
 			require.Greater(c, len(logs["r1"]), 0, "receiver r1 does not have any logs")
 			require.Greater(c, len(logs["r2"]), 0, "receiver r2 does not have any logs")
 
-			assert.Equal(c, "filebeatreceiver/r1", logs["r1"][0].Flatten()["agent.otelcol.component.id"], "expected agent.otelcol.component.id field in r1 log record")
-			assert.Equal(c, "receiver", logs["r1"][0].Flatten()["agent.otelcol.component.kind"], "expected agent.otelcol.component.kind field in r1 log record")
-			assert.Equal(c, "filebeatreceiver/r2", logs["r2"][0].Flatten()["agent.otelcol.component.id"], "expected agent.otelcol.component.id field in r2 log record")
-			assert.Equal(c, "receiver", logs["r2"][0].Flatten()["agent.otelcol.component.kind"], "expected agent.otelcol.component.kind field in r2 log record")
+			assert.Equal(c, "test", logs["r1"][0].Flatten()["message"], "expected r1 message field to be 'test'")
+			assert.Equal(c, "test", logs["r2"][0].Flatten()["message"], "expected r2  message field to be 'test'")
 
 			// Make sure that each receiver has a separate logger
 			// instance and does not interfere with others. Previously, the

x-pack/filebeat/input/gcppubsub/otel_test.go

@@ -175,11 +175,7 @@ processors:
 		"agent.ephemeral_id",
 		"agent.id",
 		"event.created",
-		// only present in beats receivers
-		"agent.otelcol.component.id",
-		"agent.otelcol.component.kind",
 	}
 
 	oteltest.AssertMapsEqual(t, filebeatDoc, otelDoc, ignoredFields, "expected documents to be equal")
-
 }

x-pack/filebeat/tests/integration/otel_lsexporter_test.go

@@ -65,7 +65,7 @@ func TestDataShapeOTelVSClassicE2E(t *testing.T) {
 
 	// Agent does not support `index` setting, while beats does.
 	//	Our focus is on agent classic vs otel mode comparison, so we do not test `index` for filebeat
-	var beatsCfgFile = `
+	beatsCfgFile := `
 filebeat.inputs:
   - type: filestream
     id: filestream-input-id
@@ -173,8 +173,6 @@ service:
 		"log.file.inode",
 		"log.file.path",
 		// only present in beats receivers
-		"agent.otelcol.component.id",
-		"agent.otelcol.component.kind",
 		"log.file.device_id",
 		"log.file.fingerprint",
 	}
@@ -268,7 +266,6 @@ service:
 	require.NoError(t, err, "failed to read otel events")
 	require.Equal(t, numEvents, len(otelEvents),
 		"different number of events: sent=%d, get=%d", numEvents, len(otelEvents))
-
 }
 
 func generateEvents(numEvents int) []string {
@@ -377,10 +374,8 @@ func compareOutputFiles(t *testing.T, fbFilePath, otelFilePath string, ignoredFi
 		oteltest.AssertMapsEqual(t, fbEvent.data, otelEvent.data, ignoredFields,
 			fmt.Sprintf("event comparison failed for ID %s (line %d)", fbEvent.id, i))
 
-		assert.Equal(t, "filebeatreceiver", otelEvent.data.Flatten()["agent.otelcol.component.id"], "expected agent.otelcol.component.id field in log record")
-		assert.Equal(t, "receiver", otelEvent.data.Flatten()["agent.otelcol.component.kind"], "expected agent.otelcol.component.kind field in log record")
-		assert.NotContains(t, fbEvent.data.Flatten(), "agent.otelcol.component.id", "expected agent.otelcol.component.id field not to be present in filebeat log record")
-		assert.NotContains(t, fbEvent.data.Flatten(), "agent.otelcol.component.kind", "expected agent.otelcol.component.kind field not to be present in filebeat log record")
+		assert.Equal(t, "filebeat", otelEvent.data.Flatten()["agent.type"], "expected agent.type to be 'filebeat' in otel data")
+		assert.Equal(t, "filebeat", fbEvent.data.Flatten()["agent.type"], "expected agent.type to be 'filebeat' in filebeat data")
 	}
 }