[OTel] Add E2E to ensure data consistency in Logstash Exporter (#46721)

Add E2E TestDataShapeOTelVSClassicE2E for Logstash Exporter
to compare the data shape of filebeat vs otel mode

Author: kaisecheng

x-pack/filebeat/docker-compose.yml

@@ -57,3 +57,33 @@ services:
       - http-server
       - --addr=:8090
       - --config=/files/config.yaml
+
+  init-logstash:
+    image: busybox:latest
+    user: "0:0"
+    command: ["sh", "-c", "chown -R 1000:1000 /data"]
+    volumes:
+      - logstash_testdata:/data
+    restart: "no"
+  logstash:
+    depends_on:
+      - init-logstash
+    extends:
+      file: ${ES_BEATS}/testing/environments/${STACK_ENVIRONMENT}.yml
+      service: logstash
+    volumes:
+      - ./tests/integration/logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml
+      - logstash_testdata:/usr/share/logstash/testdata
+    ports:
+      - 9600:9600
+  nginx:
+    image: nginx:alpine
+    depends_on:
+      - init-logstash
+    volumes:
+      - logstash_testdata:/usr/share/nginx/html:ro
+    ports:
+      - 8082:80
+
+volumes:
+  logstash_testdata:

x-pack/filebeat/tests/integration/logstash/config/pipelines.yml

@@ -0,0 +1,42 @@
+# `@metadata` is an internal field in Logstash, so we need to rename it to `metadata`
+# in order for it to be visible for comparison
+- pipeline.id: fb_classic
+  pipeline.workers: 1
+  pipeline.ordered: true
+  config.string: |
+    input {
+      beats {
+        port => "5044"
+        enrich => "none"
+      }
+    }
+    filter {
+      mutate {
+        rename => ["@metadata", "metadata"]
+      }
+    }
+    output {
+      file {
+        path => "/usr/share/logstash/testdata/%{testcase}_fb.json"
+      }
+    }
+- pipeline.id: fb_otel
+  pipeline.workers: 1
+  pipeline.ordered: true
+  config.string: |
+    input {
+      beats {
+        port => "5055"
+        enrich => "none"
+      }
+    }
+    filter {
+      mutate {
+        rename => ["@metadata", "metadata"]
+      }
+    }
+    output {
+      file {
+        path => "/usr/share/logstash/testdata/%{testcase}_otel.json"
+      }
+    }