[chore][codecov] Add code coverage to integration tests (#6146)

* [chore][codecov] Add code coverage to integration tests

* Add env variable references to integration tests to capture coverage

* Add empty newline at end of file

* Move shell command to run rather than env

Author: crobert-1

.github/workflows/integration-test.yml

@@ -65,7 +65,7 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
           cache-dependency-path: '**/go.sum'
-      - run: make binaries-linux_${{ matrix.ARCH }}
+      - run: make binaries-linux_${{ matrix.ARCH }} COVER_TESTING=true
       - uses: actions/upload-artifact@v4
         with:
           name: otelcol-${{ matrix.ARCH }}
@@ -237,11 +237,13 @@ jobs:
           if [[ "${{ matrix.PROFILE }}" = "smartagent" ]]; then
             target="smartagent-integration-test-with-cover"
           fi
+          export CONTAINER_COVER_SRC="$(realpath .)/tests/coverage"
           make $target 2>&1 | tee $TEST_OUTPUT
           exit_status=${PIPESTATUS[0]}
           echo "Exit status: $exit_status"
           exit $exit_status
         env:
+          CONTAINER_COVER_DEST: '/etc/otel/collector/coverage'
           SPLUNK_OTEL_COLLECTOR_IMAGE: 'otelcol:latest'
       # The Integration Test output is extremely large so we upload it as an artifact
       - name: Upload Integration Test Output as Artifact
@@ -251,6 +253,13 @@ jobs:
           name: ${{ env.TEST_OUTPUT }}
           path: ${{ env.TEST_OUTPUT }}
           retention-days: 5
+      - name: Upload coverage report
+        uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # 5.4.0
+        with:
+          verbose: true
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   integration-test-binary:
     runs-on: ${{ matrix.RUNNER }}
@@ -316,11 +325,13 @@ jobs:
           if [[ "${{ matrix.PROFILE }}" = "smartagent" ]]; then
             target="smartagent-integration-test-with-cover"
           fi
+          export CONTAINER_COVER_SRC="$(realpath .)/tests/coverage"
           make $target 2>&1 | tee $TEST_OUTPUT
           exit_status=${PIPESTATUS[0]}
           echo "Exit status: $exit_status"
           exit $exit_status
         env:
+          CONTAINER_COVER_DEST: '/etc/otel/collector/coverage'
           SPLUNK_OTEL_COLLECTOR_IMAGE: ""
       # The Integration Test output is extremely large so we upload it as an artifact
       - name: Upload Integration Test Output as Artifact
@@ -330,6 +341,13 @@ jobs:
           name: ${{ env.TEST_OUTPUT }}
           path: ${{ env.TEST_OUTPUT }}
           retention-days: 5
+      - name: Upload coverage report
+        uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # 5.4.0
+        with:
+          verbose: true
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   integration-test-discovery-matrix:
     runs-on: ubuntu-24.04
@@ -386,9 +404,19 @@ jobs:
         - run: ln -sf otelcol_linux_${{ matrix.ARCH }} ./bin/otelcol
         - run: chmod a+x ./bin/*
         - name: Run ${{ matrix.SERVICE }} Discovery Integration Test With Cover
-          run: make integration-test-${{ matrix.SERVICE }}-discovery-with-cover
+          run: |
+            export CONTAINER_COVER_SRC="$(realpath .)/tests/coverage"
+            make integration-test-${{ matrix.SERVICE }}-discovery-with-cover
           env:
+            CONTAINER_COVER_DEST: '/etc/otel/collector/coverage'
             SPLUNK_OTEL_COLLECTOR_IMAGE: 'otelcol:latest'
+        - name: Upload coverage report
+          uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # 5.4.0
+          with:
+            verbose: true
+            fail_ci_if_error: true
+          env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   integration-test-discovery-k8s-matrix:
     runs-on: ubuntu-24.04
@@ -448,9 +476,20 @@ jobs:
         run: |
           kind load docker-image otelcol:latest --name kind
       - name: Run ${{ matrix.SERVICE }} Discovery Kubernetes Integration Test With Cover
-        run: KUBECONFIG=$HOME/.kube/config SKIP_TEARDOWN=true make integration-test-${{ matrix.SERVICE }}-discovery-k8s-with-cover
+        run: |
+          export CONTAINER_COVER_SRC="$(realpath .)/tests/coverage"
+          KUBECONFIG=$HOME/.kube/config SKIP_TEARDOWN=true make integration-test-${{ matrix.SERVICE }}-discovery-k8s-with-cover
+        env:
+          CONTAINER_COVER_DEST: '/etc/otel/collector/coverage'
       - name: Print logs
         if: failure()
         run: |
           kubectl get pods -A 
           kubectl get pod -A -l app=otelcol -o jsonpath="{range .items[*]}{.metadata.namespace} {.metadata.name}{'\n'}{end}" | xargs -r -n2 sh -c 'kubectl logs -n $0 $1'
+      - name: Upload coverage report
+        uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # 5.4.0
+        with:
+          verbose: true
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

Makefile

@@ -43,6 +43,8 @@ SKIP_COMPILE=false
 ARCH?=amd64
 BUNDLE_SUPPORTED_ARCHS := amd64 arm64
 SKIP_BUNDLE=false
+# Used for building the collector to collect coverage information
+COVER_TESTING=false
 
 # For integration testing against local changes you can run
 # SPLUNK_OTEL_COLLECTOR_IMAGE='otelcol:latest' make -e docker-otelcol integration-test
@@ -242,7 +244,11 @@ generate-metrics:
 .PHONY: otelcol
 otelcol:
 	go generate ./...
+ifeq ($(COVER_TESTING), true)
+	GO111MODULE=on CGO_ENABLED=$(CGO_ENABLED) go build $(COVER_OPTS) -trimpath -o ./bin/otelcol_$(GOOS)_$(GOARCH)$(EXTENSION) $(BUILD_INFO) ./cmd/otelcol
+else
 	GO111MODULE=on CGO_ENABLED=$(CGO_ENABLED) go build -trimpath -o ./bin/otelcol_$(GOOS)_$(GOARCH)$(EXTENSION) $(BUILD_INFO) ./cmd/otelcol
+endif
 ifeq ($(OS), Windows_NT)
 	$(LINK_CMD) .\bin\otelcol$(EXTENSION) .\bin\otelcol_$(GOOS)_$(GOARCH)$(EXTENSION)
 else

tests/general/container_test.go

@@ -18,6 +18,7 @@ package tests
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -33,6 +34,9 @@ import (
 
 func TestDefaultContainerConfigRequiresEnvVars(t *testing.T) {
 	image := testutils.GetCollectorImageOrSkipTest(t)
+	coverDest := os.Getenv("CONTAINER_COVER_DEST")
+	coverSrc := os.Getenv("CONTAINER_COVER_SRC")
+
 	tests := []struct {
 		name    string
 		env     map[string]string
@@ -41,18 +45,27 @@ func TestDefaultContainerConfigRequiresEnvVars(t *testing.T) {
 		{"missing realm", map[string]string{
 			"SPLUNK_REALM":        "",
 			"SPLUNK_ACCESS_TOKEN": "some_token",
+			"GOCOVERDIR":          coverDest,
 		}, "SPLUNK_REALM"},
 		{"missing token", map[string]string{
 			"SPLUNK_REALM":        "some_realm",
 			"SPLUNK_ACCESS_TOKEN": "",
+			"GOCOVERDIR":          coverDest,
 		}, "SPLUNK_ACCESS_TOKEN"},
 	}
 	for _, testcase := range tests {
 		t.Run(testcase.name, func(tt *testing.T) {
 			logCore, logs := observer.New(zap.DebugLevel)
 			logger := zap.New(logCore)
 
-			collector, err := testutils.NewCollectorContainer().WithImage(image).WithEnv(testcase.env).WithLogger(logger).WillFail(true).Build()
+			collector := testutils.NewCollectorContainer().WithImage(image).WithEnv(testcase.env).WithLogger(logger).WillFail(true)
+
+			if coverSrc != "" && coverDest != "" {
+				collector = collector.WithMount(coverSrc, coverDest)
+			}
+
+			var err error
+			collector, err = collector.Build()
 			require.NoError(t, err)
 			require.NotNil(t, collector)
 			defer collector.Shutdown()

tests/internal/discoverytest/discoverytest.go

@@ -69,14 +69,22 @@ func Run(t *testing.T, receiverName string, configFilePath string, logMessageToA
 	dockerGID, err := getDockerGID()
 	require.NoError(t, err)
 
+	coverDest := os.Getenv("CONTAINER_COVER_DEST")
+	coverSrc := os.Getenv("CONTAINER_COVER_SRC")
+	var coverDirBind string
+	if coverSrc != "" && coverDest != "" {
+		coverDirBind = fmt.Sprintf("%s:%s", coverSrc, coverDest)
+	}
+
 	req := testcontainers.ContainerRequest{
 		Image: "otelcol:latest",
 		HostConfigModifier: func(hc *container.HostConfig) {
-			hc.Binds = []string{"/var/run/docker.sock:/var/run/docker.sock"}
+			hc.Binds = []string{"/var/run/docker.sock:/var/run/docker.sock", coverDirBind}
 			hc.NetworkMode = network.NetworkHost
 			hc.GroupAdd = []string{dockerGID}
 		},
 		Env: map[string]string{
+			"GOCOVERDIR":                  coverDest,
 			"SPLUNK_REALM":                "us2",
 			"SPLUNK_ACCESS_TOKEN":         "12345",
 			"SPLUNK_DISCOVERY_LOG_LEVEL":  "info",

tests/receivers/jmx/cassandra/jmx_cassandra_discovery_test.go

@@ -79,6 +79,13 @@ func jmxCassandraAutoDiscoveryHelper(t *testing.T, ctx context.Context, configFi
 	dockerGID, err := getDockerGID()
 	require.NoError(t, err)
 
+	coverDest := os.Getenv("CONTAINER_COVER_DEST")
+	coverSrc := os.Getenv("CONTAINER_COVER_SRC")
+	var coverDirBind string
+	if coverSrc != "" && coverDest != "" {
+		coverDirBind = fmt.Sprintf("%s:%s", coverSrc, coverDest)
+	}
+
 	otelConfigPath, err := filepath.Abs(filepath.Join(".", "testdata", configFile))
 	if err != nil {
 		return nil, err
@@ -95,11 +102,12 @@ func jmxCassandraAutoDiscoveryHelper(t *testing.T, ctx context.Context, configFi
 	req := testcontainers.ContainerRequest{
 		Image: "otelcol:latest",
 		HostConfigModifier: func(hc *container.HostConfig) {
-			hc.Binds = []string{"/var/run/docker.sock:/var/run/docker.sock"}
+			hc.Binds = []string{"/var/run/docker.sock:/var/run/docker.sock", coverDirBind}
 			hc.NetworkMode = network.NetworkHost
 			hc.GroupAdd = []string{dockerGID}
 		},
 		Env: map[string]string{
+			"GOCOVERDIR":                  coverDest,
 			"SPLUNK_REALM":                "us2",
 			"SPLUNK_ACCESS_TOKEN":         "12345",
 			"SPLUNK_DISCOVERY_LOG_LEVEL":  "info",

tests/testutils/golden.go

@@ -18,6 +18,7 @@ package testutils
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -100,11 +101,18 @@ func RunMetricsCollectionTest(t *testing.T, configFile string, expectedFilePath
 	if runtime.GOOS == "darwin" {
 		dockerHost = "host.docker.internal"
 	}
+
+	coverDest := os.Getenv("CONTAINER_COVER_DEST")
+	coverSrc := os.Getenv("CONTAINER_COVER_SRC")
+
 	cc := NewCollectorContainer().
 		WithImage(GetCollectorImageOrSkipTest(t)).
 		WithConfigPath(filepath.Join("testdata", configFile)).
 		WithLogger(logger).
-		WithEnv(map[string]string{"OTLP_ENDPOINT": fmt.Sprintf("%s:%d", dockerHost, port)}).
+		WithEnv(map[string]string{
+			"GOCOVERDIR":    coverDest,
+			"OTLP_ENDPOINT": fmt.Sprintf("%s:%d", dockerHost, port),
+		}).
 		WithEnv(opts.collectorEnvVars)
 	for k, v := range opts.fileMounts {
 		cc.(*CollectorContainer).Container = cc.(*CollectorContainer).Container.WithFile(testcontainers.ContainerFile{
@@ -113,6 +121,11 @@ func RunMetricsCollectionTest(t *testing.T, configFile string, expectedFilePath
 			FileMode:          0644,
 		})
 	}
+
+	if coverSrc != "" && coverDest != "" {
+		cc = cc.WithMount(coverSrc, coverDest)
+	}
+
 	p, err := cc.Build()
 	require.NoError(t, err)
 	require.NoError(t, p.Start())

tests/testutils/testcase.go

@@ -133,7 +133,12 @@ func (t *Testcase) splunkOtelCollector(configFilename string, builders ...Collec
 
 func (t *Testcase) newCollector(initial Collector, configFilename string, builders ...CollectorBuilder) (collector Collector, shutdown func()) {
 	collector = initial
+
+	coverDest := os.Getenv("CONTAINER_COVER_DEST")
+	coverSrc := os.Getenv("CONTAINER_COVER_SRC")
+
 	envVars := map[string]string{
+		"GOCOVERDIR":     coverDest,
 		"OTLP_ENDPOINT":  t.OTLPEndpointForCollector,
 		"SPLUNK_TEST_ID": t.ID,
 	}
@@ -159,6 +164,10 @@ func (t *Testcase) newCollector(initial Collector, configFilename string, builde
 	}
 	collector = collector.WithEnv(splunkEnv)
 
+	if coverSrc != "" && coverDest != "" {
+		collector = collector.WithMount(coverSrc, coverDest)
+	}
+
 	var err error
 	collector, err = collector.Build()
 	require.NoError(t, err)