[chore] Move functional test to separate directories (#1694)

Move functional test to separate directories instead of using build tags

Author: dmitryax

.github/workflows/functional_test_v2.yaml

@@ -72,7 +72,7 @@ jobs:
         env:
           K8S_VERSION: ${{ matrix.k8s-version }}
         run: |
-          TEARDOWN_BEFORE_SETUP=true UPDATE_EXPECTED_RESULTS=${{ env.UPLOAD_UPDATED_EXPECTED_RESULTS }} TAGS=${{ matrix.test-job }} make functionaltest
+          TEARDOWN_BEFORE_SETUP=true UPDATE_EXPECTED_RESULTS=${{ env.UPLOAD_UPDATED_EXPECTED_RESULTS }} SUITE=${{ matrix.test-job }} make functionaltest
       - name: Collect Kubernetes Cluster debug info on failure
         if: always() && (steps.run-functional-tests.outcome == 'failure' || env.UPLOAD_KUBERNETES_DEBUG_INFO == 'true')
         id: collect-debug-info
@@ -129,7 +129,7 @@ jobs:
         env:
           HOST_ENDPOINT: 0.0.0.0
         run: |
-          TEARDOWN_BEFORE_SETUP=true TAGS=functional make functionaltest
+          TEARDOWN_BEFORE_SETUP=true SUITE=functional make functionaltest
 
   eks-upgrade-test:
     name: Test helm upgrade in EKS - credentials needed
@@ -188,7 +188,7 @@ jobs:
         env:
           HOST_ENDPOINT: 0.0.0.0
         run: |
-          TAGS=functional make functionaltest
+          SUITE=functional make functionaltest
 
   eks-upgrade-from-release-test:
     name: Test helm upgrade from release in EKS - credentials needed
@@ -243,7 +243,7 @@ jobs:
         env:
           HOST_ENDPOINT: 0.0.0.0
         run: |
-          TAGS=functional make functionaltest
+          SUITE=functional make functionaltest
 
   gke-autopilot-test:
     name: Test helm install in GKE/Autopilot - credentials needed
@@ -281,7 +281,7 @@ jobs:
         env:
           HOST_ENDPOINT: 0.0.0.0
         run: |
-          TEARDOWN_BEFORE_SETUP=true TAGS=functional make functionaltest
+          TEARDOWN_BEFORE_SETUP=true SUITE=functional make functionaltest
 
   gke-autopilot-upgrade-test:
     name: Test helm upgrade in GKE/Autopilot - credentials needed
@@ -340,7 +340,7 @@ jobs:
         env:
           HOST_ENDPOINT: 0.0.0.0
         run: |
-          TAGS=functional make functionaltest
+          SUITE=functional make functionaltest
 
   aks-windows-test:
     name: Test helm install in AKS - credentials needed
@@ -379,7 +379,7 @@ jobs:
         env:
           HOST_ENDPOINT: 0.0.0.0
         run: |
-          TEARDOWN_BEFORE_SETUP=true TAGS=functional make functionaltest
+          TEARDOWN_BEFORE_SETUP=true SUITE=functional make functionaltest
 
   gce-autopilot-test:
     name: Test helm install in GCE (kops) - credentials needed
@@ -410,4 +410,4 @@ jobs:
           HOST_ENDPOINT: 0.0.0.0
           KUBECONFIG: /tmp/kubeconfig
         run: |
-          TEARDOWN_BEFORE_SETUP=true TAGS=functional make functionaltest
+          TEARDOWN_BEFORE_SETUP=true SUITE=functional make functionaltest

Makefile

@@ -94,11 +94,11 @@ unittest: ## Run unittests on the Helm chart
 
 # Example Usage:
 #   make functionaltest
-#   make functionaltest SKIP_SETUP=true SKIP_TEARDOWN=true SKIP_TESTS=true TEARDOWN_BEFORE_SETUP=true TAGS="functional" UPDATE_EXPECTED_RESULTS=true KUBE_TEST_ENV="kind" KUBECONFIG="/path/to/kubeconfig"
+#   make functionaltest SKIP_SETUP=true SKIP_TEARDOWN=true SKIP_TESTS=true TEARDOWN_BEFORE_SETUP=true SUITE="functional" UPDATE_EXPECTED_RESULTS=true KUBE_TEST_ENV="kind" KUBECONFIG="/path/to/kubeconfig"
 .PHONY: functionaltest
 functionaltest: ## Run functional tests for this Helm chart with optional tags and environment variables
 	@echo "Running functional tests for this helm chart..."
-	cd functional_tests && go test -v $(if $(TAGS),-tags $(TAGS)) || exit 1
+	cd functional_tests && go test -v ./$(SUITE)/... || exit 1
 
 ##@ Changelog
 # Tasks related to changelog management

functional_tests/README.md

@@ -29,7 +29,7 @@ When running tests you can use the following env vars to help with local develop
 - `SKIP_TEARDOWN`: Skip cleanup (useful to keep apps for local dev).
 - `SKIP_TESTS`: Skip tests; only set up and tear down the cluster.
 - `TEARDOWN_BEFORE_SETUP`: Clean up deployments before setting up.
-- `TAGS`: Specify which tests to run (e.g., `TAGS="functional"`).
+- `SUITE`: Specify which test suite to run (e.g., `SUITE="functional"`).
 - `UPDATE_EXPECTED_RESULTS`: Generate new golden files for test results.
 
 ## Run

functional_tests/configuration_switching_test.go renamed to functional_tests/configuration_switching/configuration_switching_test.go

@@ -1,9 +1,7 @@
 // Copyright Splunk Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-//go:build configuration_switching
-
-package functional_tests
+package configuration_switching
 
 import (
 	"bytes"
@@ -28,7 +26,6 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/consumer/consumertest"
 	"helm.sh/helm/v3/pkg/action"
-	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/kube"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -44,7 +41,7 @@ const (
 	hecMetricsReceiverPort     = 8091
 	apiPort                    = 8881
 	hecLogsObjectsReceiverPort = 8092
-	testDir                    = "testdata_configuration_switching"
+	testDir                    = "testdata"
 	valuesDir                  = "values"
 )
 
@@ -84,15 +81,13 @@ func deployChartsAndApps(t *testing.T, valuesFileName string, repl map[string]in
 	client, err := kubernetes.NewForConfig(kubeConfig)
 	require.NoError(t, err)
 
-	chartPath := filepath.Join("..", "helm-charts", "splunk-otel-collector")
-	chart, err := loader.Load(chartPath)
-	require.NoError(t, err)
+	chart := internal.LoadCollectorChart(t)
 
 	var valuesBytes []byte
 	valuesBytes, err = os.ReadFile(filepath.Join(testDir, valuesDir, valuesFileName))
 	require.NoError(t, err)
 
-	hostEp := hostEndpoint(t)
+	hostEp := internal.HostEndpoint(t)
 	if len(hostEp) == 0 {
 		require.Fail(t, "Host endpoint not found")
 	}
@@ -195,55 +190,55 @@ func testAgentLogsAndMetrics(t *testing.T) {
 	agentLogsConsumer := setupOnce(t).logsConsumer
 
 	t.Run("check logs and metrics received when both are enabled", func(t *testing.T) {
-		resetLogsSink(t, agentLogsConsumer)
-		resetMetricsSink(t, hecMetricsConsumer)
+		internal.ResetLogsSink(t, agentLogsConsumer)
+		internal.ResetMetricsSink(t, hecMetricsConsumer)
 
-		checkNoMetricsReceived(t, hecMetricsConsumer)
-		checkNoEventsReceived(t, agentLogsConsumer)
+		internal.CheckNoMetricsReceived(t, hecMetricsConsumer)
+		internal.CheckNoEventsReceived(t, agentLogsConsumer)
 
 		replacements := map[string]interface{}{
 			"MetricsEnabled": true,
 			"LogsEnabled":    true,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
 
-		waitForMetrics(t, 5, hecMetricsConsumer)
-		waitForLogs(t, 5, agentLogsConsumer)
+		internal.WaitForMetrics(t, 5, hecMetricsConsumer)
+		internal.WaitForLogs(t, 5, agentLogsConsumer)
 		uninstallDeployment(t)
 	})
 
 	t.Run("check metrics only enabled", func(t *testing.T) {
-		resetLogsSink(t, agentLogsConsumer)
-		resetMetricsSink(t, hecMetricsConsumer)
+		internal.ResetLogsSink(t, agentLogsConsumer)
+		internal.ResetMetricsSink(t, hecMetricsConsumer)
 
-		checkNoMetricsReceived(t, hecMetricsConsumer)
-		checkNoEventsReceived(t, agentLogsConsumer)
+		internal.CheckNoMetricsReceived(t, hecMetricsConsumer)
+		internal.CheckNoEventsReceived(t, agentLogsConsumer)
 
 		replacements := map[string]interface{}{
 			"MetricsEnabled": true,
 			"LogsEnabled":    false,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
 
-		waitForMetrics(t, 5, hecMetricsConsumer)
-		checkNoEventsReceived(t, agentLogsConsumer)
+		internal.WaitForMetrics(t, 5, hecMetricsConsumer)
+		internal.CheckNoEventsReceived(t, agentLogsConsumer)
 		uninstallDeployment(t)
 	})
 
 	t.Run("check logs only enabled", func(t *testing.T) {
-		resetLogsSink(t, agentLogsConsumer)
-		resetMetricsSink(t, hecMetricsConsumer)
+		internal.ResetLogsSink(t, agentLogsConsumer)
+		internal.ResetMetricsSink(t, hecMetricsConsumer)
 
 		replacements := map[string]interface{}{
 			"MetricsEnabled": false,
 			"LogsEnabled":    true,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
 
-		waitForLogs(t, 5, agentLogsConsumer)
+		internal.WaitForLogs(t, 5, agentLogsConsumer)
 		uninstallDeployment(t)
-		resetLogsSink(t, agentLogsConsumer)
-		resetMetricsSink(t, hecMetricsConsumer)
+		internal.ResetLogsSink(t, agentLogsConsumer)
+		internal.ResetMetricsSink(t, hecMetricsConsumer)
 	})
 }
 
@@ -256,9 +251,9 @@ func testIndexSwitch(t *testing.T) {
 
 	valuesFileName := "values_indexes_switching.yaml.tmpl"
 	hecMetricsConsumer := setupOnce(t).hecMetricsConsumer
-	checkNoMetricsReceived(t, hecMetricsConsumer)
+	internal.CheckNoMetricsReceived(t, hecMetricsConsumer)
 	agentLogsConsumer := setupOnce(t).logsConsumer
-	checkNoEventsReceived(t, agentLogsConsumer)
+	internal.CheckNoEventsReceived(t, agentLogsConsumer)
 
 	t.Run("check logs and metrics index switching", func(t *testing.T) {
 		replacements := map[string]interface{}{
@@ -267,8 +262,8 @@ func testIndexSwitch(t *testing.T) {
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
 
-		waitForMetrics(t, 3, hecMetricsConsumer)
-		waitForLogs(t, 3, agentLogsConsumer)
+		internal.WaitForMetrics(t, 3, hecMetricsConsumer)
+		internal.WaitForLogs(t, 3, agentLogsConsumer)
 
 		var sourcetypes []string
 		var indices []string
@@ -291,11 +286,11 @@ func testIndexSwitch(t *testing.T) {
 			"Sourcetype":           nonDefaultSourcetype,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
-		resetLogsSink(t, agentLogsConsumer)
-		resetMetricsSink(t, hecMetricsConsumer)
+		internal.ResetLogsSink(t, agentLogsConsumer)
+		internal.ResetMetricsSink(t, hecMetricsConsumer)
 
-		waitForMetrics(t, 3, hecMetricsConsumer)
-		waitForLogs(t, 3, agentLogsConsumer)
+		internal.WaitForMetrics(t, 3, hecMetricsConsumer)
+		internal.WaitForLogs(t, 3, agentLogsConsumer)
 		logs = agentLogsConsumer.AllLogs()
 		sourcetypes, indices = getLogsIndexAndSourceType(logs)
 		t.Logf("Indices: %v", indices)
@@ -308,15 +303,15 @@ func testIndexSwitch(t *testing.T) {
 		assert.True(t, mIndices[0] == newMetricsIndex)
 	})
 	uninstallDeployment(t)
-	resetLogsSink(t, agentLogsConsumer)
-	resetMetricsSink(t, hecMetricsConsumer)
+	internal.ResetLogsSink(t, agentLogsConsumer)
+	internal.ResetMetricsSink(t, hecMetricsConsumer)
 }
 
 func testClusterReceiverEnabledOrDisabled(t *testing.T) {
 	valuesFileName := "values_cluster_receiver_switching.yaml.tmpl"
 	namespace := "default"
 	logsObjectsConsumer := setupOnce(t).logsObjectsConsumer
-	hostEp := hostEndpoint(t)
+	hostEp := internal.HostEndpoint(t)
 	if len(hostEp) == 0 {
 		require.Fail(t, "Host endpoint not found")
 	}
@@ -332,30 +327,30 @@ func testClusterReceiverEnabledOrDisabled(t *testing.T) {
 		pods = listPodsInNamespace(t, namespace)
 		assert.True(t, len(pods.Items) == 1)
 		assert.True(t, strings.HasPrefix(pods.Items[0].Name, "sock-splunk-otel-collector-agent"))
-		checkNoEventsReceived(t, logsObjectsConsumer)
+		internal.CheckNoEventsReceived(t, logsObjectsConsumer)
 
 		t.Log("cluster receiver enabled")
 		replacements = map[string]interface{}{
 			"ClusterReceiverEnabled": true,
 			"LogObjectsHecEndpoint":  logsObjectsHecEndpoint,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
-		resetLogsSink(t, logsObjectsConsumer)
+		internal.ResetLogsSink(t, logsObjectsConsumer)
 
 		pods = listPodsInNamespace(t, namespace)
 		assert.True(t, len(pods.Items) == 2)
 		assert.True(t, checkPodExists(pods, "sock-splunk-otel-collector-agent"))
 		assert.True(t, checkPodExists(pods, "sock-splunk-otel-collector-k8s-cluster-receiver"))
-		waitForLogs(t, 5, logsObjectsConsumer)
+		internal.WaitForLogs(t, 5, logsObjectsConsumer)
 	})
 	uninstallDeployment(t)
-	resetLogsSink(t, logsObjectsConsumer)
+	internal.ResetLogsSink(t, logsObjectsConsumer)
 }
 
 func testVerifyLogsAndMetricsAttributes(t *testing.T) {
 	attributesList := [4]string{"k8s.node.name", "k8s.pod.name", "k8s.pod.uid", "k8s.namespace.name"}
 
-	hostEp := hostEndpoint(t)
+	hostEp := internal.HostEndpoint(t)
 	if len(hostEp) == 0 {
 		require.Fail(t, "Host endpoint not found")
 	}
@@ -370,8 +365,8 @@ func testVerifyLogsAndMetricsAttributes(t *testing.T) {
 			"LogObjectsHecEndpoint":  logsObjectsHecEndpoint,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
-		resetLogsSink(t, logsObjectsConsumer)
-		waitForLogs(t, 5, logsObjectsConsumer)
+		internal.ResetLogsSink(t, logsObjectsConsumer)
+		internal.WaitForLogs(t, 5, logsObjectsConsumer)
 		t.Logf("===> >>>> Logs: %v", len(logsObjectsConsumer.AllLogs()))
 
 		for _, attr := range attributesList {
@@ -393,10 +388,10 @@ func testVerifyLogsAndMetricsAttributes(t *testing.T) {
 			"LogObjectsHecEndpoint":  logsObjectsHecEndpoint,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
-		resetMetricsSink(t, hecMetricsConsumer)
+		internal.ResetMetricsSink(t, hecMetricsConsumer)
 		t.Logf("===> >>>> Metrics: %d", len(hecMetricsConsumer.AllMetrics()))
 
-		waitForMetrics(t, 5, hecMetricsConsumer)
+		internal.WaitForMetrics(t, 5, hecMetricsConsumer)
 		for _, attr := range attributesList {
 			t.Log("Checking attributes: ", attr)
 			attrValues, notFoundCounter := getMetricsAttributes(hecMetricsConsumer.AllMetrics(), attr)
@@ -415,9 +410,9 @@ func testVerifyLogsAndMetricsAttributes(t *testing.T) {
 			"LogsEnabled":    true,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
-		resetLogsSink(t, agentLogsConsumer)
+		internal.ResetLogsSink(t, agentLogsConsumer)
 
-		waitForLogs(t, 5, agentLogsConsumer)
+		internal.WaitForLogs(t, 5, agentLogsConsumer)
 		for _, attr := range attributesList {
 			t.Log("Checking attribute: ", attr)
 			attrValues, notFoundCounter := getLogsAttributes(agentLogsConsumer.AllLogs(), attr)
@@ -436,9 +431,9 @@ func testVerifyLogsAndMetricsAttributes(t *testing.T) {
 			"LogsEnabled":    true,
 		}
 		deployChartsAndApps(t, valuesFileName, replacements)
-		resetMetricsSink(t, hecMetricsConsumer)
+		internal.ResetMetricsSink(t, hecMetricsConsumer)
 
-		waitForMetrics(t, 5, hecMetricsConsumer)
+		internal.WaitForMetrics(t, 5, hecMetricsConsumer)
 		for _, attr := range attributesList {
 			t.Log("Checking attribute: ", attr)
 			attrValues, notFoundCounter := getMetricsAttributes(hecMetricsConsumer.AllMetrics(), attr)