[chore] Use a k8s lease to run functional test concurrently (#1704)

The Github actions concurrency feature can only ensure only allows one queued job. Any other jobs are cancelled. It means that the third EKS job is always cancelled, see https://github.com/signalfx/splunk-otel-collector-chart/actions/runs/13731622293/job/38416102735.

This change introduces another approach to handle concurrency via k8s leases. It allows queueing many jobs on the same cluster and run them locally.

Author: dmitryax

.github/workflows/functional_test_v2.yaml

@@ -101,8 +101,6 @@ jobs:
     if: |
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
       (github.ref == 'refs/heads/main')
-    concurrency:
-      group: eks-access
     env:
       KUBECONFIG: /tmp/kube-config-splunk-otel-collector-chart-functional-testing-eks
       KUBE_TEST_ENV: eks
@@ -139,8 +137,6 @@ jobs:
     if: |
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
       (github.ref == 'refs/heads/main')
-    concurrency:
-      group: eks-access
     env:
       KUBECONFIG: /tmp/kube-config-splunk-otel-collector-chart-functional-testing-eks
       KUBE_TEST_ENV: eks
@@ -180,28 +176,21 @@ jobs:
           aws eks update-kubeconfig --name rotel-eks --region us-west-1
       - name: Update dependencies
         run: |
-          cd base && make dep-update
-      - name: Deploy previous version of the chart
-        run: |
-          helm list | grep -q "^sock$" && echo "Found previous 'sock' release. Deleting..." && helm delete sock
-          cd base && helm install sock helm-charts/splunk-otel-collector --set cloudProvider=aws --set distribution=eks --set splunkObservability.realm=us0 --set splunkObservability.accessToken=xxxxx
-      - name: Update dependencies
-        run: |
-          make dep-update
+          make dep-update && cd base && make dep-update
       - name: run functional tests
         env:
           HOST_ENDPOINT: 0.0.0.0
+          UPGRADE_FROM_VALUES: aws_upgrade_from_previous_release_values.yaml
+          UPGRADE_FROM_CHART_DIR: base
         run: |
-          SUITE=functional make functionaltest
+          TEARDOWN_BEFORE_SETUP=true SUITE=functional make functionaltest
 
   gke-autopilot-test:
     name: Test helm install in GKE/Autopilot - credentials needed
     needs: kubernetes-test
     if: |
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
       (github.ref == 'refs/heads/main')
-    concurrency:
-      group: gke-autopilot-access
     env:
       KUBE_TEST_ENV: gke/autopilot
       SKIP_TESTS: "true"
@@ -240,8 +229,6 @@ jobs:
     if: |
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
       (github.ref == 'refs/heads/main')
-    concurrency:
-      group: gke-autopilot-access
     env:
       KUBE_TEST_ENV: gke/autopilot
       SKIP_TESTS: "true"
@@ -283,26 +270,21 @@ jobs:
           project_id: ${{ secrets.GKE_PROJECT }}
       - name: Update dependencies
         run: |
-          make dep-update
-      - name: Deploy previous version of the chart
-        run: |
-          cd base
-          helm dependency update helm-charts/splunk-otel-collector
-          helm install sock helm-charts/splunk-otel-collector --set cloudProvider=gcp --set distribution=gke/autopilot --set splunkObservability.realm=us0 --set splunkObservability.accessToken=xxxxx
+          make dep-update && cd base && make dep-update
       - name: run functional tests
         env:
           HOST_ENDPOINT: 0.0.0.0
+          UPGRADE_FROM_VALUES: gke_autopilot_upgrade_from_previous_release_values.yaml
+          UPGRADE_FROM_CHART_DIR: base
         run: |
-          SUITE=functional make functionaltest
+          TEARDOWN_BEFORE_SETUP=true SUITE=functional make functionaltest
 
   aks-windows-test:
     name: Test helm install in AKS - credentials needed
     needs: kubernetes-test
     if: |
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
       (github.ref == 'refs/heads/main')
-    concurrency:
-      group: aks-windows-group
     env:
       KUBE_TEST_ENV: aks
       SKIP_TESTS: "true"
@@ -342,8 +324,6 @@ jobs:
     if: |
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
       (github.ref == 'refs/heads/main')
-    concurrency:
-      group: gce-access
     env:
       KUBE_TEST_ENV: gce
       SKIP_TESTS: "true"

functional_tests/configuration_switching/configuration_switching_test.go

@@ -70,7 +70,7 @@ func deployChartsAndApps(t *testing.T, valuesFileName string, repl map[string]in
 	client, err := kubernetes.NewForConfig(kubeConfig)
 	require.NoError(t, err)
 
-	chart := internal.LoadCollectorChart(t)
+	chart := internal.LoadCollectorChart(t, "")
 
 	var valuesBytes []byte
 	valuesBytes, err = os.ReadFile(filepath.Join(testDir, valuesDir, valuesFileName))

functional_tests/functional/functional_test.go

@@ -61,6 +61,7 @@ const (
 	manifestsDir                           = "manifests"
 	eksValuesDir                           = "expected_eks_values"
 	kindValuesDir                          = "expected_kind_values"
+	helmActionTimeout                      = 5 * time.Minute
 )
 
 var archRe = regexp.MustCompile("-amd64$|-arm64$|-ppc64le$")
@@ -93,22 +94,24 @@ func setupOnce(t *testing.T) *sinks {
 				signalFxReceiverK8sClusterReceiverPort),
 			tracesConsumer: internal.SetupOTLPTracesSink(t),
 		}
+
+		testKubeConfig, setKubeConfig := os.LookupEnv("KUBECONFIG")
+		require.True(t, setKubeConfig, "the environment variable KUBECONFIG must be set")
+		internal.AcquireLeaseForTest(t, testKubeConfig)
 		if os.Getenv("TEARDOWN_BEFORE_SETUP") == "true" {
-			teardown(t)
+			teardown(t, testKubeConfig)
 		}
 		// deploy the chart and applications.
 		if os.Getenv("SKIP_SETUP") == "true" {
 			t.Log("Skipping setup as SKIP_SETUP is set to true")
 			return
 		}
-		deployChartsAndApps(t)
+		deployChartsAndApps(t, testKubeConfig)
 	})
 
 	return globalSinks
 }
-func deployChartsAndApps(t *testing.T) {
-	testKubeConfig, setKubeConfig := os.LookupEnv("KUBECONFIG")
-	require.True(t, setKubeConfig, "the environment variable KUBECONFIG must be set")
+func deployChartsAndApps(t *testing.T, testKubeConfig string) {
 	kubeTestEnv, setKubeTestEnv := os.LookupEnv("KUBE_TEST_ENV")
 	require.True(t, setKubeTestEnv, "the environment variable KUBE_TEST_ENV must be set")
 	kubeConfig, err := clientcmd.BuildConfigFromFlags("", testKubeConfig)
@@ -182,7 +185,7 @@ func deployChartsAndApps(t *testing.T) {
 		}
 	}, 1*time.Minute, 5*time.Second)
 
-	chart := internal.LoadCollectorChart(t)
+	chart := internal.LoadCollectorChart(t, "")
 
 	var valuesBytes []byte
 	switch kubeTestEnv {
@@ -235,18 +238,35 @@ func deployChartsAndApps(t *testing.T) {
 	}); err != nil {
 		require.NoError(t, err)
 	}
+
 	install := action.NewInstall(actionConfig)
 	install.Namespace = "default"
 	install.ReleaseName = "sock"
-	_, err = install.Run(chart, values)
-	if err != nil {
-		t.Logf("error reported during helm install: %v\n", err)
-		retryUpgrade := action.NewUpgrade(actionConfig)
-		retryUpgrade.Namespace = "default"
-		retryUpgrade.Install = true
-		_, err = retryUpgrade.Run("sock", chart, values)
+	install.Timeout = helmActionTimeout
+
+	// If UPGRADE_FROM_VALUES env var is set, we install the helm chart using the values. Otherwise, run helm install.
+	// UPGRADE_FROM_CHART_DIR is an optional env var that provides an alternative path for the initial helm chart.
+	upgradeFromValues := os.Getenv("UPGRADE_FROM_VALUES")
+	if upgradeFromValues != "" {
+		// install the base chart
+		initValuesBytes, rfErr := os.ReadFile(filepath.Join(testDir, valuesDir, upgradeFromValues))
+		require.NoError(t, rfErr)
+		initChart := internal.LoadCollectorChart(t, os.Getenv("UPGRADE_FROM_CHART_DIR"))
+		var initValues map[string]interface{}
+		require.NoError(t, yaml.Unmarshal(initValuesBytes, &initValues))
+		_, err = install.Run(initChart, initValues)
 		require.NoError(t, err)
+
+		// test the upgrade
+		upgrade := action.NewUpgrade(actionConfig)
+		upgrade.Namespace = "default"
+		upgrade.Install = true
+		upgrade.Timeout = helmActionTimeout
+		_, err = upgrade.Run("sock", chart, values)
+	} else {
+		_, err = install.Run(chart, values)
 	}
+	require.NoError(t, err)
 
 	waitForAllDeploymentsToStart(t, client)
 
@@ -405,14 +425,12 @@ func deployChartsAndApps(t *testing.T) {
 			return
 		}
 		t.Log("Cleaning up cluster")
-		teardown(t)
+		teardown(t, testKubeConfig)
 
 	})
 }
 
-func teardown(t *testing.T) {
-	testKubeConfig, setKubeConfig := os.LookupEnv("KUBECONFIG")
-	require.True(t, setKubeConfig, "the environment variable KUBECONFIG must be set")
+func teardown(t *testing.T, testKubeConfig string) {
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 	kubeConfig, err := clientcmd.BuildConfigFromFlags("", testKubeConfig)
 	require.NoError(t, err)
@@ -515,6 +533,7 @@ func teardown(t *testing.T) {
 	uninstall := action.NewUninstall(actionConfig)
 	uninstall.IgnoreNotFound = true
 	uninstall.Wait = true
+	uninstall.Timeout = helmActionTimeout
 	_, _ = uninstall.Run("sock")
 }
 

functional_tests/functional/testdata/values/aws_upgrade_from_previous_release_values.yaml

@@ -0,0 +1,6 @@
+cloudProvider: aws
+distribution: eks
+environment: dev
+splunkObservability:
+  realm: us0
+  accessToken: xxxxx

functional_tests/functional/testdata/values/gke_autopilot_upgrade_from_previous_release_values.yaml

@@ -0,0 +1,5 @@
+cloudProvider: gcp
+distribution: gke/autopilot
+splunkObservability:
+  realm: us0
+  accessToken: xxxxx

functional_tests/go.mod

@@ -3,21 +3,19 @@
 
 module github.com/signalfx/splunk-otel-collector-chart/functional_tests
 
-go 1.23.0
+go 1.24.0
 
 require (
 	github.com/docker/docker v27.5.1+incompatible
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.121.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.121.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/xk8stest v0.121.0
-	github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.121.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/signalfxreceiver v0.121.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/splunkhecreceiver v0.121.0
 	github.com/stretchr/testify v1.10.0
 	go.opentelemetry.io/collector/component/componenttest v0.121.0
 	go.opentelemetry.io/collector/consumer/consumertest v0.121.0
 	go.opentelemetry.io/collector/pdata v1.27.0
-	go.opentelemetry.io/collector/processor/processortest v0.121.0
 	go.opentelemetry.io/collector/receiver/otlpreceiver v0.121.0
 	go.opentelemetry.io/collector/receiver/receivertest v0.121.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -40,9 +38,6 @@ require (
 	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/alecthomas/participle/v2 v2.1.1 // indirect
-	github.com/antchfx/xmlquery v1.4.3 // indirect
-	github.com/antchfx/xpath v1.3.3 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
@@ -62,8 +57,6 @@ require (
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/elastic/go-grok v0.3.1 // indirect
-	github.com/elastic/lunes v0.1.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
@@ -80,9 +73,7 @@ require (
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
@@ -98,10 +89,8 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
-	github.com/iancoleman/strcase v0.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmoiron/sqlx v1.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -113,7 +102,6 @@ require (
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lib/pq v1.10.9 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
-	github.com/magefile/mage v1.15.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
@@ -133,11 +121,8 @@ require (
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/ackextension v0.121.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.121.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.121.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.121.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.121.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk v0.121.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.121.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.121.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/signalfx v0.121.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -159,7 +144,6 @@ require (
 	github.com/spf13/cast v1.7.0 // indirect
 	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -190,10 +174,7 @@ require (
 	go.opentelemetry.io/collector/internal/sharedcomponent v0.121.0 // indirect
 	go.opentelemetry.io/collector/internal/telemetry v0.121.0 // indirect
 	go.opentelemetry.io/collector/pdata/pprofile v0.121.0 // indirect
-	go.opentelemetry.io/collector/pdata/testdata v0.121.0 // indirect
 	go.opentelemetry.io/collector/pipeline v0.121.0 // indirect
-	go.opentelemetry.io/collector/processor v0.121.0 // indirect
-	go.opentelemetry.io/collector/processor/xprocessor v0.121.0 // indirect
 	go.opentelemetry.io/collector/receiver v0.121.0 // indirect
 	go.opentelemetry.io/collector/receiver/xreceiver v0.121.0 // indirect
 	go.opentelemetry.io/collector/semconv v0.121.0 // indirect
@@ -207,7 +188,6 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.35.0 // indirect
-	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/net v0.35.0 // indirect
 	golang.org/x/oauth2 v0.24.0 // indirect
 	golang.org/x/sync v0.11.0 // indirect
@@ -220,7 +200,6 @@ require (
 	google.golang.org/protobuf v1.36.5 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/apiserver v0.32.2 // indirect
 	k8s.io/cli-runtime v0.32.1 // indirect
 	k8s.io/component-base v0.32.2 // indirect