Simplify and speed up "kpt live" e2e tests (#2613)

* Simplify and speed up live e2e tests

* Addressed comments

Author: mortent

Makefile

@@ -87,7 +87,7 @@ test-fn-eval: build
 
 # target to run e2e tests for "kpt fn eval" command
 test-live-apply: build
-	PATH=$(GOBIN):$(PATH) go test -v -timeout=20m --tags=kind --run=TestLiveApply/testdata/live-apply/$(T)  ./e2e/
+	PATH=$(GOBIN):$(PATH) go test -v -timeout=20m --tags=kind -p 2 --run=TestLiveApply/testdata/live-apply/$(T)  ./e2e/
 
 vet:
 	go vet ./...

e2e/live_test.go

@@ -17,12 +17,12 @@
 package e2e
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
 
 	livetest "github.com/GoogleContainerTools/kpt/pkg/test/live"
+	"github.com/stretchr/testify/require"
 )
 
 func TestLiveApply(t *testing.T) {
@@ -32,18 +32,40 @@ func TestLiveApply(t *testing.T) {
 func runTests(t *testing.T, path string) {
 	testCases := scanTestCases(t, path)
 
+	livetest.RemoveKindCluster(t)
+	livetest.CreateKindCluster(t)
+
 	for p := range testCases {
+		p := p
 		c := testCases[p]
-		for format := range c.Output {
-			testName := fmt.Sprintf("%s#%s", p, format)
-			t.Run(testName, func(t *testing.T) {
-				(&livetest.Runner{
-					Config: c,
-					Path: p,
-					Format: format,
-				}).Run(t)
-			})
+
+		if !c.Parallel {
+			continue
 		}
+
+		t.Run(p, func(t *testing.T) {
+			if c.Parallel {
+				t.Parallel()
+			}
+
+			if c.NoResourceGroup {
+				require.False(t, c.Parallel, "Parallel tests can not modify the RG CRD")
+				if livetest.CheckIfResourceGroupInstalled(t) {
+					livetest.RemoveResourceGroup(t)
+				}
+			} else {
+				livetest.InstallResourceGroup(t)
+			}
+
+			ns := filepath.Base(p)
+			livetest.CreateNamespace(t, ns)
+			defer livetest.RemoveNamespace(t, ns)
+
+			(&livetest.Runner{
+				Config:    c,
+				Path:      p,
+			}).Run(t)
+		})
 	}
 }
 
@@ -68,4 +90,4 @@ func scanTestCases(t *testing.T, path string) map[string]livetest.TestCaseConfig
 		t.Fatalf("failed to scan for test cases in %s", path)
 	}
 	return testCases
-}
+}

e2e/testdata/live-apply/apply-depends-on/config.yaml

@@ -12,20 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-preinstallResourceGroup: true
-output:
-  events:
-    stdOut: |
-      deployment.apps/first-nginx created
-      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
-      deployment.apps/second-nginx created
-      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
-  json:
-    stdOut: |
-      {"eventType":"resourceApplied","group":"apps","kind":"Deployment","name":"first-nginx","namespace":"apply-depends-on","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
-      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
-      {"eventType":"resourceApplied","group":"apps","kind":"Deployment","name":"second-nginx","namespace":"apply-depends-on","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
-      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
+parallel: true
+
+stdOut: |
+  deployment.apps/first-nginx created
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+  deployment.apps/second-nginx created
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+
 inventory:
   - group: apps
     kind: Deployment

e2e/testdata/live-apply/apply-depends-on/resources/first.yaml

@@ -18,7 +18,7 @@ metadata:
   name: first-nginx
   namespace: apply-depends-on
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: first-nginx

e2e/testdata/live-apply/apply-depends-on/resources/second.yaml

@@ -20,7 +20,7 @@ metadata:
   annotations:
     config.kubernetes.io/depends-on: apps/namespaces/apply-depends-on/Deployment/first-nginx
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: second-nginx

e2e/testdata/live-apply/crd-and-cr/config.yaml

@@ -12,21 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-requiresCleanCluster: true
-preinstallResourceGroup: true
-output:
-  events:
-    stdOut: |
-      customresourcedefinition.apiextensions.k8s.io/customs.kpt.dev created
-      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
-      custom.kpt.dev/cr created
-      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
-  json:
-    stdOut: |
-      {"eventType":"resourceApplied","group":"apiextensions.k8s.io","kind":"CustomResourceDefinition","name":"customs.kpt.dev","namespace":"","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
-      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
-      {"eventType":"resourceApplied","group":"kpt.dev","kind":"Custom","name":"cr","namespace":"","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
-      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
+parallel: true
+stdOut: |
+  customresourcedefinition.apiextensions.k8s.io/customs.kpt.dev created
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+  custom.kpt.dev/cr created
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
 inventory:
   - group: apiextensions.k8s.io
     kind: CustomResourceDefinition

e2e/testdata/live-apply/dry-run-with-install-rg/config.yaml

@@ -12,21 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-requiresCleanCluster: true
+parallel: false
+noResourceGroup: true
 kptArgs:
   - "--dry-run"
   - "--install-resource-group"
-output:
-  events:
-    stdOut: |
-      deployment.apps/nginx-deployment created (dry-run)
-      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed (dry-run)
-    stdErr: |
-      installing inventory ResourceGroup CRD.
-  json:
-    stdOut: |
-      {"eventType":"resourceApplied","group":"apps","kind":"Deployment","name":"nginx-deployment","namespace":"dry-run-with-install-rg","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
-      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
-    stdErr: |
-      installing inventory ResourceGroup CRD.
+stdOut: |
+  deployment.apps/nginx-deployment created (dry-run)
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed (dry-run)
+stdErr: |
+  installing inventory ResourceGroup CRD.
 exitCode: 0

e2e/testdata/live-apply/dry-run-with-install-rg/resources/deployment.yaml

@@ -24,7 +24,7 @@ spec:
     rollingUpdate:
       maxUnavailable: 1
       maxSurge: 0
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: nginx-deployment

e2e/testdata/live-apply/dry-run-without-rg/config.yaml

@@ -12,11 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-requiresCleanCluster: true
+parallel: false
+noResourceGroup: true
 kptArgs:
   - "--dry-run"
-output:
-  events:
-    stdErr: |
-      Error: The ResourceGroup CRD was not found in the cluster. Please install it either by using the '--install-resource-group' flag or the 'kpt live install-resource-group' command.
+stdErr: |
+  Error: The ResourceGroup CRD was not found in the cluster. Please install it either by using the '--install-resource-group' flag or the 'kpt live install-resource-group' command.
 exitCode: 1

e2e/testdata/live-apply/install-rg-on-apply/config.yaml

@@ -12,20 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-requiresCleanCluster: true
-output:
-  events:
-    stdOut: |
-      deployment.apps/nginx-deployment created
-      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
-    stdErr: |
-      installing inventory ResourceGroup CRD.
-  json:
-    stdOut: |
-      {"eventType":"resourceApplied","group":"apps","kind":"Deployment","name":"nginx-deployment","namespace":"install-rg-on-apply","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
-      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
-    stdErr: |
-      installing inventory ResourceGroup CRD.
+parallel: false
+noResourceGroup: true
+stdOut: |
+  deployment.apps/nginx-deployment created
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+stdErr: |
+  installing inventory ResourceGroup CRD.
 inventory:
   - group: apps
     kind: Deployment