Add support for multiple output formats for live e2e tests (#2555)

mortent · web-flow · commit efbff3131f88 · 2021-10-25T18:41:32.000-07:00
diff --git a/Makefile b/Makefile
@@ -89,7 +89,7 @@ test-fn-eval: build
 
 # target to run e2e tests for "kpt fn eval" command
 test-live-apply: build
-	PATH=$(GOBIN):$(PATH) go test -v --tags=kind --run=TestLiveApply/testdata/live-apply/$(T)  ./e2e/
+	PATH=$(GOBIN):$(PATH) go test -v -timeout=20m --tags=kind --run=TestLiveApply/testdata/live-apply/$(T)  ./e2e/
 
 vet:
 	go vet ./...
diff --git a/e2e/live_test.go b/e2e/live_test.go
@@ -17,6 +17,7 @@
 package e2e
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -33,12 +34,16 @@ func runTests(t *testing.T, path string) {
 
 	for p := range testCases {
 		c := testCases[p]
-		t.Run(p, func(t *testing.T) {
-			(&livetest.Runner{
-				Config: c,
-				Path: p,
-			}).Run(t)
-		})
+		for format := range c.Output {
+			testName := fmt.Sprintf("%s#%s", p, format)
+			t.Run(testName, func(t *testing.T) {
+				(&livetest.Runner{
+					Config: c,
+					Path: p,
+					Format: format,
+				}).Run(t)
+			})
+		}
 	}
 }
 
diff --git a/e2e/testdata/live-apply/apply-depends-on/config.yaml b/e2e/testdata/live-apply/apply-depends-on/config.yaml
@@ -1,9 +1,17 @@
 preinstallResourceGroup: true
-stdOut: |
-  deployment.apps/first-nginx created
-  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
-  deployment.apps/second-nginx created
-  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+output:
+  events:
+    stdOut: |
+      deployment.apps/first-nginx created
+      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+      deployment.apps/second-nginx created
+      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+  json:
+    stdOut: |
+      {"eventType":"resourceApplied","group":"apps","kind":"Deployment","name":"first-nginx","namespace":"apply-depends-on","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
+      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
+      {"eventType":"resourceApplied","group":"apps","kind":"Deployment","name":"second-nginx","namespace":"apply-depends-on","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
+      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
 inventory:
   - group: apps
     kind: Deployment
diff --git a/e2e/testdata/live-apply/crd-and-cr/config.yaml b/e2e/testdata/live-apply/crd-and-cr/config.yaml
@@ -1,10 +1,18 @@
 requiresCleanCluster: true
 preinstallResourceGroup: true
-stdOut: |
-  customresourcedefinition.apiextensions.k8s.io/customs.kpt.dev created
-  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
-  custom.kpt.dev/cr created
-  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+output:
+  events:
+    stdOut: |
+      customresourcedefinition.apiextensions.k8s.io/customs.kpt.dev created
+      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+      custom.kpt.dev/cr created
+      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+  json:
+    stdOut: |
+      {"eventType":"resourceApplied","group":"apiextensions.k8s.io","kind":"CustomResourceDefinition","name":"customs.kpt.dev","namespace":"","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
+      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
+      {"eventType":"resourceApplied","group":"kpt.dev","kind":"Custom","name":"cr","namespace":"","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
+      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
 inventory:
   - group: apiextensions.k8s.io
     kind: CustomResourceDefinition
diff --git a/e2e/testdata/live-apply/dry-run-with-install-rg/config.yaml b/e2e/testdata/live-apply/dry-run-with-install-rg/config.yaml
@@ -2,9 +2,17 @@ requiresCleanCluster: true
 kptArgs:
   - "--dry-run"
   - "--install-resource-group"
-stdOut: |
-  deployment.apps/nginx-deployment created (dry-run)
-  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed (dry-run)
-stdErr: |
-  installing inventory ResourceGroup CRD.
+output:
+  events:
+    stdOut: |
+      deployment.apps/nginx-deployment created (dry-run)
+      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed (dry-run)
+    stdErr: |
+      installing inventory ResourceGroup CRD.
+  json:
+    stdOut: |
+      {"eventType":"resourceApplied","group":"apps","kind":"Deployment","name":"nginx-deployment","namespace":"dry-run-with-install-rg","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
+      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
+    stdErr: |
+      installing inventory ResourceGroup CRD.
 exitCode: 0
diff --git a/e2e/testdata/live-apply/dry-run-without-rg/config.yaml b/e2e/testdata/live-apply/dry-run-without-rg/config.yaml
@@ -1,6 +1,8 @@
 requiresCleanCluster: true
 kptArgs:
   - "--dry-run"
-stdErr: |
-  Error: The ResourceGroup CRD was not found in the cluster. Please install it either by using the '--install-resource-group' flag or the 'kpt live install-resource-group' command.
+output:
+  events:
+    stdErr: |
+      Error: The ResourceGroup CRD was not found in the cluster. Please install it either by using the '--install-resource-group' flag or the 'kpt live install-resource-group' command.
 exitCode: 1
diff --git a/e2e/testdata/live-apply/install-rg-on-apply/config.yaml b/e2e/testdata/live-apply/install-rg-on-apply/config.yaml
@@ -1,9 +1,17 @@
 requiresCleanCluster: true
-stdErr: |
-  installing inventory ResourceGroup CRD.
-stdOut: |
-  deployment.apps/nginx-deployment created
-  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+output:
+  events:
+    stdOut: |
+      deployment.apps/nginx-deployment created
+      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+    stdErr: |
+      installing inventory ResourceGroup CRD.
+  json:
+    stdOut: |
+      {"eventType":"resourceApplied","group":"apps","kind":"Deployment","name":"nginx-deployment","namespace":"install-rg-on-apply","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
+      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
+    stdErr: |
+      installing inventory ResourceGroup CRD.
 inventory:
   - group: apps
     kind: Deployment
diff --git a/e2e/testdata/live-apply/prune-depends-on/config.yaml b/e2e/testdata/live-apply/prune-depends-on/config.yaml
@@ -1,11 +1,21 @@
 preinstallResourceGroup: true
-stdOut: |
-  configmap/cm created
-  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
-  deployment.apps/second-nginx pruned
-  1 resource(s) pruned, 0 skipped, 0 failed
-  deployment.apps/first-nginx pruned
-  1 resource(s) pruned, 0 skipped, 0 failed
+output:
+  events:
+    stdOut: |
+      configmap/cm created
+      1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+      deployment.apps/second-nginx pruned
+      1 resource(s) pruned, 0 skipped, 0 failed
+      deployment.apps/first-nginx pruned
+      1 resource(s) pruned, 0 skipped, 0 failed
+  json:
+    stdOut: |
+      {"eventType":"resourceApplied","group":"","kind":"ConfigMap","name":"cm","namespace":"prune-depends-on","operation":"Created","timestamp":"<TIMESTAMP>","type":"apply"}
+      {"configuredCount":0,"count":1,"createdCount":1,"eventType":"completed","failedCount":0,"serverSideCount":0,"timestamp":"<TIMESTAMP>","type":"apply","unchangedCount":0}
+      {"eventType":"resourcePruned","group":"apps","kind":"Deployment","name":"second-nginx","namespace":"prune-depends-on","operation":"Pruned","timestamp":"<TIMESTAMP>","type":"prune"}
+      {"eventType":"completed","pruned":1,"skipped":0,"timestamp":"<TIMESTAMP>","type":"prune"}
+      {"eventType":"resourcePruned","group":"apps","kind":"Deployment","name":"first-nginx","namespace":"prune-depends-on","operation":"Pruned","timestamp":"<TIMESTAMP>","type":"prune"}
+      {"eventType":"completed","pruned":1,"skipped":0,"timestamp":"<TIMESTAMP>","type":"prune"}
 inventory:
   - kind: ConfigMap
     name: cm
diff --git a/pkg/test/live/config.go b/pkg/test/live/config.go
@@ -26,12 +26,7 @@ type TestCaseConfig struct {
 	// ExitCode is the expected exit code from the kpt commands. Default: 0
 	ExitCode int `yaml:"exitCode,omitempty"`
 
-	// StdErr is the expected standard error output. Default: ""
-	StdErr string `yaml:"stdErr,omitempty"`
-
-	// StdOut is the expected standard output from running the command.
-	// Default: ""
-	StdOut string `yaml:"stdOut,omitempty"`
+	Output map[string]Output `yaml:"output,omitempty"`
 
 	// Inventory is the expected list of resource present in the inventory.
 	Inventory []InventoryEntry `yaml:"inventory,omitempty"`
@@ -49,6 +44,15 @@ type TestCaseConfig struct {
 	KptArgs []string `yaml:"kptArgs,omitempty"`
 }
 
+type Output struct {
+	// StdErr is the expected standard error output. Default: ""
+	StdErr string `yaml:"stdErr,omitempty"`
+
+	// StdOut is the expected standard output from running the command.
+	// Default: ""
+	StdOut string `yaml:"stdOut,omitempty"`
+}
+
 // InventoryEntry defines an entry in an inventory list.
 type InventoryEntry struct {
 	Group     string `yaml:"group,omitempty"`
diff --git a/pkg/test/live/runner.go b/pkg/test/live/runner.go
@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"testing"
@@ -43,6 +44,9 @@ type Runner struct {
 
 	// Path provides the path to the test files.
 	Path string
+
+	// Format provides the output format that should be used.
+	Format string
 }
 
 // Run executes the test.
@@ -77,7 +81,7 @@ func (r *Runner) Run(t *testing.T) {
 
 	r.RunPreApply(t)
 
-	stdout, stderr, err := r.RunApply()
+	stdout, stderr, err := r.RunApply(t)
 	r.VerifyExitCode(t, err)
 	r.VerifyStdout(t, stdout)
 	r.VerifyStderr(t, stderr)
@@ -102,8 +106,12 @@ func (r *Runner) RunPreApply(t *testing.T) {
 	}
 }
 
-func (r *Runner) RunApply() (string, string, error) {
+func (r *Runner) RunApply(t *testing.T) (string, string, error) {
 	args := append([]string{"live", "apply"}, r.Config.KptArgs...)
+	if r.Format != "" {
+		args = append(args, "--output", r.Format)
+	}
+	t.Logf("Running command: kpt %s", strings.Join(args, " "))
 	cmd := exec.Command("kpt", args...)
 	cmd.Dir = filepath.Join(r.Path, "resources")
 
@@ -220,11 +228,11 @@ func (r *Runner) VerifyExitCode(t *testing.T, err error) {
 }
 
 func (r *Runner) VerifyStdout(t *testing.T, stdout string) {
-	assert.Equal(t, strings.TrimSpace(r.Config.StdOut), strings.TrimSpace(stdout))
+	assert.Equal(t, strings.TrimSpace(r.Config.Output[r.Format].StdOut), strings.TrimSpace(substituteTimestamps(stdout)))
 }
 
 func (r *Runner) VerifyStderr(t *testing.T, stderr string) {
-	assert.Equal(t, strings.TrimSpace(r.Config.StdErr), strings.TrimSpace(stderr))
+	assert.Equal(t, strings.TrimSpace(r.Config.Output[r.Format].StdErr), strings.TrimSpace(substituteTimestamps(stderr)))
 }
 
 func (r *Runner) VerifyInventory(t *testing.T, name, namespace string) {
@@ -291,3 +299,9 @@ func inventorySortFunc(inv []InventoryEntry) func(i, j int) bool {
 		return iInv.Namespace < jInv.Namespace
 	}
 }
+
+var timestampRegexp = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z`)
+
+func substituteTimestamps(text string) string {
+	return timestampRegexp.ReplaceAllString(text, "<TIMESTAMP>")
+}
