New e2e tests for kpt live (#2390)

* New e2e tests for kpt live

* Updated

Author: mortent

Makefile

@@ -84,6 +84,10 @@ test-fn-render: build
 test-fn-eval: build
 	PATH=$(GOBIN):$(PATH) go test -v --tags=docker --run=TestFnEval/testdata/fn-eval/$(T)  ./e2e/
 
+# target to run e2e tests for "kpt fn eval" command
+test-live-apply: build
+	PATH=$(GOBIN):$(PATH) go test -v --tags=kind --run=TestLiveApply/testdata/live-apply/$(T)  ./e2e/
+
 vet:
 	go vet ./...
 

e2e/README.md

@@ -0,0 +1,53 @@
+# End-to-end testing of kpt
+
+## kpt live e2e tests
+
+We currently have two different solutions for running e2e tests for the kpt
+live functionality. We are working on reconciling this into one approach that
+we can use consistently.
+
+All e2e tests for live requires that kind is available.
+
+### Testing with go
+
+We have a framework for running e2e tests based on specifying test cases
+under testdata/live-apply folder (tests for other kpt live commands will be
+added). The entry point for the test framework is in
+the `live_test.go` file.
+
+In order to run all the tests for live apply, there is a make target
+```sh
+make test-live-apply
+```
+
+It is possible to run a single test by specifying the name of the test case:
+```sh
+make test-live-apply T=crd-and-cr
+```
+
+#### Structure of a test case
+
+Each test case is a folder directly under `testdata/live-apply`. In the root
+of each test case folder there must be a `config.yaml` file that provides the
+configuration of the test case (like whether a clean cluster is required and
+the expected output). The package that will be applied with `kpt live apply` is
+provided in the `resources` folder.
+
+#### Configuration options
+
+These are the configuration options available in the `config.yaml` file:
+ * `exitCode`: Defines the expected exit code after running the kpt live command. Defaults to 0.
+ * `stdErr`: Defines the expected output to stdErr after running the command. Defaults to "".
+ * `stdOut`: Defines the expected output to stdOut after running the command. Defaults to "".
+ * `inventory`: Defines the expected inventory after running the command.
+ * `requiresCleanCluster`: Defines whether a new kind cluster should be created prior to running the test.
+ * `preinstallResourceGroup`: Defines whether the framework should make sure the RG CRD is available before running the test.
+ * `kptArgs`: Defines the arguments that will be used with executing the kpt live command.
+
+## Testing with bash
+
+This approach uses a bash script that runs through several scenarios for
+kpt live in sequence. Run it by running
+```sh
+./live/end-to-end-test.sh
+```

e2e/live_test.go

@@ -0,0 +1,66 @@
+// +build kind
+
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	livetest "github.com/GoogleContainerTools/kpt/pkg/test/live"
+)
+
+func TestLiveApply(t *testing.T) {
+	runTests(t, filepath.Join(".", "testdata", "live-apply"))
+}
+
+func runTests(t *testing.T, path string) {
+	testCases := scanTestCases(t, path)
+
+	for p := range testCases {
+		c := testCases[p]
+		t.Run(p, func(t *testing.T) {
+			(&livetest.Runner{
+				Config: c,
+				Path: p,
+			}).Run(t)
+		})
+	}
+}
+
+func scanTestCases(t *testing.T, path string) map[string]livetest.TestCaseConfig {
+	testCases := make(map[string]livetest.TestCaseConfig)
+	err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			return nil
+		}
+		if path == p {
+			return nil
+		}
+
+		testCases[p] = livetest.ReadTestCaseConfig(t, p)
+		return filepath.SkipDir
+	})
+	if err != nil {
+		t.Fatalf("failed to scan for test cases in %s", path)
+	}
+	return testCases
+}

e2e/testdata/live-apply/crd-and-cr/config.yaml

@@ -0,0 +1,14 @@
+requiresCleanCluster: true
+preinstallResourceGroup: true
+stdOut: |
+  customresourcedefinition.apiextensions.k8s.io/customs.kpt.dev created
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+  custom.kpt.dev/cr created
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed
+inventory:
+  - group: apiextensions.k8s.io
+    kind: CustomResourceDefinition
+    name: customs.kpt.dev
+  - group: kpt.dev
+    kind: Custom
+    name: cr

e2e/testdata/live-apply/crd-and-cr/resources/Kptfile

@@ -0,0 +1,8 @@
+apiVersion: kpt.dev/v1
+kind: Kptfile
+metadata:
+  name: crd-and-cr
+inventory:
+  namespace: crd-and-cr
+  name: crd-and-cr
+  inventoryID: crd-and-cr

e2e/testdata/live-apply/crd-and-cr/resources/cr.yaml

@@ -0,0 +1,4 @@
+apiVersion: kpt.dev/v1
+kind: Custom
+metadata:
+  name: cr

e2e/testdata/live-apply/crd-and-cr/resources/crd.yaml

@@ -0,0 +1,29 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: customs.kpt.dev
+spec:
+  conversion:
+    strategy: None
+  group: kpt.dev
+  names:
+    kind: Custom
+    listKind: CustomList
+    plural: customs
+    singular: custom
+  scope: Cluster
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            type: string
+          kind:
+            type: string
+          metadata:
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources: {}

e2e/testdata/live-apply/dry-run-with-install-rg/config.yaml

@@ -0,0 +1,10 @@
+requiresCleanCluster: true
+kptArgs:
+  - "--dry-run"
+  - "--install-resource-group"
+stdOut: |
+  deployment.apps/nginx-deployment created (dry-run)
+  1 resource(s) applied. 1 created, 0 unchanged, 0 configured, 0 failed (dry-run)
+stdErr: |
+  installing inventory ResourceGroup CRD.
+exitCode: 0

e2e/testdata/live-apply/dry-run-with-install-rg/resources/Kptfile

@@ -0,0 +1,8 @@
+apiVersion: kpt.dev/v1
+kind: Kptfile
+metadata:
+  name: dry-run-with-install-rg
+inventory:
+  namespace: dry-run-with-install-rg
+  name: dry-run-with-install-rg
+  inventoryID: dry-run-with-install-rg

e2e/testdata/live-apply/dry-run-with-install-rg/resources/deployment.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  namespace: dry-run-with-install-rg
+  labels:
+    app: nginx-deployment
+spec:
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
+  replicas: 2
+  selector:
+    matchLabels:
+      app: nginx-deployment
+  template:
+    metadata:
+      labels:
+        app: nginx-deployment
+        pdb: mypdb
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.10
+        ports:
+        - containerPort: 80
+          protocol: TCP