Issue #370 Add test story to check cluster health

Author: Jakub Sliacan

go.mod

@@ -5,7 +5,7 @@ require (
 	github.com/DATA-DOG/godog v0.7.13
 	github.com/YourFin/binappend v0.0.0-20181105185800-0add4bf0b9ad
 	github.com/cavaliercoder/grab v2.0.0+incompatible
-	github.com/code-ready/clicumber v0.0.0-20190503113956-2563aed4ef12
+	github.com/code-ready/clicumber v0.0.0-20190801094041-02a92c8f7ece
 	github.com/code-ready/goodhosts v0.0.0-20190712111040-f090f3f77c26
 	github.com/code-ready/machine v0.0.0-20190731093717-b6d974ad44d0
 	github.com/docker/go-units v0.4.0

go.sum

@@ -13,6 +13,8 @@ github.com/cavaliercoder/grab v2.0.0+incompatible h1:wZHbBQx56+Yxjx2TCGDcenhh3cJ
 github.com/cavaliercoder/grab v2.0.0+incompatible/go.mod h1:tTBkfNqSBfuMmMBFaO2phgyhdYhiZQ/+iXCZDzcDsMI=
 github.com/code-ready/clicumber v0.0.0-20190503113956-2563aed4ef12 h1:h7ppinXfQf2BuaKRqdgA+BjWEL5H5LICRL0Ubq920ys=
 github.com/code-ready/clicumber v0.0.0-20190503113956-2563aed4ef12/go.mod h1:bjcKpFX8OebrlwSDgDAf0KAqpdVPGQssQk+IcVwlcog=
+github.com/code-ready/clicumber v0.0.0-20190801094041-02a92c8f7ece h1:ZuvFtsdv+H0JlBkKU+M6zGZOH0JcH7hnIvVNy5PQZro=
+github.com/code-ready/clicumber v0.0.0-20190801094041-02a92c8f7ece/go.mod h1:bjcKpFX8OebrlwSDgDAf0KAqpdVPGQssQk+IcVwlcog=
 github.com/code-ready/goodhosts v0.0.0-20190712111040-f090f3f77c26 h1:rNdqNBHRAAuGv68zZCX0rQAekl0g4qwnseA8OFM2j54=
 github.com/code-ready/goodhosts v0.0.0-20190712111040-f090f3f77c26/go.mod h1:kc1Z2UDuIlxPOo7VWCIkYPOaD55KPX0t5qS5cmLBe8Y=
 github.com/code-ready/machine v0.0.0-20190513092840-66a8c3b3c494 h1:Krl/Ub9djuzrYW49kUgyroWDMlrqeagL2QrNCjgx1mA=

test/integration/crcsuite/crcsuite.go

@@ -19,13 +19,17 @@ package crcsuite
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
+	"time"
 
 	"github.com/DATA-DOG/godog"
 	"github.com/DATA-DOG/godog/gherkin"
 
 	clicumber "github.com/code-ready/clicumber/testsuite"
+	"github.com/code-ready/crc/pkg/crc/oc"
 )
 
 var (
@@ -47,6 +51,10 @@ func FeatureContext(s *godog.Suite) {
 		StartCRCWithDefaultBundleAndHypervisorSucceedsOrFails)
 	s.Step(`^setting config property "(.*)" to value "(.*)" (succeeds|fails)$`,
 		SetConfigPropertyToValueSucceedsOrFails)
+	s.Step(`^login to the oc cluster (succeeds|fails)$`,
+		LoginToOcClusterSucceedsOrFails)
+	s.Step(`^with up to "(\d+)" retries with wait period of "(\d*(?:ms|s|m))" all cluster operators are running$`,
+		CheckClusterOperatorsWithRetry)
 
 	// CRC file operations
 	s.Step(`^file "([^"]*)" exists in CRC home folder$`,
@@ -96,6 +104,28 @@ func FeatureContext(s *godog.Suite) {
 	})
 }
 
+func CheckClusterOperatorsWithRetry(retryCount int, retryWait string) error {
+
+	retryDuration, err := time.ParseDuration(retryWait)
+	if err != nil {
+		return err
+	}
+
+	ocConfig := oc.UseOCWithConfig("crc")
+	for i := 0; i < retryCount; i++ {
+		s, err := oc.GetClusterOperatorStatus(ocConfig)
+		if err != nil {
+			return err
+		}
+		if s == true {
+			return nil
+		}
+		time.Sleep(retryDuration)
+	}
+
+	return fmt.Errorf("Some cluster operators are still not running.\n")
+}
+
 func DeleteFileFromCRCHome(fileName string) error {
 
 	theFile := filepath.Join(CRCHome, fileName)
@@ -175,9 +205,31 @@ func ConfigFileInCRCHomeContainsKey(format string, configFile string, condition
 	return nil
 }
 
+func LoginToOcClusterSucceedsOrFails(expected string) error {
+
+	bundle := strings.Split(bundleName, ".crcbundle")[0]
+	pswdLocation := filepath.Join(CRCHome, "cache", bundle, "kubeadmin-password")
+
+	pswdFile, err := os.Open(pswdLocation)
+	if err != nil {
+		return err
+	}
+	defer pswdFile.Close()
+
+	pswd, err := ioutil.ReadAll(pswdFile)
+	if err != nil {
+		return err
+	}
+
+	cmd := fmt.Sprintf("oc login --insecure-skip-tls-verify -u kubeadmin -p %s https://api.crc.testing:6443", pswd)
+	err = clicumber.ExecuteCommandSucceedsOrFails(cmd, expected)
+
+	return err
+}
+
 func StartCRCWithDefaultBundleAndDefaultHypervisorSucceedsOrFails(expected string) error {
 
-	cmd := fmt.Sprintf("crc start -b %s -p '%s'", bundleName, pullSecretFile)
+	cmd := fmt.Sprintf("crc start -b %s -p '%s' --log-level debug", bundleName, pullSecretFile)
 	err := clicumber.ExecuteCommandSucceedsOrFails(cmd, expected)
 
 	return err

test/integration/features/story_health.feature

@@ -0,0 +1,61 @@
+Feature: 
+    End-to-end health check. Set-up and start CRC. Then create a
+    project and deploy an app. Check on the app and delete the
+    project. Stop and delete CRC.
+
+    Scenario Outline: Start CRC
+        Given executing "crc setup" succeeds
+        When starting CRC with default bundle and hypervisor "<vm-driver>" succeeds
+        Then stdout should contain "CodeReady Containers instance is running"
+        And executing "eval $(crc oc-env)" succeeds
+        When with up to "4" retries with wait period of "2m" command "crc status" output should contain "Running (v4.x)"
+        Then login to the oc cluster succeeds
+
+    @darwin
+        Examples:
+            | vm-driver  |
+            | hyperkit   |
+
+    @linux
+        Examples:
+            | vm-driver |
+            | libvirt   |
+
+    @darwin @linux 
+    Scenario: Check cluster health
+        Given executing "crc status" succeeds
+        And stdout contains "Running (v4.x)"
+        When executing "oc get nodes"
+        Then stdout contains "Ready" 
+        And stdout does not contain "Not ready"
+        # next line checks similar things as `crc status` except gives more informative output
+        And with up to "5" retries with wait period of "1m" all cluster operators are running
+
+    @darwin @linux
+    Scenario: Create project
+        When executing "oc new-project testproj" succeeds
+        Then stdout should contain
+            """
+            Now using project "testproj" on server "https://api.crc.testing:6443".
+            """
+        And stdout should contain "You can add applications to this project with the 'new-app' command."
+
+    @darwin @linux
+    Scenario: Create and test app
+        When executing "oc new-app centos/httpd-24-centos7~https://github.com/sclorg/httpd-ex" succeeds
+        Then stdout should contain "Creating resources"
+        And stdout should contain
+            """
+            service "httpd-ex" created
+            """
+        When executing "oc rollout status dc/httpd-ex" succeeds
+        Then stdout should contain "successfully rolled out"
+        And executing "oc expose svc/httpd-ex" succeeds
+        And with up to "2" retries with wait period of "60s" command "curl -kI http://httpd-ex-testproj.apps-crc.testing" output should contain "HTTP/1.1 200 OK"
+
+    Scenario: Clean up
+        Given executing "oc delete project testproj" succeeds
+        When executing "crc stop -f" succeeds
+        Then stdout should contain "CodeReady Containers instance stopped"
+        When executing "crc delete" succeeds
+        Then stdout should contain "CodeReady Containers instance deleted"

vendor/github.com/code-ready/clicumber/testsuite/shell.go

@@ -6,7 +6,7 @@ github.com/DATA-DOG/godog/colors
 github.com/YourFin/binappend
 # github.com/cavaliercoder/grab v2.0.0+incompatible
 github.com/cavaliercoder/grab
-# github.com/code-ready/clicumber v0.0.0-20190503113956-2563aed4ef12
+# github.com/code-ready/clicumber v0.0.0-20190801094041-02a92c8f7ece
 github.com/code-ready/clicumber/testsuite
 github.com/code-ready/clicumber/util
 # github.com/code-ready/goodhosts v0.0.0-20190712111040-f090f3f77c26