conformance testing pattern (#669)

* conformance testing pattern

* conformance experiment

Author: sriumcp

.github/workflows/knative-e2e-tests.yaml

@@ -109,8 +109,8 @@ jobs:
         export ITER8=$(pwd)
         source $ITER8/samples/knative/canary/check.sh
 
-  canary-progressive-helm:
-    name: progressive canary experiment using helm
+  conformance:
+    name: conformance experiment
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
@@ -121,29 +121,20 @@ jobs:
     - name: platform setup
       run: |
         export ITER8=$(pwd)
-        $ITER8/samples/knative/quickstart/platformsetup.sh kourier
-
-    - uses: azure/setup-helm@v1
-
-    - name: RBAC rules for helm
-      run: |
-        export ITER8=$(pwd)
-        kubectl apply -f ${ITER8}/samples/knative/canaryprogressive/helm-rbac.yaml
+        $ITER8/samples/knative/quickstart/platformsetup.sh contour
 
-    - name: create Knative app with canary
+    - name: create Knative app
       run: |
         export ITER8=$(pwd)
-        helm install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --namespace=default
-        kubectl wait ksvc/sample-app --for condition=Ready --timeout=240s
-        helm upgrade --install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --values=https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/experimental-values.yaml --namespace=default
+        kubectl apply -f $ITER8/samples/knative/conformance/baseline.yaml        
         kubectl wait --for=condition=Ready ksvc/sample-app --timeout=240s
 
     - name: Generate requests
       run: |
         export ITER8=$(pwd)
-        sed -i "s/6000s/120s/g" $ITER8/samples/knative/canaryprogressive/fortio.yaml        
+        sed -i "s/6000s/120s/g" $ITER8/samples/knative/conformance/fortio.yaml
         URL_VALUE=$(kubectl get ksvc sample-app -o json | jq .status.address.url)
-        sed "s+URL_VALUE+${URL_VALUE}+g" $ITER8/samples/knative/canaryprogressive/fortio.yaml | kubectl apply -f -
+        sed "s+URL_VALUE+${URL_VALUE}+g" $ITER8/samples/knative/conformance/fortio.yaml | kubectl apply -f -
 
         # Wait for Fortio to be ready
         pod_name=$(kubectl get pods --selector=job-name=fortio -o jsonpath='{.items[*].metadata.name}')
@@ -156,21 +147,20 @@ jobs:
         # Define metrics
         kubectl apply -f $ITER8/samples/knative/quickstart/metrics.yaml
 
-        export ITER8=$(pwd)
-        kubectl apply -f $ITER8/samples/knative/canaryprogressive/experiment.yaml
+        kubectl apply -f $ITER8/samples/knative/conformance/experiment.yaml
 
     - name: Sleep until end of experiment
       run: |
         sleep 150.0
-        kubectl get experiment canary-progressive -o yaml
+        kubectl get experiment conformance-exp -o yaml
     
     - name: Check if experiment is complete and successful
       run: |
         export ITER8=$(pwd)
-        source $ITER8/samples/knative/canaryprogressive/check.sh
-
-  canary-fixedsplit-kustomize:
-    name: fixedsplit canary experiment with kustomize
+        source $ITER8/samples/knative/conformance/check.sh
+        
+  canary-progressive-helm:
+    name: progressive canary experiment using helm
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
@@ -181,22 +171,29 @@ jobs:
     - name: platform setup
       run: |
         export ITER8=$(pwd)
-        $ITER8/samples/knative/quickstart/platformsetup.sh contour
+        $ITER8/samples/knative/quickstart/platformsetup.sh kourier
+
+    - uses: azure/setup-helm@v1
+
+    - name: RBAC rules for helm
+      run: |
+        export ITER8=$(pwd)
+        kubectl apply -f ${ITER8}/samples/knative/canaryprogressive/helm-rbac.yaml
 
     - name: create Knative app with canary
       run: |
         export ITER8=$(pwd)
-        kustomize build $ITER8/samples/knative/canaryfixedsplit/baseline | kubectl apply -f -
+        helm install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --namespace=default
         kubectl wait ksvc/sample-app --for condition=Ready --timeout=240s
-        kustomize build $ITER8/samples/knative/canaryfixedsplit/experimentalservice | kubectl apply -f -        
+        helm upgrade --install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --values=https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/experimental-values.yaml --namespace=default
+        kubectl wait --for=condition=Ready ksvc/sample-app --timeout=240s
 
     - name: Generate requests
       run: |
         export ITER8=$(pwd)
-        kubectl wait --for=condition=Ready ksvc/sample-app --timeout=240s
-        sed -i "s/6000s/120s/g" $ITER8/samples/knative/canaryfixedsplit/fortio.yaml        
+        sed -i "s/6000s/120s/g" $ITER8/samples/knative/canaryprogressive/fortio.yaml        
         URL_VALUE=$(kubectl get ksvc sample-app -o json | jq .status.address.url)
-        sed "s+URL_VALUE+${URL_VALUE}+g" $ITER8/samples/knative/canaryfixedsplit/fortio.yaml | kubectl apply -f -
+        sed "s+URL_VALUE+${URL_VALUE}+g" $ITER8/samples/knative/canaryprogressive/fortio.yaml | kubectl apply -f -
 
         # Wait for Fortio to be ready
         pod_name=$(kubectl get pods --selector=job-name=fortio -o jsonpath='{.items[*].metadata.name}')
@@ -209,20 +206,21 @@ jobs:
         # Define metrics
         kubectl apply -f $ITER8/samples/knative/quickstart/metrics.yaml
 
-        kubectl apply -f $ITER8/samples/knative/canaryfixedsplit/experiment.yaml
+        export ITER8=$(pwd)
+        kubectl apply -f $ITER8/samples/knative/canaryprogressive/experiment.yaml
 
     - name: Sleep until end of experiment
       run: |
         sleep 150.0
-        kubectl get experiment canary-fixedsplit -o yaml
+        kubectl get experiment canary-progressive -o yaml
     
     - name: Check if experiment is complete and successful
       run: |
         export ITER8=$(pwd)
-        source $ITER8/samples/knative/canaryfixedsplit/check.sh
+        source $ITER8/samples/knative/canaryprogressive/check.sh
 
-  conformance:
-    name: conformance
+  canary-fixedsplit-kustomize:
+    name: fixedsplit canary experiment with kustomize
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
@@ -235,18 +233,20 @@ jobs:
         export ITER8=$(pwd)
         $ITER8/samples/knative/quickstart/platformsetup.sh contour
 
-    - name: create Knative app
+    - name: create Knative app with canary
       run: |
         export ITER8=$(pwd)
-        kubectl apply -f $ITER8/samples/knative/conformance/baseline.yaml        
-        kubectl wait --for=condition=Ready ksvc/sample-app --timeout=240s
+        kustomize build $ITER8/samples/knative/canaryfixedsplit/baseline | kubectl apply -f -
+        kubectl wait ksvc/sample-app --for condition=Ready --timeout=240s
+        kustomize build $ITER8/samples/knative/canaryfixedsplit/experimentalservice | kubectl apply -f -        
 
     - name: Generate requests
       run: |
         export ITER8=$(pwd)
-        sed -i "s/6000s/120s/g" $ITER8/samples/knative/conformance/fortio.yaml
+        kubectl wait --for=condition=Ready ksvc/sample-app --timeout=240s
+        sed -i "s/6000s/120s/g" $ITER8/samples/knative/canaryfixedsplit/fortio.yaml        
         URL_VALUE=$(kubectl get ksvc sample-app -o json | jq .status.address.url)
-        sed "s+URL_VALUE+${URL_VALUE}+g" $ITER8/samples/knative/conformance/fortio.yaml | kubectl apply -f -
+        sed "s+URL_VALUE+${URL_VALUE}+g" $ITER8/samples/knative/canaryfixedsplit/fortio.yaml | kubectl apply -f -
 
         # Wait for Fortio to be ready
         pod_name=$(kubectl get pods --selector=job-name=fortio -o jsonpath='{.items[*].metadata.name}')
@@ -259,14 +259,15 @@ jobs:
         # Define metrics
         kubectl apply -f $ITER8/samples/knative/quickstart/metrics.yaml
 
-        kubectl apply -f $ITER8/samples/knative/conformance/experiment.yaml
+        kubectl apply -f $ITER8/samples/knative/canaryfixedsplit/experiment.yaml
 
     - name: Sleep until end of experiment
       run: |
         sleep 150.0
-        kubectl get experiment conformance-sample -o yaml
+        kubectl get experiment canary-fixedsplit -o yaml
     
     - name: Check if experiment is complete and successful
       run: |
         export ITER8=$(pwd)
-        source $ITER8/samples/knative/conformance/check.sh
+        source $ITER8/samples/knative/canaryfixedsplit/check.sh
+