Kfservingexp (#651)

* Updated experiment and e2e test

* Fixed kfserving exp

* Updated weightobjref in knative sample

* Updated fixed split experiment

Author: sriumcp

samples/kfserving/quickstart/e2etest.sh

@@ -39,25 +39,32 @@ echo "Generate requests"
 
 URL_VALUE="http://$(kubectl -n istio-system get svc istio-ingressgateway -o jsonpath='{.spec.clusterIP}'):80/productpage"
 
+# Port-forward Istio ingress
 INGRESS_GATEWAY_SERVICE=$(kubectl get svc -n istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
 kubectl port-forward -n istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
 
-# this step requires fortio
+# Get the prediction payload
 curl -o /tmp/input.json https://raw.githubusercontent.com/kubeflow/kfserving/master/docs/samples/v1beta1/rollout/input.json
-fortio load -t 310s -qps 5.0 -H "Host: example.com" -payload-file /tmp/input.json -json /tmp/fortiooutput.json localhost:8080/v1/models/flowers:predict 
+
+# Send prediction requests... this step requires fortio
+fortio load -t 360s -qps 4.0 -H "Host: example.com" -payload-file /tmp/input.json -json /tmp/fortiooutput.json http://localhost:8080/v1/models/flowers:predict
+
+# Wait
+kubectl wait experiment $EXPERIMENT --for=condition=Completed --timeout=300s
 
 # Log final experiment
 kubectl get experiment $EXPERIMENT -o yaml
 
 # Check
 source $ITER8/samples/kfserving/quickstart/check.sh
 
-# Cleanup .. not needed since cluster is getting deleted
-# included to test manual instructions
-# continue even if errors
 kubectl delete -f $ITER8/samples/kfserving/quickstart/experiment.yaml
-kubectl delete -f $ITER8/samples/kfserving/quickstart/baseline.yaml
-kubectl delete -f $ITER8/samples/kfserving/quickstart/candidate.yaml
+# Cleanup .. delete inference services (and its ns takes forever)...
+# so skipping this for now... need to be fixed later;
+# possibly by bringing this up with KFServing
+
+# kubectl delete -f $ITER8/samples/kfserving/quickstart/baseline.yaml
+# kubectl delete -f $ITER8/samples/kfserving/quickstart/candidate.yaml
 
 # delete cluster
 kind delete cluster

samples/kfserving/quickstart/experiment.yaml

@@ -21,14 +21,14 @@ spec:
       preferredDirection: High # maximize user engagement
     objectives:
     - metric: iter8-kfserving/mean-latency
-      upperLimit: 500
+      upperLimit: 1500
     - metric: iter8-kfserving/95th-percentile-tail-latency
       upperLimit: 2000
     - metric: iter8-kfserving/error-rate
       upperLimit: "0.01"
   duration:
     intervalSeconds: 10
-    iterationsPerLoop: 30
+    iterationsPerLoop: 25
   versionInfo:
     # information about model versions used in this experiment
     baseline:

samples/knative/canaryfixedsplit/experiment.yaml

@@ -17,7 +17,7 @@ spec:
           args:
           - "-c"
           - |
-           kustomize build github.com/iter8-tools/iter8/samples/knative/canaryfixedsplit/{{ .name }}?ref=master \
+           kustomize build github.com/iter8-tools/iter8/samples/knative/canaryfixedsplit/{{ .promote }}?ref=master \
             | kubectl apply -f -
   criteria:
     # mean latency of version should be under 50 milliseconds
@@ -37,5 +37,11 @@ spec:
     # information about app versions used in this experiment
     baseline:
       name: sample-app-v1
+      variables:
+      - name: promote
+        value: baseline
     candidates:
     - name: sample-app-v2
+      variables:
+      - name: promote
+        value: candidate

samples/knative/canaryprogressive/experiment.yaml

@@ -41,11 +41,23 @@ spec:
     # information about app versions used in this experiment
     baseline:
       name: sample-app-v1
+      weightObjRef:
+        apiVersion: serving.knative.dev/v1
+        kind: Service
+        name: sample-app
+        namespace: default
+        fieldPath: .spec.traffic[0].percent
       variables:
       - name: promote
         value: baseline
     candidates:
     - name: sample-app-v2
+      weightObjRef:
+        apiVersion: serving.knative.dev/v1
+        kind: Service
+        name: sample-app
+        namespace: default
+        fieldPath: .spec.traffic[1].percent
       variables:
       - name: promote
         value: candidate