Ensure that driver is deleted prior to sparkapplication resubmission (#1521)

* Ensure that driver is deleted prior to sparkapplication resubmission

Signed-off-by: Khor Shu Heng <[email protected]>

* Update app version

Signed-off-by: Khor Shu Heng <[email protected]>

* Update chart version

Signed-off-by: Khor Shu Heng <[email protected]>

Co-authored-by: Khor Shu Heng <[email protected]>

Author: khorshuheng

charts/spark-operator-chart/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 name: spark-operator
 description: A Helm chart for Spark on Kubernetes operator
-version: 1.1.20
-appVersion: v1beta2-1.3.4-3.1.1
+version: 1.1.21
+appVersion: v1beta2-1.3.5-3.1.1
 keywords:
   - spark
 home: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator

pkg/controller/sparkapplication/controller.go

@@ -579,7 +579,15 @@ func (c *Controller) syncSparkApplication(key string) error {
 			appCopy.Status.AppState.State = v1beta2.FailedState
 			c.recordSparkApplicationEvent(appCopy)
 		} else if isNextRetryDue(appCopy.Spec.RestartPolicy.OnSubmissionFailureRetryInterval, appCopy.Status.SubmissionAttempts, appCopy.Status.LastSubmissionAttemptTime) {
-			appCopy = c.submitSparkApplication(appCopy)
+			if c.validateSparkResourceDeletion(appCopy) {
+				c.submitSparkApplication(appCopy)
+			} else {
+				if err := c.deleteSparkResources(appCopy); err != nil {
+					glog.Errorf("failed to delete resources associated with SparkApplication %s/%s: %v",
+						appCopy.Namespace, appCopy.Name, err)
+					return err
+				}
+			}
 		}
 	case v1beta2.InvalidatingState:
 		// Invalidate the current run and enqueue the SparkApplication for re-execution.