[Feature] Force delete Pods which are stuck in init phase (#1181)

Author: ajanikow

CHANGELOG.md

@@ -18,6 +18,7 @@
 - (Improvement) Use inspector for ArangoMember
 - (DebugPackage) Collect logs from pods
 - (Bugfix) Move Agency CommitIndex log message to Trace
+- (Feature) Force delete Pods which are stuck in init phase
 
 ## [1.2.20](https://github.com/arangodb/kube-arangodb/tree/1.2.20) (2022-10-25)
 - (Feature) Add action progress

pkg/deployment/resources/pod_inspector.go

@@ -56,6 +56,7 @@ const (
 	// we will mark the pod as scheduled for termination
 	recheckSoonPodInspectorInterval = util.Interval(time.Second) // Time between Pod inspection if we think something will change soon
 	maxPodInspectorInterval         = util.Interval(time.Hour)   // Maximum time between Pod inspection (if nothing else happens)
+	forcePodDeletionGracePeriod     = 15 * time.Minute
 )
 
 func (r *Resources) handleRestartedPod(pod *core.Pod, memberStatus *api.MemberStatus, wasTerminated, markAsTerminated *bool) {
@@ -396,12 +397,30 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter
 
 			// Check if any additional deletion request is required
 			if !k8sutil.IsPodAlive(pod) {
+				var gps int64 = 10
+
+				forceDelete := false
+				if t := k8sutil.PodStopTime(pod); !t.IsZero() {
+					if time.Since(t) > forcePodDeletionGracePeriod {
+						forceDelete = true
+					}
+				} else if t := pod.DeletionTimestamp; t != nil {
+					if time.Since(t.Time) > forcePodDeletionGracePeriod {
+						forceDelete = true
+					}
+				}
+
+				if forceDelete {
+					gps = 0
+					log.Str("pod-name", pod.GetName()).Warn("Enforcing deletion of Pod")
+				}
+
 				// Pod is dead, but still not removed. Send additional deletion request
 				nctx, c := globals.GetGlobals().Timeouts().Kubernetes().WithTimeout(ctx)
 				defer c()
 
 				if err := cachedStatus.PodsModInterface().V1().Delete(nctx, pod.GetName(), meta.DeleteOptions{
-					GracePeriodSeconds: util.NewInt64(10),
+					GracePeriodSeconds: util.NewInt64(gps),
 					Preconditions:      meta.NewUIDPreconditions(string(pod.GetUID())),
 				}); err != nil {
 					if kerrors.IsNotFound(err) {

pkg/util/k8sutil/pods.go

@@ -384,6 +384,47 @@ func IsContainerAlive(container core.ContainerStatus) bool {
 	return container.State.Running != nil
 }
 
+// PodStopTime returns time when pod has been stopped
+func PodStopTime(pod *core.Pod) time.Time {
+	var t time.Time
+
+	if q := ContainersRecentStopTime(pod.Status.ContainerStatuses); q.After(t) {
+		t = q
+	}
+
+	if q := ContainersRecentStopTime(pod.Status.InitContainerStatuses); q.After(t) {
+		t = q
+	}
+
+	if q := ContainersRecentStopTime(pod.Status.EphemeralContainerStatuses); q.After(t) {
+		t = q
+	}
+
+	return t
+}
+
+// ContainersRecentStopTime returns most recent termination time of pods
+func ContainersRecentStopTime(containers []core.ContainerStatus) time.Time {
+	var t time.Time
+
+	for _, c := range containers {
+		if v := ContainerStopTime(c); v.After(t) {
+			t = v
+		}
+	}
+
+	return t
+}
+
+// ContainerStopTime returns time of the Container stop. If container is running, time.Zero is returned
+func ContainerStopTime(container core.ContainerStatus) time.Time {
+	if p := container.State.Terminated; p != nil {
+		return p.FinishedAt.Time
+	}
+
+	return time.Time{}
+}
+
 // ClusterJWTVolumeMount creates a volume mount structure for a cluster JWT secret (token).
 func ClusterJWTVolumeMount() core.VolumeMount {
 	return core.VolumeMount{