Add support for a new Pod Deletion Policy

Author: FireDrunk

controller/kubernetes_pod_controller.go

@@ -35,6 +35,8 @@ const (
 	controllerAgentName = "longhorn-kubernetes-pod-controller"
 
 	remountRequestDelayDuration = 5 * time.Second
+
+	podDeletionAnnotation = "longhorn.io/allow-deletion"
 )
 
 type KubernetesPodController struct {
@@ -472,7 +474,7 @@ func (kc *KubernetesPodController) getPodWithConflictedAttachment(pods []*corev1
 //
 // Force delete a pod when all of the below conditions are meet:
 // 1. NodeDownPodDeletionPolicy is different than DoNothing
-// 2. pod belongs to a StatefulSet/Deployment depend on NodeDownPodDeletionPolicy
+// 2. pod belongs to a StatefulSet/Deployment depend on NodeDownPodDeletionPolicy OR is explicitly marked for deletion.
 // 3. node containing the pod is down
 // 4. the pod is terminating and the DeletionTimestamp has passed.
 // 5. pod has a PV with provisioner driver.longhorn.io
@@ -484,7 +486,8 @@ func (kc *KubernetesPodController) handlePodDeletionIfNodeDown(pod *corev1.Pod,
 
 	shouldDelete := (deletionPolicy == types.NodeDownPodDeletionPolicyDeleteStatefulSetPod && isOwnedByStatefulSet(pod)) ||
 		(deletionPolicy == types.NodeDownPodDeletionPolicyDeleteDeploymentPod && isOwnedByDeployment(pod)) ||
-		(deletionPolicy == types.NodeDownPodDeletionPolicyDeleteBothStatefulsetAndDeploymentPod && (isOwnedByStatefulSet(pod) || isOwnedByDeployment(pod)))
+		(deletionPolicy == types.NodeDownPodDeletionPolicyDeleteBothStatefulsetAndDeploymentPod && (isOwnedByStatefulSet(pod) || isOwnedByDeployment(pod))) ||
+		(deletionPolicy == types.NodeDownPodDeletionPolicyDeleteBothStatefulsetAndDeploymentPodAndAnnotatedPods && (isOwnedByStatefulSet(pod) || isOwnedByDeployment(pod) || isAnnotatedWithDeletionAnnotation(pod)))
 
 	if !shouldDelete {
 		return nil
@@ -681,6 +684,15 @@ func isOwnedByDeployment(pod *corev1.Pod) bool {
 	return false
 }
 
+func isAnnotatedWithDeletionAnnotation(pod *corev1.Pod) bool {
+	if val, ok := pod.ObjectMeta.Annotations[podDeletionAnnotation]; ok {
+		if strings.ToLower(val) == "true" {
+			return true
+		}
+	}
+	return false // We did not find the annotation, or the annotation is something else than "true" or "True"
+}
+
 // enqueuePodChange determines if the pod requires processing based on whether the pod has a PV created by us (driver.longhorn.io)
 func (kc *KubernetesPodController) enqueuePodChange(obj interface{}) {
 	key, err := controller.KeyFunc(obj)

types/setting.go

@@ -1760,10 +1760,11 @@ var (
 type NodeDownPodDeletionPolicy string
 
 const (
-	NodeDownPodDeletionPolicyDoNothing                             = NodeDownPodDeletionPolicy("do-nothing") // Kubernetes default behavior
-	NodeDownPodDeletionPolicyDeleteStatefulSetPod                  = NodeDownPodDeletionPolicy("delete-statefulset-pod")
-	NodeDownPodDeletionPolicyDeleteDeploymentPod                   = NodeDownPodDeletionPolicy("delete-deployment-pod")
-	NodeDownPodDeletionPolicyDeleteBothStatefulsetAndDeploymentPod = NodeDownPodDeletionPolicy("delete-both-statefulset-and-deployment-pod")
+	NodeDownPodDeletionPolicyDoNothing                                             = NodeDownPodDeletionPolicy("do-nothing") // Kubernetes default behavior
+	NodeDownPodDeletionPolicyDeleteStatefulSetPod                                  = NodeDownPodDeletionPolicy("delete-statefulset-pod")
+	NodeDownPodDeletionPolicyDeleteDeploymentPod                                   = NodeDownPodDeletionPolicy("delete-deployment-pod")
+	NodeDownPodDeletionPolicyDeleteBothStatefulsetAndDeploymentPod                 = NodeDownPodDeletionPolicy("delete-both-statefulset-and-deployment-pod")
+	NodeDownPodDeletionPolicyDeleteBothStatefulsetAndDeploymentPodAndAnnotatedPods = NodeDownPodDeletionPolicy("delete-both-statefulset-and-deployment-pod-and-annotated-pods")
 )
 
 type NodeDrainPolicy string