Fix reset failing to delete PVs blocked by PDBs and respawned pods (#4100)

* fix volume destroy for kubeone reset

Signed-off-by: rajaSahil <sahilraja242@gmail.com>

* Make setup-go use the latest and greatest Go version

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

* Don't use cache in actions/setup-go

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

* use go-version: 1.26 in GH action setup-go

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

---------

Signed-off-by: rajaSahil <sahilraja242@gmail.com>
Signed-off-by: Artiom Diomin <artiom@kubermatic.com>
Co-authored-by: Artiom Diomin <artiom@kubermatic.com>

Author: rajaSahil

.github/workflows/golangci-lint.yml

@@ -16,7 +16,6 @@ jobs:
       - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
         with:
           go-version: 1.26
-          cache: true
 
       - name: golangci-lint
         uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0

.github/workflows/release.yml

@@ -27,7 +27,6 @@ jobs:
       - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
         with:
           go-version: 1.26
-          cache: true
 
       - name: fetch default stable for Kubernetes
         run: |

go.mod

@@ -1,6 +1,6 @@
 module k8c.io/kubeone
 
-go 1.26.1
+go 1.26
 
 require (
 	dario.cat/mergo v1.0.1

pkg/clientutil/service.go

@@ -36,7 +36,7 @@ func CleanupLBs(ctx context.Context, logger logrus.FieldLogger, c client.Client)
 	// Block service creation so gateway/other controllers can't recreate LB services
 	// while we're deleting them.
 	logger.Infoln("Creating ValidatingWebhookConfiguration to disable future Service creation...")
-	if err := creationPreventingWebhook(ctx, c, "", LBResources); err != nil {
+	if err := creationPreventingWebhook(ctx, c, "", LBResources, nil); err != nil {
 		return fail.KubeClient(err, "disabling future Service creation")
 	}
 

pkg/clientutil/volumes.go

@@ -27,6 +27,7 @@ import (
 	"k8c.io/kubeone/pkg/fail"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -45,13 +46,25 @@ const (
 
 var VolumeResources = []string{"persistentvolumes", "persistentvolumeclaims"}
 
+// PodResources is the list of resources blocked by the Pod creation-preventing webhook.
+var PodResources = []string{"pods"}
+
 func CleanupUnretainedVolumes(ctx context.Context, logger logrus.FieldLogger, c client.Client, restConfig *rest.Config) error {
 	// We disable the PV & PVC creation so nothing creates new PV's while we delete them
 	logger.Infoln("Creating ValidatingWebhookConfiguration to disable future PV & PVC creation...")
 	if err := disablePVCreation(ctx, c); err != nil {
 		return fail.KubeClient(err, "disabling future PV & PVC creation.")
 	}
 
+	// We also disable Pod creation so workload controllers (Deployments, StatefulSets, ...) can't
+	// recreate the Pods we delete below. A recreated Pod would re-mount the PVC and keep the
+	// kubernetes.io/pvc-protection finalizer, leaving the PVC stuck in Terminating. The webhook
+	// excludes kube-system so the CSI controllers keep running to actually delete the cloud volumes.
+	logger.Infoln("Creating ValidatingWebhookConfiguration to disable future Pod creation...")
+	if err := disablePodCreation(ctx, c); err != nil {
+		return fail.KubeClient(err, "disabling future Pod creation.")
+	}
+
 	pvcList, pvList, err := getDynamicallyProvisionedUnretainedPvs(ctx, c)
 	if err != nil {
 		return err
@@ -91,7 +104,24 @@ func CleanupUnretainedVolumes(ctx context.Context, logger logrus.FieldLogger, c
 
 func disablePVCreation(ctx context.Context, c client.Client) error {
 	// Prevent re-creation of PVs and PVCs by using an intentionally defunct admissionWebhook
-	return creationPreventingWebhook(ctx, c, "", VolumeResources)
+	return creationPreventingWebhook(ctx, c, "", VolumeResources, nil)
+}
+
+func disablePodCreation(ctx context.Context, c client.Client) error {
+	// Prevent re-creation of Pods by using an intentionally defunct admissionWebhook. kube-system is
+	// excluded so the CSI controllers (and other system components) keep running to delete the cloud
+	// volumes backing the PVCs we remove.
+	namespaceSelector := &metav1.LabelSelector{
+		MatchExpressions: []metav1.LabelSelectorRequirement{
+			{
+				Key:      corev1.LabelMetadataName,
+				Operator: metav1.LabelSelectorOpNotIn,
+				Values:   []string{metav1.NamespaceSystem},
+			},
+		},
+	}
+
+	return creationPreventingWebhook(ctx, c, "", PodResources, namespaceSelector)
 }
 
 func cleanupPVCUsingPods(ctx context.Context, c client.Client, log logrus.FieldLogger, kubeClient *kubernetes.Clientset) error {
@@ -100,11 +130,11 @@ func cleanupPVCUsingPods(ctx context.Context, c client.Client, log logrus.FieldL
 		return fail.KubeClient(err, "listing Pods from user cluster.")
 	}
 
-	var pvUsingPods []*corev1.Pod
+	var pvUsingPods []corev1.Pod
 	for idx := range podList.Items {
 		pod := &podList.Items[idx]
 		if podUsesPV(pod) {
-			pvUsingPods = append(pvUsingPods, pod)
+			pvUsingPods = append(pvUsingPods, *pod)
 		}
 	}
 
@@ -119,6 +149,10 @@ func cleanupPVCUsingPods(ctx context.Context, c client.Client, log logrus.FieldL
 		// ReplicaSet)
 		Force:              true,
 		DeleteEmptyDirData: true,
+		// DisableEviction makes drain delete pods directly instead of using the
+		// eviction API. The cluster is being torn down, so PodDisruptionBudgets
+		// must not be allowed to block (and indefinitely fail) the deletion.
+		DisableEviction:    true,
 		GracePeriodSeconds: -1,
 		Out:                os.Stdout,
 		ErrOut:             os.Stdout,
@@ -135,16 +169,9 @@ func cleanupPVCUsingPods(ctx context.Context, c client.Client, log logrus.FieldL
 			log.Infof("pod %s/%s is %s", pod.GetNamespace(), pod.GetName(), evicted)
 		},
 	}
-	evictionGroupVersion, err := drain.CheckEvictionSupport(kubeClient)
-	if err != nil {
-		return err
-	}
 
-	for _, pod := range pvUsingPods {
-		err := helper.EvictPod(*pod, evictionGroupVersion)
-		if err != nil {
-			return fail.KubeClient(err, "deleting the pod.")
-		}
+	if err := helper.DeleteOrEvictPods(pvUsingPods); err != nil {
+		return fail.KubeClient(err, "deleting the pods using PVs.")
 	}
 
 	return nil

pkg/clientutil/webhook.go

@@ -23,13 +23,15 @@ import (
 	"k8c.io/kubeone/pkg/fail"
 
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // creationPreventingWebhook returns a ValidatingWebhookConfiguration that is intentionally defunct
-// and will prevent all creation requests from succeeding.
-func creationPreventingWebhook(ctx context.Context, c client.Client, apiGroup string, resources []string) error {
+// and will prevent all creation requests from succeeding. An optional namespaceSelector limits which
+// namespaces the webhook applies to (e.g. to keep CSI controllers in kube-system running during cleanup).
+func creationPreventingWebhook(ctx context.Context, c client.Client, apiGroup string, resources []string, namespaceSelector *metav1.LabelSelector) error {
 	failurePolicy := admissionregistrationv1.Fail
 	sideEffects := admissionregistrationv1.SideEffectClassNone
 	vwc := admissionregistrationv1.ValidatingWebhookConfiguration{}
@@ -62,6 +64,7 @@ func creationPreventingWebhook(ctx context.Context, c client.Client, apiGroup st
 			},
 		},
 	}
+	vwc.Webhooks[0].NamespaceSelector = namespaceSelector
 	vwc.Webhooks[0].FailurePolicy = &failurePolicy
 	vwc.Webhooks[0].SideEffects = &sideEffects
 	vwc.Webhooks[0].AdmissionReviewVersions = []string{"v1"}

pkg/tasks/reset.go

@@ -73,6 +73,11 @@ func RemoveVolumes(s *state.State) error {
 			"kubernetes-cluster-cleanup-"+strings.Join(clientutil.VolumeResources, "-")); err != nil {
 			s.Logger.Warn("Unable to delete ValidatingWebhookConfiguration.")
 		}
+		s.Logger.Infoln("Deleting ValidatingWebhookConfiguration to enable future Pod creation...")
+		if err := clientutil.DeletePreventingWebhook(s.Context, s.DynamicClient,
+			"kubernetes-cluster-cleanup-"+strings.Join(clientutil.PodResources, "-")); err != nil {
+			s.Logger.Warn("Unable to delete ValidatingWebhookConfiguration.")
+		}
 
 		return lastErr
 	}