CloneSet supports to reuse PVC during upgrade

Signed-off-by: Siyu Wang <[email protected]>

apis/apps/v1alpha1/well_known_labels.go

@@ -10,6 +10,9 @@ const (
 	// SpecifiedDeleteKey indicates this object should be deleted, and the value could be the deletion option.
 	SpecifiedDeleteKey = "apps.kruise.io/specified-delete"
 
+	// KeepPVCForDeletionKey indicates should keep PVC for reuse when specified delete the pod.
+	KeepPVCForDeletionKey = "apps.kruise.io/keep-pvc-for-deletion"
+
 	// ImagePreDownloadCreatedKey indicates the images of this revision have been pre-downloaded
 	ImagePreDownloadCreatedKey = "apps.kruise.io/pre-predownload-created"
 

pkg/controller/cloneset/sync/cloneset_scale.go

@@ -36,6 +36,7 @@ import (
 	"github.com/openkruise/kruise/pkg/util/expectations"
 	"github.com/openkruise/kruise/pkg/util/lifecycle"
 	"github.com/openkruise/kruise/pkg/util/revision"
+	"github.com/openkruise/kruise/pkg/util/specifieddelete"
 )
 
 const (
@@ -289,6 +290,9 @@ func (r *realControl) deletePods(cs *appsv1alpha1.CloneSet, podsToDelete []*v1.P
 		modified = true
 		r.recorder.Event(cs, v1.EventTypeNormal, "SuccessfulDelete", fmt.Sprintf("succeed to delete pod %s", pod.Name))
 
+		if specifieddelete.ShouldKeepPVC(pod) {
+			continue
+		}
 		// delete pvcs which have the same instance-id
 		for _, pvc := range pvcs {
 			if pvc.Labels[appsv1alpha1.CloneSetInstanceID] != pod.Labels[appsv1alpha1.CloneSetInstanceID] {

pkg/controller/cloneset/sync/cloneset_update.go

@@ -308,7 +308,8 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc
 
 	klog.V(2).InfoS("CloneSet started to patch Pod specified-delete for update", "cloneSet", klog.KObj(cs), "pod", klog.KObj(pod), "updateRevision", klog.KObj(updateRevision))
 
-	if patched, err := specifieddelete.PatchPodSpecifiedDelete(c.Client, pod, "true"); err != nil {
+	keepPVC := !cs.Spec.ScaleStrategy.DisablePVCReuse && utilfeature.DefaultFeatureGate.Enabled(features.CloneSetPVCReuseDuringUpdate)
+	if patched, err := specifieddelete.PatchPodSpecifiedDelete(c.Client, pod, keepPVC); err != nil {
 		c.recorder.Eventf(cs, v1.EventTypeWarning, "FailedUpdatePodReCreate",
 			"failed to patch pod specified-delete %s for update(revision %s): %v", pod.Name, updateRevision.Name, err)
 		return 0, err

pkg/features/kruise_features.go

@@ -138,6 +138,10 @@ const (
 	// InPlaceWorkloadVerticalScaling enable CloneSet/Advanced StatefulSet controller to support vertical scaling
 	// of managed Pods.
 	InPlaceWorkloadVerticalScaling featuregate.Feature = "InPlaceWorkloadVerticalScaling"
+
+	// CloneSetPVCReuseDuringUpdate enables CloneSet to reuse PVC during update, and it also depends on
+	// the spec.scaleStrategy.disablePVCReuse not to be true.
+	CloneSetPVCReuseDuringUpdate featuregate.Feature = "CloneSetPVCReuseDuringUpdate"
 )
 
 var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
@@ -175,6 +179,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
 	StatefulSetAutoResizePVCGate:             {Default: false, PreRelease: featuregate.Alpha},
 	ForceDeleteTimeoutExpectationFeatureGate: {Default: false, PreRelease: featuregate.Alpha},
 	InPlaceWorkloadVerticalScaling:           {Default: false, PreRelease: featuregate.Alpha},
+	CloneSetPVCReuseDuringUpdate:             {Default: false, PreRelease: featuregate.Alpha},
 }
 
 func init() {

pkg/util/specifieddelete/specified_delete.go

@@ -18,29 +18,43 @@ package specifieddelete
 
 import (
 	"context"
-	"fmt"
 
-	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
+	"github.com/openkruise/kruise/pkg/util"
 )
 
 func IsSpecifiedDelete(obj metav1.Object) bool {
 	_, ok := obj.GetLabels()[appsv1alpha1.SpecifiedDeleteKey]
 	return ok
 }
 
-func PatchPodSpecifiedDelete(c client.Client, pod *v1.Pod, value string) (bool, error) {
+func ShouldKeepPVC(obj metav1.Object) bool {
+	return obj.GetLabels()[appsv1alpha1.KeepPVCForDeletionKey] == "true"
+}
+
+func PatchPodSpecifiedDelete(c client.Client, pod *v1.Pod, keepPVC bool) (bool, error) {
 	if _, ok := pod.Labels[appsv1alpha1.SpecifiedDeleteKey]; ok {
 		return false, nil
 	}
 
-	body := fmt.Sprintf(
-		`{"metadata":{"labels":{"%s":"%s"}}}`,
-		appsv1alpha1.SpecifiedDeleteKey,
-		value,
-	)
-	return true, c.Patch(context.TODO(), pod, client.RawPatch(types.StrategicMergePatchType, []byte(body)))
+	body := patchBody{Metadata: patchMeta{Labels: map[string]string{
+		appsv1alpha1.SpecifiedDeleteKey: "true",
+	}}}
+	if keepPVC {
+		body.Metadata.Labels[appsv1alpha1.KeepPVCForDeletionKey] = "true"
+	}
+	return true, c.Patch(context.TODO(), pod, client.RawPatch(types.StrategicMergePatchType, []byte(util.DumpJSON(body))))
+}
+
+type patchBody struct {
+	Metadata patchMeta `json:"metadata"`
+}
+
+type patchMeta struct {
+	Labels map[string]string `json:"labels"`
 }

pkg/util/specifieddelete/specified_delete_test.go

@@ -0,0 +1,73 @@
+package specifieddelete
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
+)
+
+func TestPatchPodSpecifiedDelete(t *testing.T) {
+	tests := []struct {
+		pod      *corev1.Pod
+		keepPVC  bool
+		expected *corev1.Pod
+	}{
+		{
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-pod",
+				},
+				Spec: corev1.PodSpec{},
+			},
+			keepPVC: false,
+			expected: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-pod",
+					Labels: map[string]string{
+						appsv1alpha1.SpecifiedDeleteKey: "true",
+					},
+				},
+				Spec: corev1.PodSpec{},
+			},
+		},
+		{
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-pod",
+				},
+				Spec: corev1.PodSpec{},
+			},
+			keepPVC: true,
+			expected: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-pod",
+					Labels: map[string]string{
+						appsv1alpha1.SpecifiedDeleteKey:    "true",
+						appsv1alpha1.KeepPVCForDeletionKey: "true",
+					},
+				},
+				Spec: corev1.PodSpec{},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			cli := fake.NewClientBuilder().WithObjects(test.pod).Build()
+			_, err := PatchPodSpecifiedDelete(cli, test.pod, test.keepPVC)
+			assert.NoError(t, err)
+
+			// patch will write result object into the given test.pod
+			if apiequality.Semantic.DeepEqual(test.expected, test.pod) {
+				t.Fatalf("expected %v but got %v", test.expected, test.pod)
+			}
+		})
+	}
+}

test/e2e/apps/cloneset.go

@@ -1268,6 +1268,9 @@ func testUpdateVolumeClaimTemplates(tester *framework.CloneSetTester, randStr st
 	updateStrategy := appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.RecreateCloneSetUpdateStrategyType}
 	var replicas int = 4
 	cs := tester.NewCloneSet("clone-"+randStr, int32(replicas), updateStrategy)
+	// If enable DisablePVCReuse and the CloneSetPVCReuseDuringUpdate feature-gate, CloneSet will reuse PVC during upgrade.
+	// So if user wants to recreate PVC, then he shouldn't set pvc to be reused.
+	cs.Spec.ScaleStrategy.DisablePVCReuse = true
 	imageConfig := imageutils.GetConfig(imageutils.Nginx)
 	imageConfig.SetRegistry("docker.io/library")
 	imageConfig.SetVersion("alpine")