Merge pull request #446 from Omar007/feature/multi-node-pv

[Feature] Allow additional selector terms to be defined in storage config
kubernetes-sigs · Dec 28, 2024 · d37f066 · d37f066
2 parents ad20e13 + 2729447
commit d37f066
Show file tree

Hide file tree

Showing 7 changed files with 142 additions and 195 deletions.
diff --git a/helm/provisioner/templates/configmap.yaml b/helm/provisioner/templates/configmap.yaml
@@ -26,7 +26,7 @@ data:
 {{- end }}
 {{- if .Values.useJobForCleaning }}
   useJobForCleaning: "yes"
-{{- end}}
+{{- end }}
 {{- if .Values.tolerations }}
   jobTolerations: | {{ toYaml .Values.tolerations | nindent 4 }}
 {{- end }}
@@ -35,7 +35,7 @@ data:
 {{- end }}
 {{- if .Values.minResyncPeriod }}
   minResyncPeriod: {{ .Values.minResyncPeriod | quote }}
-{{- end}}
+{{- end }}
   storageClassMap: |
     {{- range $classConfig := .Values.classes }}
     {{ $classConfig.name }}:
@@ -45,7 +45,7 @@ data:
       blockCleanerCommand:
       {{- range $val := $classConfig.blockCleanerCommand }}
         - {{ $val | quote }}
-      {{- end}}
+      {{- end }}
       {{- end }}
       {{- if $classConfig.volumeMode }}
       volumeMode: {{ $classConfig.volumeMode }}
@@ -56,4 +56,8 @@ data:
       {{- if $classConfig.namePattern }}
       namePattern: {{ $classConfig.namePattern | quote }}
       {{- end }}
+      {{- if $classConfig.selector }}
+      selector:
+      {{- toYaml $classConfig.selector | nindent 8 }}
+      {{- end }}
     {{- end }}
diff --git a/pkg/common/common.go b/pkg/common/common.go
@@ -44,7 +44,6 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
-	volumeUtil "k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/utils/mount"
 )
 
@@ -141,6 +140,9 @@ type MountConfig struct {
 	// NamePattern name pattern check
 	// only discover file name matching pattern("*" by default)
 	NamePattern string `json:"namePattern" yaml:"namePattern"`
+	// Additional selector terms to set for node affinity in addition to the provisioner node name.
+	// Useful for shared disks as affinity can not be changed after provisioning the PV.
+	Selector []v1.NodeSelectorTerm `json:"selector" yaml:"selector"`
 }
 
 // RuntimeConfig stores all the objects that the provisioner needs to run
@@ -495,44 +497,25 @@ func GetVolumeMode(volUtil util.VolumeUtil, fullPath string) (v1.PersistentVolum
 	return "", fmt.Errorf("Block device check for %q failed: %s", fullPath, errblk)
 }
 
-// NodeExists checks to see if a Node exists in the Indexer of a NodeLister.
-// It tries to get the node and if it fails, it uses the well known label
-// `kubernetes.io/hostname` to find the Node.
-func NodeExists(nodeLister corelisters.NodeLister, nodeName string) (bool, error) {
-	_, err := nodeLister.Get(nodeName)
-	if errors.IsNotFound(err) {
+// AnyNodeExists checks to see if a Node exists in the Indexer of a NodeLister.
+// If this fails, it uses the well known label `kubernetes.io/hostname` to find the Node.
+// It aborts early if an unexpected error occurs and it's uncertain if a node would exist or not.
+func AnyNodeExists(nodeLister corelisters.NodeLister, nodeNames []string) bool {
+	for _, nodeName := range nodeNames {
+		_, err := nodeLister.Get(nodeName)
+		if err == nil || !errors.IsNotFound(err) {
+			return true
+		}
 		req, err := labels.NewRequirement(NodeLabelKey, selection.Equals, []string{nodeName})
 		if err != nil {
-			return false, err
+			return true
 		}
 		nodes, err := nodeLister.List(labels.NewSelector().Add(*req))
-		if err != nil {
-			return false, err
+		if err != nil || len(nodes) > 0 {
+			return true
 		}
-		return len(nodes) > 0, nil
 	}
-	return err == nil, err
-}
-
-// NodeAttachedToLocalPV gets the name of the Node that a local PV has a NodeAffinity to.
-// It assumes that there should be only one matching Node for a local PV and that
-// the local PV follows the form:
-//
-//	nodeAffinity:
-//	  required:
-//	    nodeSelectorTerms:
-//	    - matchExpressions:
-//	      - key: kubernetes.io/hostname
-//	        operator: In
-//	        values:
-//	        - <node1>
-func NodeAttachedToLocalPV(pv *v1.PersistentVolume) (string, bool) {
-	nodeNames := volumeUtil.GetLocalPersistentVolumeNodeNames(pv)
-	// We assume that there should only be one matching node.
-	if nodeNames == nil || len(nodeNames) != 1 {
-		return "", false
-	}
-	return nodeNames[0], true
+	return false
 }
 
 // IsLocalPVWithStorageClass checks that a PV is a local PV that belongs to any of the passed in StorageClasses.

diff --git a/pkg/common/common_test.go b/pkg/common/common_test.go
@@ -236,6 +236,46 @@ func TestLoadProvisionerConfigs(t *testing.T) {
 			},
 			nil,
 		},
+		{
+			map[string]string{"storageClassMap": `local-storage:
+   hostDir: /mnt/disks
+   mountDir: /mnt/disks
+   selector:
+     - matchExpressions:
+         - key: "kubernetes.io/hostname"
+           operator: "In"
+           values:
+             - otherNode1
+`,
+			},
+			ProvisionerConfiguration{
+				StorageClassConfig: map[string]MountConfig{
+					"local-storage": {
+						HostDir:             "/mnt/disks",
+						MountDir:            "/mnt/disks",
+						BlockCleanerCommand: []string{"/scripts/quick_reset.sh"},
+						VolumeMode:          "Filesystem",
+						NamePattern:         "*",
+						Selector: []v1.NodeSelectorTerm{
+							{
+								MatchExpressions: []v1.NodeSelectorRequirement{
+									{
+										Key:      "kubernetes.io/hostname",
+										Operator: v1.NodeSelectorOpIn,
+										Values:   []string{"otherNode1"},
+									},
+								},
+							},
+						},
+					},
+				},
+				UseAlphaAPI: true,
+				MinResyncPeriod: metav1.Duration{
+					Duration: time.Hour + time.Minute*30,
+				},
+			},
+			nil,
+		},
 	}
 	for _, v := range testcases {
 		for name, value := range v.data {
@@ -477,7 +517,7 @@ func TestGetVolumeMode(t *testing.T) {
 	}
 }
 
-func TestNodeExists(t *testing.T) {
+func TestAnyNodeExists(t *testing.T) {
 	nodeName := "test-node"
 	nodeWithName := &v1.Node{
 		ObjectMeta: metav1.ObjectMeta{
@@ -495,21 +535,39 @@ func TestNodeExists(t *testing.T) {
 	tests := []struct {
 		nodeAdded *v1.Node
 		// Required.
-		nodeQueried    *v1.Node
+		nodeQueried    []string
 		expectedResult bool
 	}{
 		{
 			nodeAdded:      nodeWithName,
-			nodeQueried:    nodeWithName,
+			nodeQueried:    []string{nodeName},
 			expectedResult: true,
 		},
 		{
 			nodeAdded:      nodeWithLabel,
-			nodeQueried:    nodeWithName,
+			nodeQueried:    []string{nodeName},
 			expectedResult: true,
 		},
 		{
-			nodeQueried:    nodeWithName,
+			nodeQueried:    []string{nodeName},
+			expectedResult: false,
+		},
+		{
+			nodeAdded:      nodeWithName,
+			nodeQueried:    []string{"other-node", nodeName},
+			expectedResult: true,
+		},
+		{
+			nodeAdded:      nodeWithLabel,
+			nodeQueried:    []string{"other-node", nodeName},
+			expectedResult: true,
+		},
+		{
+			nodeQueried:    []string{},
+			expectedResult: false,
+		},
+		{
+			nodeQueried:    nil,
 			expectedResult: false,
 		},
 	}
@@ -523,62 +581,13 @@ func TestNodeExists(t *testing.T) {
 			nodeInformer.Informer().GetStore().Add(test.nodeAdded)
 		}
 
-		exists, err := NodeExists(nodeInformer.Lister(), test.nodeQueried.Name)
-		if err != nil {
-			t.Errorf("Got unexpected error: %s", err.Error())
-		}
+		exists := AnyNodeExists(nodeInformer.Lister(), test.nodeQueried)
 		if exists != test.expectedResult {
 			t.Errorf("expected result: %t, actual: %t", test.expectedResult, exists)
 		}
 	}
 }
 
-func TestNodeAttachedToLocalPV(t *testing.T) {
-	nodeName := "testNodeName"
-
-	tests := []struct {
-		name             string
-		pv               *v1.PersistentVolume
-		expectedNodeName string
-		expectedStatus   bool
-	}{
-		{
-			name:             "NodeAffinity will all necessary fields",
-			pv:               withNodeAffinity(pv(), []string{nodeName}, NodeLabelKey),
-			expectedNodeName: nodeName,
-			expectedStatus:   true,
-		},
-		{
-			name:             "empty nodeNames array",
-			pv:               withNodeAffinity(pv(), []string{}, NodeLabelKey),
-			expectedNodeName: "",
-			expectedStatus:   false,
-		},
-		{
-			name:             "multiple nodeNames",
-			pv:               withNodeAffinity(pv(), []string{nodeName, "newNode"}, NodeLabelKey),
-			expectedNodeName: "",
-			expectedStatus:   false,
-		},
-		{
-			name:             "wrong node label key",
-			pv:               withNodeAffinity(pv(), []string{nodeName}, "wrongLabel"),
-			expectedNodeName: "",
-			expectedStatus:   false,
-		},
-	}
-
-	for _, test := range tests {
-		nodeName, ok := NodeAttachedToLocalPV(test.pv)
-		if ok != test.expectedStatus {
-			t.Errorf("test: %s, status: %t, expectedStaus: %t", test.name, ok, test.expectedStatus)
-		}
-		if nodeName != test.expectedNodeName {
-			t.Errorf("test: %s, nodeName: %s, expectedNodeName: %s", test.name, nodeName, test.expectedNodeName)
-		}
-	}
-}
-
 func TestIsLocalPVWithStorageClass(t *testing.T) {
 	tests := []struct {
 		name              string