sort and choose hwameistor storage class as backend storage

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/pytool/ssh v0.0.0-20190312091242-5aaea5918db7
 	github.com/radovskyb/watcher v1.0.7
 	github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00
+	k8s.io/kubernetes v1.24.0
 	k8s.io/mount-utils v0.24.0
 )
 

go.sum

@@ -1727,6 +1727,7 @@ k8s.io/kube-scheduler v0.24.0/go.mod h1:DUq+fXaC51N1kl2YnT2EZSxOph6JOmIJe/pQe5ke
 k8s.io/kube-state-metrics v1.7.2/go.mod h1:U2Y6DRi07sS85rmVPmBFlmv+2peBcL8IWGjM+IjYA/E=
 k8s.io/kubectl v0.24.0/go.mod h1:pdXkmCyHiRTqjYfyUJiXtbVNURhv0/Q1TyRhy2d5ic0=
 k8s.io/kubelet v0.24.0/go.mod h1:p3BBacmHTCMpUf+nluhlyzuGHmONKAspqCvpu9oPAyA=
+k8s.io/kubernetes v1.24.0 h1:9qRjlCuMjooyFTXLxduMBT+MZSdROWa3idI1AXZirVs=
 k8s.io/kubernetes v1.24.0/go.mod h1:8e8maMiZzBR2/8Po5Uulx+MXZUYJuN3vtKwD4Ct1Xi0=
 k8s.io/legacy-cloud-providers v0.24.0/go.mod h1:j2gujMUYBEtbYfJaL8JUOgInzERm9fxJwEaOkZcnEUk=
 k8s.io/metrics v0.24.0/go.mod h1:jrLlFGdKl3X+szubOXPG0Lf2aVxuV3QJcbsgVRAM6fI=

pkg/dataset-manager/dataset/dataset_worker.go

@@ -8,6 +8,7 @@ import (
 	dslisters "github.com/hwameistor/datastore/pkg/apis/client/listers/datastore/v1alpha1"
 	datastore "github.com/hwameistor/datastore/pkg/apis/datastore/v1alpha1"
 	"github.com/hwameistor/datastore/pkg/utils"
+	"github.com/hwameistor/hwameistor/pkg/apis/hwameistor/v1alpha1"
 	v1 "k8s.io/api/core/v1"
 	storageapis "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -16,6 +17,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
+	util2 "k8s.io/kubernetes/pkg/apis/storage/v1/util"
 	"os"
 	"strings"
 
@@ -47,6 +49,10 @@ var (
 			},
 		},
 	}
+	volumeAttrTemplate = map[string]string{
+		"convertible": "false",
+		"volumeKind":  "LVM",
+	}
 	minStorageCapacity = int64(4194304)
 	minStorageQuantity = resource.NewQuantity(minStorageCapacity, resource.BinarySI)
 )
@@ -180,9 +186,9 @@ func (ctr *dsController) SyncNewOrUpdatedDatasource(ds *datastore.DataSet) {
 			return
 		}
 		// PV not found, create it
-		poolClass, err := ctr.selectPoolClassAsStorage()
+		poolClass, err := ctr.choosePoolClassAsStorage()
 		if err != nil {
-			klog.Errorf("Failed to select pool class as storage: %v", err)
+			klog.Errorf("Failed to choose one pool class as storage: %v", err)
 			return
 		}
 
@@ -200,13 +206,11 @@ func (ctr *dsController) deleteRelatedPersistentVolume(pvName string) error {
 func (ctr *dsController) createRelatedPersistentVolume(pvName, poolClass string, capacityBytes int64) (err error) {
 	newPV := persistentVolumeTemplate.DeepCopy()
 
-	volumeAttr := make(map[string]string)
-	volumeAttr["convertible"] = "false"
+	volumeMode := v1.PersistentVolumeFilesystem
+	volumeAttr := volumeAttrTemplate
 	volumeAttr["csi.storage.k8s.io/pv/name"] = pvName
-	volumeAttr["volumeKind"] = "LVM"
-	volumeAttr["poolClass"] = poolClass
+	volumeAttr[v1alpha1.VolumeParameterPoolClassKey] = poolClass
 
-	volumeMode := v1.PersistentVolumeFilesystem
 	newPV.Name = pvName
 	newPV.Spec.VolumeMode = &volumeMode
 	newPV.Spec.CSI.VolumeAttributes = volumeAttr
@@ -258,7 +262,7 @@ func (ctr *dsController) updateDatasetCapacity(ds *datastore.DataSet) error {
 	return err
 }
 
-func (ctr *dsController) selectPoolClassAsStorage() (string, error) {
+func (ctr *dsController) choosePoolClassAsStorage() (string, error) {
 	// default pool class(HDD, SSD, NVMe, etc.)
 	selectedPoolClass, ok := os.LookupEnv(poolClassEnv)
 	if !ok {
@@ -272,7 +276,7 @@ func (ctr *dsController) selectPoolClassAsStorage() (string, error) {
 		if len(hwStorageClasses) == 0 {
 			return "", fmt.Errorf("both default storage poolClass and storageclass not found")
 		}
-		selectedPoolClass = hwStorageClasses[0].Parameters["poolClass"]
+		selectedPoolClass = hwStorageClasses[0].Parameters[v1alpha1.VolumeParameterPoolClassKey]
 
 		klog.V(4).Infof("Found %d hwameistor storageclasses, choose %s as backend storage class", len(hwStorageClasses), selectedPoolClass)
 		return selectedPoolClass, nil
@@ -282,13 +286,33 @@ func (ctr *dsController) selectPoolClassAsStorage() (string, error) {
 }
 
 func sortHwameiStorageClasses(storageClasses []storageapis.StorageClass) []storageapis.StorageClass {
-	var hwameiStorStorageClasses []storageapis.StorageClass
+	var sortedHwStorageClasses, hwDefaultSC, hwNVMeSC, hwSSDSC, hwHDDSC []storageapis.StorageClass
 	for _, sc := range storageClasses {
-		if sc.Provisioner == "lvm.hwameistor.io" && sc.Parameters != nil && len(sc.Parameters["poolClass"]) > 0 {
-			hwameiStorStorageClasses = append(hwameiStorStorageClasses, sc)
+		// only validated hwameistor storageclass will be sorted
+		if sc.Provisioner == "lvm.hwameistor.io" && sc.Parameters != nil && len(sc.Parameters[v1alpha1.VolumeParameterPoolClassKey]) > 0 {
+			if util2.IsDefaultAnnotation(sc.ObjectMeta) {
+				hwDefaultSC = append(hwDefaultSC, sc)
+				continue
+			}
+
+			switch sc.Parameters[v1alpha1.VolumeParameterPoolClassKey] {
+			case v1alpha1.DiskClassNameHDD:
+				hwHDDSC = append(hwHDDSC, sc)
+			case v1alpha1.DiskClassNameSSD:
+				hwSSDSC = append(hwSSDSC, sc)
+			case v1alpha1.DiskClassNameNVMe:
+				hwNVMeSC = append(hwNVMeSC, sc)
+			default:
+				klog.V(4).Infof("Unknown poolClass %s, skip it", sc.Parameters[v1alpha1.VolumeParameterPoolClassKey])
+			}
 		}
 	}
 
-	// todo(ming) - order storageClasses: defaultStorageClass > NVMe > SSD > HDD
-	return hwameiStorStorageClasses
+	sortedHwStorageClasses = append(sortedHwStorageClasses, hwDefaultSC...)
+	sortedHwStorageClasses = append(sortedHwStorageClasses, hwNVMeSC...)
+	sortedHwStorageClasses = append(sortedHwStorageClasses, hwSSDSC...)
+	sortedHwStorageClasses = append(sortedHwStorageClasses, hwHDDSC...)
+
+	klog.V(4).Infof("Sorted hwameistor storageclasses: %d, default storageclass: %d", len(sortedHwStorageClasses), len(hwDefaultSC))
+	return sortedHwStorageClasses
 }

vendor/modules.txt

@@ -742,6 +742,9 @@ k8s.io/kube-openapi/pkg/spec3
 k8s.io/kube-openapi/pkg/util/proto
 k8s.io/kube-openapi/pkg/util/sets
 k8s.io/kube-openapi/pkg/validation/spec
+# k8s.io/kubernetes v1.24.0 => k8s.io/kubernetes v1.24.0
+## explicit; go 1.16
+k8s.io/kubernetes/pkg/apis/storage/v1/util
 # k8s.io/mount-utils v0.24.0 => k8s.io/mount-utils v0.24.0
 ## explicit; go 1.16
 k8s.io/mount-utils