feat: add comprehensive mount pod configuration system with DaemonSet support

- Implement three mount modes: per-pvc, shared-pod, and daemonset
- Add ConfigMap-based configuration for per-StorageClass mount settings
- Support DaemonSet deployment with flexible nodeAffinity configuration
- Add automatic fallback from DaemonSet to shared pod when node scheduling fails
- Rename daemonset_config to mount_config for clarity
- Add node compatibility checking before DaemonSet scheduling
- Include comprehensive tests for mount selection and fallback behavior
- Update documentation with configuration examples and best practices

This allows users to optimize mount pod deployment based on their specific
needs without modifying existing StorageClasses, with automatic fallback
ensuring workloads never fail due to mount pod scheduling issues.

Author: elijah-rou

deploy/kubernetes/csi-daemonset-mount/daemonset-config.yaml renamed to deploy/kubernetes/csi-daemonset-mount/mount-config.yaml

@@ -1,19 +1,19 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: juicefs-daemonset-config
+  name: juicefs-mount-config
   namespace: kube-system
 data:
-  # Default configuration for all StorageClasses (if no specific config exists)
+  # Default configuration for all StorageClasses
   default: |
     nodeAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
         nodeSelectorTerms:
         - matchExpressions:
           - key: node-role.kubernetes.io/control-plane
             operator: DoesNotExist
-  
-  # Specific configuration for a StorageClass named "juicefs-sc"
+
+  # StorageClass "juicefs-sc" configuration
   juicefs-sc: |
     nodeAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
@@ -23,16 +23,3 @@ data:
             operator: In
             values:
             - "true"
-  
-  # Another StorageClass with different affinity
-  juicefs-sc-performance: |
-    nodeAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 100
-        preference:
-          matchExpressions:
-          - key: node.kubernetes.io/instance-type
-            operator: In
-            values:
-            - m5.xlarge
-            - m5.2xlarge

docs/en/guide/daemonset-mount.md

@@ -38,7 +38,7 @@ Create a ConfigMap to define node affinity for your StorageClasses without modif
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: juicefs-daemonset-config
+  name: juicefs-mount-config
   namespace: kube-system
 data:
   # Default configuration for all StorageClasses

docs/en/guide/mount-pod-configuration.md

@@ -57,6 +57,16 @@ data:
             - "true"
 ```
 
+## Automatic Fallback Behavior
+
+When using DaemonSet mode, the CSI Driver automatically falls back to shared pod mode if:
+
+1. The DaemonSet cannot schedule a pod on the node due to nodeAffinity restrictions
+2. The node has taints that prevent the DaemonSet pod from being scheduled
+3. The DaemonSet pod fails to become ready within the timeout period
+
+This ensures that workloads can still mount volumes even if the DaemonSet configuration prevents pods from running on certain nodes.
+
 ## Mount Modes Explained
 
 ### Per-PVC Mode

pkg/config/mount_config.go

@@ -30,7 +30,7 @@ import (
 
 const (
 	MountConfigMapName = "juicefs-mount-config"
-	// Use the same DefaultConfigKey from daemonset_config.go
+	DefaultConfigKey   = "default"
 )
 
 // MountMode defines how mount pods are deployed

pkg/config/daemonset_config.go renamed to pkg/config/mount_config_helper.go

@@ -28,65 +28,57 @@ import (
 	"github.com/juicedata/juicefs-csi-driver/pkg/k8sclient"
 )
 
-const (
-	DaemonSetConfigMapName = "juicefs-daemonset-config"
-	DefaultConfigKey       = "default"
-)
-
-// DaemonSetConfig represents the configuration for a DaemonSet deployment
-type DaemonSetConfig struct {
-	Enabled      *bool                `yaml:"enabled,omitempty"`     // Explicitly enable/disable DaemonSet for this StorageClass
-	NodeAffinity *corev1.NodeAffinity `yaml:"nodeAffinity,omitempty"`
-}
+// Helper functions for DaemonSet node affinity configuration
+// The main MountConfig struct is defined in mount_config.go
 
-// GetDaemonSetConfig retrieves the DaemonSet configuration for a given StorageClass
+// GetDaemonSetNodeAffinity retrieves the DaemonSet node affinity configuration for a given StorageClass
 // It first checks for a StorageClass-specific configuration, then falls back to default
-func GetDaemonSetConfig(ctx context.Context, client *k8sclient.K8sClient, storageClassName string) (*corev1.NodeAffinity, error) {
-	log := klog.NewKlogr().WithName("daemonset-config")
+func GetDaemonSetNodeAffinity(ctx context.Context, client *k8sclient.K8sClient, storageClassName string) (*corev1.NodeAffinity, error) {
+	log := klog.NewKlogr().WithName("mount-config")
 
 	// Try to get the ConfigMap
-	configMap, err := client.GetConfigMap(ctx, DaemonSetConfigMapName, Namespace)
+	configMap, err := client.GetConfigMap(ctx, MountConfigMapName, Namespace)
 	if err != nil {
 		if k8serrors.IsNotFound(err) {
-			log.V(1).Info("DaemonSet ConfigMap not found, using no node affinity", 
-				"configMap", DaemonSetConfigMapName, "namespace", Namespace)
+			log.V(1).Info("Mount ConfigMap not found, using no node affinity", 
+				"configMap", MountConfigMapName, "namespace", Namespace)
 			return nil, nil // No ConfigMap means no node affinity restrictions
 		}
-		return nil, fmt.Errorf("failed to get DaemonSet ConfigMap: %v", err)
+		return nil, fmt.Errorf("failed to get Mount ConfigMap: %v", err)
 	}
 
 	// Try to get StorageClass-specific configuration
 	if configData, exists := configMap.Data[storageClassName]; exists {
-		log.V(1).Info("Found StorageClass-specific DaemonSet configuration", 
+		log.V(1).Info("Found StorageClass-specific mount configuration", 
 			"storageClass", storageClassName)
-		return parseDaemonSetConfig(configData)
+		return parseDaemonSetNodeAffinity(configData)
 	}
 
 	// Fall back to default configuration
 	if configData, exists := configMap.Data[DefaultConfigKey]; exists {
-		log.V(1).Info("Using default DaemonSet configuration for StorageClass", 
+		log.V(1).Info("Using default mount configuration for StorageClass", 
 			"storageClass", storageClassName)
-		return parseDaemonSetConfig(configData)
+		return parseDaemonSetNodeAffinity(configData)
 	}
 
-	log.V(1).Info("No DaemonSet configuration found for StorageClass", 
+	log.V(1).Info("No mount configuration found for StorageClass", 
 		"storageClass", storageClassName)
 	return nil, nil
 }
 
-// parseDaemonSetConfig parses the configuration string into a NodeAffinity
-func parseDaemonSetConfig(configData string) (*corev1.NodeAffinity, error) {
-	config := &DaemonSetConfig{}
+// parseDaemonSetNodeAffinity parses the configuration string to extract NodeAffinity
+func parseDaemonSetNodeAffinity(configData string) (*corev1.NodeAffinity, error) {
+	config := &MountConfig{}
 	if err := yaml.Unmarshal([]byte(configData), config); err != nil {
-		return nil, fmt.Errorf("failed to parse DaemonSet configuration: %v", err)
+		return nil, fmt.Errorf("failed to parse mount configuration: %v", err)
 	}
 	return config.NodeAffinity, nil
 }
 
 // LoadDaemonSetNodeAffinity loads node affinity for a StorageClass from ConfigMap
 // This is called when creating or updating a DaemonSet for mount pods
 func LoadDaemonSetNodeAffinity(ctx context.Context, client *k8sclient.K8sClient, jfsSetting *JfsSetting) error {
-	log := klog.NewKlogr().WithName("daemonset-config")
+	log := klog.NewKlogr().WithName("mount-config")
 
 	// Skip if not using DaemonSet deployment
 	if !StorageClassShareMount || !StorageClassDaemonSet {
@@ -109,7 +101,7 @@ func LoadDaemonSetNodeAffinity(ctx context.Context, client *k8sclient.K8sClient,
 		storageClassName = jfsSetting.UniqueId
 	}
 
-	nodeAffinity, err := GetDaemonSetConfig(ctx, client, storageClassName)
+	nodeAffinity, err := GetDaemonSetNodeAffinity(ctx, client, storageClassName)
 	if err != nil {
 		log.Error(err, "Failed to get DaemonSet configuration", 
 			"storageClass", storageClassName)

pkg/juicefs/mount/daemonset_mount.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -37,6 +38,23 @@ import (
 	"github.com/juicedata/juicefs-csi-driver/pkg/util/resource"
 )
 
+// DaemonSetSchedulingError indicates that a DaemonSet cannot schedule on a specific node
+type DaemonSetSchedulingError struct {
+	DaemonSetName string
+	NodeName      string
+	Message       string
+}
+
+func (e *DaemonSetSchedulingError) Error() string {
+	return e.Message
+}
+
+// IsDaemonSetSchedulingError checks if the error is a DaemonSet scheduling error
+func IsDaemonSetSchedulingError(err error) bool {
+	_, ok := err.(*DaemonSetSchedulingError)
+	return ok
+}
+
 type DaemonSetMount struct {
 	log klog.Logger
 	k8sMount.SafeFormatAndMount
@@ -294,6 +312,25 @@ func (d *DaemonSetMount) createOrUpdateDaemonSet(ctx context.Context, dsName str
 func (d *DaemonSetMount) waitUntilDaemonSetReady(ctx context.Context, dsName string, jfsSetting *jfsConfig.JfsSetting) error {
 	log := util.GenLog(ctx, d.log, "waitUntilDaemonSetReady")
 
+	// First, check if the DaemonSet can schedule a pod on this node
+	canSchedule, err := d.canScheduleOnNode(ctx, dsName)
+	if err != nil {
+		log.Error(err, "Failed to check if DaemonSet can schedule on node")
+		// Continue anyway, might be a transient error
+	}
+	
+	if !canSchedule {
+		// DaemonSet cannot schedule on this node due to nodeAffinity
+		// Return a specific error that can be handled by the caller
+		log.Info("DaemonSet cannot schedule on this node due to nodeAffinity, need fallback", 
+			"dsName", dsName, "nodeName", jfsConfig.NodeName)
+		return &DaemonSetSchedulingError{
+			DaemonSetName: dsName,
+			NodeName:      jfsConfig.NodeName,
+			Message:       "DaemonSet cannot schedule on this node due to nodeAffinity restrictions",
+		}
+	}
+	
 	// Wait for DaemonSet to have pods ready on current node
 	timeout := 5 * time.Minute
 	waitCtx, cancel := context.WithTimeout(ctx, timeout)
@@ -302,7 +339,12 @@ func (d *DaemonSetMount) waitUntilDaemonSetReady(ctx context.Context, dsName str
 	for {
 		select {
 		case <-waitCtx.Done():
-			return fmt.Errorf("timeout waiting for DaemonSet %s to be ready on node %s", dsName, jfsConfig.NodeName)
+			// Timeout - could be because pod cannot be scheduled on this node
+			return &DaemonSetSchedulingError{
+				DaemonSetName: dsName,
+				NodeName:      jfsConfig.NodeName,
+				Message:       fmt.Sprintf("timeout waiting for DaemonSet pod to be ready on node %s", jfsConfig.NodeName),
+			}
 		default:
 			ds, err := d.K8sClient.GetDaemonSet(waitCtx, dsName, jfsConfig.Namespace)
 			if err != nil {
@@ -416,4 +458,145 @@ func (d *DaemonSetMount) getDaemonSetNameFromTarget(ctx context.Context, target
 	}
 
 	return ""
+}
+
+// canScheduleOnNode checks if a DaemonSet can schedule a pod on the current node
+func (d *DaemonSetMount) canScheduleOnNode(ctx context.Context, dsName string) (bool, error) {
+	log := util.GenLog(ctx, d.log, "canScheduleOnNode")
+	
+	// Get the DaemonSet
+	ds, err := d.K8sClient.GetDaemonSet(ctx, dsName, jfsConfig.Namespace)
+	if err != nil {
+		return false, err
+	}
+	
+	// Get the current node
+	node, err := d.K8sClient.GetNode(ctx, jfsConfig.NodeName)
+	if err != nil {
+		log.Error(err, "Failed to get node", "nodeName", jfsConfig.NodeName)
+		return false, err
+	}
+	
+	// Check if the node matches the DaemonSet's nodeAffinity
+	if ds.Spec.Template.Spec.Affinity != nil && ds.Spec.Template.Spec.Affinity.NodeAffinity != nil {
+		nodeAffinity := ds.Spec.Template.Spec.Affinity.NodeAffinity
+		
+		// Check required node affinity
+		if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
+			matches := false
+			for _, term := range nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
+				if nodeMatchesSelectorTerm(node, &term) {
+					matches = true
+					break
+				}
+			}
+			if !matches {
+				log.Info("Node does not match DaemonSet's required node affinity", 
+					"nodeName", jfsConfig.NodeName, "dsName", dsName)
+				return false, nil
+			}
+		}
+	}
+	
+	// Check if the node has any taints that would prevent scheduling
+	// (This is a simplified check - a full implementation would need to check tolerations)
+	if len(node.Spec.Taints) > 0 && len(ds.Spec.Template.Spec.Tolerations) == 0 {
+		for _, taint := range node.Spec.Taints {
+			if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute {
+				log.Info("Node has taints that prevent scheduling", 
+					"nodeName", jfsConfig.NodeName, "taint", taint)
+				return false, nil
+			}
+		}
+	}
+	
+	return true, nil
+}
+
+// nodeMatchesSelectorTerm checks if a node matches a node selector term
+func nodeMatchesSelectorTerm(node *corev1.Node, term *corev1.NodeSelectorTerm) bool {
+	// Check match expressions
+	for _, expr := range term.MatchExpressions {
+		if !nodeMatchesExpression(node, &expr) {
+			return false
+		}
+	}
+	
+	// Check match fields
+	for _, field := range term.MatchFields {
+		if !nodeMatchesFieldSelector(node, &field) {
+			return false
+		}
+	}
+	
+	return true
+}
+
+// nodeMatchesExpression checks if a node matches a label selector requirement
+func nodeMatchesExpression(node *corev1.Node, expr *corev1.NodeSelectorRequirement) bool {
+	value, exists := node.Labels[expr.Key]
+	
+	switch expr.Operator {
+	case corev1.NodeSelectorOpIn:
+		if !exists {
+			return false
+		}
+		for _, v := range expr.Values {
+			if value == v {
+				return true
+			}
+		}
+		return false
+	case corev1.NodeSelectorOpNotIn:
+		if !exists {
+			return true
+		}
+		for _, v := range expr.Values {
+			if value == v {
+				return false
+			}
+		}
+		return true
+	case corev1.NodeSelectorOpExists:
+		return exists
+	case corev1.NodeSelectorOpDoesNotExist:
+		return !exists
+	case corev1.NodeSelectorOpGt, corev1.NodeSelectorOpLt:
+		// These operators are typically used for numeric comparisons
+		// For simplicity, we're not implementing them here
+		return true
+	default:
+		return false
+	}
+}
+
+// nodeMatchesFieldSelector checks if a node matches a field selector
+func nodeMatchesFieldSelector(node *corev1.Node, field *corev1.NodeSelectorRequirement) bool {
+	var value string
+	switch field.Key {
+	case "metadata.name":
+		value = node.Name
+	// Add more field selectors as needed
+	default:
+		return false
+	}
+	
+	switch field.Operator {
+	case corev1.NodeSelectorOpIn:
+		for _, v := range field.Values {
+			if value == v {
+				return true
+			}
+		}
+		return false
+	case corev1.NodeSelectorOpNotIn:
+		for _, v := range field.Values {
+			if value == v {
+				return false
+			}
+		}
+		return true
+	default:
+		return false
+	}
 }