chore: refine the settings of the manager

cmd/manager/main.go

@@ -41,8 +41,10 @@ import (
 	discoverycli "k8s.io/client-go/discovery"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -85,10 +87,13 @@ import (
 const (
 	appName = "kubeblocks"
 
-	probeAddrFlagKey     flagName = "health-probe-bind-address"
-	metricsAddrFlagKey   flagName = "metrics-bind-address"
-	leaderElectFlagKey   flagName = "leader-elect"
-	leaderElectIDFlagKey flagName = "leader-elect-id"
+	probeAddrFlagKey                flagName = "health-probe-bind-address"
+	metricsAddrFlagKey              flagName = "metrics-bind-address"
+	leaderElectFlagKey              flagName = "leader-elect"
+	leaderElectIDFlagKey            flagName = "leader-elect-id"
+	leaderElectLeaseDurationFlagKey flagName = "leader-elect-lease-duration"
+	leaderElectRenewDeadlineFlagKey flagName = "leader-elect-renew-deadline"
+	leaderElectRetryPeriodFlagKey   flagName = "leader-elect-retry-period"
 
 	// switch flags key for API groups
 	appsFlagKey         flagName = "apps"
@@ -151,11 +156,14 @@ func init() {
 	viper.SetDefault(instanceset.MaxPlainRevisionCount, 1024)
 	viper.SetDefault(instanceset.FeatureGateIgnorePodVerticalScaling, false)
 	viper.SetDefault(intctrlutil.FeatureGateEnableRuntimeMetrics, false)
-	viper.SetDefault(constant.CfgKBReconcileWorkers, 8)
 	viper.SetDefault(constant.FeatureGateIgnoreConfigTemplateDefaultMode, false)
 	viper.SetDefault(constant.FeatureGateInPlacePodVerticalScaling, false)
 	viper.SetDefault(constant.I18nResourcesName, "kubeblocks-i18n-resources")
 	viper.SetDefault(constant.APIVersionSupported, "")
+	viper.SetDefault(constant.CfgKBReconcileWorkers, 32)
+	viper.SetDefault(constant.CfgCacheSyncTimeout, 300)
+	viper.SetDefault(constant.CfgClientQPS, 128)
+	viper.SetDefault(constant.CfgClientBurst, 256)
 }
 
 type flagName string
@@ -171,13 +179,19 @@ func (r flagName) viperName() string {
 func setupFlags() {
 	flag.String(metricsAddrFlagKey.String(), ":8080", "The address the metric endpoint binds to.")
 	flag.String(probeAddrFlagKey.String(), ":8081", "The address the probe endpoint binds to.")
+
 	flag.Bool(leaderElectFlagKey.String(), false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-
 	flag.String(leaderElectIDFlagKey.String(), "001c317f",
 		"The leader election ID prefix for controller manager. "+
 			"This ID must be unique to controller manager.")
+	flag.Int(leaderElectLeaseDurationFlagKey.String(), 30,
+		"The duration (in seconds) that non-leader candidates will wait to force acquire leadership.")
+	flag.Int(leaderElectRenewDeadlineFlagKey.String(), 25,
+		"The duration (in seconds) that the acting control plane will retry refreshing leadership before giving up.")
+	flag.Int(leaderElectRetryPeriodFlagKey.String(), 5,
+		"The duration (in seconds) the LeaderElector clients should wait between tries of actions.")
 
 	flag.Bool(appsFlagKey.String(), true,
 		"Enable the apps controller manager.")
@@ -356,7 +370,6 @@ func main() {
 		// pattern of '{{ hashFNV .Repo }}.{{ .Domain }}', make sure regenerate this ID
 		// if you have forked from this project template.
 		LeaderElectionID: enableLeaderElectionID + ".kubeblocks.io",
-
 		// NOTES:
 		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
 		// when the Manager ends. This requires the binary to immediately end when the
@@ -369,6 +382,9 @@ func main() {
 		// if you are doing or intending to do any operation such as performing cleanups
 		// after the manager stops then its usage might be unsafe.
 		LeaderElectionReleaseOnCancel: true,
+		LeaseDuration:                 ptr.To(time.Duration(viper.GetInt(leaderElectLeaseDurationFlagKey.viperName())) * time.Second),
+		RenewDeadline:                 ptr.To(time.Duration(viper.GetInt(leaderElectRenewDeadlineFlagKey.viperName())) * time.Second),
+		RetryPeriod:                   ptr.To(time.Duration(viper.GetInt(leaderElectRetryPeriodFlagKey.viperName())) * time.Second),
 
 		WebhookServer: webhook.NewServer(webhook.Options{
 			Port:    9443,
@@ -379,6 +395,9 @@ func main() {
 				DisableFor: append(intctrlutil.GetUncachedObjects(), &parametersv1alpha1.ComponentParameter{}),
 			},
 		},
+		Controller: config.Controller{
+			CacheSyncTimeout: viper.GetDuration(constant.CfgCacheSyncTimeout) * time.Second,
+		},
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

controllers/apps/cluster/cluster_controller.go

@@ -21,7 +21,6 @@ package cluster
 
 import (
 	"context"
-	"math"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -176,7 +175,7 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return intctrlutil.NewControllerManagedBy(mgr).
 		For(&appsv1.Cluster{}).
 		WithOptions(controller.Options{
-			MaxConcurrentReconciles: int(math.Ceil(viper.GetFloat64(constant.CfgKBReconcileWorkers) / 4)),
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers),
 		}).
 		Owns(&appsv1.Component{}).
 		Owns(&corev1.Service{}). // cluster services

controllers/k8score/event_controller.go

@@ -29,13 +29,16 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
+	"github.com/apecloud/kubeblocks/pkg/constant"
 	"github.com/apecloud/kubeblocks/pkg/controller/component"
 	"github.com/apecloud/kubeblocks/pkg/controller/instanceset"
 	"github.com/apecloud/kubeblocks/pkg/controller/multicluster"
 	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+	viper "github.com/apecloud/kubeblocks/pkg/viperx"
 )
 
 const (
@@ -99,7 +102,10 @@ func (r *EventReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 // SetupWithManager sets up the controller with the Manager.
 func (r *EventReconciler) SetupWithManager(mgr ctrl.Manager, multiClusterMgr multicluster.Manager) error {
 	b := intctrlutil.NewControllerManagedBy(mgr).
-		For(&corev1.Event{})
+		For(&corev1.Event{}).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers) / 4,
+		})
 
 	if multiClusterMgr != nil {
 		multiClusterMgr.Watch(b, &corev1.Event{}, &handler.EnqueueRequestForObject{})

controllers/operations/opsrequest_controller.go

@@ -21,7 +21,6 @@ package operations
 
 import (
 	"context"
-	"math"
 	"reflect"
 	"slices"
 	"strings"
@@ -94,7 +93,7 @@ func (r *OpsRequestReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return intctrlutil.NewControllerManagedBy(mgr).
 		For(&opsv1alpha1.OpsRequest{}).
 		WithOptions(controller.Options{
-			MaxConcurrentReconciles: int(math.Ceil(viper.GetFloat64(constant.CfgKBReconcileWorkers) / 2)),
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers) / 2,
 		}).
 		Watches(&appsv1.Cluster{}, handler.EnqueueRequestsFromMapFunc(r.parseRunningOpsRequests)).
 		Watches(&workloads.InstanceSet{}, handler.EnqueueRequestsFromMapFunc(r.parseRunningOpsRequestsForInstanceSet)).

controllers/parameters/componentparameter_controller.go

@@ -22,7 +22,6 @@ package parameters
 import (
 	"context"
 	"fmt"
-	"math"
 	"strconv"
 
 	"github.com/pkg/errors"
@@ -88,7 +87,7 @@ func (r *ComponentParameterReconciler) SetupWithManager(mgr ctrl.Manager, multiC
 	builder := intctrlutil.NewControllerManagedBy(mgr).
 		For(&parametersv1alpha1.ComponentParameter{}).
 		WithOptions(controller.Options{
-			MaxConcurrentReconciles: int(math.Ceil(viper.GetFloat64(constant.CfgKBReconcileWorkers) / 2)),
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers) / 4,
 		}).
 		Owns(&corev1.ConfigMap{})
 	if multiClusterMgr != nil {

controllers/parameters/reconfigure_controller.go

@@ -23,7 +23,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"math"
 	"time"
 
 	"github.com/pkg/errors"
@@ -143,7 +142,7 @@ func (r *ReconfigureReconciler) SetupWithManager(mgr ctrl.Manager, multiClusterM
 	b := intctrlutil.NewControllerManagedBy(mgr).
 		For(&corev1.ConfigMap{}).
 		WithOptions(controller.Options{
-			MaxConcurrentReconciles: int(math.Ceil(viper.GetFloat64(constant.CfgKBReconcileWorkers) / 4)),
+			MaxConcurrentReconciles: viper.GetInt(constant.CfgKBReconcileWorkers) / 4,
 		})
 
 	if multiClusterMgr != nil {

deploy/helm/templates/dataprotection.yaml

@@ -52,7 +52,12 @@ spec:
           image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.tools.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            limits:
+              cpu: 50m
+              memory: 64Mi
+            requests:
+              cpu: 50m
+              memory: 64Mi
           command:
             - /bin/true
       containers:
@@ -185,7 +190,12 @@ spec:
             initialDelaySeconds: 5
             periodSeconds: 10
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            limits:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
+            requests:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
           volumeMounts:
             - mountPath: /etc/kubeblocks
               name: manager-config

deploy/helm/templates/deployment.yaml

@@ -50,7 +50,12 @@ spec:
           image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.tools.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            limits:
+              cpu: 50m
+              memory: 64Mi
+            requests:
+              cpu: 50m
+              memory: 64Mi
           command:
             - /bin/true
       containers:
@@ -59,8 +64,17 @@ spec:
             - "--health-probe-bind-address=:8081"
             - "--metrics-bind-address=:8080"
             - "--leader-elect"
-            {{- with .Values.manager }}
-            - "--leader-elect-id={{- default "001c317f" .leaderElectId }}"
+            {{- with .Values.manager.leaderElection }}
+            - "--leader-elect-id={{- default "001c317f" .Id }}"
+            {{- end }}
+            {{- if .Values.manager.leaderElection.leaseDuration }}
+            - "--leader-elect-lease-duration={{ .Values.manager.leaderElection.leaseDuration }}"
+            {{- end }}
+            {{- if .Values.manager.leaderElection.renewDeadline }}
+            - "--leader-elect-renew-deadline={{ .Values.manager.leaderElection.renewDeadline }}"
+            {{- end }}
+            {{- if .Values.manager.leaderElection.retryPeriod }}
+            - "--leader-elect-retry-period={{ .Values.manager.leaderElection.retryPeriod }}"
             {{- end }}
             - "--zap-devel={{- default "false" .Values.loggerSettings.developmentMode }}"
             - "--zap-time-encoding={{- default "iso8601" .Values.loggerSettings.timeEncoding }}"
@@ -102,6 +116,10 @@ spec:
             - name: KUBEBLOCKS_RECONCILE_WORKERS
               value: {{ .Values.reconcileWorkers | quote }}
             {{- end }}
+            {{- if .Values.cache.syncTimeout }}
+            - name: CACHE_SYNC_TIMEOUT
+              value: {{ .Values.cache.syncTimeout | quote }}
+            {{- end }}
             {{- if .Values.client.qps }}
             - name: CLIENT_QPS
               value: {{ .Values.client.qps | quote }}
@@ -219,7 +237,12 @@ spec:
             initialDelaySeconds: 5
             periodSeconds: 10
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            limits:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
+            requests:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
           volumeMounts:
             - mountPath: /etc/kubeblocks
               name: manager-config

deploy/helm/values.yaml

@@ -25,15 +25,20 @@ image:
 ##
 replicaCount: 1
 
-## MaxConcurrentReconciles for component, instanceSet and opsRequest controllers.
-##
+## MaxConcurrentReconciles for cluster, component and instanceSet controllers.
+## default is 32
 reconcileWorkers: ""
 
+## k8s cache configuration.
+cache:
+  # default is 300 seconds
+  syncTimeout: ""
+
 ## k8s client configuration.
 client:
-  # default is 20
+  # default is 128
   qps: ""
-  # default is 30
+  # default is 256
   burst: ""
 
 ## @param nameOverride
@@ -44,7 +49,6 @@ nameOverride: ""
 ##
 fullnameOverride: ""
 
-
 ## KubeBlocks RBAC access priority setting
 ##
 ## @param rbac.enabled is used to enable or disable KubeBlocks RBAC access priority.
@@ -238,24 +242,15 @@ serviceMonitor:
 ## @param topologySpreadConstraints
 topologySpreadConstraints: []
 
-
 ## Resource settings
 ##
-## @param resources.limits
-## @param resources.requests
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+## @param resources.cpu
+## @param resources.memory
+resources:
   # TODO(user): Configure the resources accordingly based on the project requirements.
   # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
-  # limits:
-  #   cpu: 500m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 10m
-#   memory: 64Mi
+  cpu: 500m
+  memory: 1Gi
 
 ## @param priorityClassName
 ##
@@ -348,7 +343,16 @@ webhooks:
 
 ## manager server settings
 manager:
-  leaderElectId: ""
+  ## leader election
+  leaderElection:
+    # default is "001c317f"
+    Id: ""
+    # default is 30 seconds
+    leaseDuration:
+    # default is 25 seconds
+    renewDeadline:
+    # default is 5 seconds
+    retryPeriod:
 
 ## Data protection settings
 ##

pkg/constant/viper_config.go

@@ -53,6 +53,7 @@ const (
 	CfgKeyDPBackupEncryptionAlgorithm    = "DP_BACKUP_ENCRYPTION_ALGORITHM"
 
 	CfgKBReconcileWorkers = "KUBEBLOCKS_RECONCILE_WORKERS"
+	CfgCacheSyncTimeout   = "CACHE_SYNC_TIMEOUT"
 	CfgClientQPS          = "CLIENT_QPS"
 	CfgClientBurst        = "CLIENT_BURST"