From a789478747bf98ca9217b942f8e8caf48e6d813b Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Wed, 15 Oct 2025 18:08:01 +0000 Subject: [PATCH 01/11] azure: add azuredisk-csi-driver addon --- .../k8s-1.21.yaml.template | 736 ++++++++++++++++++ 1 file changed, 736 insertions(+) create mode 100644 upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template new file mode 100644 index 0000000000000..99425a63e8a39 --- /dev/null +++ b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template @@ -0,0 +1,736 @@ +# Azure Disk CSI Driver +# Based on https://github.com/kubernetes-sigs/azuredisk-csi-driver + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-controller-sa + namespace: kube-system + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-node-sa + namespace: kube-system + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-provisioner-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-attacher-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-snapshotter-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-resizer-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-provisioner-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-attacher-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-attacher-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-snapshotter-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-resizer-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-resizer-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-node-getter-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-node-role + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: kube-system + name: azuredisk-external-provisioner-cfg + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-provisioner-role-cfg + namespace: kube-system + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: Role + name: azuredisk-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: csi-azuredisk-controller + namespace: kube-system + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +spec: + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-azuredisk-controller + template: + metadata: + labels: + app: csi-azuredisk-controller + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver + spec: + hostNetwork: true + serviceAccountName: csi-azuredisk-controller-sa + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "true" + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v4.0.1 + args: + - "--feature-gates=Topology=true" + - "--csi-address=$(ADDRESS)" + - "--v=2" + - "--timeout=15s" + - "--leader-election" + - "--leader-election-namespace=kube-system" + - "--worker-threads=40" + - "--extra-create-metadata=true" + - "--strict-topology=true" + - "--kube-api-qps=50" + - "--kube-api-burst=100" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.5.1 + args: + - "-v=2" + - "-csi-address=$(ADDRESS)" + - "-timeout=120s" + - "-leader-election" + - "-leader-election-namespace=kube-system" + - "-worker-threads=1000" + - "-kube-api-qps=50" + - "-kube-api-burst=100" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v7.0.2 + args: + - "-csi-address=$(ADDRESS)" + - "-leader-election" + - "-leader-election-namespace=kube-system" + - "-v=2" + - "-timeout=1200s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.10.1 + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - "-leader-election-namespace=kube-system" + - "-timeout=120s" + - "-handle-volume-inuse-error=false" + - "-feature-gates=RecoverVolumeExpansionFailure=true" + - "-kube-api-qps=50" + - "-kube-api-burst=100" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --http-endpoint=localhost:29602 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + - name: azuredisk + image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.5 + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:29604" + - "--disable-avset-nodes=false" + - "--vm-type=" + - "--drivername=disk.csi.azure.com" + - "--cloud-config-secret-name=azure-cloud-provider" + - "--cloud-config-secret-namespace=kube-system" + - "--custom-user-agent=" + - "--user-agent-suffix=OSS-kubectl" + - "--allow-empty-cloud-config=true" + - "--vmss-cache-ttl-seconds=-1" + - "--enable-traffic-manager=false" + - "--traffic-manager-port=7788" + ports: + - containerPort: 29602 + name: healthz + protocol: TCP + - containerPort: 29604 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: AZURE_GO_SDK_LOG_LEVEL + value: "" + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-node + namespace: kube-system + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-azuredisk-node + template: + metadata: + labels: + app: csi-azuredisk-node + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver + spec: + hostNetwork: true + dnsPolicy: Default + serviceAccountName: csi-azuredisk-node-sa + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - operator: Exists + containers: + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --http-endpoint=localhost:29603 + - --v=2 + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + - name: node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1 + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + - name: azuredisk + image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.5 + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:29605" + - "--drivername=disk.csi.azure.com" + - "--cloud-config-secret-name=azure-cloud-provider" + - "--cloud-config-secret-namespace=kube-system" + - "--custom-user-agent=" + - "--user-agent-suffix=OSS-kubectl" + - "--allow-empty-cloud-config=true" + - "--support-zone=true" + - "--get-node-info-from-labels=false" + - "--enable-traffic-manager=false" + - "--traffic-manager-port=7788" + ports: + - containerPort: 29603 + name: healthz + protocol: TCP + - containerPort: 29605 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AZURE_GO_SDK_LOG_LEVEL + value: "" + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /dev + name: device-dir + - mountPath: /sys/bus/scsi/devices + name: sys-devices-dir + - mountPath: /sys/class/ + name: sys-class-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + add: + - SYS_ADMIN + privileged: true + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/disk.csi.azure.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/ + type: DirectoryOrCreate + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /sys/bus/scsi/devices + type: DirectoryOrCreate + name: sys-devices-dir + - hostPath: + path: /sys/class/ + type: DirectoryOrCreate + name: sys-class-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: disk.csi.azure.com + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +spec: + attachRequired: true + podInfoOnMount: false + fsGroupPolicy: File +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: managed-csi + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: disk.csi.azure.com +parameters: + skuName: StandardSSD_LRS +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: managed-csi-premium + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +provisioner: disk.csi.azure.com +parameters: + skuName: Premium_LRS +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true \ No newline at end of file From ca24d06c971306b33e10ce15b4fbd4e350185e78 Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Wed, 15 Oct 2025 18:40:28 +0000 Subject: [PATCH 02/11] azure: add azuredisk-csi-driver bootstrap integration --- .../bootstrapchannelbuilder.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index dabb60d3d098b..df502b4c904d9 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -770,6 +770,21 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } } + if b.Cluster.IsKubernetesGTE("1.21") && b.Cluster.GetCloudProvider() == kops.CloudProviderAzure { + { + key := "azuredisk-csi-driver.addons.k8s.io" + id := "k8s-1.21" + location := key + "/" + id + ".yaml" + + addons.Add(&channelsapi.AddonSpec{ + Name: fi.PtrTo(key), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.PtrTo(location), + Id: id, + }) + } + } + if b.Cluster.GetCloudProvider() == kops.CloudProviderGCE { if fi.ValueOf(b.Cluster.Spec.CloudConfig.ManageStorageClasses) { key := "storage-gce.addons.k8s.io" From 76d76291c18f6f5ee26a88b6dec29bdd2f9823fa Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 13:12:45 +0000 Subject: [PATCH 03/11] resolved initial suggestions --- .../k8s-1.21.yaml.template | 3 ++- .../bootstrapchannelbuilder/bootstrapchannelbuilder.go | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template index 99425a63e8a39..cb3bc0980bfa8 100644 --- a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template +++ b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template @@ -1,6 +1,7 @@ # Azure Disk CSI Driver # Based on https://github.com/kubernetes-sigs/azuredisk-csi-driver +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -733,4 +734,4 @@ parameters: skuName: Premium_LRS reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true \ No newline at end of file +allowVolumeExpansion: true diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index df502b4c904d9..645af20c90fde 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -768,12 +768,9 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) Id: id, }) } - } - - if b.Cluster.IsKubernetesGTE("1.21") && b.Cluster.GetCloudProvider() == kops.CloudProviderAzure { { key := "azuredisk-csi-driver.addons.k8s.io" - id := "k8s-1.21" + id := "k8s-1.31" location := key + "/" + id + ".yaml" addons.Add(&channelsapi.AddonSpec{ @@ -1140,7 +1137,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } if b.Cluster.Spec.CloudProvider.AWS != nil && (b.Cluster.Spec.CloudProvider.AWS.EBSCSIDriver.Managed == nil || fi.ValueOf(b.Cluster.Spec.CloudProvider.AWS.EBSCSIDriver.Managed)) { - key := "aws-ebs-csi-driver.addons.k8s.io" + key := "aws-ebs-csi-driver.addons.k8s.iyuo" { id := "k8s-1.17" From bf6fcef4d21ebc4bea9c667529812cb879dd7057 Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 13:25:45 +0000 Subject: [PATCH 04/11] typo --- .../cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 645af20c90fde..8a4e25a28cd9f 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -1137,7 +1137,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } if b.Cluster.Spec.CloudProvider.AWS != nil && (b.Cluster.Spec.CloudProvider.AWS.EBSCSIDriver.Managed == nil || fi.ValueOf(b.Cluster.Spec.CloudProvider.AWS.EBSCSIDriver.Managed)) { - key := "aws-ebs-csi-driver.addons.k8s.iyuo" + key := "aws-ebs-csi-driver.addons.k8s.io" { id := "k8s-1.17" From 38253e21cada92ca00be6e1357a561b46da03e2b Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 13:48:27 +0000 Subject: [PATCH 05/11] azure: changed template version --- .../{k8s-1.21.yaml.template => k8s-1.31.yaml.template} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/{k8s-1.21.yaml.template => k8s-1.31.yaml.template} (100%) diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template similarity index 100% rename from upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.21.yaml.template rename to upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template From 2104e71d1da35dce5ad7e7f6c22745515432c245 Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 14:08:48 +0000 Subject: [PATCH 06/11] azure: changed template version to 1.31 + content --- .../k8s-1.31.yaml.template | 338 +++++------------- 1 file changed, 92 insertions(+), 246 deletions(-) diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template index cb3bc0980bfa8..63daa10f4836c 100644 --- a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template +++ b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template @@ -1,5 +1,5 @@ -# Azure Disk CSI Driver -# Based on https://github.com/kubernetes-sigs/azuredisk-csi-driver +# Azure Disk CSI Driver v1.31.0 +# Based on https://github.com/kubernetes-sigs/azuredisk-csi-driver/tree/master/deploy/v1.31.0 --- apiVersion: v1 @@ -52,89 +52,26 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["get", "list"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: azuredisk-external-attacher-role + name: csi-azuredisk-node-role labels: app.kubernetes.io/name: azuredisk-csi-driver app.kubernetes.io/component: csi-driver rules: - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "patch"] + resources: ["secrets"] + verbs: ["get"] - apiGroups: [""] resources: ["nodes"] - verbs: ["get", "list", "watch"] + verbs: ["get", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: azuredisk-external-snapshotter-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: azuredisk-external-resizer-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-azuredisk-node-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -rules: - - apiGroups: [""] - resources: ["nodes"] verbs: ["get"] --- kind: ClusterRoleBinding @@ -156,55 +93,7 @@ roleRef: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: azuredisk-csi-attacher-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -subjects: - - kind: ServiceAccount - name: csi-azuredisk-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: azuredisk-external-attacher-role - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: azuredisk-csi-snapshotter-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -subjects: - - kind: ServiceAccount - name: csi-azuredisk-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: azuredisk-external-snapshotter-role - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: azuredisk-csi-resizer-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -subjects: - - kind: ServiceAccount - name: csi-azuredisk-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: azuredisk-external-resizer-role - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: azuredisk-csi-node-getter-binding + name: csi-azuredisk-node-secret-binding labels: app.kubernetes.io/name: azuredisk-csi-driver app.kubernetes.io/component: csi-driver @@ -217,38 +106,8 @@ roleRef: name: csi-azuredisk-node-role apiGroup: rbac.authorization.k8s.io --- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: kube-system - name: azuredisk-external-provisioner-cfg - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: azuredisk-csi-provisioner-role-cfg - namespace: kube-system - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -subjects: - - kind: ServiceAccount - name: csi-azuredisk-controller-sa - namespace: kube-system -roleRef: - kind: Role - name: azuredisk-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 kind: Deployment +apiVersion: apps/v1 metadata: name: csi-azuredisk-controller namespace: kube-system @@ -257,7 +116,6 @@ metadata: app.kubernetes.io/component: csi-driver spec: replicas: 2 - revisionHistoryLimit: 10 selector: matchLabels: app: csi-azuredisk-controller @@ -277,27 +135,29 @@ spec: seccompProfile: type: RuntimeDefault tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Equal - value: "true" - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Equal - value: "true" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + effect: "NoSchedule" containers: - name: csi-provisioner - image: registry.k8s.io/sig-storage/csi-provisioner:v4.0.1 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v5.1.0 args: - - "--feature-gates=Topology=true" + - "--feature-gates=Topology=true,HonorPVReclaimPolicy=true" - "--csi-address=$(ADDRESS)" - "--v=2" - - "--timeout=15s" + - "--timeout=30s" - "--leader-election" - "--leader-election-namespace=kube-system" - - "--worker-threads=40" + - "--worker-threads=100" - "--extra-create-metadata=true" - "--strict-topology=true" - "--kube-api-qps=50" @@ -321,16 +181,16 @@ spec: - ALL readOnlyRootFilesystem: true - name: csi-attacher - image: registry.k8s.io/sig-storage/csi-attacher:v4.5.1 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-attacher:v4.7.0 args: - "-v=2" - "-csi-address=$(ADDRESS)" - - "-timeout=120s" + - "-timeout=1200s" - "-leader-election" - - "-leader-election-namespace=kube-system" + - "--leader-election-namespace=kube-system" - "-worker-threads=1000" - - "-kube-api-qps=50" - - "-kube-api-burst=100" + - "-kube-api-qps=200" + - "-kube-api-burst=400" env: - name: ADDRESS value: /csi/csi.sock @@ -350,13 +210,14 @@ spec: - ALL readOnlyRootFilesystem: true - name: csi-snapshotter - image: registry.k8s.io/sig-storage/csi-snapshotter:v7.0.2 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v8.1.0 args: - "-csi-address=$(ADDRESS)" - "-leader-election" - - "-leader-election-namespace=kube-system" - - "-v=2" - - "-timeout=1200s" + - "--leader-election-namespace=kube-system" + - "--v=2" + - "--timeout=1200s" + - "--extra-create-metadata=true" env: - name: ADDRESS value: /csi/csi.sock @@ -376,17 +237,15 @@ spec: - ALL readOnlyRootFilesystem: true - name: csi-resizer - image: registry.k8s.io/sig-storage/csi-resizer:v1.10.1 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v1.12.0 args: - "-csi-address=$(ADDRESS)" - "-v=2" - "-leader-election" - - "-leader-election-namespace=kube-system" - - "-timeout=120s" - - "-handle-volume-inuse-error=false" - - "-feature-gates=RecoverVolumeExpansionFailure=true" - - "-kube-api-qps=50" - - "-kube-api-burst=100" + - "--leader-election-namespace=kube-system" + - '-handle-volume-inuse-error=false' + - '-feature-gates=RecoverVolumeExpansionFailure=true' + - "-timeout=240s" env: - name: ADDRESS value: /csi/csi.sock @@ -406,7 +265,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: liveness-probe - image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.14.0 args: - --csi-address=/csi/csi.sock - --probe-timeout=3s @@ -428,34 +287,25 @@ spec: - ALL readOnlyRootFilesystem: true - name: azuredisk - image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.5 + image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.31.0 + imagePullPolicy: IfNotPresent args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--metrics-address=0.0.0.0:29604" - - "--disable-avset-nodes=false" - - "--vm-type=" - - "--drivername=disk.csi.azure.com" - - "--cloud-config-secret-name=azure-cloud-provider" - - "--cloud-config-secret-namespace=kube-system" - - "--custom-user-agent=" - "--user-agent-suffix=OSS-kubectl" - - "--allow-empty-cloud-config=true" - - "--vmss-cache-ttl-seconds=-1" - - "--enable-traffic-manager=false" - - "--traffic-manager-port=7788" + - "--disable-avset-nodes=false" + - "--allow-empty-cloud-config=false" ports: - - containerPort: 29602 - name: healthz - protocol: TCP - containerPort: 29604 name: metrics protocol: TCP livenessProbe: failureThreshold: 5 httpGet: + host: localhost path: /healthz - port: healthz + port: 29602 initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 30 @@ -468,8 +318,6 @@ spec: optional: true - name: CSI_ENDPOINT value: unix:///csi/csi.sock - - name: AZURE_GO_SDK_LOG_LEVEL - value: "" volumeMounts: - mountPath: /csi name: socket-dir @@ -523,18 +371,27 @@ spec: serviceAccountName: csi-azuredisk-node-sa nodeSelector: kubernetes.io/os: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet priorityClassName: system-node-critical securityContext: seccompProfile: type: RuntimeDefault tolerations: - - operator: Exists + - operator: "Exists" containers: - name: liveness-probe volumeMounts: - mountPath: /csi - name: plugin-dir - image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + name: socket-dir + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.14.0 args: - --csi-address=/csi/csi.sock - --probe-timeout=3s @@ -553,7 +410,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: node-driver-registrar - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.12.0 args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) @@ -572,7 +429,7 @@ spec: - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock volumeMounts: - - name: plugin-dir + - name: socket-dir mountPath: /csi - name: registration-dir mountPath: /registration @@ -589,34 +446,25 @@ spec: - ALL readOnlyRootFilesystem: true - name: azuredisk - image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.5 + image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.31.0 + imagePullPolicy: IfNotPresent args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" - - "--metrics-address=0.0.0.0:29605" - - "--drivername=disk.csi.azure.com" - - "--cloud-config-secret-name=azure-cloud-provider" - - "--cloud-config-secret-namespace=kube-system" - - "--custom-user-agent=" - - "--user-agent-suffix=OSS-kubectl" + - "--enable-perf-optimization=true" - "--allow-empty-cloud-config=true" - - "--support-zone=true" - "--get-node-info-from-labels=false" - - "--enable-traffic-manager=false" - - "--traffic-manager-port=7788" ports: - containerPort: 29603 name: healthz protocol: TCP - - containerPort: 29605 - name: metrics - protocol: TCP livenessProbe: failureThreshold: 5 httpGet: + host: localhost path: /healthz - port: healthz + port: 29603 initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 30 @@ -634,65 +482,60 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: AZURE_GO_SDK_LOG_LEVEL - value: "" + securityContext: + privileged: true + capabilities: + drop: + - ALL volumeMounts: - mountPath: /csi - name: plugin-dir + name: socket-dir - mountPath: /var/lib/kubelet/ mountPropagation: Bidirectional - name: kubelet-dir + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred - mountPath: /dev name: device-dir - mountPath: /sys/bus/scsi/devices name: sys-devices-dir - mountPath: /sys/class/ - name: sys-class-dir - - mountPath: /etc/kubernetes/ - name: azure-cred + name: sys-class resources: limits: - memory: 200Mi + memory: 600Mi requests: cpu: 10m memory: 20Mi - securityContext: - allowPrivilegeEscalation: true - capabilities: - drop: - - ALL - add: - - SYS_ADMIN - privileged: true volumes: - hostPath: path: /var/lib/kubelet/plugins/disk.csi.azure.com type: DirectoryOrCreate - name: plugin-dir + name: socket-dir - hostPath: path: /var/lib/kubelet/ type: DirectoryOrCreate - name: kubelet-dir + name: mountpoint-dir - hostPath: path: /var/lib/kubelet/plugins_registry/ type: DirectoryOrCreate name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred - hostPath: path: /dev type: Directory name: device-dir - hostPath: path: /sys/bus/scsi/devices - type: DirectoryOrCreate + type: Directory name: sys-devices-dir - hostPath: path: /sys/class/ - type: DirectoryOrCreate - name: sys-class-dir - - hostPath: - path: /etc/kubernetes/ - type: DirectoryOrCreate - name: azure-cred + type: Directory + name: sys-class --- apiVersion: storage.k8s.io/v1 kind: CSIDriver @@ -701,6 +544,9 @@ metadata: labels: app.kubernetes.io/name: azuredisk-csi-driver app.kubernetes.io/component: csi-driver + annotations: + csiDriver: v1.31.0 + snapshot: v6.2.1 spec: attachRequired: true podInfoOnMount: false @@ -734,4 +580,4 @@ parameters: skuName: Premium_LRS reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true +allowVolumeExpansion: true \ No newline at end of file From 61048e591eba08f2d196a809af69f9ba7189b051 Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 15:31:53 +0000 Subject: [PATCH 07/11] azure: added nodeSelector kubernetes.io/role: master to csi-azuredisk-controller --- .../azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template index 63daa10f4836c..e591a998ca5e1 100644 --- a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template +++ b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template @@ -130,6 +130,7 @@ spec: serviceAccountName: csi-azuredisk-controller-sa nodeSelector: kubernetes.io/os: linux + kubernetes.io/role: master priorityClassName: system-cluster-critical securityContext: seccompProfile: @@ -580,4 +581,4 @@ parameters: skuName: Premium_LRS reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true \ No newline at end of file +allowVolumeExpansion: true From 921823439b86e514016091a7044e82390b090269 Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 13:17:37 -0400 Subject: [PATCH 08/11] azure: upgrade template to v1.33.5 --- .../k8s-1.31.yaml.template | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template index e591a998ca5e1..2443342d4b2c8 100644 --- a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template +++ b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template @@ -1,5 +1,5 @@ -# Azure Disk CSI Driver v1.31.0 -# Based on https://github.com/kubernetes-sigs/azuredisk-csi-driver/tree/master/deploy/v1.31.0 +# Azure Disk CSI Driver v1.33.5 +# Based on https://github.com/kubernetes-sigs/azuredisk-csi-driver/tree/master/deploy/v1.33.5 --- apiVersion: v1 @@ -130,7 +130,7 @@ spec: serviceAccountName: csi-azuredisk-controller-sa nodeSelector: kubernetes.io/os: linux - kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" priorityClassName: system-cluster-critical securityContext: seccompProfile: @@ -150,7 +150,7 @@ spec: effect: "NoSchedule" containers: - name: csi-provisioner - image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v5.1.0 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v5.3.0 args: - "--feature-gates=Topology=true,HonorPVReclaimPolicy=true" - "--csi-address=$(ADDRESS)" @@ -182,7 +182,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: csi-attacher - image: mcr.microsoft.com/oss/kubernetes-csi/csi-attacher:v4.7.0 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-attacher:v4.10.0 args: - "-v=2" - "-csi-address=$(ADDRESS)" @@ -211,7 +211,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: csi-snapshotter - image: mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v8.1.0 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v8.3.0 args: - "-csi-address=$(ADDRESS)" - "-leader-election" @@ -238,7 +238,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: csi-resizer - image: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v1.12.0 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v1.14.0 args: - "-csi-address=$(ADDRESS)" - "-v=2" @@ -266,7 +266,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: liveness-probe - image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.14.0 + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.17.0 args: - --csi-address=/csi/csi.sock - --probe-timeout=3s @@ -288,7 +288,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: azuredisk - image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.31.0 + image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.5 imagePullPolicy: IfNotPresent args: - "--v=5" @@ -392,7 +392,7 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.14.0 + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.17.0 args: - --csi-address=/csi/csi.sock - --probe-timeout=3s @@ -411,7 +411,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: node-driver-registrar - image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.12.0 + image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.14.0 args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) @@ -447,7 +447,7 @@ spec: - ALL readOnlyRootFilesystem: true - name: azuredisk - image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.31.0 + image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.5 imagePullPolicy: IfNotPresent args: - "--v=5" @@ -546,7 +546,7 @@ metadata: app.kubernetes.io/name: azuredisk-csi-driver app.kubernetes.io/component: csi-driver annotations: - csiDriver: v1.31.0 + csiDriver: v1.33.5 snapshot: v6.2.1 spec: attachRequired: true From 473517a1b9778e799ca647770c1bc8f6568b7b74 Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 14:05:10 -0400 Subject: [PATCH 09/11] azure: missing RBAC --- .../k8s-1.31.yaml.template | 214 +++++++++++++++--- 1 file changed, 180 insertions(+), 34 deletions(-) diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template index 2443342d4b2c8..b5828d9ae6318 100644 --- a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template +++ b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template @@ -73,6 +73,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -106,6 +109,154 @@ roleRef: name: csi-azuredisk-node-role apiGroup: rbac.authorization.k8s.io --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-attacher-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-attacher-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-attacher-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-snapshotter-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-snapshotter-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-resizer-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-resizer-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-resizer-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-controller-secret-role + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-controller-secret-binding + labels: + app.kubernetes.io/name: azuredisk-csi-driver + app.kubernetes.io/component: csi-driver +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-controller-secret-role + apiGroup: rbac.authorization.k8s.io +--- kind: Deployment apiVersion: apps/v1 metadata: @@ -130,7 +281,6 @@ spec: serviceAccountName: csi-azuredisk-controller-sa nodeSelector: kubernetes.io/os: linux - node-role.kubernetes.io/control-plane: "" priorityClassName: system-cluster-critical securityContext: seccompProfile: @@ -150,9 +300,9 @@ spec: effect: "NoSchedule" containers: - name: csi-provisioner - image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v5.3.0 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/csi-provisioner:v5.3.0 args: - - "--feature-gates=Topology=true,HonorPVReclaimPolicy=true" + - "--feature-gates=Topology=true,HonorPVReclaimPolicy=true,VolumeAttributesClass=true" - "--csi-address=$(ADDRESS)" - "--v=2" - "--timeout=30s" @@ -163,6 +313,7 @@ spec: - "--strict-topology=true" - "--kube-api-qps=50" - "--kube-api-burst=100" + - "--retry-interval-max=30m" env: - name: ADDRESS value: /csi/csi.sock @@ -176,13 +327,11 @@ spec: cpu: 10m memory: 20Mi securityContext: - allowPrivilegeEscalation: false capabilities: drop: - ALL - readOnlyRootFilesystem: true - name: csi-attacher - image: mcr.microsoft.com/oss/kubernetes-csi/csi-attacher:v4.10.0 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/csi-attacher:v4.10.0 args: - "-v=2" - "-csi-address=$(ADDRESS)" @@ -205,13 +354,11 @@ spec: cpu: 10m memory: 20Mi securityContext: - allowPrivilegeEscalation: false capabilities: drop: - ALL - readOnlyRootFilesystem: true - name: csi-snapshotter - image: mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v8.3.0 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/csi-snapshotter:v8.3.0 args: - "-csi-address=$(ADDRESS)" - "-leader-election" @@ -219,6 +366,8 @@ spec: - "--v=2" - "--timeout=1200s" - "--extra-create-metadata=true" + - "--retry-interval-max=30m" + - "--worker-threads=250" env: - name: ADDRESS value: /csi/csi.sock @@ -227,26 +376,25 @@ spec: mountPath: /csi resources: limits: - memory: 200Mi + memory: 400Mi requests: cpu: 10m memory: 20Mi securityContext: - allowPrivilegeEscalation: false capabilities: drop: - ALL - readOnlyRootFilesystem: true - name: csi-resizer - image: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v1.14.0 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/csi-resizer:v1.14.0 args: - "-csi-address=$(ADDRESS)" - "-v=2" - "-leader-election" - "--leader-election-namespace=kube-system" - '-handle-volume-inuse-error=false' - - '-feature-gates=RecoverVolumeExpansionFailure=true' + - '-feature-gates=RecoverVolumeExpansionFailure=true,VolumeAttributesClass=true' - "-timeout=240s" + - "--retry-interval-max=30m" env: - name: ADDRESS value: /csi/csi.sock @@ -260,16 +408,14 @@ spec: cpu: 10m memory: 20Mi securityContext: - allowPrivilegeEscalation: false capabilities: drop: - ALL - readOnlyRootFilesystem: true - name: liveness-probe - image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.17.0 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/livenessprobe:v2.17.0 args: - --csi-address=/csi/csi.sock - - --probe-timeout=3s + - --probe-timeout=10s - --http-endpoint=localhost:29602 - --v=2 volumeMounts: @@ -282,13 +428,11 @@ spec: cpu: 10m memory: 20Mi securityContext: - allowPrivilegeEscalation: false capabilities: drop: - ALL - readOnlyRootFilesystem: true - name: azuredisk - image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.5 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/azuredisk-csi:v1.33.5 imagePullPolicy: IfNotPresent args: - "--v=5" @@ -331,11 +475,9 @@ spec: cpu: 10m memory: 20Mi securityContext: - allowPrivilegeEscalation: false capabilities: drop: - ALL - readOnlyRootFilesystem: true volumes: - name: socket-dir emptyDir: {} @@ -392,10 +534,10 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.17.0 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/livenessprobe:v2.17.0 args: - --csi-address=/csi/csi.sock - - --probe-timeout=3s + - --probe-timeout=10s - --http-endpoint=localhost:29603 - --v=2 resources: @@ -405,13 +547,11 @@ spec: cpu: 10m memory: 20Mi securityContext: - allowPrivilegeEscalation: false capabilities: drop: - ALL - readOnlyRootFilesystem: true - name: node-driver-registrar - image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.14.0 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/csi-node-driver-registrar:v2.15.0 args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) @@ -441,13 +581,11 @@ spec: cpu: 10m memory: 20Mi securityContext: - allowPrivilegeEscalation: false capabilities: drop: - ALL - readOnlyRootFilesystem: true - name: azuredisk - image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.5 + image: mcr.microsoft.com/oss/v2/kubernetes-csi/azuredisk-csi:v1.33.5 imagePullPolicy: IfNotPresent args: - "--v=5" @@ -456,10 +594,14 @@ spec: - "--enable-perf-optimization=true" - "--allow-empty-cloud-config=true" - "--get-node-info-from-labels=false" + - "--metrics-address=0.0.0.0:29605" ports: - containerPort: 29603 name: healthz protocol: TCP + - containerPort: 29605 + name: metrics + protocol: TCP livenessProbe: failureThreshold: 5 httpGet: @@ -467,7 +609,7 @@ spec: path: /healthz port: 29603 initialDelaySeconds: 30 - timeoutSeconds: 10 + timeoutSeconds: 30 periodSeconds: 30 env: - name: AZURE_CREDENTIAL_FILE @@ -488,6 +630,10 @@ spec: capabilities: drop: - ALL + lifecycle: + preStop: + exec: + command: ["/azurediskplugin", "--pre-stop-hook=true"] volumeMounts: - mountPath: /csi name: socket-dir @@ -504,7 +650,7 @@ spec: name: sys-class resources: limits: - memory: 600Mi + memory: 1000Mi requests: cpu: 10m memory: 20Mi @@ -546,7 +692,7 @@ metadata: app.kubernetes.io/name: azuredisk-csi-driver app.kubernetes.io/component: csi-driver annotations: - csiDriver: v1.33.5 + csiDriver: v1.33.0 snapshot: v6.2.1 spec: attachRequired: true From a281207dde5d9facbbc6bc0365f5bb76f27354cc Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 18:03:09 -0400 Subject: [PATCH 10/11] azure: update template based on https://github.com/kubernetes-sigs/azuredisk-csi-driver/tree/master/deploy/v1.33.5 --- .../k8s-1.31.yaml.template | 611 ++++++++---------- 1 file changed, 275 insertions(+), 336 deletions(-) diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template index b5828d9ae6318..2925510fca960 100644 --- a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template +++ b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template @@ -2,31 +2,30 @@ # Based on https://github.com/kubernetes-sigs/azuredisk-csi-driver/tree/master/deploy/v1.33.5 --- -apiVersion: v1 -kind: ServiceAccount +apiVersion: storage.k8s.io/v1 +kind: CSIDriver metadata: - name: csi-azuredisk-controller-sa - namespace: kube-system - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver + name: disk.csi.azure.com + annotations: + csiDriver: v1.33.0 + snapshot: v6.2.1 +spec: + attachRequired: true + podInfoOnMount: false + fsGroupPolicy: File +--- --- apiVersion: v1 kind: ServiceAccount metadata: - name: csi-azuredisk-node-sa + name: csi-azuredisk-controller-sa namespace: kube-system - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver --- + kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-provisioner-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -56,34 +55,11 @@ rules: resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] --- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-azuredisk-node-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch"] ---- + kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-csi-provisioner-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa @@ -92,54 +68,41 @@ roleRef: kind: ClusterRole name: azuredisk-external-provisioner-role apiGroup: rbac.authorization.k8s.io + --- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-azuredisk-node-secret-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -subjects: - - kind: ServiceAccount - name: csi-azuredisk-node-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-azuredisk-node-role - apiGroup: rbac.authorization.k8s.io ---- + kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-attacher-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "patch"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] - verbs: ["patch"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] --- + kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-csi-attacher-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa @@ -148,18 +111,20 @@ roleRef: kind: ClusterRole name: azuredisk-external-attacher-role apiGroup: rbac.authorization.k8s.io + --- + kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-snapshotter-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] @@ -173,13 +138,11 @@ rules: resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] --- + kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-csi-snapshotter-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa @@ -189,37 +152,38 @@ roleRef: name: azuredisk-external-snapshotter-role apiGroup: rbac.authorization.k8s.io --- + kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-resizer-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "patch"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] + verbs: ["update", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: azuredisk-csi-resizer-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver + name: azuredisk-csi-resizer-role subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa @@ -228,26 +192,22 @@ roleRef: kind: ClusterRole name: azuredisk-external-resizer-role apiGroup: rbac.authorization.k8s.io + --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-azuredisk-controller-secret-role - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] + --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-azuredisk-controller-secret-binding - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa @@ -257,14 +217,237 @@ roleRef: name: csi-azuredisk-controller-secret-role apiGroup: rbac.authorization.k8s.io --- +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-node-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-node-role + apiGroup: rbac.authorization.k8s.io +--- +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-node + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-azuredisk-node + template: + metadata: + labels: + app: csi-azuredisk-node + spec: + hostNetwork: true + dnsPolicy: Default + serviceAccountName: csi-azuredisk-node-sa + nodeSelector: + kubernetes.io/os: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - operator: "Exists" + containers: + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: mcr.microsoft.com/oss/v2/kubernetes-csi/livenessprobe:v2.17.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=10s + - --http-endpoint=localhost:29603 + - --v=2 + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL + - name: node-driver-registrar + image: mcr.microsoft.com/oss/v2/kubernetes-csi/csi-node-driver-registrar:v2.15.0 + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL + - name: azuredisk + image: mcr.microsoft.com/oss/v2/kubernetes-csi/azuredisk-csi:v1.33.5 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--enable-perf-optimization=true" + - "--allow-empty-cloud-config=true" + - "--get-node-info-from-labels=false" + - "--metrics-address=0.0.0.0:29605" + ports: + - containerPort: 29603 + name: healthz + protocol: TCP + - containerPort: 29605 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + host: localhost + path: /healthz + port: 29603 + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + capabilities: + drop: + - ALL + lifecycle: + preStop: + exec: + command: ["/azurediskplugin", "--pre-stop-hook=true"] + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /dev + name: device-dir + - mountPath: /sys/bus/scsi/devices + name: sys-devices-dir + - mountPath: /sys/class/ + name: sys-class + resources: + limits: + memory: 1000Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/disk.csi.azure.com + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /sys/bus/scsi/devices + type: Directory + name: sys-devices-dir + - hostPath: + path: /sys/class/ + type: Directory + name: sys-class +--- +--- kind: Deployment apiVersion: apps/v1 metadata: name: csi-azuredisk-controller namespace: kube-system - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver spec: replicas: 2 selector: @@ -274,13 +457,11 @@ spec: metadata: labels: app: csi-azuredisk-controller - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver spec: hostNetwork: true serviceAccountName: csi-azuredisk-controller-sa nodeSelector: - kubernetes.io/os: linux + kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node priorityClassName: system-cluster-critical securityContext: seccompProfile: @@ -415,7 +596,7 @@ spec: image: mcr.microsoft.com/oss/v2/kubernetes-csi/livenessprobe:v2.17.0 args: - --csi-address=/csi/csi.sock - - --probe-timeout=10s + - --probe-timeout=3s - --http-endpoint=localhost:29602 - --v=2 volumeMounts: @@ -485,246 +666,4 @@ spec: hostPath: path: /etc/kubernetes/ type: DirectoryOrCreate ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-azuredisk-node - namespace: kube-system - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -spec: - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - app: csi-azuredisk-node - template: - metadata: - labels: - app: csi-azuredisk-node - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver - spec: - hostNetwork: true - dnsPolicy: Default - serviceAccountName: csi-azuredisk-node-sa - nodeSelector: - kubernetes.io/os: linux - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: type - operator: NotIn - values: - - virtual-kubelet - priorityClassName: system-node-critical - securityContext: - seccompProfile: - type: RuntimeDefault - tolerations: - - operator: "Exists" - containers: - - name: liveness-probe - volumeMounts: - - mountPath: /csi - name: socket-dir - image: mcr.microsoft.com/oss/v2/kubernetes-csi/livenessprobe:v2.17.0 - args: - - --csi-address=/csi/csi.sock - - --probe-timeout=10s - - --http-endpoint=localhost:29603 - - --v=2 - resources: - limits: - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi - securityContext: - capabilities: - drop: - - ALL - - name: node-driver-registrar - image: mcr.microsoft.com/oss/v2/kubernetes-csi/csi-node-driver-registrar:v2.15.0 - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=2 - livenessProbe: - exec: - command: - - /csi-node-driver-registrar - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --mode=kubelet-registration-probe - initialDelaySeconds: 30 - timeoutSeconds: 15 - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - resources: - limits: - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi - securityContext: - capabilities: - drop: - - ALL - - name: azuredisk - image: mcr.microsoft.com/oss/v2/kubernetes-csi/azuredisk-csi:v1.33.5 - imagePullPolicy: IfNotPresent - args: - - "--v=5" - - "--endpoint=$(CSI_ENDPOINT)" - - "--nodeid=$(KUBE_NODE_NAME)" - - "--enable-perf-optimization=true" - - "--allow-empty-cloud-config=true" - - "--get-node-info-from-labels=false" - - "--metrics-address=0.0.0.0:29605" - ports: - - containerPort: 29603 - name: healthz - protocol: TCP - - containerPort: 29605 - name: metrics - protocol: TCP - livenessProbe: - failureThreshold: 5 - httpGet: - host: localhost - path: /healthz - port: 29603 - initialDelaySeconds: 30 - timeoutSeconds: 30 - periodSeconds: 30 - env: - - name: AZURE_CREDENTIAL_FILE - valueFrom: - configMapKeyRef: - name: azure-cred-file - key: path - optional: true - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - securityContext: - privileged: true - capabilities: - drop: - - ALL - lifecycle: - preStop: - exec: - command: ["/azurediskplugin", "--pre-stop-hook=true"] - volumeMounts: - - mountPath: /csi - name: socket-dir - - mountPath: /var/lib/kubelet/ - mountPropagation: Bidirectional - name: mountpoint-dir - - mountPath: /etc/kubernetes/ - name: azure-cred - - mountPath: /dev - name: device-dir - - mountPath: /sys/bus/scsi/devices - name: sys-devices-dir - - mountPath: /sys/class/ - name: sys-class - resources: - limits: - memory: 1000Mi - requests: - cpu: 10m - memory: 20Mi - volumes: - - hostPath: - path: /var/lib/kubelet/plugins/disk.csi.azure.com - type: DirectoryOrCreate - name: socket-dir - - hostPath: - path: /var/lib/kubelet/ - type: DirectoryOrCreate - name: mountpoint-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - name: registration-dir - - hostPath: - path: /etc/kubernetes/ - type: DirectoryOrCreate - name: azure-cred - - hostPath: - path: /dev - type: Directory - name: device-dir - - hostPath: - path: /sys/bus/scsi/devices - type: Directory - name: sys-devices-dir - - hostPath: - path: /sys/class/ - type: Directory - name: sys-class ---- -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: disk.csi.azure.com - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver - annotations: - csiDriver: v1.33.0 - snapshot: v6.2.1 -spec: - attachRequired: true - podInfoOnMount: false - fsGroupPolicy: File ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: managed-csi - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver - annotations: - storageclass.kubernetes.io/is-default-class: "true" -provisioner: disk.csi.azure.com -parameters: - skuName: StandardSSD_LRS -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: managed-csi-premium - labels: - app.kubernetes.io/name: azuredisk-csi-driver - app.kubernetes.io/component: csi-driver -provisioner: disk.csi.azure.com -parameters: - skuName: Premium_LRS -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true +--- \ No newline at end of file From eed90f14419113edc213e52d0386eafb21d72fde Mon Sep 17 00:00:00 2001 From: ThomasCardin Date: Thu, 16 Oct 2025 19:21:50 -0400 Subject: [PATCH 11/11] azure: test csi-azuredisk-controller, azuredisk with --allow-empty-cloud-config=true --- .../azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template index 2925510fca960..218f42ceeaaf2 100644 --- a/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template +++ b/upup/models/cloudup/resources/addons/azuredisk-csi-driver.addons.k8s.io/k8s-1.31.yaml.template @@ -461,7 +461,7 @@ spec: hostNetwork: true serviceAccountName: csi-azuredisk-controller-sa nodeSelector: - kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node + kubernetes.io/os: linux # add "node-role.kubernetes.io/control-plane: "" " to run controller on master node priorityClassName: system-cluster-critical securityContext: seccompProfile: @@ -621,7 +621,7 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=OSS-kubectl" - "--disable-avset-nodes=false" - - "--allow-empty-cloud-config=false" + - "--allow-empty-cloud-config=true" ports: - containerPort: 29604 name: metrics