ARO-25108: Add v1api20251223preview API support for HcpOpenShift resources (#145)

marek-veber · web-flow · commit c452227553c3 · 2026-04-08T09:41:27.000Z
* use 2025-12-23-preview as a storage for HCPOpenShift* resources

* add CHANGELOG/v1.22.1-mce-217.md
diff --git a/CHANGELOG/v1.22.1-mce-217.md b/CHANGELOG/v1.22.1-mce-217.md
@@ -0,0 +1,56 @@
+## Changes by Kind
+
+### Feature
+
+- **ARO HCP API v1api20251223preview support**: Full support for the new `2025-12-23-preview` ARM API version for HcpOpenShiftClusters, HcpOpenShiftClustersNodePools, and HcpOpenShiftClustersExternalAuth resources
+  - Both `v1api20240610preview` and `v1api20251223preview` API versions are supported simultaneously
+  - CAPZ auto-detects the API version used and handles structural differences (e.g., KMS `vaultName` field location)
+  - Updated ASO CRDs with `v1api20251223previewstorage` as the new storage version for HcpOpenShift resources
+
+### Bug Fixes
+
+- **ASO resource reconciliation conflict fix (ARO-25529)**: Stop continuously re-patching ASO resources that are already ready, preventing conflicts with ASO controllers (etcd commit failures and generation inflation)
+  - Skip resource reconciliation when all resources are ready and the spec hasn't changed
+  - Use `ObservedGeneration` on the `ResourcesReady` condition to detect spec changes
+  - Add `ASOReadyChangedPredicate` to watch for ASO Ready condition changes on managed resources
+  - Remove unnecessary `RequeueAfter` polling in favor of event-driven watches
+- **ExternalAuth deletion fix**: Fix ExternalAuth deletion when NodePool is in failed state
+- **KeyVault bombardment fix**: Add predicate to dynamic watches to prevent excessive KeyVault reconciliation
+- **AROMachinePool autoscaler fix**: Fix autoscaling and API version handling for AROMachinePool
+- **Encryption key readiness fix**: Properly defer HcpOpenShiftCluster apply until encryption key version is available, preventing CRD validation errors
+- **Graceful error handling**: Handle NotFound and NoMatch errors gracefully in AROMachinePool and AROControlPlane reconcilers
+
+### Dependencies
+
+- Updated ASO CRDs to include `v1api20251223preview` and `v1api20251223previewstorage` versions
+- Migrated HcpOpenShift resource storage version from `v1api20240610previewstorage` to `v1api20251223previewstorage`
+
+## API Changes (v1api20240610preview vs v1api20251223preview)
+
+### HcpOpenShiftCluster - KMS encryption structure
+
+**v1api20240610preview** (old):
+```yaml
+kms:
+  activeKey:
+    name: "key-name"
+    vaultName: "vault-name"    # vaultName inside activeKey
+    version: "key-version"
+```
+
+**v1api20251223preview** (new):
+```yaml
+kms:
+  activeKey:
+    name: "key-name"
+    version: "key-version"
+  vaultName: "vault-name"      # vaultName moved to kms level
+  visibility: "Public"         # new required field
+```
+
+## Details
+
+This release is based on upstream v1.22.0 with Red Hat MCE-specific additions for ARO-HCP support.
+
+Base upstream version: v1.22.0
+MCE version: 2.17
diff --git a/Makefile b/Makefile
@@ -175,8 +175,8 @@ CRD_ROOT ?= $(MANIFEST_ROOT)/crd/bases
 WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
 RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml
-ASO_VERSION := v2.13.0-hcpclusters.3
-ASO_WORKSPACE := marek-veber
+ASO_VERSION := v2.13.0-hcpclusters.7
+ASO_WORKSPACE := stolostron
 ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com fleetsmembers.containerservice.azure.com extensions.kubernetesconfiguration.azure.com userassignedidentities.managedidentity.azure.com roleassignments.authorization.azure.com networksecuritygroups.network.azure.com vaults.keyvault.azure.com hcpopenshiftclusters.redhatopenshift.azure.com hcpopenshiftclustersnodepools.redhatopenshift.azure.com hcpopenshiftclustersexternalauths.redhatopenshift.azure.com
 
 # Allow overriding the imagePullPolicy
diff --git a/config/aso/crds.yaml b/config/aso/crds.yaml
@@ -6,7 +6,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: bastionhosts.network.azure.com
 spec:
   conversion:
@@ -1703,7 +1703,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: extensions.kubernetesconfiguration.azure.com
 spec:
   conversion:
@@ -4176,7 +4176,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: fleetsmembers.containerservice.azure.com
 spec:
   conversion:
@@ -4738,7 +4738,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: hcpopenshiftclusters.redhatopenshift.azure.com
 spec:
   conversion:
@@ -6671,7 +6671,7 @@ spec:
               type: object
           type: object
       served: true
-      storage: true
+      storage: false
       subresources:
         status: {}
     - additionalPrinterColumns:
@@ -7036,15 +7036,45 @@ spec:
                           pulls targeting the specified source registries.
                         properties:
                           mirrors:
-                            description: 'Mirrors: Mirrors are one or more image repositories that may also contain the same images.'
+                            description: |-
+                              Mirrors: mirrors is zero or more locations that may also contain the same images. No mirror will
+                              be configured if not specified. Images can be pulled from these mirrors only if they are
+                              referenced by their digests. The mirrored location is obtained by replacing the part of
+                              the input reference that matches source by the mirrors entry, e.g. for
+                              registry.redhat.io/product/repo reference, a (source, mirror) pair *.redhat.io,
+                              mirror.local/redhat causes a mirror.local/redhat/product/repo repository to be used.
+                              The order of mirrors in this list is treated as the user's desired priority, while source
+                              is by default considered lower priority than all mirrors.
+                              If no mirror is specified or all image pulls from the mirror list fail, the image will
+                              continue to be pulled from the repository in the pull spec.
+                              Other cluster configuration, including (but not limited to) other imageDigestMirrors
+                              objects, may impact the exact order mirrors are contacted in, or some mirrors may be
+                              contacted in parallel, so this should be considered a preference rather than a guarantee
+                              of ordering.
+                              mirrors uses one of the following formats:
+                              * host[:port]
+                              * host[:port]/namespace[/namespace...]
+                              * host[:port]/namespace[/namespace...]/repo
+                              for more information about the format, see:
+                              https://github.com/containers/image/blob/main/docs/containers-registries.conf.5.md#choosing-a-registry-toml-table
                             items:
                               maxLength: 255
                               type: string
                             maxItems: 255
                             minItems: 1
                             type: array
                           source:
-                            description: 'Source: Source is the image repository that users refer to, e.g. in image pull specifications.'
+                            description: |-
+                              Source: source matches the repository that users refer to, e.g. in image pull specifications.
+                              Setting source to a registry hostname, e.g. docker.io, quay.io, or registry.redhat.io,
+                              will match the image pull specification of the corresponding registry.
+                              source uses one of the following formats:
+                              * host[:port]
+                              * host[:port]/namespace[/namespace...]
+                              * host[:port]/namespace[/namespace...]/repo
+                              * [*.]host
+                              for more information about the format, see:
+                              https://github.com/containers/image/blob/main/docs/containers-registries.conf.5.md#choosing-a-registry-toml-table
                             maxLength: 255
                             type: string
                         required:
@@ -7543,15 +7573,45 @@ spec:
                           pulls targeting the specified source registries.
                         properties:
                           mirrors:
-                            description: 'Mirrors: Mirrors are one or more image repositories that may also contain the same images.'
+                            description: |-
+                              Mirrors: mirrors is zero or more locations that may also contain the same images. No mirror will
+                              be configured if not specified. Images can be pulled from these mirrors only if they are
+                              referenced by their digests. The mirrored location is obtained by replacing the part of
+                              the input reference that matches source by the mirrors entry, e.g. for
+                              registry.redhat.io/product/repo reference, a (source, mirror) pair *.redhat.io,
+                              mirror.local/redhat causes a mirror.local/redhat/product/repo repository to be used.
+                              The order of mirrors in this list is treated as the user's desired priority, while source
+                              is by default considered lower priority than all mirrors.
+                              If no mirror is specified or all image pulls from the mirror list fail, the image will
+                              continue to be pulled from the repository in the pull spec.
+                              Other cluster configuration, including (but not limited to) other imageDigestMirrors
+                              objects, may impact the exact order mirrors are contacted in, or some mirrors may be
+                              contacted in parallel, so this should be considered a preference rather than a guarantee
+                              of ordering.
+                              mirrors uses one of the following formats:
+                              * host[:port]
+                              * host[:port]/namespace[/namespace...]
+                              * host[:port]/namespace[/namespace...]/repo
+                              for more information about the format, see:
+                              https://github.com/containers/image/blob/main/docs/containers-registries.conf.5.md#choosing-a-registry-toml-table
                             items:
                               maxLength: 255
                               type: string
                             maxItems: 255
                             minItems: 1
                             type: array
                           source:
-                            description: 'Source: Source is the image repository that users refer to, e.g. in image pull specifications.'
+                            description: |-
+                              Source: source matches the repository that users refer to, e.g. in image pull specifications.
+                              Setting source to a registry hostname, e.g. docker.io, quay.io, or registry.redhat.io,
+                              will match the image pull specification of the corresponding registry.
+                              source uses one of the following formats:
+                              * host[:port]
+                              * host[:port]/namespace[/namespace...]
+                              * host[:port]/namespace[/namespace...]/repo
+                              * [*.]host
+                              for more information about the format, see:
+                              https://github.com/containers/image/blob/main/docs/containers-registries.conf.5.md#choosing-a-registry-toml-table
                             maxLength: 255
                             type: string
                         required:
@@ -8805,7 +8865,7 @@ spec:
               type: object
           type: object
       served: true
-      storage: false
+      storage: true
       subresources:
         status: {}
 ---
@@ -8817,7 +8877,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: hcpopenshiftclustersexternalauths.redhatopenshift.azure.com
 spec:
   conversion:
@@ -10088,7 +10148,7 @@ spec:
               type: object
           type: object
       served: true
-      storage: true
+      storage: false
       subresources:
         status: {}
     - additionalPrinterColumns:
@@ -11341,7 +11401,7 @@ spec:
               type: object
           type: object
       served: true
-      storage: false
+      storage: true
       subresources:
         status: {}
 ---
@@ -11353,7 +11413,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: hcpopenshiftclustersnodepools.redhatopenshift.azure.com
 spec:
   conversion:
@@ -12636,7 +12696,7 @@ spec:
               type: object
           type: object
       served: true
-      storage: true
+      storage: false
       subresources:
         status: {}
     - additionalPrinterColumns:
@@ -13960,7 +14020,7 @@ spec:
               type: object
           type: object
       served: true
-      storage: false
+      storage: true
       subresources:
         status: {}
 ---
@@ -13972,7 +14032,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: managedclusters.containerservice.azure.com
 spec:
   conversion:
@@ -54955,7 +55015,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: managedclustersagentpools.containerservice.azure.com
 spec:
   conversion:
@@ -67852,7 +67912,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: natgateways.network.azure.com
 spec:
   conversion:
@@ -69326,7 +69386,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: networksecuritygroups.network.azure.com
 spec:
   conversion:
@@ -70445,7 +70505,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: privateendpoints.network.azure.com
 spec:
   conversion:
@@ -73125,7 +73185,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: resourcegroups.resources.azure.com
 spec:
   conversion:
@@ -73594,7 +73654,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: roleassignments.authorization.azure.com
 spec:
   conversion:
@@ -74893,7 +74953,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: userassignedidentities.managedidentity.azure.com
 spec:
   conversion:
@@ -76200,7 +76260,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: vaults.keyvault.azure.com
 spec:
   conversion:
@@ -79177,7 +79237,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: virtualnetworks.network.azure.com
 spec:
   conversion:
@@ -81023,7 +81083,7 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.19.0
   labels:
     app.kubernetes.io/name: azure-service-operator
-    app.kubernetes.io/version: v2.13.0-hcpclusters.3
+    app.kubernetes.io/version: v2.13.0-hcpclusters.7
   name: virtualnetworkssubnets.network.azure.com
 spec:
   conversion:
