Fix ODCR usage with Flexible Instance Types

1. Move the code to generate RunInstances with ODCR policies to a better place to clean up the code.

2. Add CreateFleet with Capacity Reservation Resource Groups ARN policy and ListGroupResources policy to head node. These policies are required to create_fleet with Capacity Reservation Resource Group.

3. Only run validator against the first instance type of the InstanceTypes

4. Make test_efa test to use Capacity Reservation Resource Groups ARN

Signed-off-by: Hanwen <[email protected]>

Author: hanwen-cluster

cli/src/pcluster/config/cluster_config.py

@@ -2464,13 +2464,17 @@ def _register_validators(self):
                     self._register_validator(
                         CapacityReservationValidator,
                         capacity_reservation_id=cr_target.capacity_reservation_id,
-                        instance_type=compute_resource.instance_type,
+                        # ToDo: This validator is only correct for single instance type
+                        #  Add more validators to be check ODCR with flexible instance types
+                        instance_type=compute_resource.instance_types[0],
                         subnet=queue.networking.subnet_ids[0],
                     )
                     self._register_validator(
                         CapacityReservationResourceGroupValidator,
                         capacity_reservation_resource_group_arn=cr_target.capacity_reservation_resource_group_arn,
-                        instance_type=compute_resource.instance_type,
+                        # ToDo: This validator is only correct for single instance type
+                        #  Add more validators to be check ODCR with flexible instance types
+                        instance_type=compute_resource.instance_types[0],
                         subnet=queue.networking.subnet_ids[0],
                     )
 

cli/src/pcluster/templates/cdk_builder_utils.py

@@ -625,6 +625,42 @@ def _build_policy(self) -> List[iam.PolicyStatement]:
                     ),
                 ]
             )
+            capacity_reservation_ids = self._config.capacity_reservation_ids
+            if self._config.capacity_reservation_ids:
+                policy.append(
+                    iam.PolicyStatement(
+                        actions=["ec2:RunInstances"],
+                        effect=iam.Effect.ALLOW,
+                        resources=[
+                            self._format_arn(
+                                service="ec2",
+                                resource=f"capacity-reservation/{capacity_reservation_id}",
+                            )
+                            for capacity_reservation_id in capacity_reservation_ids
+                        ],
+                    )
+                )
+            capacity_reservation_resource_group_arns = self._config.capacity_reservation_resource_group_arns
+            if capacity_reservation_resource_group_arns:
+                policy.extend(
+                    [
+                        iam.PolicyStatement(
+                            actions=["ec2:RunInstances"],
+                            effect=iam.Effect.ALLOW,
+                            resources=capacity_reservation_resource_group_arns,
+                        ),
+                        iam.PolicyStatement(
+                            actions=["ec2:CreateFleet"],
+                            effect=iam.Effect.ALLOW,
+                            resources=capacity_reservation_resource_group_arns,
+                        ),
+                        iam.PolicyStatement(
+                            actions=["resource-groups:ListGroupResources"],
+                            effect=iam.Effect.ALLOW,
+                            resources=capacity_reservation_resource_group_arns,
+                        ),
+                    ]
+                )
 
         if self._config.scheduling.scheduler == "plugin":
             cluster_shared_artifacts = get_attr(
@@ -654,32 +690,6 @@ def _build_policy(self) -> List[iam.PolicyStatement]:
                             ]
                         )
 
-        if self._config.scheduling.scheduler != "awsbatch":
-            capacity_reservation_ids = self._config.capacity_reservation_ids
-            if self._config.capacity_reservation_ids:
-                policy.append(
-                    iam.PolicyStatement(
-                        actions=["ec2:RunInstances"],
-                        effect=iam.Effect.ALLOW,
-                        resources=[
-                            self._format_arn(
-                                service="ec2",
-                                resource=f"capacity-reservation/{capacity_reservation_id}",
-                            )
-                            for capacity_reservation_id in capacity_reservation_ids
-                        ],
-                    )
-                )
-            capacity_reservation_resource_group_arns = self._config.capacity_reservation_resource_group_arns
-            if capacity_reservation_resource_group_arns:
-                policy.append(
-                    iam.PolicyStatement(
-                        actions=["ec2:RunInstances"],
-                        effect=iam.Effect.ALLOW,
-                        resources=capacity_reservation_resource_group_arns,
-                    )
-                )
-
         if self._config.directory_service:
             policy.append(
                 iam.PolicyStatement(

tests/integration-tests/tests/efa/test_efa/test_efa/pcluster.config.yaml

@@ -30,7 +30,7 @@ Scheduling:
             {% if instance == "p4d.24xlarge" %}GdrSupport: true{% endif %}
 {% if instance == "p4d.24xlarge" %}
           CapacityReservationTarget:
-            CapacityReservationId: cr-0fa65fcdbd597f551
+            CapacityReservationResourceGroupArn: arn:aws:resource-groups:us-east-1:447714826191:group/EC2CRGroup
 {% endif %}
 SharedStorage:
   - MountDir: /shared