CI: testing AWS confidential VMs

.github/workflows/e2e_aws.yaml

@@ -68,6 +68,8 @@ jobs:
     needs: aws-credentials
     if: needs.aws-credentials.outputs.has_secrets == 'true'
     runs-on: ubuntu-22.04
+    # Remove this when the job becomes stable
+    continue-on-error: ${{ inputs.cluster_type == 'eks' }}
     defaults:
       run:
         working-directory: src/cloud-api-adaptor
@@ -107,6 +109,11 @@ jobs:
           echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV"
           echo "ORAS_VERSION=$(yq -e '.tools.oras' versions.yaml)" >> "$GITHUB_ENV"
 
+      - name: Install qemu-img
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y qemu-utils
+
       - name: Setup Golang version ${{ env.GO_VERSION }}
         uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
         with:
@@ -152,12 +159,29 @@ jobs:
           cat kustomization.yaml
           echo "::endgroup::"
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
+        with:
+          aws-region: us-east-1
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }}
+          role-duration-seconds: 7200
+
       - name: Config aws
         run: |
+          if [[ "${CLUSTER_TYPE}" = "eks" ]]; then
+            DISABLE_CVM="false"
+            AWS_REGION="us-east-2"
+            echo "AWS_REGION=${AWS_REGION}" >> "$GITHUB_ENV"
+            # m6a.large (AMD SEV-SNP) is available in us-east-2
+            echo "aws_region=\"${AWS_REGION}\"" >> aws.properties
+            echo "podvm_aws_instance_type=\"m6a.large\"" >> aws.properties
+          else
+            DISABLE_CVM="true"
+          fi
           cat <<EOF>>aws.properties
           CAA_IMAGE="${CAA_IMAGE}"
           container_runtime="${CONTAINER_RUNTIME}"
-          disablecvm="true"
+          disablecvm="${DISABLE_CVM}"
           cluster_type="${CLUSTER_TYPE}"
           ssh_kp_name="caa-e2e-test"
           resources_basename="${RESOURCES_BASENAME}"
@@ -167,13 +191,6 @@ jobs:
           cat aws.properties
           echo "::endgroup::"
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
-        with:
-          aws-region: us-east-1
-          role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }}
-          role-duration-seconds: 7200
-
       - name: Create on-prem cluster
         if: inputs.cluster_type == 'onprem'
         run: |
@@ -188,6 +205,13 @@ jobs:
           echo "KUBECONFIG=$HOME/.kcli/clusters/peer-pods/auth/kubeconfig" >> "$GITHUB_ENV"
           echo "::endgroup::"
 
+      - name: Install eksctl
+        if: inputs.cluster_type == 'eks'
+        run: |
+          curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz"
+          tar -xzf eksctl_Linux_amd64.tar.gz -C /usr/local/bin && rm eksctl_Linux_amd64.tar.gz
+          eksctl version
+
       - name: run tests
         id: runTests
         run: |

.github/workflows/e2e_run_all.yaml

@@ -223,21 +223,25 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        container_runtime:
-          - crio
-        os:
-          - ubuntu
-        provider:
-          - generic
-        arch:
-          - amd64
+        include:
+          - container_runtime: crio
+            cluster_type: onprem
+            os: ubuntu
+            provider: generic
+            arch: amd64
+          - container_runtime: containerd
+            cluster_type: eks
+            os: ubuntu
+            provider: generic
+            arch: amd64
     permissions:
       id-token: write # Required by aws-actions/configure-aws-credentials
       contents: read  # Required by aws-actions/configure-aws-credentials
     uses: ./.github/workflows/e2e_aws.yaml
     with:
       caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}
       container_runtime: ${{ matrix.container_runtime }}
+      cluster_type: ${{ matrix.cluster_type }}
       podvm_image: ${{ inputs.registry }}/podvm-${{ matrix.provider }}-${{ matrix.os }}-${{ matrix.arch }}:${{ inputs.podvm_image_tag }}
       git_ref: ${{ inputs.git_ref }}
       oras: false

hack/ci-e2e-aws-cleanup.sh

@@ -13,35 +13,46 @@ if [ -z "${RESOURCES_BASENAME:-}" ]; then
   exit 1
 fi
 
+AWS_REGION=${AWS_REGION:-"us-east-1"}
+CLUSTER_TYPE=${CLUSTER_TYPE:-onprem}
+
 delete_vpcs() {
+  if [ "${CLUSTER_TYPE}" = "eks" ]; then
+    local cluster_name="${RESOURCES_BASENAME}-k8s"
+    if aws eks describe-cluster --name "$cluster_name" --region "${AWS_REGION}" >/dev/null 2>&1; then
+      echo "cluster_type=\"eks\"" >> "$TEST_PROVISION_FILE"
+      echo "eks_name=\"${cluster_name}\"" >> "$TEST_PROVISION_FILE"
+    fi
+  fi
+
   local tag_vpc="${RESOURCES_BASENAME}-vpc"
-  read -r -a vpcs <<< "$(aws  ec2 describe-vpcs --filters Name=tag:Name,Values=$tag_vpc --query 'Vpcs[*].VpcId' --output text)"
+  read -r -a vpcs <<< "$(aws  ec2 describe-vpcs --filters Name=tag:Name,Values=$tag_vpc --query 'Vpcs[*].VpcId' --region "${AWS_REGION}" --output text)"
 
   if [ ${#vpcs[@]} -eq 0 ]; then
-    echo "There aren't VPCs to delete"
+    echo "There aren't VPCs to delete in ${AWS_REGION}"
     return
   fi
 
   for vpc in "${vpcs[@]}"; do
-    echo "aws_vpc_id=\"$vpc\"" > "$TEST_PROVISION_FILE"
+    echo "aws_vpc_id=\"$vpc\"" >> "$TEST_PROVISION_FILE"
 
     # Find related subnets
-    read -r -a subnets <<< "$(aws ec2 describe-subnets --filter "Name=vpc-id,Values=$vpc" --query 'Subnets[*].SubnetId' --output text)"
-    for net in "${subnets[@]}"; do
-      echo "aws_vpc_subnet_id=\"$net\"" >> "$TEST_PROVISION_FILE"
-    done
+    read -r -a subnets <<< "$(aws ec2 describe-subnets --filter "Name=vpc-id,Values=$vpc" --query 'Subnets[*].SubnetId' --region "${AWS_REGION}" --output text)"
+    if [ ${#subnets[@]} -gt 0 ]; then
+      echo "aws_vpc_subnet_id=\"$(echo "${subnets[*]}" | tr ' ' ',')\"" >> "$TEST_PROVISION_FILE"
+    fi
 
     # Find related security groups
-    read -r -a sgs <<< "$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$vpc" "Name=tag:Name,Values=${RESOURCES_BASENAME}-sg" --query 'SecurityGroups[*].GroupId' --output text)"
+    read -r -a sgs <<< "$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$vpc" "Name=tag:Name,Values=${RESOURCES_BASENAME}-sg" --query 'SecurityGroups[*].GroupId' --region "${AWS_REGION}" --output text)"
     for sg in "${sgs[@]}"; do
       echo "aws_vpc_sg_id=\"$sg\"" >> "$TEST_PROVISION_FILE"
     done
 
     # Find related route tables and internet gateways
-    read -r -a rtbs <<< "$(aws ec2 describe-route-tables --filters "Name=vpc-id,Values=$vpc" "Name=tag:Name,Values=${RESOURCES_BASENAME}-rtb" --query 'RouteTables[*].RouteTableId' --output text)"
+    read -r -a rtbs <<< "$(aws ec2 describe-route-tables --filters "Name=vpc-id,Values=$vpc" "Name=tag:Name,Values=${RESOURCES_BASENAME}-rtb" --query 'RouteTables[*].RouteTableId' --region "${AWS_REGION}" --output text)"
     for rtb in "${rtbs[@]}"; do
       echo "aws_vpc_rt_id=\"$rtb\"" >> "$TEST_PROVISION_FILE"
-      read -r -a igws <<< "$(aws ec2 describe-route-tables --filter "Name=route-table-id,Values=$rtb" --query 'RouteTables[0].Routes[*].GatewayId' --output text)"
+      read -r -a igws <<< "$(aws ec2 describe-route-tables --filter "Name=route-table-id,Values=$rtb" --query 'RouteTables[0].Routes[*].GatewayId' --region "${AWS_REGION}" --output text)"
       for igw in "${igws[@]}"; do
         [ "$igw" != "local" ] && echo "aws_vpc_igw_id=\"$igw\"" >> "$TEST_PROVISION_FILE"
       done
@@ -55,52 +66,52 @@ delete_vpcs() {
 delete_amis() {
   local tag_ami="${RESOURCES_BASENAME}-img"
 
-  read -r -a amis <<< "$(aws ec2 describe-images --owners self --filters "Name=tag:Name,Values=$tag_ami" --query 'Images[*].ImageId' --output text)"
+  read -r -a amis <<< "$(aws ec2 describe-images --owners self --filters "Name=tag:Name,Values=$tag_ami" --query 'Images[*].ImageId' --region "${AWS_REGION}" --output text)"
 
   if [ ${#amis[@]} -eq 0 ]; then
-    echo "There aren't AMIs to delete."
+    echo "There aren't AMIs to delete in ${AWS_REGION}."
     return
   fi
 
   for ami in "${amis[@]}"; do
     echo "Deregistering AMI: $ami"
     # Find related snapshots
-    snap_ids=$(aws ec2 describe-images --image-ids "$ami" --query 'Images[*].BlockDeviceMappings[*].Ebs.SnapshotId' --output text)
-    aws ec2 deregister-image --image-id "$ami"
+    snap_ids=$(aws ec2 describe-images --image-ids "$ami" --query 'Images[*].BlockDeviceMappings[*].Ebs.SnapshotId' --region "${AWS_REGION}" --output text)
+    aws ec2 deregister-image --image-id "$ami" --region "${AWS_REGION}"
     for snap in $snap_ids; do
       echo "Deleting snapshot: $snap"
-      aws ec2 delete-snapshot --snapshot-id "$snap"
+      aws ec2 delete-snapshot --snapshot-id "$snap" --region "${AWS_REGION}"
     done
   done
 
   # Delete the vmimport role if it exists
   local vmimport_role="${RESOURCES_BASENAME}-vmimport"
-  if aws iam get-role --role-name "$vmimport_role" >/dev/null 2>&1; then
+  if aws iam get-role --role-name "$vmimport_role" --region "${AWS_REGION}" >/dev/null 2>&1; then
     echo "Deleting vmimport role: $vmimport_role"
     # First delete the role policy
-    aws iam delete-role-policy --role-name "$vmimport_role" --policy-name "vmimport" 2>/dev/null || true
+    aws iam delete-role-policy --role-name "$vmimport_role" --policy-name "vmimport" --region "${AWS_REGION}" 2>/dev/null || true
     # Then delete the role
-    aws iam delete-role --role-name "$vmimport_role" 2>/dev/null || true
+    aws iam delete-role --role-name "$vmimport_role" --region "${AWS_REGION}" 2>/dev/null || true
   fi
 }
 
 delete_s3_buckets() {
   local tag_bucket="${RESOURCES_BASENAME}-bucket"
 
   # List all buckets and find ones that match our naming pattern
-  read -r -a buckets <<< "$(aws s3api list-buckets --query "Buckets[?contains(Name, '${tag_bucket}')].Name" --output text)"
+  read -r -a buckets <<< "$(aws s3api list-buckets --query "Buckets[?contains(Name, '${tag_bucket}')].Name" --region "${AWS_REGION}" --output text)"
 
   if [ ${#buckets[@]} -eq 0 ]; then
-    echo "There aren't S3 buckets to delete."
+    echo "There aren't S3 buckets to delete in ${AWS_REGION}."
     return
   fi
 
   for bucket in "${buckets[@]}"; do
     echo "Deleting S3 bucket: $bucket"
     # First, delete all objects in the bucket
-    aws s3 rm "s3://$bucket" --recursive 2>/dev/null || true
+    aws s3 rm "s3://$bucket" --recursive --region "${AWS_REGION}" 2>/dev/null || true
     # Then delete the bucket
-    aws s3api delete-bucket --bucket "$bucket" 2>/dev/null || true
+    aws s3api delete-bucket --bucket "$bucket" --region "${AWS_REGION}" 2>/dev/null || true
   done
 }
 
@@ -113,7 +124,7 @@ main() {
 
   echo "Build the caa-provisioner-cli tool"
   cd "${script_dir}/../src/cloud-api-adaptor/test/tools" || exit 1
-  make
+  make BUILTIN_CLOUD_PROVIDERS=aws
 
   delete_vpcs
   delete_amis

src/cloud-api-adaptor/go.mod

@@ -67,7 +67,6 @@ require (
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/service/eks v1.76.3
 	github.com/aws/aws-sdk-go-v2/service/iam v1.53.1
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2
 	github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers v0.0.0-00010101000000-000000000000

src/cloud-api-adaptor/go.sum

@@ -168,8 +168,6 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.1 h1:P7db/Z55pXvwnueLuHUuVlxnqjbAtiadm01+QIC42OA=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.1/go.mod h1:Wg68QRgy2gEGGdmTPU/UbVpdv8sM14bUZmF64KFwAsY=
-github.com/aws/aws-sdk-go-v2/service/eks v1.76.3 h1:840uwcJTIwrMPLuEUQVFKZbPgwnYzc5WDyXMiMYm5Ts=
-github.com/aws/aws-sdk-go-v2/service/eks v1.76.3/go.mod h1:7IU8o/Snul26xioEWN5tgoOas1ISPGsiq5gME5rPh3o=
 github.com/aws/aws-sdk-go-v2/service/iam v1.53.1 h1:xNCUk9XN6Pa9PyzbEfzgRpvEIVlqtth402yjaWvNMu4=
 github.com/aws/aws-sdk-go-v2/service/iam v1.53.1/go.mod h1:GNQZL4JRSGH6L0/SNGOtffaB1vmlToYp3KtcUIB0NhI=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=

src/cloud-api-adaptor/test/e2e/README.md

@@ -129,12 +129,14 @@ Use the properties on the table below for AWS:
 |aws_vpc_igw_id|AWS VPC Internet Gateway ID||
 |aws_vpc_rt_id|AWS VPC Route Table ID||
 |aws_vpc_sg_id|AWS VPC Security Groups ID||
-|aws_vpc_subnet_id|AWS VPC Subnet ID||
+|aws_vpc_subnet_id|AWS VPC Subnet ID. Use comma to pass a secondary subnet for EKS (e.g. 'subnet_id,secondary_subnet_id')||
 |cluster_type|Kubernetes cluster type. Either **onprem** or **eks** (see Notes below) |onprem|
 |container_runtime|Test cluster configured container runtime. Either **containerd** or **crio** |containerd|
 |disablecvm|Set to `true` to disable confidential VM||
+|eks_name|Name an existing EKS (don't provision a new)||
 |pause_image|Kubernetes pause image||
 |podvm_aws_ami_id|AWS AMI ID of the podvm||
+|podvm_aws_instance_type|AWS instance type of the podvm|t2.medium|
 |ssh_kp_name|AWS SSH key-pair name ||
 |use_public_ip|Set `true` to instantiate VMs with public IP. If `cluster_type=onprem` then this property is implictly applied||
 |tunnel_type|Tunnel type||
@@ -144,7 +146,7 @@ Use the properties on the table below for AWS:
  * The AWS credentials are obtained from the CLI [configuration files](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). **Important**: the access key and secret are recorded in plain-text in [install/overlays/aws/kustomization.yaml](../../install/overlays/aws/kustomization.yaml)
  * The subnet is created with CIDR IPv4 block 10.0.0.0/25. In case of deploying an EKS cluster,
 a secondary (private) subnet is created with CIDR IPv4 block 10.0.0.128/25
- * The cluster type **onprem** assumes Kubernetes is already provisioned and its kubeconfig file path can be found at the `KUBECONFIG` environment variable or in the `~/.kube/config` file. Whereas **eks** type instructs to create an [AWS EKS](https://aws.amazon.com/eks/) cluster on the VPC
+ * The cluster type **onprem** assumes Kubernetes is already provisioned and its kubeconfig file path can be found at the `KUBECONFIG` environment variable or in the `~/.kube/config` file. Whereas **eks** type instructs to create an [AWS EKS](https://aws.amazon.com/eks/) cluster on the VPC. You must have `eksctl` installed in your workstation or CI runner because it is used to create/destroy EKS clusters.
  * You must have `qemu-img` installed in your workstation or CI runner because it is used to convert an qcow2 disk to raw.
 
 ### Libvirt provision properties