template.yaml

AWSTemplateFormatVersion: "2010-09-09"
Description: Hypeman host on EC2 with nested virtualization enabled.

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: Network
        Parameters:
          - VpcId
          - SubnetId
          - AllowedApiCidr
          - ApiPort
          - EnableHttpIngress
          - EnableHttpsIngress
          - AllowedIngressCidr
      - Label:
          default: Instance
        Parameters:
          - InstanceType
          - RootVolumeSize
          - DataVolumeSize
          - DataVolumeIops
          - DataVolumeThroughput
          - AmiSsmParameter
      - Label:
          default: Access
        Parameters:
          - EnableSSH
          - AllowedSshCidr
          - KeyName
      - Label:
          default: Hypeman
        Parameters:
          - HypemanVersion
          - HypemanCliVersion
          - HypemanBranch
    ParameterLabels:
      VpcId:
        default: VPC
      SubnetId:
        default: Subnet
      InstanceType:
        default: EC2 instance type
      AllowedApiCidr:
        default: Hypeman API access CIDR
      ApiPort:
        default: Hypeman API port
      EnableHttpIngress:
        default: Enable HTTP ingress
      EnableHttpsIngress:
        default: Enable HTTPS ingress
      AllowedIngressCidr:
        default: Hypeman ingress access CIDR
      EnableSSH:
        default: Enable SSH
      AllowedSshCidr:
        default: SSH access CIDR
      KeyName:
        default: EC2 key pair
      RootVolumeSize:
        default: Root volume size
      DataVolumeSize:
        default: Hypeman data volume size
      DataVolumeIops:
        default: Hypeman data volume IOPS
      DataVolumeThroughput:
        default: Hypeman data volume throughput
      HypemanVersion:
        default: Hypeman release
      HypemanBranch:
        default: Hypeman source branch
      HypemanCliVersion:
        default: Hypeman CLI release
      AmiSsmParameter:
        default: Ubuntu AMI SSM parameter

Parameters:
  VpcId:
    Type: AWS::EC2::VPC::Id
    Description: Existing VPC for the Hypeman host.
  SubnetId:
    Type: AWS::EC2::Subnet::Id
    Description: Existing subnet for the Hypeman host. The subnet needs outbound internet or VPC endpoint access for package downloads, AWS APIs, and image pulls.
  InstanceType:
    Type: String
    Default: c8i.2xlarge
    AllowedPattern: "^(c8i|m8i|r8i)\\.(large|xlarge|2xlarge|4xlarge|8xlarge|12xlarge|16xlarge|24xlarge|32xlarge|48xlarge|96xlarge)$"
    Description: Intel C8i, M8i, or R8i instance type that supports EC2 nested virtualization. The default c8i.2xlarge has enough memory for the quickstart workload.
  AllowedApiCidr:
    Type: String
    Default: 127.0.0.1/32
    Description: Client CIDR allowed to reach the Hypeman API port. Use your current public IP /32 or a trusted VPN CIDR; avoid 0.0.0.0/0.
    AllowedPattern: "^([0-9]{1,3}\\.){3}[0-9]{1,3}/([0-9]|[1-2][0-9]|3[0-2])$"
  ApiPort:
    Type: Number
    Default: 8080
    MinValue: 1
    MaxValue: 65535
    Description: Hypeman API port exposed to AllowedApiCidr.
  EnableHttpIngress:
    Type: String
    Default: "false"
    AllowedValues: ["true", "false"]
    Description: Open port 80 from AllowedIngressCidr for Hypeman HTTP ingress traffic.
  EnableHttpsIngress:
    Type: String
    Default: "false"
    AllowedValues: ["true", "false"]
    Description: Open port 443 from AllowedIngressCidr for Hypeman HTTPS ingress traffic.
  AllowedIngressCidr:
    Type: String
    Default: 127.0.0.1/32
    Description: Client CIDR allowed to reach Hypeman ingress ports when enabled. Use your current public IP /32 or a trusted VPN CIDR; avoid 0.0.0.0/0.
    AllowedPattern: "^([0-9]{1,3}\\.){3}[0-9]{1,3}/([0-9]|[1-2][0-9]|3[0-2])$"
  EnableSSH:
    Type: String
    Default: "false"
    AllowedValues: ["true", "false"]
    Description: Open SSH from AllowedSshCidr and pass KeyName to the instance. Session Manager is available by default, so SSH is usually unnecessary.
  AllowedSshCidr:
    Type: String
    Default: 127.0.0.1/32
    Description: CIDR allowed to reach SSH when EnableSSH is true.
    AllowedPattern: "^([0-9]{1,3}\\.){3}[0-9]{1,3}/([0-9]|[1-2][0-9]|3[0-2])$"
  KeyName:
    Type: String
    Default: ""
    Description: Optional EC2 key pair name. Only used when EnableSSH is true.
  RootVolumeSize:
    Type: Number
    Default: 30
    MinValue: 20
    MaxValue: 16384
    Description: Root EBS volume size in GiB.
  DataVolumeSize:
    Type: Number
    Default: 100
    MinValue: 50
    MaxValue: 16384
    Description: Hypeman data EBS volume size in GiB. This volume is formatted as XFS and mounted at /var/lib/hypeman.
  DataVolumeIops:
    Type: Number
    Default: 3000
    MinValue: 3000
    MaxValue: 80000
    Description: Provisioned IOPS for the Hypeman data gp3 EBS volume. High values may require a larger DataVolumeSize.
  DataVolumeThroughput:
    Type: Number
    Default: 125
    MinValue: 125
    MaxValue: 2000
    Description: Provisioned throughput in MiB/s for the Hypeman data gp3 EBS volume. High values may require higher DataVolumeIops.
  HypemanVersion:
    Type: String
    Default: latest
    Description: Hypeman API release tag, or latest. Ignored when HypemanBranch is set.
  HypemanBranch:
    Type: String
    Default: ""
    Description: Optional Hypeman git branch to build from source for development testing. Leave empty for release installs.
  HypemanCliVersion:
    Type: String
    Default: latest
    Description: Hypeman CLI release tag, or latest.
  AmiSsmParameter:
    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
    Default: /aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id
    Description: SSM public parameter containing the Ubuntu AMI ID.

Conditions:
  UseSSH: !Equals [!Ref EnableSSH, "true"]
  UseHttpIngress: !Equals [!Ref EnableHttpIngress, "true"]
  UseHttpsIngress: !Equals [!Ref EnableHttpsIngress, "true"]

Resources:
  HypemanSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Hypeman access
      VpcId: !Ref VpcId
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: !Ref ApiPort
          ToPort: !Ref ApiPort
          CidrIp: !Ref AllowedApiCidr
          Description: Hypeman API
        - !If
          - UseHttpIngress
          - IpProtocol: tcp
            FromPort: 80
            ToPort: 80
            CidrIp: !Ref AllowedIngressCidr
            Description: Hypeman HTTP ingress
          - !Ref AWS::NoValue
        - !If
          - UseHttpsIngress
          - IpProtocol: tcp
            FromPort: 443
            ToPort: 443
            CidrIp: !Ref AllowedIngressCidr
            Description: Hypeman HTTPS ingress
          - !Ref AWS::NoValue
        - !If
          - UseSSH
          - IpProtocol: tcp
            FromPort: 22
            ToPort: 22
            CidrIp: !Ref AllowedSshCidr
            Description: SSH
          - !Ref AWS::NoValue
      SecurityGroupEgress:
        - IpProtocol: -1
          CidrIp: 0.0.0.0/0
          Description: Outbound access for package, image, and AWS API downloads
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-hypeman

  HypemanInstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
      Policies:
        - PolicyName: hypeman-describe-self
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - ec2:DescribeInstances
                  - ec2:DescribeInstanceStatus
                Resource: "*"

  HypemanInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref HypemanInstanceRole

  # CloudFormation's typed EC2 instance block device mapping does not expose gp3
  # throughput, and EC2 resources do not expose CpuOptions.NestedVirtualization
  # yet. This helper creates a minimal launch template for those fields; the
  # Hypeman EC2 instance itself remains a normal stack-managed resource.
  NestedVirtualizationLaunchTemplateRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: hypeman-nested-virtualization-launch-template
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - ec2:CreateLaunchTemplate
                  - ec2:DeleteLaunchTemplate
                  - ec2:DescribeLaunchTemplates
                  - ec2:CreateTags
                Resource: "*"

  NestedVirtualizationLaunchTemplateFunction:
    Type: AWS::Lambda::Function
    Properties:
      Runtime: python3.12
      Handler: index.handler
      Timeout: 60
      Role: !GetAtt NestedVirtualizationLaunchTemplateRole.Arn
      Code:
        ZipFile: |
          import json
          import os
          import traceback
          import urllib.parse
          import urllib.request
          import xml.etree.ElementTree as ET
          from urllib.error import HTTPError

          from botocore.auth import SigV4Auth
          from botocore.awsrequest import AWSRequest
          from botocore.session import Session

          def send(event, context, status, data, physical_id=None, reason=None):
              body = {
                  "Status": status,
                  "Reason": reason or ("See CloudWatch Logs: " + context.log_stream_name),
                  "PhysicalResourceId": physical_id or context.log_stream_name,
                  "StackId": event["StackId"],
                  "RequestId": event["RequestId"],
                  "LogicalResourceId": event["LogicalResourceId"],
                  "NoEcho": False,
                  "Data": data,
              }
              encoded = json.dumps(body).encode("utf-8")
              req = urllib.request.Request(
                  event["ResponseURL"],
                  data=encoded,
                  headers={"content-type": "", "content-length": str(len(encoded))},
                  method="PUT",
              )
              urllib.request.urlopen(req, timeout=10).read()

          def ec2_query(params):
              region = os.environ["AWS_REGION"]
              endpoint = f"https://ec2.{region}.amazonaws.com/"
              body = urllib.parse.urlencode(params).encode("utf-8")
              req = AWSRequest(
                  method="POST",
                  url=endpoint,
                  data=body,
                  headers={"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"},
              )
              credentials = Session().get_credentials().get_frozen_credentials()
              SigV4Auth(credentials, "ec2", region).add_auth(req)
              prepared = req.prepare()
              http_req = urllib.request.Request(
                  endpoint,
                  data=body,
                  headers=dict(prepared.headers),
                  method="POST",
              )
              try:
                  with urllib.request.urlopen(http_req, timeout=60) as res:
                      return res.read()
              except HTTPError as err:
                  detail = err.read().decode("utf-8", errors="replace")
                  raise RuntimeError(f"EC2 query failed with HTTP {err.code}: {detail}") from err

          def launch_template_name(event):
              stack_uuid = event["StackId"].rsplit("/", 1)[-1]
              return f"{event['ResourceProperties']['NamePrefix']}-{stack_uuid}"

          def create_launch_template(name, props):
              payload = {
                  "Action": "CreateLaunchTemplate",
                  "Version": "2016-11-15",
                  "LaunchTemplateName": name,
                  "LaunchTemplateData.CpuOptions.NestedVirtualization": "enabled",
                  "LaunchTemplateData.BlockDeviceMapping.1.DeviceName": "/dev/sda1",
                  "LaunchTemplateData.BlockDeviceMapping.1.Ebs.VolumeSize": props["RootVolumeSize"],
                  "LaunchTemplateData.BlockDeviceMapping.1.Ebs.VolumeType": "gp3",
                  "LaunchTemplateData.BlockDeviceMapping.1.Ebs.Encrypted": "true",
                  "LaunchTemplateData.BlockDeviceMapping.1.Ebs.DeleteOnTermination": "true",
                  "LaunchTemplateData.BlockDeviceMapping.2.DeviceName": "/dev/sdf",
                  "LaunchTemplateData.BlockDeviceMapping.2.Ebs.VolumeSize": props["DataVolumeSize"],
                  "LaunchTemplateData.BlockDeviceMapping.2.Ebs.VolumeType": "gp3",
                  "LaunchTemplateData.BlockDeviceMapping.2.Ebs.Iops": props["DataVolumeIops"],
                  "LaunchTemplateData.BlockDeviceMapping.2.Ebs.Throughput": props["DataVolumeThroughput"],
                  "LaunchTemplateData.BlockDeviceMapping.2.Ebs.Encrypted": "true",
                  "LaunchTemplateData.BlockDeviceMapping.2.Ebs.DeleteOnTermination": "true",
                  "TagSpecification.1.ResourceType": "launch-template",
                  "TagSpecification.1.Tag.1.Key": "Name",
                  "TagSpecification.1.Tag.1.Value": name,
                  "TagSpecification.1.Tag.2.Key": "hypeman:deployment",
                  "TagSpecification.1.Tag.2.Value": "aws",
              }
              xml = ec2_query(payload)
              root = ET.fromstring(xml)
              launch_template_id = root.find(".//{*}launchTemplateId")
              version = root.find(".//{*}latestVersionNumber")
              if launch_template_id is None or not launch_template_id.text:
                  raise RuntimeError("CreateLaunchTemplate response did not include launchTemplateId")
              return {
                  "LaunchTemplateId": launch_template_id.text,
                  "VersionNumber": version.text if version is not None and version.text else "1",
              }

          def delete_launch_template(launch_template_id):
              if launch_template_id and launch_template_id.startswith("lt-"):
                  ec2_query({
                      "Action": "DeleteLaunchTemplate",
                      "Version": "2016-11-15",
                      "LaunchTemplateId": launch_template_id,
                  })

          def handler(event, context):
              try:
                  request_type = event["RequestType"]
                  physical_id = event.get("PhysicalResourceId")
                  if request_type == "Delete":
                      delete_launch_template(physical_id)
                      send(event, context, "SUCCESS", {}, physical_id=physical_id)
                      return
                  if request_type == "Update":
                      delete_launch_template(physical_id)
                  data = create_launch_template(launch_template_name(event), event["ResourceProperties"])
                  send(event, context, "SUCCESS", data, physical_id=data["LaunchTemplateId"])
              except Exception as exc:
                  traceback.print_exc()
                  send(event, context, "FAILED", {}, physical_id=event.get("PhysicalResourceId"), reason=str(exc))

  NestedVirtualizationLaunchTemplate:
    Type: Custom::NestedVirtualizationLaunchTemplate
    Properties:
      ServiceToken: !GetAtt NestedVirtualizationLaunchTemplateFunction.Arn
      NamePrefix: hypeman
      RootVolumeSize: !Ref RootVolumeSize
      DataVolumeSize: !Ref DataVolumeSize
      DataVolumeIops: !Ref DataVolumeIops
      DataVolumeThroughput: !Ref DataVolumeThroughput

  HypemanHost:
    Type: AWS::EC2::Instance
    Properties:
      LaunchTemplate:
        LaunchTemplateId: !GetAtt NestedVirtualizationLaunchTemplate.LaunchTemplateId
        Version: !GetAtt NestedVirtualizationLaunchTemplate.VersionNumber
      ImageId: !Ref AmiSsmParameter
      InstanceType: !Ref InstanceType
      SubnetId: !Ref SubnetId
      SecurityGroupIds:
        - !Ref HypemanSecurityGroup
      IamInstanceProfile: !Ref HypemanInstanceProfile
      KeyName: !If [UseSSH, !Ref KeyName, !Ref AWS::NoValue]
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-hypeman
        - Key: hypeman:deployment
          Value: aws
      UserData:
        Fn::Base64: !Sub |
          #!/usr/bin/env bash
          set -euxo pipefail
          exec > >(tee /var/log/hypeman-bootstrap.log | logger -t hypeman-bootstrap -s 2>/dev/console) 2>&1

          export DEBIAN_FRONTEND=noninteractive
          apt-get update
          apt-get install -y ca-certificates curl docker.io e2fsprogs erofs-utils iproute2 iptables jq openssl qemu-system-x86 qemu-utils tar xfsprogs

          setup_hypeman_data_volume() {
            local data_device=""
            for _ in $(seq 1 60); do
              while read -r device type; do
                if [ "$type" = "disk" ] && ! lsblk -nrpo MOUNTPOINT "$device" | grep -qx "/"; then
                  data_device="$device"
                  break
                fi
              done < <(lsblk -dnpo NAME,TYPE)
              if [ -n "$data_device" ]; then
                break
              fi
              sleep 2
            done
            if [ -z "$data_device" ]; then
              echo "hypeman data volume was not found" >&2
              exit 1
            fi
            if [ "$(blkid -s TYPE -o value "$data_device" 2>/dev/null || true)" != "xfs" ]; then
              mkfs.xfs -f "$data_device"
            fi
            data_uuid="$(blkid -s UUID -o value "$data_device")"
            install -d -m 755 /var/lib/hypeman
            if ! grep -q " /var/lib/hypeman " /etc/fstab; then
              echo "UUID=$data_uuid /var/lib/hypeman xfs defaults,nofail 0 2" >> /etc/fstab
            fi
            mount /var/lib/hypeman || mount -a
            if [ "$(findmnt -n -o FSTYPE /var/lib/hypeman)" != "xfs" ]; then
              echo "/var/lib/hypeman is not mounted as xfs" >&2
              exit 1
            fi
          }
          setup_hypeman_data_volume

          if [ -n "${HypemanBranch}" ]; then
            apt-get install -y build-essential git make
            curl -fsSL https://go.dev/dl/go1.25.4.linux-amd64.tar.gz | tar -C /usr/local -xzf -
            export PATH="/usr/local/go/bin:$PATH"
            export HOME=/root
            export GOPATH=/root/go
            export GOMODCACHE=/root/go/pkg/mod
            export GOCACHE=/root/.cache/go-build
            install -d -m 755 "$GOPATH" "$GOMODCACHE" "$GOCACHE"
          fi
          systemctl enable --now docker

          if ! systemctl is-active --quiet snap.amazon-ssm-agent.amazon-ssm-agent && ! systemctl is-active --quiet amazon-ssm-agent; then
            snap install amazon-ssm-agent --classic || true
            systemctl enable --now snap.amazon-ssm-agent.amazon-ssm-agent || systemctl enable --now amazon-ssm-agent || true
          fi

          if [ -n "${HypemanBranch}" ]; then
            export BRANCH="${HypemanBranch}"
            unset VERSION
          elif [ "${HypemanVersion}" = "latest" ]; then
            unset VERSION
          else
            export VERSION="${HypemanVersion}"
          fi
          if [ "${HypemanCliVersion}" = "latest" ]; then
            unset CLI_VERSION
          else
            export CLI_VERSION="${HypemanCliVersion}"
          fi
          curl -fsSL https://raw.githubusercontent.com/kernel/hypeman/main/scripts/install.sh | bash

          install -d -m 755 /opt/hypeman/deploy
          cat >/usr/local/bin/hypeman-create-token <<'SCRIPT'
          #!/usr/bin/env bash
          set -euo pipefail
          user_id="remote-user"
          duration="8760h"
          if [ "$#" -ge 1 ]; then
            user_id="$1"
          fi
          if [ "$#" -ge 2 ]; then
            duration="$2"
          fi
          CONFIG_PATH=/etc/hypeman/config.yaml /opt/hypeman/bin/hypeman-token -user-id "$user_id" -duration "$duration"
          SCRIPT
          chmod 755 /usr/local/bin/hypeman-create-token

          install -d -m 700 /root/.config/hypeman
          local_token="$(hypeman-create-token local-cli 8760h)"
          cat >/root/.config/hypeman/cli.yaml <<EOF
          base_url: http://localhost:${ApiPort}
          api_key: "$local_token"
          EOF
          chmod 600 /root/.config/hypeman/cli.yaml

          cat >/opt/hypeman/deploy/validate.sh <<'SCRIPT'
          #!/usr/bin/env bash
          set -euo pipefail
          test -e /dev/kvm
          grep -Eq '(^flags|^Features).* (vmx|svm)( |$)' /proc/cpuinfo
          test "$(findmnt -n -o FSTYPE /var/lib/hypeman)" = "xfs"
          systemctl is-active --quiet hypeman
          token="$(hypeman-create-token validation 1h)"
          curl -fsS -H "Authorization: Bearer $token" http://127.0.0.1:${ApiPort}/health >/dev/null
          echo "hypeman aws validation passed"
          SCRIPT
          chmod 755 /opt/hypeman/deploy/validate.sh

          for _ in $(seq 1 60); do
            if /opt/hypeman/deploy/validate.sh; then
              exit 0
            fi
            sleep 5
          done
          /opt/hypeman/deploy/validate.sh

Outputs:
  InstanceId:
    Description: EC2 instance running Hypeman.
    Value: !Ref HypemanHost
  PublicIp:
    Description: Public IP address, if the subnet assigns one.
    Value: !GetAtt HypemanHost.PublicIp
  PrivateIp:
    Description: Private IP address.
    Value: !GetAtt HypemanHost.PrivateIp
  HypemanEndpoint:
    Description: Hypeman API endpoint.
    Value: !Sub
      - http://${Host}:${Port}
      - Host: !GetAtt HypemanHost.PublicIp
        Port: !Ref ApiPort
  SsmSessionCommand:
    Description: Command to start a Session Manager shell.
    Value: !Sub
      - aws ssm start-session --region ${AWS::Region} --target ${InstanceId}
      - InstanceId: !Ref HypemanHost
  CreateTokenCommand:
    Description: Command to generate a JWT on the instance.
    Value: sudo hypeman-create-token remote-user 8760h