Skip to content

Commit e0af99a

Browse files
DanilBaibakDanilBaibak
authored
Added linux arm64 runners to terraform (#4419)
Added linux arm64 runners to terraform
1 parent d7e0423 commit e0af99a

File tree

14 files changed

+235
-25
lines changed

14 files changed

+235
-25
lines changed

‎terraform-aws-github-runner/main.tf‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@ locals {
1717
Environment = var.environment
1818
})
1919

20-
s3_action_runner_url_linux = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key_linux}"
21-
s3_action_runner_url_windows = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key_windows}"
22-
runner_architecture = substr(var.instance_type, 0, 2) == "a1" || substr(var.instance_type, 1, 2) == "6g" ? "arm64" : "x64"
20+
s3_action_runner_url_linux = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key_linux}"
21+
s3_action_runner_url_linux_arm64 = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key_linux_arm64}"
22+
s3_action_runner_url_windows = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key_windows}"
23+
runner_architecture = substr(var.instance_type, 0, 2) == "a1" || substr(var.instance_type, 1, 2) == "6g" ? "arm64" : "x64"
2324
}
2425

2526
resource "random_string" "random" {
@@ -137,10 +138,12 @@ module "runners" {
137138

138139
launch_template_name_linux = module.runners_instances.launch_template_name_linux
139140
launch_template_name_linux_nvidia = module.runners_instances.launch_template_name_linux_nvidia
141+
launch_template_name_linux_arm64 = module.runners_instances.launch_template_name_linux_arm64
140142
launch_template_name_windows = module.runners_instances.launch_template_name_windows
141143
launch_template_version_linux = module.runners_instances.launch_template_version_linux
142144
launch_template_version_windows = module.runners_instances.launch_template_version_windows
143145
launch_template_version_linux_nvidia = module.runners_instances.launch_template_version_linux_nvidia
146+
launch_template_version_linux_arm64 = module.runners_instances.launch_template_version_linux_arm64
144147

145148
logging_retention_in_days = var.logging_retention_in_days
146149
scale_up_lambda_concurrency = var.scale_up_lambda_concurrency
@@ -167,9 +170,10 @@ module "runners_instances" {
167170
encrypt = var.encrypt_secrets
168171
}
169172

170-
s3_bucket_runner_binaries = module.runner_binaries.bucket
171-
s3_location_runner_binaries_linux = local.s3_action_runner_url_linux
172-
s3_location_runner_binaries_windows = local.s3_action_runner_url_windows
173+
s3_bucket_runner_binaries = module.runner_binaries.bucket
174+
s3_location_runner_binaries_linux = local.s3_action_runner_url_linux
175+
s3_location_runner_binaries_linux_arm64 = local.s3_action_runner_url_linux_arm64
176+
s3_location_runner_binaries_windows = local.s3_action_runner_url_windows
173177

174178
instance_type = var.instance_type
175179
block_device_mappings = var.block_device_mappings

‎terraform-aws-github-runner/modules/runner-binaries-syncer/main.tf‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ terraform {
99
}
1010

1111
locals {
12-
action_runner_distribution_object_key_linux = "actions-runner-linux-x64.tar.gz"
13-
action_runner_distribution_object_key_windows = "actions-runner-windows-x64.zip"
12+
action_runner_distribution_object_key_linux = "actions-runner-linux-x64.tar.gz"
13+
action_runner_distribution_object_key_linux_arm64 = "actions-runner-linux-arm64-2.307.1.tar.gz"
14+
action_runner_distribution_object_key_windows = "actions-runner-windows-x64.zip"
1415
}
1516

1617
resource "aws_s3_bucket" "action_dist" {

‎terraform-aws-github-runner/modules/runner-binaries-syncer/outputs.tf‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ output "runner_distribution_object_key_linux" {
66
value = local.action_runner_distribution_object_key_linux
77
}
88

9+
output "runner_distribution_object_key_linux_arm64" {
10+
value = local.action_runner_distribution_object_key_linux_arm64
11+
}
12+
913
output "runner_distribution_object_key_windows" {
1014
value = local.action_runner_distribution_object_key_windows
1115
}

‎terraform-aws-github-runner/modules/runners-instances/launch-template.tf‎

Lines changed: 77 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,27 @@
11
locals {
2-
instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path
3-
name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"]
4-
role_path = var.role_path == null ? "/${var.environment}/" : var.role_path
5-
userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl"
6-
userdata_install_config_runner_linux = "${path.module}/templates/install-config-runner.sh"
2+
instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path
3+
name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"]
4+
role_path = var.role_path == null ? "/${var.environment}/" : var.role_path
5+
userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl"
6+
userdata_install_config_runner_linux = "${path.module}/templates/install-config-runner.sh"
77
userdata_install_config_runner_windows = "${path.module}/templates/install-config-runner.ps1"
88
userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template
99
userdata_template_windows = "${path.module}/templates/user-data.ps1"
1010

1111
arm_patch = var.runner_architecture == "arm64" ? templatefile(local.userdata_arm_patch, {}) : ""
12+
1213
install_config_runner_linux = templatefile(local.userdata_install_config_runner_linux, {
1314
environment = var.environment
1415
s3_location_runner_distribution = var.s3_location_runner_binaries_linux
1516
run_as_root_user = var.runner_as_root ? "root" : ""
1617
arm_patch = local.arm_patch
1718
})
19+
install_config_runner_linux_arm64 = templatefile(local.userdata_install_config_runner_linux, {
20+
environment = var.environment
21+
s3_location_runner_distribution = var.s3_location_runner_binaries_linux_arm64
22+
run_as_root_user = var.runner_as_root ? "root" : ""
23+
arm_patch = local.arm_patch
24+
})
1825
install_config_runner_windows = templatefile(local.userdata_install_config_runner_windows, {
1926
environment = var.environment
2027
s3_location_runner_distribution = var.s3_location_runner_binaries_windows
@@ -37,6 +44,20 @@ data "aws_ami" "runner_ami_linux" {
3744
owners = var.ami_owners_linux
3845
}
3946

47+
data "aws_ami" "runner_ami_linux_arm64" {
48+
most_recent = "true"
49+
50+
dynamic "filter" {
51+
for_each = var.ami_filter_linux_arm64
52+
content {
53+
name = filter.key
54+
values = filter.value
55+
}
56+
}
57+
58+
owners = var.ami_owners_linux_arm64
59+
}
60+
4061
data "aws_ami" "runner_ami_windows" {
4162
most_recent = "true"
4263

@@ -91,7 +112,7 @@ resource "aws_launch_template" "linux_runner" {
91112
pre_install = var.userdata_pre_install
92113
post_install = var.userdata_post_install
93114
enable_cloudwatch_agent = var.enable_cloudwatch_agent
94-
nvidia_driver_install = var.nvidia_driver_install
115+
nvidia_driver_install = false
95116
ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner_linux[0].name : ""
96117
ghes_url = var.ghes_url
97118
install_config_runner = local.install_config_runner_linux
@@ -149,6 +170,55 @@ resource "aws_launch_template" "linux_runner_nvidia" {
149170
tags = local.tags
150171
}
151172

173+
resource "aws_launch_template" "linux_arm64_runner" {
174+
name = "${var.environment}-action-linux-arm64-runner"
175+
176+
iam_instance_profile {
177+
name = aws_iam_instance_profile.runner.name
178+
}
179+
180+
instance_initiated_shutdown_behavior = "terminate"
181+
182+
image_id = data.aws_ami.runner_ami_linux_arm64.id
183+
instance_type = var.instance_type
184+
key_name = var.key_name
185+
186+
tag_specifications {
187+
resource_type = "instance"
188+
tags = merge(
189+
local.tags,
190+
{
191+
"Name" = format("%s", local.name_runner),
192+
"InstanceManagement" = "dynamic"
193+
},
194+
)
195+
}
196+
197+
tag_specifications {
198+
resource_type = "volume"
199+
tags = merge(
200+
local.tags,
201+
{
202+
"Name" = format("%s", local.name_runner)
203+
"InstanceManagement" = "dynamic"
204+
},
205+
)
206+
}
207+
208+
user_data = base64encode(templatefile(local.userdata_template, {
209+
environment = var.environment
210+
pre_install = var.userdata_pre_install
211+
post_install = var.userdata_post_install
212+
enable_cloudwatch_agent = var.enable_cloudwatch_agent
213+
nvidia_driver_install = false
214+
ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner_linux_arm64[0].name : ""
215+
ghes_url = var.ghes_url
216+
install_config_runner = local.install_config_runner_linux_arm64
217+
}))
218+
219+
tags = local.tags
220+
}
221+
152222
resource "aws_launch_template" "windows_runner" {
153223
name = "${var.environment}-action-windows-runner"
154224

@@ -204,7 +274,7 @@ resource "aws_launch_template" "windows_runner" {
204274
pre_install = var.userdata_pre_install
205275
post_install = var.userdata_post_install
206276
enable_cloudwatch_agent = var.enable_cloudwatch_agent
207-
nvidia_driver_install = var.nvidia_driver_install
277+
nvidia_driver_install = false
208278
ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner_windows[0].name : ""
209279
ghes_url = var.ghes_url
210280
install_config_runner = local.install_config_runner_windows

‎terraform-aws-github-runner/modules/runners-instances/logging.tf‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,26 @@ resource "aws_ssm_parameter" "cloudwatch_agent_config_runner_linux_nvidia" {
9797
tags = local.tags
9898
}
9999

100+
resource "aws_ssm_parameter" "cloudwatch_agent_config_runner_linux_arm64" {
101+
count = var.enable_cloudwatch_agent ? 1 : 0
102+
name = "${var.environment}-cloudwatch_agent_config_runner_linux_arm64"
103+
type = "String"
104+
value = jsonencode(
105+
jsondecode(
106+
templatefile(
107+
"${path.module}/templates/cloudwatch_config.json",
108+
{
109+
aws_region = var.aws_region
110+
environment = var.environment
111+
logfiles = jsonencode(local.logfiles_linux)
112+
metrics_collected = templatefile("${path.module}/templates/cloudwatch_config_linux_arm64.json", {})
113+
}
114+
)
115+
)
116+
)
117+
tags = local.tags
118+
}
119+
100120
resource "aws_cloudwatch_log_group" "gh_runners_linux" {
101121
count = length(local.loggroups_names_linux)
102122
name = local.loggroups_names_linux[count.index]
@@ -126,6 +146,17 @@ resource "aws_iam_role_policy" "cloudwatch_linux_nvidia" {
126146
)
127147
}
128148

149+
resource "aws_iam_role_policy" "cloudwatch_linux_arm64" {
150+
count = var.enable_ssm_on_runners ? 1 : 0
151+
name = "CloudWatchLogginAndMetricsLinuxARM64"
152+
role = aws_iam_role.runner.name
153+
policy = templatefile("${path.module}/policies/instance-cloudwatch-policy.json",
154+
{
155+
ssm_parameter_arn = aws_ssm_parameter.cloudwatch_agent_config_runner_linux_arm64[0].arn
156+
}
157+
)
158+
}
159+
129160
resource "aws_ssm_parameter" "cloudwatch_agent_config_runner_windows" {
130161
count = var.enable_cloudwatch_agent ? 1 : 0
131162
name = "${var.environment}-cloudwatch_agent_config_runner_windows"

‎terraform-aws-github-runner/modules/runners-instances/outputs.tf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ output "launch_template_name_linux_nvidia" {
3838
value = aws_launch_template.linux_runner_nvidia.name
3939
}
4040

41+
output "launch_template_name_linux_arm64" {
42+
value = aws_launch_template.linux_arm64_runner.name
43+
}
44+
4145
output "launch_template_name_windows" {
4246
value = aws_launch_template.windows_runner.name
4347
}
@@ -50,6 +54,10 @@ output "launch_template_version_linux_nvidia" {
5054
value = aws_launch_template.linux_runner_nvidia.latest_version
5155
}
5256

57+
output "launch_template_version_linux_arm64" {
58+
value = aws_launch_template.linux_arm64_runner.latest_version
59+
}
60+
5361
output "launch_template_version_windows" {
5462
value = aws_launch_template.windows_runner.latest_version
5563
}

‎terraform-aws-github-runner/modules/runners-instances/templates/cloudwatch_config_linux_arm64.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"cpu": {
3+
"measurement": [
4+
"cpu_usage_idle",
5+
"cpu_usage_iowait",
6+
"cpu_usage_user",
7+
"cpu_usage_system"
8+
],
9+
"metrics_collection_interval": 10
10+
},
11+
"disk": {
12+
"measurement": [
13+
"free",
14+
"total",
15+
"used",
16+
"used_percent",
17+
"inodes_free",
18+
"inodes_total"
19+
],
20+
"metrics_collection_interval": 10,
21+
"resources": [
22+
"*"
23+
]
24+
},
25+
"diskio": {
26+
"measurement": [
27+
"io_time"
28+
],
29+
"metrics_collection_interval": 10,
30+
"resources": [
31+
"/"
32+
]
33+
},
34+
"mem": {
35+
"measurement": [
36+
"total",
37+
"used",
38+
"free",
39+
"used_percent"
40+
],
41+
"metrics_collection_interval": 10
42+
},
43+
"swap": {
44+
"measurement": [
45+
"swap_used_percent"
46+
],
47+
"metrics_collection_interval": 10
48+
}
49+
}

‎terraform-aws-github-runner/modules/runners-instances/templates/install-config-runner.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,14 @@ rm -rf actions-runner.tar.gz
88

99
${arm_patch}
1010

11-
INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
12-
REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
11+
if [ "$(uname -m)" == "aarch64" ]; then
12+
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
13+
INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id -H "X-aws-ec2-metadata-token: $TOKEN")
14+
REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document -H "X-aws-ec2-metadata-token: $TOKEN" | jq -r .region)
15+
else
16+
INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
17+
REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
18+
fi
1319

1420
echo wait for configuration
1521
while [[ $(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") == null ]]; do

‎terraform-aws-github-runner/modules/runners-instances/templates/user-data.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@ amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${ssm_key_cloudwatc
1212
%{ endif ~}
1313

1414
# Install docker
15-
amazon-linux-extras install docker
15+
if [ "$(uname -m)" == "aarch64" ]; then
16+
yum install -y docker
17+
else
18+
amazon-linux-extras install docker
19+
fi
20+
1621
service docker start
1722
usermod -a -G docker ec2-user
1823

‎terraform-aws-github-runner/modules/runners-instances/variables.tf‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ variable "s3_location_runner_binaries_linux" {
4040
type = string
4141
}
4242

43+
variable "s3_location_runner_binaries_linux_arm64" {
44+
description = "S3 location of runner distribution."
45+
type = string
46+
}
47+
4348
variable "s3_location_runner_binaries_windows" {
4449
description = "S3 location of runner distribution."
4550
type = string
@@ -66,6 +71,21 @@ variable "ami_filter_linux" {
6671
}
6772
}
6873

74+
variable "ami_filter_linux_arm64" {
75+
description = "List of maps used to create the AMI filter for the action runner AMI."
76+
type = map(list(string))
77+
78+
default = {
79+
name = ["amzn2-ami-hvm-2.*-arm64-gp2"]
80+
}
81+
}
82+
83+
variable "ami_owners_linux_arm64" {
84+
description = "The list of owners used to select the AMI of linux action runner instances."
85+
type = list(string)
86+
default = ["amazon"]
87+
}
88+
6989
variable "ami_filter_windows" {
7090
description = "List of maps used to create the AMI filter for the action runner AMI."
7191
type = map(list(string))
@@ -176,12 +196,6 @@ variable "enable_cloudwatch_agent" {
176196
default = true
177197
}
178198

179-
variable "nvidia_driver_install" {
180-
description = "Preinstall nvidia driver on GPU machines."
181-
type = bool
182-
default = false
183-
}
184-
185199
variable "ghes_url" {
186200
description = "GitHub Enterprise Server URL. DO NOT SET IF USING PUBLIC GITHUB"
187201
type = string

0 commit comments

Comments
 (0)