Skip to content

Commit da2cbd7

Browse files
neoakrisneoakris
committed
sigificantly improved karpenter UX: Simplified config and fixed issue where cdk destroy would fail
1 parent ce4ab11 commit da2cbd7

8 files changed

Lines changed: 124 additions & 68 deletions

‎config/eks/higher_envs_eks_config.ts‎

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ import * as eks from 'aws-cdk-lib/aws-eks';
44
import * as iam from 'aws-cdk-lib/aws-iam';
55
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31'; //npm install @aws-cdk/lambda-layer-kubectl-v31
66
import request from 'sync-request-curl'; //npm install sync-request-curl (cdk requires sync functions, async not allowed)
7-
import { Karpenter } from 'cdk-eks-karpenter' //npm install cdk-eks-karpenter
8-
import { Karpenter_YAML_Generator } from '../../lib/Karpenter_Manifests';
7+
import { Karpenter_Helm_Config, Karpenter_YAML_Generator, Apply_Karpenter_YAMLs_with_fixes } from '../../lib/Karpenter_Manifests';
98
//Intended Use:
109
//EasyEKS Admins: edit this file with config to apply to all lower environment eks cluster's in your org.
1110

@@ -92,21 +91,16 @@ export function deploy_workloads(config: Easy_EKS_Config_Data, stack: cdk.Stack,
9291

9392
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
9493
// Install Karpenter.sh
95-
const karpenter = new Karpenter(stack, 'Karpenter', {
96-
cluster: cluster,
97-
namespace: 'kube-system',
98-
version: '1.3.3', //https://gallery.ecr.aws/karpenter/karpenter
99-
nodeRole: config.baselineNodeRole, //custom NodeRole to pass to Karpenter Nodes
100-
helmExtraValues: { //https://github.com/aws/karpenter-provider-aws/blob/v1.2.0/charts/karpenter/values.yaml
94+
const karpenter_helm_config: Karpenter_Helm_Config = {
95+
helm_chart_version: '1.4.0', //https://gallery.ecr.aws/karpenter/karpenter
96+
helm_chart_values: { //https://github.com/aws/karpenter-provider-aws/blob/v1.4.0/charts/karpenter/values.yaml
10197
replicas: 2,
10298
},
103-
});
104-
karpenter.node.addDependency(cluster.awsAuth); //editing order of operations to say deploy karpenter after cluster exists
105-
106-
const karpenter_YAML_generator = new Karpenter_YAML_Generator({
99+
};
100+
const karpenter_YAMLs = (new Karpenter_YAML_Generator({
107101
cluster: cluster,
108102
config: config,
109-
amiSelectorTerms_alias: "bottlerocket@v1.31.0", /* <-- Bottlerocket alias always ends in a zero
103+
amiSelectorTerms_alias: "bottlerocket@v1.31.0", /* <-- Bottlerocket alias always ends in a zero, below is proof by command output
110104
export K8S_VERSION="1.31"
111105
aws ssm get-parameters-by-path --path "/aws/service/bottlerocket/aws-k8s-$K8S_VERSION" --recursive | jq -cr '.Parameters[].Name' | grep -v "latest" | awk -F '/' '{print $7}' | sort | uniq
112106
*/
@@ -117,17 +111,8 @@ export function deploy_workloads(config: Easy_EKS_Config_Data, stack: cdk.Stack,
117111
{ type: "spot", arch: "arm64", nodepools_cpu_limit: 1000, weight: 2, },
118112
{ type: "spot", arch: "amd64", nodepools_cpu_limit: 1000, weight: 1, },
119113
]
120-
});
121-
const karpenter_YAMLs = karpenter_YAML_generator.generate_manifests();
122-
const apply_kaprenter_YAML = new eks.KubernetesManifest(stack, 'karpenter_YAMLs',
123-
{
124-
cluster: cluster,
125-
manifest: karpenter_YAMLs,
126-
overwrite: true,
127-
prune: false,
128-
});
129-
//cluster.addManifest('karpenter_YAML', ...karpenter_YAMLs); //... is a TypeScript operation to convert array to CSV.
130-
apply_kaprenter_YAML.node.addDependency(karpenter); //Inform cdk of order of operations
114+
})).generate_manifests();
115+
Apply_Karpenter_YAMLs_with_fixes(stack, cluster, config, karpenter_helm_config, karpenter_YAMLs, awsLoadBalancerController);
131116
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
132117

133118
}//end deploy_workloads()

‎config/eks/lower_envs_eks_config.ts‎

Lines changed: 10 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ import * as eks from 'aws-cdk-lib/aws-eks';
44
import * as iam from 'aws-cdk-lib/aws-iam';
55
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31'; //npm install @aws-cdk/lambda-layer-kubectl-v31
66
import request from 'sync-request-curl'; //npm install sync-request-curl (cdk requires sync functions, async not allowed)
7-
import { Karpenter } from 'cdk-eks-karpenter' //npm install cdk-eks-karpenter
8-
import { Karpenter_YAML_Generator } from '../../lib/Karpenter_Manifests';
7+
import { Karpenter_Helm_Config, Karpenter_YAML_Generator, Apply_Karpenter_YAMLs_with_fixes } from '../../lib/Karpenter_Manifests';
98
//Intended Use:
109
//EasyEKS Admins: edit this file with config to apply to all lower environment eks cluster's in your org.
1110

@@ -50,7 +49,7 @@ export function deploy_workloads(config: Easy_EKS_Config_Data, stack: cdk.Stack,
5049
const ALBC_Version = 'v2.12.0'; //April 9th, 2025 latest from https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases
5150
const ALBC_IAM_Policy_Url = `https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/refs/tags/${ALBC_Version}/docs/install/iam_policy.json`
5251
const ALBC_IAM_Policy_JSON = JSON.parse(request("GET", ALBC_IAM_Policy_Url).body.toString());
53-
const ALBC_IAM_Policy = new iam.Policy(stack, `${config.id}_AWS_LB_Controller_policy_for_EKS`, {
52+
const ALBC_IAM_Policy = new iam.Policy(stack, 'AWS_LB_Controller_IAM_policy_for_EKS', {
5453
document: iam.PolicyDocument.fromJson( ALBC_IAM_Policy_JSON ),
5554
});
5655
const ALBC_Kube_SA = new eks.ServiceAccount(stack, 'aws-load-balancer-controller_kube-sa', {
@@ -92,20 +91,16 @@ export function deploy_workloads(config: Easy_EKS_Config_Data, stack: cdk.Stack,
9291

9392
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
9493
// Install Karpenter.sh
95-
const karpenter = new Karpenter(stack, 'Karpenter', {
96-
cluster: cluster,
97-
namespace: 'kube-system',
98-
version: '1.3.3', //https://gallery.ecr.aws/karpenter/karpenter
99-
nodeRole: config.baselineNodeRole, //custom NodeRole to pass to Karpenter Nodes
100-
helmExtraValues: { //https://github.com/aws/karpenter-provider-aws/blob/v1.2.0/charts/karpenter/values.yaml
94+
const karpenter_helm_config: Karpenter_Helm_Config = {
95+
helm_chart_version: '1.4.0', //https://gallery.ecr.aws/karpenter/karpenter
96+
helm_chart_values: { //https://github.com/aws/karpenter-provider-aws/blob/v1.4.0/charts/karpenter/values.yaml
10197
replicas: 1,
10298
},
103-
});
104-
karpenter.node.addDependency(cluster.awsAuth); //editing order of operations to say deploy karpenter after cluster exists
105-
const karpenter_YAML_generator = new Karpenter_YAML_Generator({
99+
};
100+
const karpenter_YAMLs = (new Karpenter_YAML_Generator({
106101
cluster: cluster,
107102
config: config,
108-
amiSelectorTerms_alias: "bottlerocket@v1.31.0", /* <-- Bottlerocket alias always ends in a zero
103+
amiSelectorTerms_alias: "bottlerocket@v1.31.0", /* <-- Bottlerocket alias always ends in a zero, below is proof by command output
109104
export K8S_VERSION="1.31"
110105
aws ssm get-parameters-by-path --path "/aws/service/bottlerocket/aws-k8s-$K8S_VERSION" --recursive | jq -cr '.Parameters[].Name' | grep -v "latest" | awk -F '/' '{print $7}' | sort | uniq
111106
*/
@@ -116,17 +111,8 @@ export function deploy_workloads(config: Easy_EKS_Config_Data, stack: cdk.Stack,
116111
{ type: "on-demand", arch: "arm64", nodepools_cpu_limit: 1000, weight: 2, },
117112
{ type: "on-demand", arch: "amd64", nodepools_cpu_limit: 1000, weight: 1, },
118113
]
119-
});
120-
const karpenter_YAMLs = karpenter_YAML_generator.generate_manifests();
121-
const apply_kaprenter_YAML = new eks.KubernetesManifest(stack, 'karpenter_YAMLs',
122-
{
123-
cluster: cluster,
124-
manifest: karpenter_YAMLs,
125-
overwrite: true,
126-
prune: false,
127-
});
128-
//cluster.addManifest('karpenter_YAML', ...karpenter_YAMLs); //... is a TypeScript operation to convert array to CSV.
129-
apply_kaprenter_YAML.node.addDependency(karpenter); //Inform cdk of order of operations
114+
})).generate_manifests();
115+
Apply_Karpenter_YAMLs_with_fixes(stack, cluster, config, karpenter_helm_config, karpenter_YAMLs, awsLoadBalancerController);
130116
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
131117

132118
}//end deploy_workloads()

‎config/eks/my_orgs_baseline_eks_config.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,13 +238,12 @@ export function deploy_workloads(config: Easy_EKS_Config_Data, stack: cdk.Stack,
238238
managedPolicies: [ iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AmazonEBSCSIDriverPolicy') ],
239239
assumedBy: (new cdk.aws_iam.ServicePrincipal("pods.eks.amazonaws.com")),
240240
});
241-
ebs_csi_addon_iam_role.assumeRolePolicy!.addStatements(
241+
ebs_csi_addon_iam_role.assumeRolePolicy!.addStatements( //<-- ! is TypeScript for "I know this variable isn't null"
242242
new iam.PolicyStatement({
243243
actions: ['sts:AssumeRole', 'sts:TagSession'],
244244
principals: [new iam.ServicePrincipal('pods.eks.amazonaws.com')],
245245
})
246246
);
247-
248247
const ebs_csi_addon = new eks.CfnAddon(stack, 'aws-ebs-csi-driver', {
249248
clusterName: cluster.clusterName,
250249
addonName: 'aws-ebs-csi-driver',

‎lib/Easy_EKS.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ export class Easy_EKS{ //purposefully don't extend stack, to implement builder p
7070
deploy_eks_construct_into_this_objects_stack(){
7171
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
7272
//Logic to define baseline Managed Node Group
73-
this.config.baselineNodeRole = initialize_baselineNodeRole(this.stack);
73+
this.config.workerNodeRole = initialize_WorkerNodeRole(this.stack);
74+
//^-- Both have the same IAM rights, but 2 objects were needed to avoid a cdk destroy error
7475
const baseline_LT_Spec = initalize_baseline_LT_Spec(this.stack, this.config);
7576
const baseline_MNG: eks.NodegroupOptions = {
7677
subnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS },
@@ -80,7 +81,7 @@ export class Easy_EKS{ //purposefully don't extend stack, to implement builder p
8081
desiredSize: this.config.baselineNodesNumber,
8182
minSize: 0,
8283
maxSize: 50,
83-
nodeRole: this.config.baselineNodeRole,
84+
nodeRole: this.config.workerNodeRole,
8485
launchTemplateSpec: baseline_LT_Spec, //<-- necessary to add tags to EC2 instances
8586
};
8687
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -206,7 +207,7 @@ export class Easy_EKS{ //purposefully don't extend stack, to implement builder p
206207

207208

208209
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
209-
function initialize_baselineNodeRole(stack: cdk.Stack){
210+
function initialize_WorkerNodeRole(stack: cdk.Stack){
210211
const ipv6_support_iam_policy = new iam.PolicyDocument({
211212
statements: [
212213
new iam.PolicyStatement({
@@ -219,7 +220,7 @@ function initialize_baselineNodeRole(stack: cdk.Stack){
219220
}),
220221
],
221222
});
222-
const EKS_Node_Role = new iam.Role(stack, `EKS_Node_Role`, {
223+
const EKS_Worker_Node_Role = new iam.Role(stack, 'EKS_Worker_Node_Role', {
223224
//roleName: //cdk isn't great about cleaning up resources, so leting it generate name is more reliable
224225
assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'),
225226
managedPolicies: [
@@ -234,7 +235,8 @@ function initialize_baselineNodeRole(stack: cdk.Stack){
234235
ipv6_support_iam_policy,
235236
},
236237
});
237-
return EKS_Node_Role
238+
EKS_Worker_Node_Role.applyRemovalPolicy(cdk.RemovalPolicy.RETAIN); //Workaround to avoid cdk destroy bug
239+
return EKS_Worker_Node_Role
238240
}
239241
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
240242

‎lib/Easy_EKS_Config_Data.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ export class Easy_EKS_Config_Data { //This object just holds config data.
2525
kmsKeyAlias: string; //kms key with this alias will be created or reused if pre-existing
2626
baselineNodesNumber: number;
2727
baselineNodesType: eks.CapacityType; //enum eks.CapacityType.SPOT or eks.CapacityType.ON_DEMAND
28-
baselineNodeRole: iam.Role; //used by baselineMNG & Karpenter
28+
workerNodeRole: iam.Role; //used by baselineMNG & Karpenter
2929
constructor(id_for_stack_and_eks: string){
3030
this.id = id_for_stack_and_eks; /*
3131
Constructor with minimal args is on purpose for desired UX of "builder pattern".

‎lib/Karpenter_Manifests.ts‎

Lines changed: 87 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
import { Easy_EKS_Config_Data } from './Easy_EKS_Config_Data';
22
import * as cdk from 'aws-cdk-lib';
33
import * as eks from 'aws-cdk-lib/aws-eks';
4+
import * as iam from 'aws-cdk-lib/aws-iam';
5+
import { IConstruct } from 'constructs';
6+
import { Karpenter } from 'cdk-eks-karpenter' //npm install cdk-eks-karpenter
7+
8+
export interface Karpenter_Helm_Config {
9+
helm_chart_version: string,
10+
helm_chart_values?: Record<string, any> | undefined,
11+
}
412

513
export interface Karpenter_Manifest_Loop_Inputs {
614
arch: string,
@@ -9,6 +17,8 @@ export interface Karpenter_Manifest_Loop_Inputs {
917
nodepools_cpu_limit: number,
1018
}
1119

20+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
21+
1222
export class Karpenter_YAML_Generator{
1323

1424
config: Easy_EKS_Config_Data;
@@ -54,7 +64,7 @@ export class Karpenter_YAML_Generator{
5464
},
5565
"spec": {
5666
"amiFamily": "Bottlerocket",
57-
"role": `${config.baselineNodeRole.roleName}`,
67+
"role": `${config.workerNodeRole.roleName}`,
5868
"subnetSelectorTerms": subnetSelectorTerms,
5969
"securityGroupSelectorTerms": [ { "tags": { "aws:eks:cluster-name": `${cluster.clusterName}` } } ],
6070
"tags": {//ARM64-bottlerocket-spot
@@ -132,9 +142,83 @@ export class Karpenter_YAML_Generator{
132142
array_of_yaml_manifests_to_return.push(karpenter_bottlerocket_NodePool);
133143
}//end for
134144

135-
136-
137145
return array_of_yaml_manifests_to_return;
138146
} //end generate_manifests
139147

140148
} //end class Karepnter_Manifests
149+
150+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
151+
152+
export function Apply_Karpenter_YAMLs_with_fixes(stack: cdk.Stack, cluster: eks.Cluster, config: Easy_EKS_Config_Data,
153+
karpenter_helm_config: Karpenter_Helm_Config,
154+
karpenter_YAMLs: {[key: string]: any;}[],
155+
readiness_dependency: IConstruct){
156+
157+
//v-- Creates a ton of prerequisites and a release/instance of karpenter helm chart
158+
const karpenter = new Karpenter(stack, 'Karpenter', {
159+
cluster: cluster,
160+
namespace: 'kube-system',
161+
version: karpenter_helm_config.helm_chart_version,
162+
nodeRole: config.workerNodeRole,
163+
helmExtraValues: karpenter_helm_config.helm_chart_values,
164+
});
165+
//v-- The following 2 lines update cdk's order of operations to wait to deploy karpenter, until cluster is ready
166+
karpenter.node.addDependency(readiness_dependency); //Expected value of awsLoadBalancerController Helm Release
167+
karpenter.node.addDependency(cluster.awsAuth);
168+
//v-- Patch fix for https://github.com/aws-samples/cdk-eks-karpenter/issues/231
169+
Patch_Karpenters_IAM_Role(stack, config);
170+
//v-- The following 2 lines help prevent cdk destroy issue
171+
const karpenter_helm_chart_CFR = (stack.node.tryFindChild(config.id)?.node.tryFindChild('chart-karpenter')?.node.defaultChild as cdk.CfnResource);
172+
if(karpenter_helm_chart_CFR){ karpenter_helm_chart_CFR.applyRemovalPolicy(cdk.RemovalPolicy.RETAIN); }
173+
174+
//v-- kubectl apply -f karpenter_YAMLs
175+
const apply_karpenter_YAML = new eks.KubernetesManifest(stack, 'karpenter_YAMLs',
176+
{
177+
cluster: cluster,
178+
manifest: karpenter_YAMLs,
179+
overwrite: true,
180+
prune: true,
181+
}
182+
);
183+
//v-- Inform cdk of order of operations
184+
apply_karpenter_YAML.node.addDependency(karpenter);
185+
//v-- The following 2 lines prevent cdk destroy issue
186+
const apply_karpenter_YAML_CFR = (apply_karpenter_YAML.node.defaultChild as cdk.CfnResource);
187+
if(apply_karpenter_YAML_CFR){ apply_karpenter_YAML_CFR.applyRemovalPolicy(cdk.RemovalPolicy.RETAIN); }
188+
189+
} //end function Apply_Karpenter_YAMLs_with_fixes
190+
191+
function Patch_Karpenters_IAM_Role(stack: cdk.Stack, config: Easy_EKS_Config_Data){
192+
//Patch fix for https://github.com/aws-samples/cdk-eks-karpenter/issues/231
193+
let karpenter_controller_pods_role = stack.node.tryFindChild(config.id)?.node.tryFindChild('karpenter')?.node.tryFindChild('Role') as iam.Role;
194+
const karpenter_IAM_Policy_JSON = {
195+
"Version": "2012-10-17",
196+
"Statement": [
197+
{
198+
"Sid": "AllowInstanceProfileActions",
199+
"Effect": "Allow",
200+
"Resource": `arn:aws:iam::${process.env.CDK_DEFAULT_ACCOUNT!}:instance-profile/*`,
201+
"Action": [
202+
"iam:GetInstanceProfile",
203+
"iam:AddRoleToInstanceProfile",
204+
"iam:RemoveRoleFromInstanceProfile",
205+
],
206+
},
207+
{ //v-- This isn't a hard requirement, but enables faster convergence
208+
// without it karpenter logs temporarily mention an IAM rights failure, that fixes itself within 11 mins.
209+
"Sid": "LessRestrictivePassRoleForFasterConvergence",
210+
"Effect": "Allow",
211+
"Resource": `arn:aws:iam::${process.env.CDK_DEFAULT_ACCOUNT!}:role/*`,
212+
"Action": [
213+
"iam:PassRole",
214+
],
215+
},
216+
]
217+
};
218+
const karpenter_IAM_Policy = new iam.Policy(stack, `karpenter_controller_pod_IAM_policy_for_EKS`, {
219+
document: iam.PolicyDocument.fromJson( karpenter_IAM_Policy_JSON ),
220+
});
221+
karpenter_controller_pods_role.attachInlinePolicy(karpenter_IAM_Policy);
222+
}
223+
224+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

‎package-lock.json‎

Lines changed: 8 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
"dependencies": {
2222
"@aws-cdk/lambda-layer-kubectl-v31": "^2.0.3",
2323
"aws-cdk": "^2.1010.0",
24-
"cdk-eks-karpenter": "^1.0.13",
24+
"cdk-eks-karpenter": "^1.0.16",
2525
"cdk-fck-nat": "^1.5.6",
2626
"constructs": "^10.0.0",
2727
"i": "^0.3.7",

0 commit comments

Comments
 (0)