Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647
github.com/awslabs/operatorpkg v0.0.0-20240518001059-1e35978ba21b
github.com/go-logr/zapr v1.3.0
github.com/google/uuid v1.6.0
github.com/imdario/mergo v0.3.16
github.com/mitchellh/hashstructure/v2 v2.0.2
github.com/onsi/ginkgo/v2 v2.20.0
Expand Down Expand Up @@ -64,7 +65,6 @@ require (
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
Expand Down
30 changes: 19 additions & 11 deletions pkg/cloudprovider/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -618,7 +618,7 @@ var _ = Describe("CloudProvider", func() {
},
},
})
awsEnv.EC2API.DescribeSecurityGroupsOutput.Set(&ec2.DescribeSecurityGroupsOutput{
awsEnv.EC2API.DescribeSecurityGroupsBehavior.Output.Set(&ec2.DescribeSecurityGroupsOutput{
SecurityGroups: []*ec2.SecurityGroup{
{
GroupId: aws.String(validSecurityGroup),
Expand All @@ -632,7 +632,7 @@ var _ = Describe("CloudProvider", func() {
},
},
})
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{
Subnets: []*ec2.Subnet{
{
SubnetId: aws.String(validSubnet1),
Expand Down Expand Up @@ -1084,7 +1084,7 @@ var _ = Describe("CloudProvider", func() {
})
It("should launch instances into subnet with the most available IP addresses", func() {
awsEnv.SubnetCache.Flush()
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
{SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(10),
Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}},
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(100),
Expand All @@ -1101,7 +1101,7 @@ var _ = Describe("CloudProvider", func() {
})
It("should launch instances into subnet with the most available IP addresses in-between cache refreshes", func() {
awsEnv.SubnetCache.Flush()
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
{SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(10),
Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}},
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(11),
Expand All @@ -1126,7 +1126,7 @@ var _ = Describe("CloudProvider", func() {
Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-1"))
})
It("should update in-flight IPs when a CreateFleet error occurs", func() {
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
{SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10),
Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}},
}})
Expand All @@ -1137,12 +1137,20 @@ var _ = Describe("CloudProvider", func() {
Expect(len(bindings)).To(Equal(0))
})
It("should launch instances into subnets that are excluded by another NodePool", func() {
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
{SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(10),
Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}},
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1b"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(100),
Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}},
}})
awsEnv.EC2API.Subnets.Store("test-zone-1a", &ec2.Subnet{
SubnetId: aws.String("test-subnet-1"),
AvailabilityZone: aws.String("test-zone-1a"),
AvailabilityZoneId: aws.String("tstz1-1a"),
AvailableIpAddressCount: aws.Int64(10),
Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}},
})
awsEnv.EC2API.Subnets.Store("test-zone-1b", &ec2.Subnet{
SubnetId: aws.String("test-subnet-2"),
AvailabilityZone: aws.String("test-zone-1b"),
AvailabilityZoneId: aws.String("tstz1-1a"),
AvailableIpAddressCount: aws.Int64(100),
Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}},
})
nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1"}}}
ExpectApplied(ctx, env.Client, nodePool, nodeClass)
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/nodeclass/status/subnet_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ var _ = Describe("NodeClass Subnet Status Controller", func() {
}))
})
It("Should have the correct ordering for the Subnets", func() {
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{
{SubnetId: aws.String("subnet-test1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int64(20)},
{SubnetId: aws.String("subnet-test2"), AvailabilityZone: aws.String("test-zone-1b"), AvailabilityZoneId: aws.String("tstz1-1b"), AvailableIpAddressCount: aws.Int64(100)},
{SubnetId: aws.String("subnet-test3"), AvailabilityZone: aws.String("test-zone-1c"), AvailabilityZoneId: aws.String("tstz1-1c"), AvailableIpAddressCount: aws.Int64(50)},
Expand Down
6 changes: 6 additions & 0 deletions pkg/controllers/providers/ssm/invalidation/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ import (
"sigs.k8s.io/karpenter/pkg/operator/controller"
"sigs.k8s.io/karpenter/pkg/operator/injection"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"

"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
"github.com/aws/karpenter-provider-aws/pkg/providers/ssm"
Expand Down Expand Up @@ -65,6 +68,9 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc
amis := []amifamily.AMI{}
for _, nodeClass := range lo.Map(lo.Keys(amiIDsToParameters), func(amiID string, _ int) *v1beta1.EC2NodeClass {
return &v1beta1.EC2NodeClass{
ObjectMeta: metav1.ObjectMeta{
UID: uuid.NewUUID(), // ensures that this doesn't hit the AMI cache.
},
Spec: v1beta1.EC2NodeClassSpec{
AMISelectorTerms: []v1beta1.AMISelectorTerm{{ID: amiID}},
},
Expand Down
4 changes: 2 additions & 2 deletions pkg/controllers/providers/ssm/invalidation/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ var _ = Describe("SSM Invalidation Controller", func() {
Expect(err).To(BeNil())
currentEntries := getSSMCacheEntries()
Expect(len(currentEntries)).To(Equal(2))
awsEnv.EC2Cache.Flush()
awsEnv.AMICache.Flush()
ExpectReconcileSucceeded(ctx, invalidationController, client.ObjectKey{})
awsEnv.SSMAPI.Reset()
_, err = awsEnv.AMIProvider.List(ctx, nodeClass)
Expand All @@ -106,7 +106,7 @@ var _ = Describe("SSM Invalidation Controller", func() {
currentEntries := getSSMCacheEntries()
deprecateAMIs(lo.Values(currentEntries)...)
Expect(len(currentEntries)).To(Equal(2))
awsEnv.EC2Cache.Flush()
awsEnv.AMICache.Flush()
ExpectReconcileSucceeded(ctx, invalidationController, client.ObjectKey{})
awsEnv.SSMAPI.Reset()
_, err = awsEnv.AMIProvider.List(ctx, nodeClass)
Expand Down
7 changes: 7 additions & 0 deletions pkg/fake/atomic.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,13 @@ func (a *AtomicPtrSlice[T]) Pop() *T {
return last
}

func (a *AtomicPtrSlice[T]) At(index int) *T {
a.mu.Lock()
defer a.mu.Unlock()

return clone(a.values[index])
}

func (a *AtomicPtrSlice[T]) ForEach(fn func(*T)) {
a.mu.RLock()
defer a.mu.RUnlock()
Expand Down
193 changes: 98 additions & 95 deletions pkg/fake/ec2api.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ type CapacityPool struct {
type EC2Behavior struct {
DescribeImagesOutput AtomicPtr[ec2.DescribeImagesOutput]
DescribeLaunchTemplatesOutput AtomicPtr[ec2.DescribeLaunchTemplatesOutput]
DescribeSubnetsOutput AtomicPtr[ec2.DescribeSubnetsOutput]
DescribeSecurityGroupsOutput AtomicPtr[ec2.DescribeSecurityGroupsOutput]
DescribeSubnetsBehavior MockedFunction[ec2.DescribeSubnetsInput, ec2.DescribeSubnetsOutput]
DescribeSecurityGroupsBehavior MockedFunction[ec2.DescribeSecurityGroupsInput, ec2.DescribeSecurityGroupsOutput]
DescribeInstanceTypesOutput AtomicPtr[ec2.DescribeInstanceTypesOutput]
DescribeInstanceTypeOfferingsOutput AtomicPtr[ec2.DescribeInstanceTypeOfferingsOutput]
DescribeAvailabilityZonesOutput AtomicPtr[ec2.DescribeAvailabilityZonesOutput]
Expand All @@ -61,6 +61,7 @@ type EC2Behavior struct {
CreateTagsBehavior MockedFunction[ec2.CreateTagsInput, ec2.CreateTagsOutput]
CalledWithCreateLaunchTemplateInput AtomicPtrSlice[ec2.CreateLaunchTemplateInput]
CalledWithDescribeImagesInput AtomicPtrSlice[ec2.DescribeImagesInput]
Subnets sync.Map
Instances sync.Map
LaunchTemplates sync.Map
InsufficientCapacityPools atomic.Slice[CapacityPool]
Expand All @@ -84,8 +85,8 @@ var DefaultSupportedUsageClasses = aws.StringSlice([]string{"on-demand", "spot"}
func (e *EC2API) Reset() {
e.DescribeImagesOutput.Reset()
e.DescribeLaunchTemplatesOutput.Reset()
e.DescribeSubnetsOutput.Reset()
e.DescribeSecurityGroupsOutput.Reset()
e.DescribeSubnetsBehavior.Reset()
e.DescribeSecurityGroupsBehavior.Reset()
e.DescribeInstanceTypesOutput.Reset()
e.DescribeInstanceTypeOfferingsOutput.Reset()
e.DescribeAvailabilityZonesOutput.Reset()
Expand Down Expand Up @@ -405,107 +406,109 @@ func (e *EC2API) DeleteLaunchTemplateWithContext(_ context.Context, input *ec2.D
}

func (e *EC2API) DescribeSubnetsWithContext(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...request.Option) (*ec2.DescribeSubnetsOutput, error) {
if !e.NextError.IsNil() {
defer e.NextError.Reset()
return nil, e.NextError.Get()
}
if !e.DescribeSubnetsOutput.IsNil() {
describeSubnetsOutput := e.DescribeSubnetsOutput.Clone()
describeSubnetsOutput.Subnets = FilterDescribeSubnets(describeSubnetsOutput.Subnets, input.Filters)
return describeSubnetsOutput, nil
}
subnets := []*ec2.Subnet{
{
SubnetId: aws.String("subnet-test1"),
AvailabilityZone: aws.String("test-zone-1a"),
AvailabilityZoneId: aws.String("tstz1-1a"),
AvailableIpAddressCount: aws.Int64(100),
MapPublicIpOnLaunch: aws.Bool(false),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-1")},
{Key: aws.String("foo"), Value: aws.String("bar")},
return e.DescribeSubnetsBehavior.Invoke(input, func(input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) {
output := &ec2.DescribeSubnetsOutput{}
e.Subnets.Range(func(key, value any) bool {
subnet := value.(*ec2.Subnet)
if lo.Contains(lo.Map(input.SubnetIds, func(s *string, _ int) string { return lo.FromPtr(s) }), lo.FromPtr(subnet.SubnetId)) || len(input.Filters) != 0 && len(FilterDescribeSubnets([]*ec2.Subnet{subnet}, input.Filters)) != 0 {
output.Subnets = append(output.Subnets, subnet)
}
return true
})
if len(output.Subnets) != 0 {
return output, nil
}

defaultSubnets := []*ec2.Subnet{
{
SubnetId: aws.String("subnet-test1"),
AvailabilityZone: aws.String("test-zone-1a"),
AvailabilityZoneId: aws.String("tstz1-1a"),
AvailableIpAddressCount: aws.Int64(100),
MapPublicIpOnLaunch: aws.Bool(false),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-1")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
VpcId: aws.String("vpc-test1"),
},
},
{
SubnetId: aws.String("subnet-test2"),
AvailabilityZone: aws.String("test-zone-1b"),
AvailabilityZoneId: aws.String("tstz1-1b"),
AvailableIpAddressCount: aws.Int64(100),
MapPublicIpOnLaunch: aws.Bool(true),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-2")},
{Key: aws.String("foo"), Value: aws.String("bar")},
{
SubnetId: aws.String("subnet-test2"),
AvailabilityZone: aws.String("test-zone-1b"),
AvailabilityZoneId: aws.String("tstz1-1b"),
AvailableIpAddressCount: aws.Int64(100),
MapPublicIpOnLaunch: aws.Bool(true),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-2")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
VpcId: aws.String("vpc-test1"),
},
},
{
SubnetId: aws.String("subnet-test3"),
AvailabilityZone: aws.String("test-zone-1c"),
AvailabilityZoneId: aws.String("tstz1-1c"),
AvailableIpAddressCount: aws.Int64(100),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-3")},
{Key: aws.String("TestTag")},
{Key: aws.String("foo"), Value: aws.String("bar")},
{
SubnetId: aws.String("subnet-test3"),
AvailabilityZone: aws.String("test-zone-1c"),
AvailabilityZoneId: aws.String("tstz1-1c"),
AvailableIpAddressCount: aws.Int64(100),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-3")},
{Key: aws.String("TestTag")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
VpcId: aws.String("vpc-test1"),
},
},
{
SubnetId: aws.String("subnet-test4"),
AvailabilityZone: aws.String("test-zone-1a-local"),
AvailabilityZoneId: aws.String("tstz1-1alocal"),
AvailableIpAddressCount: aws.Int64(100),
MapPublicIpOnLaunch: aws.Bool(true),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-4")},
{
SubnetId: aws.String("subnet-test4"),
AvailabilityZone: aws.String("test-zone-1a-local"),
AvailabilityZoneId: aws.String("tstz1-1alocal"),
AvailableIpAddressCount: aws.Int64(100),
MapPublicIpOnLaunch: aws.Bool(true),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-4")},
},
VpcId: aws.String("vpc-test1"),
},
},
}
if len(input.Filters) == 0 {
return nil, fmt.Errorf("InvalidParameterValue: The filter 'null' is invalid")
}
return &ec2.DescribeSubnetsOutput{Subnets: FilterDescribeSubnets(subnets, input.Filters)}, nil
}
if len(input.Filters) == 0 {
return nil, fmt.Errorf("InvalidParameterValue: The filter 'null' is invalid")
}
return &ec2.DescribeSubnetsOutput{Subnets: FilterDescribeSubnets(defaultSubnets, input.Filters)}, nil
})
}

func (e *EC2API) DescribeSecurityGroupsWithContext(_ context.Context, input *ec2.DescribeSecurityGroupsInput, _ ...request.Option) (*ec2.DescribeSecurityGroupsOutput, error) {
if !e.NextError.IsNil() {
defer e.NextError.Reset()
return nil, e.NextError.Get()
}
if !e.DescribeSecurityGroupsOutput.IsNil() {
describeSecurityGroupsOutput := e.DescribeSecurityGroupsOutput.Clone()
describeSecurityGroupsOutput.SecurityGroups = FilterDescribeSecurtyGroups(describeSecurityGroupsOutput.SecurityGroups, input.Filters)
return e.DescribeSecurityGroupsOutput.Clone(), nil
}
sgs := []*ec2.SecurityGroup{
{
GroupId: aws.String("sg-test1"),
GroupName: aws.String("securityGroup-test1"),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-1")},
{Key: aws.String("foo"), Value: aws.String("bar")},
return e.DescribeSecurityGroupsBehavior.Invoke(input, func(input *ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) {
defaultSecurityGroups := []*ec2.SecurityGroup{
{
GroupId: aws.String("sg-test1"),
GroupName: aws.String("securityGroup-test1"),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-1")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
},
},
{
GroupId: aws.String("sg-test2"),
GroupName: aws.String("securityGroup-test2"),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-2")},
{Key: aws.String("foo"), Value: aws.String("bar")},
{
GroupId: aws.String("sg-test2"),
GroupName: aws.String("securityGroup-test2"),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-2")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
},
},
{
GroupId: aws.String("sg-test3"),
GroupName: aws.String("securityGroup-test3"),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-3")},
{Key: aws.String("TestTag")},
{Key: aws.String("foo"), Value: aws.String("bar")},
{
GroupId: aws.String("sg-test3"),
GroupName: aws.String("securityGroup-test3"),
Tags: []*ec2.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-3")},
{Key: aws.String("TestTag")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
},
},
}
if len(input.Filters) == 0 {
return nil, fmt.Errorf("InvalidParameterValue: The filter 'null' is invalid")
}
return &ec2.DescribeSecurityGroupsOutput{SecurityGroups: FilterDescribeSecurtyGroups(sgs, input.Filters)}, nil
}
if len(input.Filters) == 0 {
return nil, fmt.Errorf("InvalidParameterValue: The filter 'null' is invalid")
}
return &ec2.DescribeSecurityGroupsOutput{SecurityGroups: FilterDescribeSecurtyGroups(defaultSecurityGroups, input.Filters)}, nil
})
}

func (e *EC2API) DescribeAvailabilityZonesWithContext(context.Context, *ec2.DescribeAvailabilityZonesInput, ...request.Option) (*ec2.DescribeAvailabilityZonesOutput, error) {
Expand Down
Loading
Loading