Skip to content

(Backport v1.1.x)fix: prevent hash collisions while resolving subnets, security groups and AMIs from nodeclass selectors #8661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 23, 2025
Merged

(Backport v1.1.x)fix: prevent hash collisions while resolving subnets, security groups and AMIs from nodeclass selectors #8661

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions pkg/cloudprovider/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,7 @@ var _ = Describe("CloudProvider", func() {
},
},
})
awsEnv.EC2API.DescribeSecurityGroupsOutput.Set(&ec2.DescribeSecurityGroupsOutput{
awsEnv.EC2API.DescribeSecurityGroupsBehavior.Output.Set(&ec2.DescribeSecurityGroupsOutput{
SecurityGroups: []ec2types.SecurityGroup{
{
GroupId: aws.String(validSecurityGroup),
Expand All @@ -660,7 +660,7 @@ var _ = Describe("CloudProvider", func() {
},
},
})
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{
Subnets: []ec2types.Subnet{
{
SubnetId: aws.String(validSubnet1),
Expand Down Expand Up @@ -1142,7 +1142,7 @@ var _ = Describe("CloudProvider", func() {
})
It("should launch instances into subnet with the most available IP addresses", func() {
awsEnv.SubnetCache.Flush()
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
{SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(10),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}},
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(100),
Expand All @@ -1159,7 +1159,7 @@ var _ = Describe("CloudProvider", func() {
})
It("should launch instances into subnet with the most available IP addresses in-between cache refreshes", func() {
awsEnv.SubnetCache.Flush()
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
{SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(10),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}},
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(11),
Expand Down Expand Up @@ -1187,7 +1187,7 @@ var _ = Describe("CloudProvider", func() {
Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-1"))
})
It("should update in-flight IPs when a CreateFleet error occurs", func() {
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
{SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int32(10),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}},
}})
Expand All @@ -1198,12 +1198,20 @@ var _ = Describe("CloudProvider", func() {
Expect(len(bindings)).To(Equal(0))
})
It("should launch instances into subnets that are excluded by another NodePool", func() {
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
{SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(10),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}},
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1b"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(100),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}},
}})
awsEnv.EC2API.Subnets.Store("test-zone-1a", ec2types.Subnet{
SubnetId: aws.String("test-subnet-1"),
AvailabilityZone: aws.String("test-zone-1a"),
AvailabilityZoneId: aws.String("tstz1-1a"),
AvailableIpAddressCount: aws.Int32(10),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}},
})
awsEnv.EC2API.Subnets.Store("test-zone-1b", ec2types.Subnet{
SubnetId: aws.String("test-subnet-2"),
AvailabilityZone: aws.String("test-zone-1b"),
AvailabilityZoneId: aws.String("tstz1-1a"),
AvailableIpAddressCount: aws.Int32(100),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}},
})
nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1"}}}
ExpectApplied(ctx, env.Client, nodePool, nodeClass)
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider)
Expand Down
4 changes: 2 additions & 2 deletions pkg/controllers/nodeclass/status/ami_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,7 @@ var _ = Describe("NodeClass AMI Status Controller", func() {
awsEnv.Clock.Step(40 * time.Minute)

// Flush Cache
awsEnv.EC2Cache.Flush()
awsEnv.AMICache.Flush()

ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass)
nodeClass = ExpectExists(ctx, env.Client, nodeClass)
Expand Down Expand Up @@ -706,7 +706,7 @@ var _ = Describe("NodeClass AMI Status Controller", func() {
},
})

awsEnv.EC2Cache.Flush()
awsEnv.AMICache.Flush()

ExpectApplied(ctx, env.Client, nodeClass)
ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/nodeclass/status/subnet_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ var _ = Describe("NodeClass Subnet Status Controller", func() {
Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeSubnetsReady)).To(BeTrue())
})
It("Should have the correct ordering for the Subnets", func() {
awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
awsEnv.EC2API.DescribeSubnetsBehavior.Output.Set(&ec2.DescribeSubnetsOutput{Subnets: []ec2types.Subnet{
{SubnetId: aws.String("subnet-test1"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(20)},
{SubnetId: aws.String("subnet-test2"), AvailabilityZone: aws.String("test-zone-1b"), AvailabilityZoneId: aws.String("tstz1-1b"), AvailableIpAddressCount: aws.Int32(100)},
{SubnetId: aws.String("subnet-test3"), AvailabilityZone: aws.String("test-zone-1c"), AvailabilityZoneId: aws.String("tstz1-1c"), AvailableIpAddressCount: aws.Int32(50)},
Expand Down
6 changes: 6 additions & 0 deletions pkg/controllers/providers/ssm/invalidation/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ import (
v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
"github.com/aws/karpenter-provider-aws/pkg/providers/ssm"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
)

// The SSM Invalidation controller is responsible for invalidating "latest" SSM parameters when they point to deprecated
Expand Down Expand Up @@ -66,6 +69,9 @@ func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) {
amis := []amifamily.AMI{}
for _, nodeClass := range lo.Map(lo.Keys(amiIDsToParameters), func(amiID string, _ int) *v1.EC2NodeClass {
return &v1.EC2NodeClass{
ObjectMeta: metav1.ObjectMeta{
UID: uuid.NewUUID(), // ensures that this doesn't hit the AMI cache.
},
Spec: v1.EC2NodeClassSpec{
AMISelectorTerms: []v1.AMISelectorTerm{{ID: amiID}},
},
Expand Down
193 changes: 98 additions & 95 deletions pkg/fake/ec2api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,18 +48,19 @@ type CapacityPool struct {
type EC2Behavior struct {
DescribeImagesOutput AtomicPtr[ec2.DescribeImagesOutput]
DescribeLaunchTemplatesOutput AtomicPtr[ec2.DescribeLaunchTemplatesOutput]
DescribeSubnetsOutput AtomicPtr[ec2.DescribeSubnetsOutput]
DescribeSecurityGroupsOutput AtomicPtr[ec2.DescribeSecurityGroupsOutput]
DescribeInstanceTypesOutput AtomicPtr[ec2.DescribeInstanceTypesOutput]
DescribeInstanceTypeOfferingsOutput AtomicPtr[ec2.DescribeInstanceTypeOfferingsOutput]
DescribeAvailabilityZonesOutput AtomicPtr[ec2.DescribeAvailabilityZonesOutput]
DescribeSpotPriceHistoryBehavior MockedFunction[ec2.DescribeSpotPriceHistoryInput, ec2.DescribeSpotPriceHistoryOutput]
CreateFleetBehavior MockedFunction[ec2.CreateFleetInput, ec2.CreateFleetOutput]
TerminateInstancesBehavior MockedFunction[ec2.TerminateInstancesInput, ec2.TerminateInstancesOutput]
DescribeInstancesBehavior MockedFunction[ec2.DescribeInstancesInput, ec2.DescribeInstancesOutput]
DescribeSubnetsBehavior MockedFunction[ec2.DescribeSubnetsInput, ec2.DescribeSubnetsOutput]
DescribeSecurityGroupsBehavior MockedFunction[ec2.DescribeSecurityGroupsInput, ec2.DescribeSecurityGroupsOutput]
CreateTagsBehavior MockedFunction[ec2.CreateTagsInput, ec2.CreateTagsOutput]
CalledWithCreateLaunchTemplateInput AtomicPtrSlice[ec2.CreateLaunchTemplateInput]
CalledWithDescribeImagesInput AtomicPtrSlice[ec2.DescribeImagesInput]
Subnets sync.Map
Instances sync.Map
LaunchTemplates sync.Map
InsufficientCapacityPools atomic.Slice[CapacityPool]
Expand All @@ -83,8 +84,8 @@ var DefaultSupportedUsageClasses = []ec2types.UsageClassType{ec2types.UsageClass
func (e *EC2API) Reset() {
e.DescribeImagesOutput.Reset()
e.DescribeLaunchTemplatesOutput.Reset()
e.DescribeSubnetsOutput.Reset()
e.DescribeSecurityGroupsOutput.Reset()
e.DescribeSubnetsBehavior.Reset()
e.DescribeSecurityGroupsBehavior.Reset()
e.DescribeInstanceTypesOutput.Reset()
e.DescribeInstanceTypeOfferingsOutput.Reset()
e.DescribeAvailabilityZonesOutput.Reset()
Expand Down Expand Up @@ -379,107 +380,109 @@ func (e *EC2API) DeleteLaunchTemplate(_ context.Context, input *ec2.DeleteLaunch
}

func (e *EC2API) DescribeSubnets(_ context.Context, input *ec2.DescribeSubnetsInput, _ ...func(*ec2.Options)) (*ec2.DescribeSubnetsOutput, error) {
if !e.NextError.IsNil() {
defer e.NextError.Reset()
return nil, e.NextError.Get()
}
if !e.DescribeSubnetsOutput.IsNil() {
describeSubnetsOutput := e.DescribeSubnetsOutput.Clone()
describeSubnetsOutput.Subnets = FilterDescribeSubnets(describeSubnetsOutput.Subnets, input.Filters)
return describeSubnetsOutput, nil
}
subnets := []ec2types.Subnet{
{
SubnetId: aws.String("subnet-test1"),
AvailabilityZone: aws.String("test-zone-1a"),
AvailabilityZoneId: aws.String("tstz1-1a"),
AvailableIpAddressCount: aws.Int32(100),
MapPublicIpOnLaunch: aws.Bool(false),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-1")},
{Key: aws.String("foo"), Value: aws.String("bar")},
return e.DescribeSubnetsBehavior.Invoke(input, func(input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) {
output := &ec2.DescribeSubnetsOutput{}
e.Subnets.Range(func(key, value any) bool {
subnet := value.(ec2types.Subnet)
if lo.Contains(input.SubnetIds, lo.FromPtr(subnet.SubnetId)) || len(input.Filters) != 0 && len(FilterDescribeSubnets([]ec2types.Subnet{subnet}, input.Filters)) != 0 {
output.Subnets = append(output.Subnets, subnet)
}
return true
})
if len(output.Subnets) != 0 {
return output, nil
}

defaultSubnets := []ec2types.Subnet{
{
SubnetId: aws.String("subnet-test1"),
AvailabilityZone: aws.String("test-zone-1a"),
AvailabilityZoneId: aws.String("tstz1-1a"),
AvailableIpAddressCount: aws.Int32(100),
MapPublicIpOnLaunch: aws.Bool(false),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-1")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
VpcId: aws.String("vpc-test1"),
},
},
{
SubnetId: aws.String("subnet-test2"),
AvailabilityZone: aws.String("test-zone-1b"),
AvailabilityZoneId: aws.String("tstz1-1b"),
AvailableIpAddressCount: aws.Int32(100),
MapPublicIpOnLaunch: aws.Bool(true),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-2")},
{Key: aws.String("foo"), Value: aws.String("bar")},
{
SubnetId: aws.String("subnet-test2"),
AvailabilityZone: aws.String("test-zone-1b"),
AvailabilityZoneId: aws.String("tstz1-1b"),
AvailableIpAddressCount: aws.Int32(100),
MapPublicIpOnLaunch: aws.Bool(true),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-2")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
VpcId: aws.String("vpc-test1"),
},
},
{
SubnetId: aws.String("subnet-test3"),
AvailabilityZone: aws.String("test-zone-1c"),
AvailabilityZoneId: aws.String("tstz1-1c"),
AvailableIpAddressCount: aws.Int32(100),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-3")},
{Key: aws.String("TestTag")},
{Key: aws.String("foo"), Value: aws.String("bar")},
{
SubnetId: aws.String("subnet-test3"),
AvailabilityZone: aws.String("test-zone-1c"),
AvailabilityZoneId: aws.String("tstz1-1c"),
AvailableIpAddressCount: aws.Int32(100),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-3")},
{Key: aws.String("TestTag")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
VpcId: aws.String("vpc-test1"),
},
},
{
SubnetId: aws.String("subnet-test4"),
AvailabilityZone: aws.String("test-zone-1a-local"),
AvailabilityZoneId: aws.String("tstz1-1alocal"),
AvailableIpAddressCount: aws.Int32(100),
MapPublicIpOnLaunch: aws.Bool(true),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-4")},
{
SubnetId: aws.String("subnet-test4"),
AvailabilityZone: aws.String("test-zone-1a-local"),
AvailabilityZoneId: aws.String("tstz1-1alocal"),
AvailableIpAddressCount: aws.Int32(100),
MapPublicIpOnLaunch: aws.Bool(true),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-subnet-4")},
},
VpcId: aws.String("vpc-test1"),
},
},
}
if len(input.Filters) == 0 {
return nil, fmt.Errorf("InvalidParameterValue: The filter 'null' is invalid")
}
return &ec2.DescribeSubnetsOutput{Subnets: FilterDescribeSubnets(subnets, input.Filters)}, nil
}
if len(input.Filters) == 0 {
return nil, fmt.Errorf("InvalidParameterValue: The filter 'null' is invalid")
}
return &ec2.DescribeSubnetsOutput{Subnets: FilterDescribeSubnets(defaultSubnets, input.Filters)}, nil
})
}

func (e *EC2API) DescribeSecurityGroups(_ context.Context, input *ec2.DescribeSecurityGroupsInput, _ ...func(*ec2.Options)) (*ec2.DescribeSecurityGroupsOutput, error) {
if !e.NextError.IsNil() {
defer e.NextError.Reset()
return nil, e.NextError.Get()
}
if !e.DescribeSecurityGroupsOutput.IsNil() {
describeSecurityGroupsOutput := e.DescribeSecurityGroupsOutput.Clone()
describeSecurityGroupsOutput.SecurityGroups = FilterDescribeSecurtyGroups(describeSecurityGroupsOutput.SecurityGroups, input.Filters)
return e.DescribeSecurityGroupsOutput.Clone(), nil
}
sgs := []ec2types.SecurityGroup{
{
GroupId: aws.String("sg-test1"),
GroupName: aws.String("securityGroup-test1"),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-1")},
{Key: aws.String("foo"), Value: aws.String("bar")},
return e.DescribeSecurityGroupsBehavior.Invoke(input, func(input *ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) {
defaultSecurityGroups := []ec2types.SecurityGroup{
{
GroupId: aws.String("sg-test1"),
GroupName: aws.String("securityGroup-test1"),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-1")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
},
},
{
GroupId: aws.String("sg-test2"),
GroupName: aws.String("securityGroup-test2"),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-2")},
{Key: aws.String("foo"), Value: aws.String("bar")},
{
GroupId: aws.String("sg-test2"),
GroupName: aws.String("securityGroup-test2"),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-2")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
},
},
{
GroupId: aws.String("sg-test3"),
GroupName: aws.String("securityGroup-test3"),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-3")},
{Key: aws.String("TestTag")},
{Key: aws.String("foo"), Value: aws.String("bar")},
{
GroupId: aws.String("sg-test3"),
GroupName: aws.String("securityGroup-test3"),
Tags: []ec2types.Tag{
{Key: aws.String("Name"), Value: aws.String("test-security-group-3")},
{Key: aws.String("TestTag")},
{Key: aws.String("foo"), Value: aws.String("bar")},
},
},
},
}
if len(input.Filters) == 0 {
return nil, fmt.Errorf("InvalidParameterValue: The filter 'null' is invalid")
}
return &ec2.DescribeSecurityGroupsOutput{SecurityGroups: FilterDescribeSecurtyGroups(sgs, input.Filters)}, nil
}
if len(input.Filters) == 0 {
return nil, fmt.Errorf("InvalidParameterValue: The filter 'null' is invalid")
}
return &ec2.DescribeSecurityGroupsOutput{SecurityGroups: FilterDescribeSecurtyGroups(defaultSecurityGroups, input.Filters)}, nil
})
}

func (e *EC2API) DescribeAvailabilityZones(context.Context, *ec2.DescribeAvailabilityZonesInput, ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) {
Expand Down
7 changes: 7 additions & 0 deletions pkg/fake/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

type MockedFunction[I any, O any] struct {
Output AtomicPtr[O] // Output to return on call to this function
MultiOut AtomicPtrSlice[O]
OutputPages AtomicPtrSlice[O]
CalledWithInput AtomicPtrSlice[I] // Slice used to keep track of passed input to this function
Error AtomicError // Error to return a certain number of times defined by custom error options
Expand All @@ -38,6 +39,7 @@ type MockedFunction[I any, O any] struct {
// each other.
func (m *MockedFunction[I, O]) Reset() {
m.Output.Reset()
m.MultiOut.Reset()
m.OutputPages.Reset()
m.CalledWithInput.Reset()
m.Error.Reset()
Expand All @@ -59,6 +61,11 @@ func (m *MockedFunction[I, O]) Invoke(input *I, defaultTransformer func(*I) (*O,
m.successfulCalls.Add(1)
return m.Output.Clone(), nil
}

if m.MultiOut.Len() > 0 {
m.successfulCalls.Add(1)
return m.MultiOut.Pop(), nil
}
// This output pages multi-threaded handling isn't perfect
// It will fail if pages are asynchronously requested from the same NextToken
if m.OutputPages.Len() > 0 {
Expand Down
Loading
Loading