Skip to content

feat: add Fleet ID to error logs for CloudTrail correlation #8547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
Open

feat: add Fleet ID to error logs for CloudTrail correlation #8547

Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
a91dd28
feat: add Fleet ID to logs for CloudTrail correlation
moko-poi Sep 30, 2025
fa9a892
feat: add Fleet ID to unavailable offerings logs for CloudTrail corre…
moko-poi Oct 2, 2025
8c0d4a8
feat: add Fleet ID to IsUnfulfillableCapacity error logs only
moko-poi Oct 2, 2025
0c1f951
refactor: consolidate Fleet ID logging in MarkUnavailable method
moko-poi Oct 4, 2025
b088159
refactor: change MarkUnavailable parameter type and improve naming
moko-poi Oct 12, 2025
6462ccd
refactor: remove unnecessary checks in MarkUnavailable logging
moko-poi Oct 12, 2025
e45c7d8
refactor: improve MarkUnavailable function based on reviewer feedback
moko-poi Oct 12, 2025
c845300
refactor: remove unnecessary MarkUnavailableForFleetErr function
moko-poi Oct 12, 2025
ee20b25
refactor: cleanup MarkUnavailable based on code review
moko-poi Oct 12, 2025
5bffe50
refactor: use map[string]string and rename extraContext to unavailabl…
moko-poi Oct 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions pkg/providers/instance/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,8 @@ func (p *DefaultProvider) launchInstance(
}
return ec2types.CreateFleetInstance{}, cloudprovider.NewCreateError(fmt.Errorf("creating fleet request, %w", err), reason, fmt.Sprintf("Error creating fleet request: %s", message))
}
p.updateUnavailableOfferingsCache(ctx, createFleetOutput.Errors, capacityType, nodeClaim, instanceTypes)
fleetID := aws.ToString(createFleetOutput.FleetId)
Copy link
Contributor

@DerekFrank DerekFrank Oct 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I prefer to inline these

Copy link
Author

@moko-poi moko-poi Oct 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point! You're right, that variable was only used once.

p.updateUnavailableOfferingsCache(ctx, createFleetOutput.Errors, capacityType, nodeClaim, instanceTypes, fleetID)
if len(createFleetOutput.Instances) == 0 || len(createFleetOutput.Instances[0].InstanceIds) == 0 {
requestID, _ := awsmiddleware.GetRequestIDMetadata(createFleetOutput.ResultMetadata)
return ec2types.CreateFleetInstance{}, serrors.Wrap(
Expand Down Expand Up @@ -481,6 +482,7 @@ func (p *DefaultProvider) updateUnavailableOfferingsCache(
capacityType string,
nodeClaim *karpv1.NodeClaim,
instanceTypes []*cloudprovider.InstanceType,
fleetID string,
) {
for _, err := range errs {
zone := lo.FromPtr(err.LaunchTemplateAndOverrides.Overrides.AvailabilityZone)
Expand All @@ -492,7 +494,21 @@ func (p *DefaultProvider) updateUnavailableOfferingsCache(
if capacityType != karpv1.CapacityTypeReserved {
for _, err := range errs {
if awserrors.IsUnfulfillableCapacity(err) {
p.unavailableOfferings.MarkUnavailableForFleetErr(ctx, err, capacityType)
// Custom logging for unfulfillable capacity errors with Fleet ID
instanceType := err.LaunchTemplateAndOverrides.Overrides.InstanceType
zone := aws.ToString(err.LaunchTemplateAndOverrides.Overrides.AvailabilityZone)

log.FromContext(ctx).WithValues(
"reason", lo.FromPtr(err.ErrorCode),
"instance-type", instanceType,
"zone", zone,
"capacity-type", capacityType,
"fleet-id", fleetID,
"ttl", awscache.UnavailableOfferingsTTL,
).V(1).Info("removing offering from offerings")

// Mark unavailable without Fleet ID in cache
p.unavailableOfferings.MarkUnavailable(ctx, lo.FromPtr(err.ErrorCode), instanceType, zone, capacityType)
}
if awserrors.IsServiceLinkedRoleCreationNotPermitted(err) {
p.unavailableOfferings.MarkCapacityTypeUnavailable(karpv1.CapacityTypeSpot)
Expand Down