Skip to content

stats/opentelemetry: record retry attempts from clientStream #8342

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 52 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
3ba457e
Fixed retry attempts in HandleRPC
vinothkumarr227 May 19, 2025
5d19779
Fixed the review changes
vinothkumarr227 May 20, 2025
4245950
Fixed vet issues
vinothkumarr227 May 20, 2025
5347db1
Fixed the review changes
vinothkumarr227 May 21, 2025
99e88d8
Fixed the review changes
vinothkumarr227 May 22, 2025
586cf63
Fixed the review changes
vinothkumarr227 May 26, 2025
1bdad7e
Fixed the test cases
vinothkumarr227 May 26, 2025
39c5f0d
Fixed the review changes
vinothkumarr227 May 30, 2025
11523b6
small tweaks
vinothkumarr227 May 30, 2025
3720f4e
Fixed the review changes
vinothkumarr227 Jun 3, 2025
b97a2da
Fixed the test cases
vinothkumarr227 Jun 3, 2025
a0ef86c
Fixed the test cases pick issues
vinothkumarr227 Jun 3, 2025
ac79ad2
Fixed the event ignore issues
vinothkumarr227 Jun 4, 2025
10c6a90
Fixed the picker event issues
vinothkumarr227 Jun 4, 2025
1048040
Fixed the test cases
vinothkumarr227 Jun 4, 2025
05e2cc8
Fixed the review changes
vinothkumarr227 Jun 6, 2025
ba08688
small tweaks
vinothkumarr227 Jun 6, 2025
fe1831f
Fixed the review changes
vinothkumarr227 Jun 12, 2025
4f76f19
Fixed the review changes
vinothkumarr227 Jun 16, 2025
65da5d8
Merge remote-tracking branch 'origin/master' into stats-retry-attempt…
vinothkumarr227 Jun 20, 2025
d489c91
small tweaks
vinothkumarr227 Jun 20, 2025
8de4d5e
small tweaks
vinothkumarr227 Jun 25, 2025
1654ba1
Fixed the review changes
vinothkumarr227 Jul 2, 2025
06f350c
Fixed the server trace issues
vinothkumarr227 Jul 2, 2025
b944353
Fixed the review changes
vinothkumarr227 Jul 3, 2025
daef268
Fixed the review changes
vinothkumarr227 Jul 8, 2025
33f89a5
Fixed the review changes
vinothkumarr227 Jul 9, 2025
2b6ff10
Fixed the sort issues
vinothkumarr227 Jul 11, 2025
08b5e7c
Fixed the vet issue
vinothkumarr227 Jul 11, 2025
e84407b
Fixed the issues
vinothkumarr227 Jul 11, 2025
fcb1279
Fixed the LB pick test issues
vinothkumarr227 Jul 16, 2025
c6a254b
small tweaks
vinothkumarr227 Jul 16, 2025
a3da45c
Fixed the issues
vinothkumarr227 Jul 16, 2025
41a5eb4
Fixed the LB Pick issues
vinothkumarr227 Jul 16, 2025
7db3821
Fixed the test
vinothkumarr227 Jul 16, 2025
6fc9f84
small tweaks
vinothkumarr227 Jul 16, 2025
eec064f
Fixed the LB pick issues
vinothkumarr227 Jul 16, 2025
99a1b6e
Fixed duplice Lb pick event
vinothkumarr227 Jul 17, 2025
bb239e3
Fixed the test issues
vinothkumarr227 Jul 17, 2025
72ff228
Fixed the test issues
vinothkumarr227 Jul 22, 2025
4144b46
small tweaks
vinothkumarr227 Jul 22, 2025
2bec1a5
Fixed the review changes
vinothkumarr227 Jul 28, 2025
abf0f8b
small tweaks
vinothkumarr227 Jul 28, 2025
8bc283f
Fixed the review changes
vinothkumarr227 Jul 29, 2025
9e2647d
small tweaks
vinothkumarr227 Jul 29, 2025
c9c6f4c
small tweaks
vinothkumarr227 Jul 30, 2025
919caa0
Merge remote-tracking branch 'origin/master' into stats-retry-attempt…
vinothkumarr227 Jul 30, 2025
2141cd5
Merge remote-tracking branch 'origin/master' into stats-retry-attempt…
vinothkumarr227 Aug 1, 2025
9b13b21
Fixed the review changes
vinothkumarr227 Aug 6, 2025
9f59e51
Fixed the review changes
vinothkumarr227 Aug 10, 2025
1660fcb
Fixed the review changes
vinothkumarr227 Aug 13, 2025
992343e
Fixed the review changes
vinothkumarr227 Aug 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions stats/opentelemetry/client_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,9 @@ func getOrCreateCallInfo(ctx context.Context, cc *grpc.ClientConn, method string
logger.Info("Creating new CallInfo since its not present in context")
}
ci = &callInfo{
target: cc.CanonicalTarget(),
method: determineMethod(method, opts...),
target: cc.CanonicalTarget(),
method: determineMethod(method, opts...),
previousRPCAttempts: new(atomic.Uint32),
}
ctx = setCallInfo(ctx, ci)
}
Expand Down
25 changes: 22 additions & 3 deletions stats/opentelemetry/client_tracing.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"log"
"strings"

"go.opentelemetry.io/otel/attribute"
otelcodes "go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
Expand Down Expand Up @@ -83,7 +84,10 @@ func (h *clientTracingHandler) finishTrace(err error, ts trace.Span) {
// It creates a new outgoing carrier which serializes information about this
// span into gRPC Metadata, if TextMapPropagator is provided in the trace
// options. if TextMapPropagator is not provided, it returns the context as is.
func (h *clientTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo, nameResolutionDelayed bool) (context.Context, *attemptInfo) {
//
// Note: The passed attemptInfo pointer (ai) is mutated in-place. Fields such as
// ai.traceSpan are updated directly. No new attemptInfo is returned.
func (h *clientTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo, nameResolutionDelayed bool) context.Context {
// Add a "Delayed name resolution complete" event to the call span
// if there was name resolution delay. In case of multiple retry attempts,
// ensure that event is added only once.
Expand All @@ -98,7 +102,7 @@ func (h *clientTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo,
carrier := otelinternaltracing.NewOutgoingCarrier(ctx)
h.options.TraceOptions.TextMapPropagator.Inject(ctx, carrier)
ai.traceSpan = span
return carrier.Context(), ai
return carrier.Context()
}

// createCallTraceSpan creates a call span to put in the provided context using
Expand All @@ -121,7 +125,12 @@ func (h *clientTracingHandler) HandleConn(context.Context, stats.ConnStats) {}
// TagRPC implements per RPC attempt context management for traces.
func (h *clientTracingHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
ctx, ai := getOrCreateRPCAttemptInfo(ctx)
ctx, ai = h.traceTagRPC(ctx, ai, info.NameResolutionDelay)
ci := getCallInfo(ctx)
if ci == nil {
logger.Error("context passed into client side stats handler (TagRPC) has no call info")
return ctx
}
ctx = h.traceTagRPC(ctx, ai, info.NameResolutionDelay)
return setRPCInfo(ctx, &rpcInfo{ai: ai})
}

Expand All @@ -132,5 +141,15 @@ func (h *clientTracingHandler) HandleRPC(ctx context.Context, rs stats.RPCStats)
logger.Error("ctx passed into client side tracing handler trace event handling has no client attempt data present")
return
}

// Client-specific Begin attributes.
if begin, ok := rs.(*stats.Begin); ok {
ci := getCallInfo(ctx)
previousRPCAttempts := ci.previousRPCAttempts.Add(1) - 1
ri.ai.traceSpan.SetAttributes(
attribute.Int64("previous-rpc-attempts", int64(previousRPCAttempts)),
attribute.Bool("transparent-retry", begin.IsTransparentRetryAttempt),
)
}
populateSpan(rs, ri.ai)
}
Loading