[Instrumentation.AWS]: always add context propagation data to requests

src/OpenTelemetry.Instrumentation.AWS/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+* Context propagation data is always added to SQS and SNS requests regardless of sampling decision.
+  This enables downstream services to make consistent sampling decisions and prevents incomplete traces.
+  ([#2447](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2447))
+
 ## 1.10.0-beta.3
 
 Released 2024-Dec-20

src/OpenTelemetry.Instrumentation.AWS/Implementation/AWSTracingPipelineHandler.cs

@@ -46,6 +46,22 @@ public override async Task<T> InvokeAsync<T>(IExecutionContext executionContext)
         return ret;
     }
 
+    private static void AddPropagationDataToRequest(Activity activity, IRequestContext requestContext)
+    {
+        var service = requestContext.ServiceMetaData.ServiceId;
+
+        if (AWSServiceType.IsSqsService(service))
+        {
+            SqsRequestContextHelper.AddAttributes(
+                requestContext, AWSMessagingUtils.InjectIntoDictionary(new PropagationContext(activity.Context, Baggage.Current)));
+        }
+        else if (AWSServiceType.IsSnsService(service))
+        {
+            SnsRequestContextHelper.AddAttributes(
+                requestContext, AWSMessagingUtils.InjectIntoDictionary(new PropagationContext(activity.Context, Baggage.Current)));
+        }
+    }
+
 #if NET
     [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage(
         "Trimming",
@@ -167,16 +183,6 @@ private void AddRequestSpecificInformation(Activity activity, IRequestContext re
         {
             this.awsSemanticConventions.TagBuilder.SetTagAttributeDbSystemToDynamoDb(activity);
         }
-        else if (AWSServiceType.IsSqsService(service))
-        {
-            SqsRequestContextHelper.AddAttributes(
-                requestContext, AWSMessagingUtils.InjectIntoDictionary(new PropagationContext(activity.Context, Baggage.Current)));
-        }
-        else if (AWSServiceType.IsSnsService(service))
-        {
-            SnsRequestContextHelper.AddAttributes(
-                requestContext, AWSMessagingUtils.InjectIntoDictionary(new PropagationContext(activity.Context, Baggage.Current)));
-        }
         else if (AWSServiceType.IsBedrockRuntimeService(service))
         {
             this.awsSemanticConventions.TagBuilder.SetTagAttributeGenAiSystemToBedrock(activity);
@@ -202,14 +208,21 @@ private void ProcessEndRequest(Activity? activity, IExecutionContext executionCo
 
         var currentActivity = Activity.Current;
 
-        if (currentActivity == null
-            || !currentActivity.Source.Name.StartsWith(TelemetryConstants.TelemetryScopePrefix, StringComparison.Ordinal)
-            || !currentActivity.IsAllDataRequested)
+        if (currentActivity == null)
         {
             return null;
         }
 
-        this.AddRequestSpecificInformation(currentActivity, executionContext.RequestContext);
+        if (currentActivity.IsAllDataRequested
+            && currentActivity.Source.Name.StartsWith(TelemetryConstants.TelemetryScopePrefix, StringComparison.Ordinal))
+        {
+            this.AddRequestSpecificInformation(currentActivity, executionContext.RequestContext);
+        }
+
+        // Context propagation should always happen regardless of sampling decision (which affects Activity.IsAllDataRequested and Activity.Source).
+        // Otherwise, downstream services can make inconsistent sampling decisions and create incomplete traces.
+        AddPropagationDataToRequest(currentActivity, executionContext.RequestContext);
+
         return currentActivity;
     }
 }

test/OpenTelemetry.Instrumentation.AWS.Tests/TestAWSClientInstrumentation.cs

@@ -232,6 +232,56 @@ public async Task TestSQSSendMessageSuccessful()
         Assert.Equal(requestId, Utils.GetTagValue(awssdk_activity, "aws.request_id"));
     }
 
+    [Fact]
+#if NETFRAMEWORK
+    public void TestSQSSendMessageSuccessfulNotSampled()
+#else
+    public async Task TestSQSSendMessageSuccessfulNotSampled()
+#endif
+    {
+        var exportedItems = new List<Activity>();
+
+        var parent = new Activity("parent").Start();
+        var requestId = @"fakerequ-esti-dfak-ereq-uestidfakere";
+
+        SendMessageRequest send_msg_req;
+
+        using (Sdk.CreateTracerProviderBuilder()
+                   .AddXRayTraceId()
+                   .SetSampler(new AlwaysOffSampler())
+                   .AddAWSInstrumentation(o =>
+                   {
+                       o.SemanticConventionVersion = SemanticConventionVersion.Latest;
+                   })
+                   .AddInMemoryExporter(exportedItems)
+                   .Build())
+        {
+            var sqs = new AmazonSQSClient(new AnonymousAWSCredentials(), RegionEndpoint.USEast1);
+            var dummyResponse = "{}";
+            CustomResponses.SetResponse(sqs, dummyResponse, requestId, true);
+            send_msg_req = new SendMessageRequest
+            {
+                QueueUrl = "https://sqs.us-east-1.amazonaws.com/123456789/MyTestQueue",
+                MessageBody = "Hello from OT",
+            };
+            send_msg_req.MessageAttributes.Add("Custom", new MessageAttributeValue { StringValue = "Value", DataType = "String" });
+#if NETFRAMEWORK
+            sqs.SendMessage(send_msg_req);
+#else
+            await sqs.SendMessageAsync(send_msg_req);
+#endif
+        }
+
+        Assert.Empty(exportedItems);
+        Assert.Equal(2, send_msg_req.MessageAttributes.Count);
+        Assert.Contains(
+            send_msg_req.MessageAttributes,
+            kv => kv.Key == "traceparent" && kv.Value.StringValue.EndsWith("00", StringComparison.Ordinal));
+        Assert.Contains(
+            send_msg_req.MessageAttributes,
+            kv => kv.Key == "Custom" && kv.Value.StringValue == "Value");
+    }
+
     [Fact]
 #if NETFRAMEWORK
     public void TestBedrockGetGuardrailSuccessful()