[Tracer] Fix resource-based sampling for ASP.NET spans in IIS (#6936)

## Summary of changes

Set's a preliminary `Span.ResourceName` for `ASP.NET` spans in
`OnBeginRequest` as well as in `OnEndRequest` to ensure that
resource-based sampling will work.

## Reason for change

Within `TracingHttpModule` we do not set a `ResourceName` for the span
in `OnBeginRequest`, this is to try and ensure that the resource names
from the parent `ASP.NET` span and it's child span will be equivalent.
We instead set it in the `OnEndRequest`.

This poses an issue though in IIS as we then (commonly) call `Inject`
which ends up making a sampling decision _without_ a
`Span.ResourceName`. What this ends up causing is that resource-based
sampling decisions / rules will never be applied.
Once a sampling decision has been made it is not changed at a later
point.

The following code path gets hit for `ASP.NET` in IIS which causes the
behavior.
```csharp
// Decorate the incoming HTTP Request with distributed tracing headers
// in case the next processor cannot access the stored Scope
// (e.g. WCF being hosted in IIS)
if (HttpRuntime.UsingIntegratedPipeline)
{
    var injectedContext = new PropagationContext(scope.Span.Context, Baggage.Current);
    tracer.TracerManager.SpanContextPropagator.Inject(injectedContext, requestHeaders.Wrap());
}
```

## Implementation details

I've extracted _a part_ of the logic to build the resource name in the
`OnEndRequest` and am using it in the `OnBeginRequest` phase. This will
allow us to have some semblance of a relatively accurate resource name
at the time that we are making the sampling decision.

The part that is different is that we do not in the `OnBeginRequest`
have access to the child span resource name, which is the path ->

```csharp
if (app.Context.Items[SharedItems.HttpContextPropagatedResourceNameKey] is string resourceName
 && !string.IsNullOrEmpty(resourceName))
{
    currentSpan.ResourceName = resourceName;
}
```

So while I _think_ that the resource name computed now will be
relatively accurate there may be instances where the child HTTP span
from it has a different resource name that overrides it.

## Test coverage

Updated MVC4 ASP net tests to have a resource that should be sampled out
(see the snapshot sampling value) based on its `aspnet` span (not the
`MVC`) span resource name.

## Other details

It may be beneficial to tag the original resource name as it may look a
bit different to help determine what resource was used in the sampling
decision (if any), but we can come back to that later.

<!-- Fixes #{issue} -->

<!-- ⚠️ Note: where possible, please obtain 2 approvals prior to
merging. Unless CODEOWNERS specifies otherwise, for external teams it is
typically best to have one review from a team member, and one review
from apm-dotnet. Trivial changes do not require 2 reviews. -->

Author: bouwkast

tracer/src/Datadog.Trace/AspNet/TracingHttpModule.cs

@@ -85,6 +85,22 @@ public void Dispose()
         {
         }
 
+        private static string BuildResourceName(Tracer tracer, HttpRequest httpRequest)
+        {
+            var url = tracer.Settings.BypassHttpRequestUrlCachingEnabled
+                               ? RequestDataHelper.BuildUrl(httpRequest)
+                               : RequestDataHelper.GetUrl(httpRequest);
+            if (url is not null)
+            {
+                var path = UriHelpers.GetCleanUriPath(url, httpRequest.ApplicationPath);
+                return $"{httpRequest.HttpMethod.ToUpperInvariant()} {path.ToLowerInvariant()}";
+            }
+            else
+            {
+                return $"{httpRequest.HttpMethod.ToUpperInvariant()}";
+            }
+        }
+
         internal static void AddHeaderTagsFromHttpResponse(HttpContext httpContext, Scope scope)
         {
             if (!Tracer.Instance.Settings.HeaderTags.IsNullOrEmpty() &&
@@ -174,7 +190,12 @@ private void OnBeginRequest(object sender, EventArgs eventArgs)
                 var tags = new WebTags();
                 scope = tracer.StartActiveInternal(_requestOperationName, extractedContext.SpanContext, tags: tags);
                 // Leave resourceName blank for now - we'll update it in OnEndRequest
-                scope.Span.DecorateWebServerSpan(resourceName: null, httpMethod, host, url, userAgent, tags);
+
+                // Attempt to set Resource Name to something that will be close to what is expected
+                // Note: we will go and re-do it in OnEndRequest, but doing it here will allow for resource-based sampling
+                // this likely won't be perfect - but we need something to try and allow resource-based sampling to function
+                var resourceName = BuildResourceName(tracer, httpRequest);
+                scope.Span.DecorateWebServerSpan(resourceName: resourceName, httpMethod, host, url, userAgent, tags);
                 tracer.TracerManager.SpanContextPropagator.AddHeadersToSpanAsTags(scope.Span, headers, tracer.Settings.HeaderTags, defaultTagPrefix: SpanContextPropagator.HttpRequestHeadersTagPrefix);
 
                 if (tracer.Settings.IpHeaderEnabled || Security.Instance.AppsecEnabled)
@@ -386,18 +407,7 @@ private void OnEndRequest(object sender, EventArgs eventArgs)
                         }
                         else
                         {
-                            var url = tracer.Settings.BypassHttpRequestUrlCachingEnabled
-                                ? RequestDataHelper.BuildUrl(app.Request)
-                                : RequestDataHelper.GetUrl(app.Request);
-                            if (url is not null)
-                            {
-                                string path = UriHelpers.GetCleanUriPath(url, app.Request.ApplicationPath);
-                                currentSpan.ResourceName = $"{app.Request.HttpMethod.ToUpperInvariant()} {path.ToLowerInvariant()}";
-                            }
-                            else
-                            {
-                                currentSpan.ResourceName = $"{app.Request.HttpMethod.ToUpperInvariant()}";
-                            }
+                            currentSpan.ResourceName = BuildResourceName(tracer, app.Request);
                         }
                     }
                     finally

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/AspNetMvc4Tests.cs

@@ -13,6 +13,7 @@
 using System.Text;
 using System.Threading.Tasks;
 using Datadog.Trace.Configuration;
+using Datadog.Trace.Sampling;
 using Datadog.Trace.TestHelpers;
 using FluentAssertions;
 using VerifyXunit;
@@ -80,6 +81,11 @@ public AspNetMvc4Tests(IisFixture iisFixture, ITestOutputHelper output, bool cla
             SetEnvironmentVariable(ConfigurationKeys.FeatureFlags.RouteTemplateResourceNamesEnabled, enableRouteTemplateResourceNames.ToString());
             SetEnvironmentVariable(ConfigurationKeys.ExpandRouteTemplatesEnabled, enableRouteTemplateExpansion.ToString());
 
+            // these two are to test resource-based sampling on the parent ASP.NET span (non-MVC) one.
+            // it should not show up at all
+            SetEnvironmentVariable(ConfigurationKeys.CustomSamplingRules, """[{"sample_rate":0.0, "service":"*", "resource":"GET /home/pleasesamplemeout/?"}]""");
+            SetEnvironmentVariable(ConfigurationKeys.CustomSamplingRulesFormat, SamplingRulesFormat.Glob.ToString()); // for ease of use
+
             _classicMode = classicMode;
             _iisFixture = iisFixture;
             _iisFixture.ShutdownPath = "/home/shutdown";
@@ -110,6 +116,7 @@ public AspNetMvc4Tests(IisFixture iisFixture, ITestOutputHelper output, bool cla
               { "/Home/BadRequestWithStatusCode?statuscode=401&TransferRequest=true", 401 },
               { "/Home/BadRequestWithStatusCode?statuscode=503&TransferRequest=true", 503 },
               { "/graphql/GetAllFoo", 200 }, // Slug in route template
+              { "/Home/PleaseSampleMeOut/5555", 200 }, // test resource-based sampling for parent ASP.NET span (non-MVC) "GET /home/pleaseSampleMeOut/?" <- note the ? it isn't {id}
         };
 
         public override Result ValidateIntegrationSpan(MockSpan span, string metadataSchemaVersion) =>

tracer/test/snapshots/AspNetMvc4Tests.Classic.NoFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt

@@ -0,0 +1,53 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/?,
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/?,
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.rule_psr: 0.0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: -1.0
+    }
+  }
+]

tracer/test/snapshots/AspNetMvc4Tests.Classic.WithFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt

@@ -0,0 +1,52 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: 1.0
+    }
+  }
+]

tracer/test/snapshots/AspNetMvc4Tests.Integrated.NoFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt

@@ -0,0 +1,53 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/?,
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/?,
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.rule_psr: 0.0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: -1.0
+    }
+  }
+]

tracer/test/snapshots/AspNetMvc4Tests.Integrated.WithExpansion.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt

@@ -0,0 +1,53 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.rule_psr: 0.0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: -1.0
+    }
+  }
+]

tracer/test/snapshots/AspNetMvc4Tests.Integrated.WithFF.__path=_Home_PleaseSampleMeOut_5555_statusCode=200.verified.txt

@@ -0,0 +1,53 @@
+[
+  {
+    TraceId: Id_1,
+    SpanId: Id_2,
+    Name: aspnet-mvc.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    ParentId: Id_3,
+    Tags: {
+      aspnet.action: pleasesamplemeout,
+      aspnet.controller: home,
+      aspnet.route: {controller}/{action}/{id},
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      span.kind: server,
+      version: 1.0.0
+    }
+  },
+  {
+    TraceId: Id_1,
+    SpanId: Id_3,
+    Name: aspnet.request,
+    Resource: GET /home/pleasesamplemeout/{id},
+    Service: sample,
+    Type: web,
+    Tags: {
+      env: integration_tests,
+      http.method: GET,
+      http.request.headers.host: localhost:00000,
+      http.route: {controller}/{action}/{id},
+      http.status_code: 200,
+      http.url: http://localhost:00000/Home/PleaseSampleMeOut/5555,
+      http.useragent: testhelper,
+      language: dotnet,
+      runtime-id: Guid_1,
+      span.kind: server,
+      version: 1.0.0
+    },
+    Metrics: {
+      process_id: 0,
+      _dd.rule_psr: 0.0,
+      _dd.top_level: 1.0,
+      _dd.tracer_kr: 1.0,
+      _sampling_priority_v1: -1.0
+    }
+  }
+]

tracer/test/test-applications/aspnet/Samples.AspNetMvc4/Controllers/HomeController.cs

@@ -66,5 +66,14 @@ public ActionResult OptionalIdentifier(int? id)
             ViewBag.Message = "Identifier set to " + id;
             return View("About");
         }
+
+        public ActionResult PleaseSampleMeOut(int? id)
+        {
+            // this should be sampled out - the resource name sampled is /home/pleaseSampleMeOut/?
+            // however the one that will be shown is /home/pleaseSampleMeOut/{id} after MVC routes are replaced
+            // but we make the sampling decision BEFORE the MVC routes are known
+            ViewBag.Message = "This should be sampled out based on the ASP.NET Request resource name " + id;
+            return View("About");
+        }
     }
 }