fix: integrate mutual TLS scenario with underlying http.sys server (#2044)

Author: DeagleGross

build/trend-scenarios.yml

@@ -108,6 +108,9 @@ parameters:
   - displayName: "HttpSys Windows: mTLS Handshakes"
     arguments: --scenario mTls-handshakes-httpsys $(tlsJobs) --property scenario=HttpSysMutualTLSHandshakes --application.options.requiredOperatingSystem windows
 
+  - displayName: "HttpSys Windows: TLS Renegotiation"
+    arguments: --scenario tls-renegotiation-httpsys $(tlsJobs) --property scenario=HttpSysTLSRenegotiation --application.options.requiredOperatingSystem windows
+
   - displayName: "Kestrel Linux: TLS Handshakes"
     arguments: --scenario tls-handshakes-kestrel $(tlsJobs) --property scenario=KestrelTLSHandshakes --application.options.requiredOperatingSystem linux
 

scenarios/tls.benchmarks.yml

@@ -15,10 +15,12 @@ jobs:
       project: src/BenchmarksApps/TLS/HttpSys/HttpSys.csproj
     readyStateText: Application started.
     variables:
-      mTLS: false
+      mTLS: false # enables settings on http.sys to negotiate client cert on connections
+      tlsRenegotiation: false # enables client cert validation
       certValidationConsoleEnabled: false
+      httpSysLogs: false
       statsEnabled: false
-    arguments: "--urls https://{{serverAddress}}:{{serverPort}} --mTLS {{mTLS}} --certValidationConsoleEnabled {{certValidationConsoleEnabled}} --statsEnabled {{statsEnabled}}"
+    arguments: "--urls https://{{serverAddress}}:{{serverPort}} --mTLS {{mTLS}} --certValidationConsoleEnabled {{certValidationConsoleEnabled}} --statsEnabled {{statsEnabled}} --tlsRenegotiation {{tlsRenegotiation}} --httpSysLogs {{httpSysLogs}}"
 
   kestrelServer:
     source:
@@ -52,7 +54,29 @@ scenarios:
     application:
       job: httpSysServer
       variables:
-        mTLS: true
+        mTLS: true # enables settings on http.sys to negotiate client cert on connections
+        tlsRenegotiation: true # enables client cert validation
+        httpSysLogs: false # only for debug purposes
+        certValidationConsoleEnabled: false # only for debug purposes
+        serverPort: 8080 # IMPORTANT: not to intersect with other tests in case http.sys configuration impacts other benchmarks
+    load:
+      job: httpclient
+      variables:
+        serverPort: 8080 # in sync with server
+        path: /hello-world
+        presetHeaders: connectionclose
+        connections: 32
+        serverScheme: https
+        certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/refs/heads/main/src/BenchmarksApps/TLS/HttpSys/testCert.pfx
+        certPwd: testPassword
+
+  tls-renegotiation-httpsys:
+    application:
+      job: httpSysServer
+      variables:
+        mTLS: false
+        tlsRenegotiation: true
+        httpSysLogs: false # only for debug purposes
         certValidationConsoleEnabled: false # only for debug purposes
     load:
       job: httpclient

src/BenchmarksApps/TLS/HttpSys/HttpSys.csproj

@@ -6,4 +6,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
+  <ItemGroup>
+    <None Update="testCert.pfx">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>

src/BenchmarksApps/TLS/HttpSys/NetShWrapper.cs

@@ -0,0 +1,89 @@
+using System.Diagnostics;
+using System.Security.Cryptography.X509Certificates;
+
+namespace HttpSys
+{
+    public static class NetShWrapper
+    {
+        public static void DisableHttpSysMutualTlsIfExists(string ipPort)
+        {
+            try
+            {
+                DisableHttpSysMutualTls(ipPort);
+            }
+            catch
+            {
+                // ignore
+            }
+        }
+
+        public static void DisableHttpSysMutualTls(string ipPort)
+        {
+            Console.WriteLine("Disabling mTLS for http.sys");
+
+            string command = $"http delete sslcert ipport={ipPort}";
+            ExecuteNetShCommand(command);
+
+            Console.WriteLine("Disabled http.sys settings for mTLS");
+        }
+
+        public static void Show()
+        {
+            ExecuteNetShCommand("http show sslcert", alwaysLogOutput: true);
+        }
+
+        public static void EnableHttpSysMutualTls(string ipPort)
+        {
+            Console.WriteLine("Setting up mTLS for http.sys");
+
+            var certificate = LoadCertificate();
+            Console.WriteLine("Loaded `testCert.pfx` from local file system");
+            using (var store = new X509Store(StoreName.My, StoreLocation.LocalMachine))
+            {
+                store.Open(OpenFlags.ReadWrite);
+                store.Add(certificate);
+                Console.WriteLine("Added `testCert.pfx` to localMachine cert store");
+                store.Close();
+            }
+
+            string certThumbprint = certificate.Thumbprint;
+            string appId = Guid.NewGuid().ToString();
+
+            string command = $"http add sslcert ipport={ipPort} certstorename=MY certhash={certThumbprint} appid={{{appId}}} clientcertnegotiation=enable";
+            ExecuteNetShCommand(command);
+
+            Console.WriteLine("Configured http.sys settings for mTLS");
+        }
+
+        private static void ExecuteNetShCommand(string command, bool alwaysLogOutput = false)
+        {
+            ProcessStartInfo processInfo = new ProcessStartInfo("netsh", command)
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            };
+
+            Console.WriteLine($"Executing command: `netsh {command}`");
+            using Process process = Process.Start(processInfo)!;
+            string output = process.StandardOutput.ReadToEnd();
+            process.WaitForExit();
+
+            if (alwaysLogOutput)
+            {
+                Console.WriteLine(output);
+            }
+
+            if (process.ExitCode != 0)
+            {
+                throw new InvalidOperationException($"netsh command execution failure: {output}");
+            }
+        }
+
+        private static X509Certificate2 LoadCertificate()
+            => File.Exists("testCert.pfx")
+            ? X509CertificateLoader.LoadPkcs12FromFile("testCert.pfx", "testPassword", X509KeyStorageFlags.MachineKeySet | X509KeyStorageFlags.Exportable)
+            : X509CertificateLoader.LoadPkcs12FromFile("../testCert.pfx", "testPassword", X509KeyStorageFlags.MachineKeySet | X509KeyStorageFlags.Exportable);
+    }
+}

src/BenchmarksApps/TLS/HttpSys/Program.cs

@@ -1,13 +1,16 @@
-using Microsoft.AspNetCore.Mvc;
+using HttpSys;
 using Microsoft.AspNetCore.Server.HttpSys;
 
 var builder = WebApplication.CreateBuilder(args);
 builder.Logging.ClearProviders();
 
 var writeCertValidationEventsToConsole = bool.TryParse(builder.Configuration["certValidationConsoleEnabled"], out var certValidationConsoleEnabled) && certValidationConsoleEnabled;
+var httpSysLoggingEnabled = bool.TryParse(builder.Configuration["httpSysLogs"], out var httpSysLogsEnabled) && httpSysLogsEnabled;
 var statsEnabled = bool.TryParse(builder.Configuration["statsEnabled"], out var connectionStatsEnabledConfig) && connectionStatsEnabledConfig;
 var mTlsEnabled = bool.TryParse(builder.Configuration["mTLS"], out var mTlsEnabledConfig) && mTlsEnabledConfig;
+var tlsRenegotiationEnabled = bool.TryParse(builder.Configuration["tlsRenegotiation"], out var tlsRenegotiationEnabledConfig) && tlsRenegotiationEnabledConfig;
 var listeningEndpoints = builder.Configuration["urls"] ?? "https://localhost:5000/";
+var httpsIpPort = listeningEndpoints.Split(";").First(x => x.Contains("https")).Replace("https://", "");
 
 #pragma warning disable CA1416 // Can be launched only on Windows (HttpSys)
 builder.WebHost.UseHttpSys(options =>
@@ -17,7 +20,7 @@
 });
 #pragma warning restore CA1416 // Can be launched only on Windows (HttpSys)
 
-var app = builder.Build();  
+var app = builder.Build();
 
 app.MapGet("/hello-world", () =>
 {
@@ -40,6 +43,40 @@
 }
 
 if (mTlsEnabled)
+{
+    var hostAppLifetime = app.Services.GetRequiredService<IHostApplicationLifetime>();
+    hostAppLifetime!.ApplicationStopping.Register(OnShutdown);
+
+    void OnShutdown()
+    {
+        Console.WriteLine("Application shutdown started.");
+
+        try
+        {
+            NetShWrapper.DisableHttpSysMutualTls(ipPort: httpsIpPort);
+        }
+        catch
+        {
+            Console.WriteLine("Failed to disable HTTP.SYS mTLS settings");
+            throw;
+        }
+    }
+
+    try
+    {
+        // if not executed, following command (enable http.sys mutual tls) will fail because binding exists
+        NetShWrapper.DisableHttpSysMutualTlsIfExists(ipPort: httpsIpPort);
+
+        NetShWrapper.EnableHttpSysMutualTls(ipPort: httpsIpPort);
+    }
+    catch
+    {
+        Console.WriteLine($"Http.Sys configuration for mTLS failed");
+        throw;
+    }
+}
+
+if (tlsRenegotiationEnabled)
 {
     // this is an http.sys middleware to get a cert
     Console.WriteLine("Registered client cert validation middleware");
@@ -72,6 +109,12 @@
 }
 
 await app.StartAsync();
+
+if (httpSysLoggingEnabled)
+{
+    NetShWrapper.Show();
+}
+
 Console.WriteLine("Application Info:");
 if (mTlsEnabled)
 {

src/BenchmarksApps/TLS/HttpSys/appsettings.Development.json

@@ -5,6 +5,8 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "mTLS": "true",
+  "mTLS": "false",
+  "httpSysLogs": "true",
+  "tlsRenegotiation": "true",
   "certValidationConsoleEnabled": "true"
 }