Improve Benchmark Accuracy

BenchmarkDotNet.sln

@@ -59,6 +59,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.P
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Weaver", "src\BenchmarkDotNet.Weaver\BenchmarkDotNet.Weaver.csproj", "{5731DE42-16FE-430E-BA90-0EBE714CB221}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -161,6 +163,10 @@ Global
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5731DE42-16FE-430E-BA90-0EBE714CB221}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5731DE42-16FE-430E-BA90-0EBE714CB221}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5731DE42-16FE-430E-BA90-0EBE714CB221}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5731DE42-16FE-430E-BA90-0EBE714CB221}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -190,6 +196,7 @@ Global
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9}
+		{5731DE42-16FE-430E-BA90-0EBE714CB221} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

NuGet.Config

@@ -8,11 +8,13 @@
     <clear />
 
     <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
-    <!-- reuquired to run Mono AOT benchmarks -->
+    <!-- required to run Mono AOT benchmarks -->
     <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
     <add key="dotnet7" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
     <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
     <add key="dotnet9" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json" />
     <add key="dotnet10" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json" />
+
+    <add key="benchmarkdotnet.weaver" value="src/BenchmarkDotNet.Weaver/packages" />
   </packageSources>
 </configuration>

build/BenchmarkDotNet.Build/Program.cs

@@ -16,8 +16,29 @@ public static int Main(string[] args)
     }
 }
 
+[TaskName(Name)]
+[TaskDescription("Pack Weaver")]
+public class PackWeaverTask : FrostingTask<BuildContext>, IHelpProvider
+{
+    private const string Name = "pack-weaver";
+
+    public override void Run(BuildContext context) => context.BuildRunner.PackWeaver();
+
+    public HelpInfo GetHelp()
+    {
+        return new HelpInfo
+        {
+            Examples = new[]
+            {
+                new Example(Name)
+            }
+        };
+    }
+}
+
 [TaskName(Name)]
 [TaskDescription("Restore NuGet packages")]
+[IsDependentOn(typeof(PackWeaverTask))]
 public class RestoreTask : FrostingTask<BuildContext>, IHelpProvider
 {
     private const string Name = "restore";

build/BenchmarkDotNet.Build/Runners/BuildRunner.cs

@@ -8,6 +8,8 @@
 using Cake.Common.Tools.DotNet.Workload.Install;
 using Cake.Core;
 using Cake.Core.IO;
+using System.IO;
+using System.Linq;
 
 namespace BenchmarkDotNet.Build.Runners;
 
@@ -20,6 +22,41 @@ public BuildRunner(BuildContext context)
         this.context = context;
     }
 
+    public void PackWeaver()
+    {
+        var weaverPath = context.AllPackableSrcProjects.Single(p => p.GetFilename() == "BenchmarkDotNet.Weaver.csproj");
+        var outputPackageDir = weaverPath.GetDirectory().Combine("packages");
+
+        // Delete old package.
+        foreach (var file in Directory.EnumerateFiles(outputPackageDir.FullPath))
+        {
+            File.Delete(file);
+        }
+
+        context.DotNetRestore(weaverPath.GetDirectory().FullPath,
+            new DotNetRestoreSettings
+            {
+                MSBuildSettings = context.MsBuildSettingsRestore
+            });
+
+        context.Information("BuildSystemProvider: " + context.BuildSystem().Provider);
+        context.DotNetBuild(weaverPath.FullPath, new DotNetBuildSettings
+        {
+            NoRestore = true,
+            DiagnosticOutput = true,
+            MSBuildSettings = context.MsBuildSettingsBuild,
+            Configuration = context.BuildConfiguration,
+            Verbosity = context.BuildVerbosity
+        });
+
+        context.DotNetPack(weaverPath.FullPath, new DotNetPackSettings
+        {
+            OutputDirectory = outputPackageDir,
+            MSBuildSettings = context.MsBuildSettingsPack,
+            Configuration = context.BuildConfiguration
+        });
+    }
+
     public void Restore()
     {
         context.DotNetRestore(context.SolutionFile.FullPath,
@@ -71,7 +108,7 @@ public void Pack()
         var settingsSrc = new DotNetPackSettings
         {
             OutputDirectory = context.ArtifactsDirectory,
-            ArgumentCustomization = args => args.Append("--include-symbols").Append("-p:SymbolPackageFormat=snupkg"),
+            ArgumentCustomization = args => args.Append("--include-symbols").Append("-p:SymbolPackageFormat=snupkg").Append("-p:IsFullPack=true"),
             MSBuildSettings = context.MsBuildSettingsPack,
             Configuration = context.BuildConfiguration
         };

build/common.props

@@ -42,6 +42,7 @@
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(VersionPrefix)' == '' ">
+    <!-- When this is changed, optionally reset WeaverVersionSuffix, then run `build.cmd pack-weaver`. -->
     <VersionPrefix>0.15.2</VersionPrefix>
   </PropertyGroup>
 
@@ -57,6 +58,11 @@
     <PackageVersion>$(Version)</PackageVersion>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <!-- Increment this when the BenchmarkDotNet.Weaver package needs to be re-packed. -->
+    <WeaverVersionSuffix>-1</WeaverVersionSuffix>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.3">
       <PrivateAssets>all</PrivateAssets>

samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj

@@ -3,6 +3,7 @@
     <!-- MSBuild was complaing about InformationalVersion from common.props file, so I excluded them in conditional way -->
     <IsFsharp>true</IsFsharp>
   </PropertyGroup>
+  <Import Project="..\..\build\common.props" />
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net462;net8.0</TargetFrameworks>

samples/BenchmarkDotNet.Samples/IntroSmokeStringBuilder.cs

@@ -0,0 +1,34 @@
+using BenchmarkDotNet.Attributes;
+using System.Text;
+
+namespace BenchmarkDotNet.Samples
+{
+    [MemoryDiagnoser(false)]
+    public class IntroSmokeStringBuilder
+    {
+        [Benchmark]
+        [Arguments(1)]
+        [Arguments(1_000)]
+        public StringBuilder Append_Strings(int repeat)
+        {
+            StringBuilder builder = new StringBuilder();
+
+            // strings are not sorted by length to mimic real input
+            for (int i = 0; i < repeat; i++)
+            {
+                builder.Append("12345");
+                builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN");
+                builder.Append("1234567890abcdefghijklmnopqrstuvwxy");
+                builder.Append("1234567890");
+                builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHI");
+                builder.Append("1234567890abcde");
+                builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCD");
+                builder.Append("1234567890abcdefghijklmnopqrst");
+                builder.Append("1234567890abcdefghij");
+                builder.Append("1234567890abcdefghijklmno");
+            }
+
+            return builder;
+        }
+    }
+}

src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <AssemblyTitle>Basic BenchmarkDotNet attributes that can be used to annotate your benchmarks</AssemblyTitle>
     <TargetFrameworks>netstandard2.0</TargetFrameworks>
-    <NoWarn>$(NoWarn);1701;1702;1705;1591;3005;NU1702;CA1825</NoWarn>
+    <NoWarn>$(NoWarn);1701;1702;1705;1591;3005;NU1702;NU5100;CA1825</NoWarn>
     <AssemblyName>BenchmarkDotNet.Annotations</AssemblyName>
     <PackageId>BenchmarkDotNet.Annotations</PackageId>
     <RootNamespace>BenchmarkDotNet</RootNamespace>
@@ -14,4 +14,20 @@
   <ItemGroup>
     <Compile Include="..\BenchmarkDotNet\Helpers\CodeAnnotations.cs" Link="Attributes\CodeAnnotations.cs" />
   </ItemGroup>
+
+  <Choose>
+    <When Condition="'$(IsFullPack)' == 'true'">
+      <ItemGroup>
+        <!-- Include the BenchmarkDotNet.Weaver dlls in tasks without making it an explicit dependency. -->
+        <Content Include="$(MSBuildThisFileDirectory)..\BenchmarkDotNet.Weaver\bin\Release\$(TargetFramework)\**\*.dll" Pack="true" PackagePath="tasks/$(TargetFramework)" />
+        <Content Include="$(MSBuildThisFileDirectory)buildTransitive\**\*.targets" Pack="true" PackagePath="buildTransitive" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <!-- PackageReference for transitive weaver dependency for ProjectReferences to this. -->
+      <ItemGroup>
+        <PackageReference Include="BenchmarkDotNet.Weaver" Version="$(Version)$(WeaverVersionSuffix)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
 </Project>

src/BenchmarkDotNet.Annotations/buildTransitive/netstandard2.0/BenchmarkDotNet.Annotations.targets

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Reference Include="BenchmarkDotNet.Annotations">
+      <HintPath>$(MSBuildThisFileDirectory)..\..\lib\netstandard2.0\BenchmarkDotNet.Annotations.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+
+  <UsingTask TaskName="BenchmarkDotNet.Weaver.WeaveAssemblyTask" AssemblyFile="$(MSBuildThisFileDirectory)..\..\tasks\netstandard2.0\BenchmarkDotNet.Weaver.dll" />
+
+  <Target Name="WeaveAssemblies" AfterTargets="AfterBuild" >
+    <WeaveAssemblyTask TargetAssembly="$(TargetDir)$(TargetFileName)" />
+  </Target>
+</Project>

src/BenchmarkDotNet.Weaver/.gitignore

@@ -0,0 +1 @@
+!packages/

src/BenchmarkDotNet.Weaver/BenchmarkDotNet.Weaver.csproj

@@ -0,0 +1,34 @@
+<!--
+If any changes are made to this project, increment the WeaverVersionSuffix in the common.props file,
+then run `build.cmd pack-weaver`.
+-->
+
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\build\common.props" />
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
+    <VersionSuffix Condition="'$(IsFullPack)' != 'true'">$(VersionSuffix)$(WeaverVersionSuffix)</VersionSuffix>
+    <OutputPath>$(MSBuildThisFileDirectory)bin\$(Configuration)</OutputPath>
+    <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+    <IncludeBuildOutput>false</IncludeBuildOutput>
+    <NoWarn>$(NoWarn);NU5100;NU5128</NoWarn>
+    <!-- AsmResolver is not signed. -->
+    <SignAssembly>false</SignAssembly>
+    <DelaySign>false</DelaySign>
+    <!-- Fix error NU5017: Cannot create a package that has no dependencies nor content. -->
+    <IsPackable Condition="'$(IsFullPack)' == 'true'">false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="AsmResolver.DotNet" Version="6.0.0-beta.3" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.Build.Framework" Version="17.14.8" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="17.14.8" PrivateAssets="all" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <!-- Include .targets file and all DLLs in the output directory in the NuGet package -->
+    <Content Include="$(MSBuildThisFileDirectory)buildTransitive\**\*.targets" Pack="true" PackagePath="buildTransitive" />
+    <Content Include="$(OutputPath)**\*.dll" Pack="true" PackagePath="tasks/$(TargetFramework)" />
+    <None Remove="packages\**" />
+  </ItemGroup>
+</Project>

src/BenchmarkDotNet.Weaver/buildTransitive/netstandard2.0/BenchmarkDotNet.Weaver.targets

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <UsingTask TaskName="BenchmarkDotNet.Weaver.WeaveAssemblyTask" AssemblyFile="$(MSBuildThisFileDirectory)..\..\tasks\netstandard2.0\BenchmarkDotNet.Weaver.dll" />
+
+  <Target Name="WeaveAssemblies" AfterTargets="AfterBuild" >
+    <WeaveAssemblyTask TargetAssembly="$(TargetDir)$(TargetFileName)" />
+  </Target>
+</Project>

src/BenchmarkDotNet.Weaver/src/WeaveAssemblyTask.cs

@@ -0,0 +1,92 @@
+// *****
+// If any changes are made to this file, increment the WeaverVersionSuffix in the common.props file,
+// then run `build.cmd pack-weaver`.
+// *****
+
+using AsmResolver.DotNet;
+using AsmResolver.PE.DotNet.Metadata.Tables;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace BenchmarkDotNet.Weaver;
+
+/// <summary>
+/// The Task used by MSBuild to weave the assembly.
+/// </summary>
+public sealed class WeaveAssemblyTask : Task
+{
+    /// <summary>
+    /// The path of the target assembly.
+    /// </summary>
+    [Required]
+    public string TargetAssembly { get; set; }
+
+    /// <summary>
+    /// Runs the weave assembly task.
+    /// </summary>
+    /// <returns><see langword="true"/> if successful; <see langword="false"/> otherwise.</returns>
+    public override bool Execute()
+    {
+        if (!File.Exists(TargetAssembly))
+        {
+            Log.LogError($"Assembly not found: {TargetAssembly}");
+            return false;
+        }
+
+
+        bool benchmarkMethodsImplAdjusted = false;
+        try
+        {
+            var module = ModuleDefinition.FromFile(TargetAssembly);
+
+            foreach (var type in module.GetAllTypes())
+            {
+                // We can skip non-public types as they are not valid for benchmarks.
+                if (type.IsNotPublic)
+                {
+                    continue;
+                }
+
+                foreach (var method in type.Methods)
+                {
+                    if (method.CustomAttributes.Any(IsBenchmarkAttribute))
+                    {
+                        var oldImpl = method.ImplAttributes;
+                        // Remove AggressiveInlining and add NoInlining.
+                        method.ImplAttributes = (oldImpl & ~MethodImplAttributes.AggressiveInlining) | MethodImplAttributes.NoInlining;
+                        benchmarkMethodsImplAdjusted |= (oldImpl & MethodImplAttributes.NoInlining) == 0;
+                    }
+                }
+            }
+
+            if (benchmarkMethodsImplAdjusted)
+            {
+                // Write to a null stream before overwriting the original file in case an exception occurs during the write (like unsupported platform).
+                // https://github.com/Washi1337/AsmResolver/issues/640
+                module.Write(Stream.Null);
+                module.Write(TargetAssembly);
+            }
+        }
+        catch (Exception e)
+        {
+            Log.LogWarning($"Assembly weaving failed. Benchmark methods found requiring NoInlining: {benchmarkMethodsImplAdjusted}. Error:{Environment.NewLine}{e}");
+        }
+        return true;
+    }
+
+    private static bool IsBenchmarkAttribute(CustomAttribute attribute)
+    {
+        // BenchmarkAttribute is unsealed, so we need to walk its hierarchy.
+        for (var attr = attribute.Constructor.DeclaringType; attr != null; attr = attr.Resolve()?.BaseType)
+        {
+            if (attr.FullName == "BenchmarkDotNet.Attributes.BenchmarkAttribute")
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+}