diff --git a/src/BenchmarkDotNet/Code/CodeGenBenchmarkRunCallType.cs b/src/BenchmarkDotNet/Code/CodeGenBenchmarkRunCallType.cs new file mode 100644 index 0000000000..96297181a4 --- /dev/null +++ b/src/BenchmarkDotNet/Code/CodeGenBenchmarkRunCallType.cs @@ -0,0 +1,25 @@ +namespace BenchmarkDotNet.Code; + +/// +/// Specifies how to generate the code that calls the benchmark's Run method. +/// +public enum CodeGenBenchmarkRunCallType +{ + /// + /// Use reflection to call the benchmark's Run method indirectly. + /// + /// + /// This is to avoid strong dependency Main-to-Runnable + /// which could cause the jitting/assembly loading to happen before we do anything. + /// We have some jitting diagnosers and we want them to catch all the informations. + /// + Reflection, + /// + /// Uses a switch to select the benchmark to call Run directly. + /// + /// + /// This is for AOT runtimes where reflection may not exist or the benchmark types + /// could be trimmed out when they are not directly referenced. + /// + Direct +} diff --git a/src/BenchmarkDotNet/Code/CodeGenerator.cs b/src/BenchmarkDotNet/Code/CodeGenerator.cs index b75f7641c9..36db0d5770 100644 --- a/src/BenchmarkDotNet/Code/CodeGenerator.cs +++ b/src/BenchmarkDotNet/Code/CodeGenerator.cs @@ -21,14 +21,12 @@ namespace BenchmarkDotNet.Code { internal static class CodeGenerator { - internal static string Generate(BuildPartition buildPartition) + internal static string Generate(BuildPartition buildPartition, CodeGenBenchmarkRunCallType benchmarkRunCallType) { (bool useShadowCopy, string shadowCopyFolderPath) = GetShadowCopySettings(); var benchmarksCode = new List(buildPartition.Benchmarks.Length); - var extraDefines = new List(); - foreach (var buildInfo in buildPartition.Benchmarks) { var benchmark = buildInfo.BenchmarkCase; @@ -64,19 +62,12 @@ internal static string Generate(BuildPartition buildPartition) benchmarksCode.Add(benchmarkTypeCode); } - if (buildPartition.IsNativeAot) - extraDefines.Add("#define NATIVEAOT"); - else if (buildPartition.IsNetFramework) - extraDefines.Add("#define NETFRAMEWORK"); - else if (buildPartition.IsWasm) - extraDefines.Add("#define WASM"); - string benchmarkProgramContent = new SmartStringBuilder(ResourceHelper.LoadTemplate("BenchmarkProgram.txt")) .Replace("$ShadowCopyDefines$", useShadowCopy ? "#define SHADOWCOPY" : null).Replace("$ShadowCopyFolderPath$", shadowCopyFolderPath) - .Replace("$ExtraDefines$", string.Join(Environment.NewLine, extraDefines)) + .Replace("$ExtraDefines$", buildPartition.IsNetFramework ? "#define NETFRAMEWORK" : string.Empty) .Replace("$DerivedTypes$", string.Join(Environment.NewLine, benchmarksCode)) .Replace("$ExtraAttribute$", GetExtraAttributes(buildPartition.RepresentativeBenchmarkCase.Descriptor)) - .Replace("$NativeAotSwitch$", GetNativeAotSwitch(buildPartition)) + .Replace("$BenchmarkRunCall$", GetBenchmarkRunCall(buildPartition, benchmarkRunCallType)) .ToString(); return benchmarkProgramContent; @@ -251,14 +242,20 @@ private static string GetParameterModifier(ParameterInfo parameterInfo) return "ref"; } - /// - /// for NativeAOT we can't use reflection to load type and run a method, so we simply generate a switch for all types.. - /// - private static string GetNativeAotSwitch(BuildPartition buildPartition) + private static string GetBenchmarkRunCall(BuildPartition buildPartition, CodeGenBenchmarkRunCallType runCallType) { - if (!buildPartition.IsNativeAot) - return default; + if (runCallType == CodeGenBenchmarkRunCallType.Reflection) + { + // Use reflection to call benchmark's Run method indirectly. + return """ + typeof(global::BenchmarkDotNet.Autogenerated.UniqueProgramName).Assembly + .GetType($"BenchmarkDotNet.Autogenerated.Runnable_{id}") + .GetMethod("Run", global::System.Reflection.BindingFlags.Public | global::System.Reflection.BindingFlags.Static) + .Invoke(null, new global::System.Object[] { host, benchmarkName, diagnoserRunMode }); + """; + } + // Generate a switch to call benchmark's Run method directly. var @switch = new StringBuilder(buildPartition.Benchmarks.Length * 30); @switch.AppendLine("switch (id) {"); diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index 2ebe5ee44c..1a4c424e06 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -12,7 +12,6 @@ using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Toolchains.Roslyn; using JetBrains.Annotations; @@ -57,9 +56,6 @@ public BuildPartition(BenchmarkBuildInfo[] benchmarks, IResolver resolver) public bool IsNativeAot => RepresentativeBenchmarkCase.Job.IsNativeAOT(); - public bool IsWasm => Runtime is WasmRuntime // given job can have Wasm toolchain set, but Runtime == default ;) - || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain is WasmToolchain); - public bool IsNetFramework => Runtime is ClrRuntime || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && (toolchain is RoslynToolchain || toolchain is CsProjClassicNetToolchain)); diff --git a/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt b/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt index dfc5bbfcca..b282d28d28 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt @@ -37,7 +37,7 @@ namespace BenchmarkDotNet.Autogenerated private static global::System.Int32 AfterAssemblyLoadingAttached(global::System.String[] args) { - global::BenchmarkDotNet.Engines.IHost host; // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it + global::BenchmarkDotNet.Engines.IHost host; // this variable name is used by CodeGenerator.GetBenchmarkRunCall, do NOT change it if (global::BenchmarkDotNet.Engines.AnonymousPipesHost.TryGetFileHandles(args, out global::System.String writeHandle, out global::System.String readHandle)) host = new global::BenchmarkDotNet.Engines.AnonymousPipesHost(writeHandle, readHandle); else @@ -49,27 +49,18 @@ namespace BenchmarkDotNet.Autogenerated try { - // we are not using Runnable here in any direct way in order to avoid strong dependency Main<=>Runnable - // which could cause the jitting/assembly loading to happen before we do anything - // we have some jitting diagnosers and we want them to catch all the informations!! - - // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it + // These variable names are used by CodeGenerator.GetBenchmarkRunCall, do NOT change them! global::System.String benchmarkName = global::System.Linq.Enumerable.FirstOrDefault(global::System.Linq.Enumerable.Skip(global::System.Linq.Enumerable.SkipWhile(args, arg => arg != "--benchmarkName"), 1)) ?? "not provided"; global::BenchmarkDotNet.Diagnosers.RunMode diagnoserRunMode = (global::BenchmarkDotNet.Diagnosers.RunMode) global::System.Int32.Parse(global::System.Linq.Enumerable.FirstOrDefault(global::System.Linq.Enumerable.Skip(global::System.Linq.Enumerable.SkipWhile(args, arg => arg != "--diagnoserRunMode"), 1)) ?? "0"); global::System.Int32 id = args.Length > 0 - ? global::System.Int32.Parse(args[args.Length - 1]) // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it + ? global::System.Int32.Parse(args[args.Length - 1]) : 0; // used when re-using generated exe without BDN (typically to repro a bug) if (args.Length == 0) { host.WriteLine("You have not specified benchmark id (an integer) so the first benchmark will be executed."); } -#if NATIVEAOT - $NativeAotSwitch$ -#else - global::System.Type type = typeof(global::BenchmarkDotNet.Autogenerated.UniqueProgramName).Assembly.GetType($"BenchmarkDotNet.Autogenerated.Runnable_{id}"); - type.GetMethod("Run", global::System.Reflection.BindingFlags.Public | global::System.Reflection.BindingFlags.Static).Invoke(null, new global::System.Object[] { host, benchmarkName, diagnoserRunMode }); -#endif + $BenchmarkRunCall$ return 0; } catch (global::System.Exception oom) when (oom is global::System.OutOfMemoryException || oom is global::System.Reflection.TargetInvocationException reflection && reflection.InnerException is global::System.OutOfMemoryException) diff --git a/src/BenchmarkDotNet/Templates/WasmLinkerDescription.xml b/src/BenchmarkDotNet/Templates/WasmLinkerDescription.xml index 6c3f5a5c16..56899a3b8a 100644 --- a/src/BenchmarkDotNet/Templates/WasmLinkerDescription.xml +++ b/src/BenchmarkDotNet/Templates/WasmLinkerDescription.xml @@ -2,5 +2,4 @@ - diff --git a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs index 165dedcbba..b846af7037 100644 --- a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs +++ b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs @@ -5,7 +5,6 @@ using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; using JetBrains.Annotations; @@ -16,6 +15,9 @@ namespace BenchmarkDotNet.Toolchains [PublicAPI] public abstract class GeneratorBase : IGenerator { + /// + public CodeGenBenchmarkRunCallType BenchmarkRunCallType { get; init; } + public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger logger, string rootArtifactsFolderPath) { var artifactsPaths = ArtifactsPaths.Empty; @@ -119,7 +121,7 @@ [PublicAPI] protected virtual void GenerateAppConfig(BuildPartition buildPartiti /// You most probably do NOT need to override this method!! /// [PublicAPI] protected virtual void GenerateCode(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) - => File.WriteAllText(artifactsPaths.ProgramCodePath, CodeGenerator.Generate(buildPartition)); + => File.WriteAllText(artifactsPaths.ProgramCodePath, CodeGenerator.Generate(buildPartition, BenchmarkRunCallType)); protected virtual string GetExecutablePath(string binariesDirectoryPath, string programName) => Path.Combine(binariesDirectoryPath, $"{programName}{GetExecutableExtension()}"); diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs index ac58acd3d0..c81e14f669 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs @@ -23,6 +23,7 @@ public MonoAotLLVMGenerator(string targetFrameworkMoniker, string cliPath, strin CustomRuntimePack = customRuntimePack; AotCompilerPath = aotCompilerPath; AotCompilerMode = aotCompilerMode; + BenchmarkRunCallType = Code.CodeGenBenchmarkRunCallType.Direct; } protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs index d1c6b41bb4..7aea40aaf4 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs @@ -13,15 +13,14 @@ namespace BenchmarkDotNet.Toolchains.MonoWasm public class WasmGenerator : CsProjGenerator { private readonly string CustomRuntimePack; - private readonly bool Aot; private readonly string MainJS; public WasmGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, string customRuntimePack, bool aot) : base(targetFrameworkMoniker, cliPath, packagesPath, runtimeFrameworkVersion: null) { - Aot = aot; CustomRuntimePack = customRuntimePack; MainJS = (targetFrameworkMoniker == "net5.0" || targetFrameworkMoniker == "net6.0") ? "main.js" : "test-main.js"; + BenchmarkRunCallType = aot ? Code.CodeGenBenchmarkRunCallType.Direct : Code.CodeGenBenchmarkRunCallType.Reflection; } protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) @@ -29,9 +28,10 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts if (((WasmRuntime)buildPartition.Runtime).Aot) { GenerateProjectFile(buildPartition, artifactsPaths, aot: true, logger); - GenerateLinkerDescriptionFile(artifactsPaths); - } - else + + var linkDescriptionFileName = "WasmLinkerDescription.xml"; + File.WriteAllText(Path.Combine(Path.GetDirectoryName(artifactsPaths.ProjectFilePath), linkDescriptionFileName), ResourceHelper.LoadTemplate(linkDescriptionFileName)); + } else { GenerateProjectFile(buildPartition, artifactsPaths, aot: false, logger); } @@ -66,16 +66,6 @@ protected void GenerateProjectFile(BuildPartition buildPartition, ArtifactsPaths GatherReferences(buildPartition, artifactsPaths, logger); } - protected void GenerateLinkerDescriptionFile(ArtifactsPaths artifactsPaths) - { - const string linkDescriptionFileName = "WasmLinkerDescription.xml"; - - var content = ResourceHelper.LoadTemplate(linkDescriptionFileName) - .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName); - - File.WriteAllText(Path.Combine(Path.GetDirectoryName(artifactsPaths.ProjectFilePath)!, linkDescriptionFileName), content); - } - protected override string GetExecutablePath(string binariesDirectoryPath, string programName) => Path.Combine(binariesDirectoryPath, "AppBundle", MainJS); protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs index 9428245127..cc25e0ea13 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs @@ -46,6 +46,7 @@ internal Generator(string ilCompilerVersion, this.ilcGenerateStackTraceData = ilcGenerateStackTraceData; this.ilcOptimizationPreference = ilcOptimizationPreference; this.ilcInstructionSet = ilcInstructionSet; + BenchmarkRunCallType = Code.CodeGenBenchmarkRunCallType.Direct; } internal readonly IReadOnlyDictionary Feeds; diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index 72ff886eb2..2f4493c99d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.MonoAotLLVM; using BenchmarkDotNet.Toolchains.MonoWasm; using Xunit; using Xunit.Abstractions; @@ -27,11 +28,11 @@ namespace BenchmarkDotNet.IntegrationTests /// public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { - private ManualConfig GetConfig() + private ManualConfig GetConfig(MonoAotCompilerMode aotCompilerMode) { var dotnetVersion = "net8.0"; var logger = new OutputLogger(Output); - var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm"); + var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm", aotCompilerMode: aotCompilerMode); var mainJsPath = Path.Combine(AppContext.BaseDirectory, "AppBundle", "test-main.js"); return ManualConfig.CreateEmpty() @@ -44,8 +45,10 @@ private ManualConfig GetConfig() .WithOption(ConfigOptions.GenerateMSBuildBinLog, true); } - [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] - public void WasmIsSupported() + [TheoryEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] + [InlineData(MonoAotCompilerMode.mini)] + [InlineData(MonoAotCompilerMode.wasm)] + public void WasmIsSupported(MonoAotCompilerMode aotCompilerMode) { // Test fails on Linux non-x64. if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64) @@ -53,11 +56,13 @@ public void WasmIsSupported() return; } - CanExecute(GetConfig()); + CanExecute(GetConfig(aotCompilerMode)); } - [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] - public void WasmSupportsInProcessDiagnosers() + [TheoryEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] + [InlineData(MonoAotCompilerMode.mini)] + [InlineData(MonoAotCompilerMode.wasm)] + public void WasmSupportsInProcessDiagnosers(MonoAotCompilerMode aotCompilerMode) { // Test fails on Linux non-x64. if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64) @@ -68,7 +73,7 @@ public void WasmSupportsInProcessDiagnosers() try { var diagnoser = new MockInProcessDiagnoser1(BenchmarkDotNet.Diagnosers.RunMode.NoOverhead); - var config = GetConfig().AddDiagnoser(diagnoser); + var config = GetConfig(aotCompilerMode).AddDiagnoser(diagnoser); CanExecute(config); diff --git a/tests/BenchmarkDotNet.Tests/CodeGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/CodeGeneratorTests.cs index f009f065b9..e5f9cbd0d5 100644 --- a/tests/BenchmarkDotNet.Tests/CodeGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/CodeGeneratorTests.cs @@ -14,8 +14,11 @@ namespace BenchmarkDotNet.Tests { public class CodeGeneratorTests { - [Fact] - public void AsyncVoidIsNotSupported() + public static TheoryData CodeGenBenchmarkRunCallTypes => [.. (CodeGenBenchmarkRunCallType[]) Enum.GetValues(typeof(CodeGenBenchmarkRunCallType))]; + + [Theory] + [MemberData(nameof(CodeGenBenchmarkRunCallTypes))] + public void AsyncVoidIsNotSupported(CodeGenBenchmarkRunCallType benchmarkRunCallType) { var asyncVoidMethod = typeof(CodeGeneratorTests) @@ -28,12 +31,14 @@ public void AsyncVoidIsNotSupported() Assert.Throws(() => CodeGenerator.Generate(new BuildPartition( [new BenchmarkBuildInfo(benchmark, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0, new([]))], - BenchmarkRunnerClean.DefaultResolver) + BenchmarkRunnerClean.DefaultResolver), + benchmarkRunCallType )); } - [Fact] - public void UsingStatementsInTheAutoGeneratedCodeAreProhibited() + [Theory] + [MemberData(nameof(CodeGenBenchmarkRunCallTypes))] + public void UsingStatementsInTheAutoGeneratedCodeAreProhibited(CodeGenBenchmarkRunCallType benchmarkRunCallType) { var fineMethod = typeof(CodeGeneratorTests) @@ -46,7 +51,8 @@ public void UsingStatementsInTheAutoGeneratedCodeAreProhibited() var generatedSourceFile = CodeGenerator.Generate(new BuildPartition( [new BenchmarkBuildInfo(benchmark, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0, new([]))], - BenchmarkRunnerClean.DefaultResolver) + BenchmarkRunnerClean.DefaultResolver), + benchmarkRunCallType ); using (StringReader stringReader = new StringReader(generatedSourceFile))