diff --git a/docs/articles/samples/IntroNuGet.md b/docs/articles/samples/IntroNuGet.md index 53d9ac54f8..1cc9af84a2 100644 --- a/docs/articles/samples/IntroNuGet.md +++ b/docs/articles/samples/IntroNuGet.md @@ -1,10 +1,10 @@ ---- +--- uid: BenchmarkDotNet.Samples.IntroNuGet --- ## Sample: IntroNuGet -You can set specific versions of NuGet dependencies for each job. +You can set specific versions of NuGet dependencies for each job using MsBuild properties in your csproj. It allows comparing different versions of the same package (if there are no breaking changes in API). ### Source code @@ -13,14 +13,11 @@ It allows comparing different versions of the same package (if there are no brea ### Output -| Method | Job | NuGetReferences | Mean | Error | StdDev | -|------------------------- |------- |----------------------- |---------:|----------:|----------:| -| SerializeAnonymousObject | 10.0.1 | Newtonsoft.Json 10.0.1 | 2.926 us | 0.0795 us | 0.0283 us | -| SerializeAnonymousObject | 10.0.2 | Newtonsoft.Json 10.0.2 | 2.877 us | 0.5928 us | 0.2114 us | -| SerializeAnonymousObject | 10.0.3 | Newtonsoft.Json 10.0.3 | 2.706 us | 0.1251 us | 0.0446 us | -| SerializeAnonymousObject | 11.0.1 | Newtonsoft.Json 11.0.1 | 2.778 us | 0.5037 us | 0.1796 us | -| SerializeAnonymousObject | 11.0.2 | Newtonsoft.Json 11.0.2 | 2.644 us | 0.0609 us | 0.0217 us | -| SerializeAnonymousObject | 9.0.1 | Newtonsoft.Json 9.0.1 | 2.722 us | 0.3552 us | 0.1267 us | +| Method | Job | Arguments | Mean | Error | StdDev | +|-------------------------- |------- |-------------------- |---------:|----------:|----------:| +| ToImmutableArrayBenchmark | v9.0.0 | /p:SciVersion=9.0.0 | 1.173 μs | 0.0057 μs | 0.0086 μs | +| ToImmutableArrayBenchmark | v9.0.3 | /p:SciVersion=9.0.3 | 1.173 μs | 0.0038 μs | 0.0058 μs | +| ToImmutableArrayBenchmark | v9.0.5 | /p:SciVersion=9.0.5 | 1.172 μs | 0.0107 μs | 0.0157 μs | ### Links diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 6c861ed1f6..5451fd721d 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -17,9 +17,12 @@ + + + 9.0.0 + - - + diff --git a/samples/BenchmarkDotNet.Samples/IntroNuGet.cs b/samples/BenchmarkDotNet.Samples/IntroNuGet.cs index a368b62b56..850a9ceebf 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNuGet.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNuGet.cs @@ -11,21 +11,26 @@ namespace BenchmarkDotNet.Samples /// Benchmarks between various versions of a NuGet package /// /// - /// Only supported with the CsProjCoreToolchain toolchain + /// Only supported with CsProj toolchains. /// [Config(typeof(Config))] public class IntroNuGet { - // Specify jobs with different versions of the same NuGet package to benchmark. - // The NuGet versions referenced on these jobs must be greater or equal to the - // same NuGet version referenced in this benchmark project. - // Example: This benchmark project references Newtonsoft.Json 13.0.1 + // Setup your csproj like this: + /* + + + 9.0.0 + + + + + */ + // All versions of the package must be source-compatible with your benchmark code. private class Config : ManualConfig { public Config() { - var baseJob = Job.MediumRun; - string[] targetVersions = [ "9.0.0", "9.0.3", @@ -34,8 +39,10 @@ public Config() foreach (var version in targetVersions) { - AddJob(baseJob.WithNuGet("System.Collections.Immutable", version) - .WithId($"v{version}")); + AddJob(Job.MediumRun + .WithMsBuildArguments($"/p:SciVersion={version}") + .WithId($"v{version}") + ); } } } diff --git a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs index 865c6f64f3..dfeaf3fd47 100644 --- a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs +++ b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs @@ -17,7 +17,10 @@ private InfrastructureResolver() Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName); Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + +#pragma warning disable CS0618 // Type or member is obsolete Register(InfrastructureMode.NuGetReferencesCharacteristic, Array.Empty); +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs index 5e5f8b5369..277627d2e9 100644 --- a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs +++ b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; @@ -18,6 +20,9 @@ public sealed class InfrastructureMode : JobMode public static readonly Characteristic EngineFactoryCharacteristic = CreateCharacteristic(nameof(EngineFactory)); public static readonly Characteristic BuildConfigurationCharacteristic = CreateCharacteristic(nameof(BuildConfiguration)); public static readonly Characteristic> ArgumentsCharacteristic = CreateCharacteristic>(nameof(Arguments)); + + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This will soon be removed")] public static readonly Characteristic> NuGetReferencesCharacteristic = CreateCharacteristic>(nameof(NuGetReferences)); public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessEmitToolchain.Instance); @@ -64,6 +69,8 @@ public IReadOnlyList Arguments set => ArgumentsCharacteristic[this] = value; } + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This will soon be removed")] public IReadOnlyCollection NuGetReferences { get => NuGetReferencesCharacteristic[this]; diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 2c778df589..0424528ad4 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -339,6 +339,8 @@ public static Job WithEnvironmentVariable(this Job job, string key, string value /// (optional)Indicate the URI of the NuGet package source to use during the restore operation. /// (optional)Allows prerelease packages to be installed. /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed, please start using .WithMsBuildArguments() instead.")] public static Job WithNuGet(this Job job, string packageName, string? packageVersion = null, Uri? source = null, bool prerelease = false) => job.WithCore(j => j.Infrastructure.NuGetReferences = new NuGetReferenceList(j.Infrastructure.NuGetReferences ?? Array.Empty()) @@ -352,9 +354,14 @@ public static Job WithNuGet(this Job job, string packageName, string? packageVer /// /// A collection of NuGet dependencies /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed, please start using .WithMsBuildArguments() instead.")] public static Job WithNuGet(this Job job, NuGetReferenceList nuGetReferences) => job.WithCore(j => j.Infrastructure.NuGetReferences = nuGetReferences); + public static Job WithMsBuildArguments(this Job job, params string[] msBuildArguments) + => job.WithArguments([.. msBuildArguments.Select(a => new MsBuildArgument(a))]); + // Accuracy /// /// Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error). @@ -437,8 +444,10 @@ private static Job WithCore(this Job job, Action updateCallback) return newJob; } - internal static bool HasDynamicBuildCharacteristic(this Job job) => - job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic) + internal static bool HasDynamicBuildCharacteristic(this Job job) +#pragma warning disable CS0618 // Type or member is obsolete + => job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic) +#pragma warning restore CS0618 // Type or member is obsolete || job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) || job.HasValue(InfrastructureMode.ArgumentsCharacteristic); } diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index 0db3a028de..89dc4c3734 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -1,8 +1,11 @@ using System; +using System.ComponentModel; using System.Text.RegularExpressions; namespace BenchmarkDotNet.Jobs { + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This type will soon be removed")] public class NuGetReference : IEquatable { public NuGetReference(string packageName, string packageVersion, Uri? source = null, bool prerelease = false) diff --git a/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs b/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs index 13c727c28f..025353b24f 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs @@ -1,12 +1,15 @@ using System; using System.Collections; using System.Collections.Generic; +using System.ComponentModel; namespace BenchmarkDotNet.Jobs { /// /// An ordered list of NuGet references. Does not allow duplicate references with the same PackageName. /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This type will soon be removed")] public class NuGetReferenceList : IReadOnlyCollection { private readonly List references = new List(); diff --git a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs index ada1b2b78e..4892de7f0c 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs @@ -47,8 +47,10 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y) return false; if (AreDifferent(jobX.Infrastructure.Arguments, jobY.Infrastructure.Arguments)) // arguments can be anything (Mono runtime settings or MsBuild parameters) return false; +#pragma warning disable CS0618 // Type or member is obsolete if (AreDifferent(jobX.Infrastructure.NuGetReferences, jobY.Infrastructure.NuGetReferences)) return false; +#pragma warning restore CS0618 // Type or member is obsolete if (!jobX.Environment.Gc.Equals(jobY.Environment.Gc)) // GC settings are per .config/.csproj return false; @@ -81,8 +83,10 @@ public int GetHashCode(BenchmarkCase obj) hashCode.Add(job.Infrastructure.BuildConfiguration); foreach (var arg in job.Infrastructure.Arguments ?? Array.Empty()) hashCode.Add(arg); +#pragma warning disable CS0618 // Type or member is obsolete foreach (var reference in job.Infrastructure.NuGetReferences ?? Array.Empty()) hashCode.Add(reference); +#pragma warning restore CS0618 // Type or member is obsolete return hashCode.ToHashCode(); } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index bcd921fdd1..607ff6f974 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.IO; using System.Linq; using System.Text; @@ -57,7 +58,9 @@ public BuildResult RestoreThenBuild() { DotNetCliCommandExecutor.LogEnvVars(WithArguments(null)); +#pragma warning disable CS0618 // Type or member is obsolete var packagesResult = AddPackages(); +#pragma warning restore CS0618 // Type or member is obsolete if (!packagesResult.IsSuccess) return BuildResult.Failure(GenerateResult, packagesResult.AllInformation); @@ -97,7 +100,9 @@ public BuildResult RestoreThenBuildThenPublish() { DotNetCliCommandExecutor.LogEnvVars(WithArguments(null)); +#pragma warning disable CS0618 // Type or member is obsolete var packagesResult = AddPackages(); +#pragma warning restore CS0618 // Type or member is obsolete if (!packagesResult.IsSuccess) return BuildResult.Failure(GenerateResult, packagesResult.AllInformation); @@ -115,6 +120,8 @@ public BuildResult RestoreThenBuildThenPublish() return PublishNoRestore().ToBuildResult(GenerateResult); } + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed")] public DotNetCliCommandResult AddPackages() { var executionTime = new TimeSpan(0); @@ -150,6 +157,7 @@ public DotNetCliCommandResult PublishNoRestore() => DotNetCliCommandExecutor.Execute(WithArguments( GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore", "publish-no-restore"))); + [Obsolete] internal static IEnumerable GetAddPackagesCommands(BuildPartition buildPartition) => GetNuGetAddPackageCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); @@ -204,6 +212,7 @@ private static string GetCustomMsBuildArguments(BenchmarkCase benchmarkCase, IRe return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation)); } + [Obsolete] private static IEnumerable GetNuGetAddPackageCommands(BenchmarkCase benchmarkCase, IResolver resolver) { if (!benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) @@ -229,6 +238,7 @@ private static string GetMandatoryMsBuildSettings(string buildConfiguration) return $"{NoMsBuildZombieProcesses} {EnforceOptimizations}"; } + [Obsolete] private static string BuildAddPackageCommand(NuGetReference reference) { var commandBuilder = new StringBuilder(); diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs index acbeeffe38..30135c15cd 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs @@ -48,12 +48,14 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas benchmarkCase); } +#pragma warning disable CS0618 // Type or member is obsolete if (benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) { yield return new ValidationError(true, "The MonoAOT toolchain does not allow specifying NuGet package dependencies", benchmarkCase); } +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs index 31c4d68c29..6774268dcc 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs @@ -51,12 +51,14 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas benchmarkCase); } +#pragma warning disable CS0618 // Type or member is obsolete if (benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) { yield return new ValidationError(true, "The Roslyn toolchain does not allow specifying NuGet package dependencies", benchmarkCase); } +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs index 65655bf3f6..e9f9469f13 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -21,6 +21,7 @@ public NuGetReferenceTests(ITestOutputHelper output) : base(output) } [Fact] + [Obsolete] public void UserCanSpecifyCustomNuGetPackageDependency() { var toolchain = RuntimeInformation.GetCurrentRuntime().GetToolchain(preferMsBuildToolchains: true); @@ -38,6 +39,7 @@ public void UserCanSpecifyCustomNuGetPackageDependency() } [FactEnvSpecific("Roslyn toolchain does not support .NET Core", EnvRequirement.FullFrameworkOnly)] + [Obsolete] public void RoslynToolchainDoesNotSupportNuGetPackageDependency() { var toolchain = RoslynToolchain.Instance; diff --git a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs index fd47e9fd83..d7fb7519f3 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs @@ -441,6 +441,7 @@ public static void AllJobModesPropertyNamesMatchCharacteristicNames() // it's ma } [Fact] + [Obsolete] public static void WithNuGet() { var j = new Job("SomeId"); diff --git a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs index e04409beb9..4ace61168b 100644 --- a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs @@ -95,6 +95,7 @@ public void CustomClrBuildJobsAreGroupedByVersion() } [Fact] + [System.Obsolete] public void CustomNuGetJobsAreGroupedByPackageVersion() { var config = ManualConfig.Create(DefaultConfig.Instance)