Deprecate WithNuget (#2812)

* Deprecated .WithNuget.
Added .WithMsBuildArguments.
Updated IntroNuGet.

* Simplify csproj version selection.

Author: timcassell

docs/articles/samples/IntroNuGet.md

@@ -1,10 +1,10 @@
----
+---
 uid: BenchmarkDotNet.Samples.IntroNuGet
 ---
 
 ## Sample: IntroNuGet
 
-You can set specific versions of NuGet dependencies for each job.
+You can set specific versions of NuGet dependencies for each job using MsBuild properties in your csproj.
 It allows comparing different versions of the same package (if there are no breaking changes in API).
 
 ### Source code
@@ -13,14 +13,11 @@ It allows comparing different versions of the same package (if there are no brea
 
 ### Output
 
-|                   Method |    Job |        NuGetReferences |     Mean |     Error |    StdDev |
-|------------------------- |------- |----------------------- |---------:|----------:|----------:|
-| SerializeAnonymousObject | 10.0.1 | Newtonsoft.Json 10.0.1 | 2.926 us | 0.0795 us | 0.0283 us |
-| SerializeAnonymousObject | 10.0.2 | Newtonsoft.Json 10.0.2 | 2.877 us | 0.5928 us | 0.2114 us |
-| SerializeAnonymousObject | 10.0.3 | Newtonsoft.Json 10.0.3 | 2.706 us | 0.1251 us | 0.0446 us |
-| SerializeAnonymousObject | 11.0.1 | Newtonsoft.Json 11.0.1 | 2.778 us | 0.5037 us | 0.1796 us |
-| SerializeAnonymousObject | 11.0.2 | Newtonsoft.Json 11.0.2 | 2.644 us | 0.0609 us | 0.0217 us |
-| SerializeAnonymousObject |  9.0.1 |  Newtonsoft.Json 9.0.1 | 2.722 us | 0.3552 us | 0.1267 us |
+| Method                    | Job    | Arguments           | Mean     | Error     | StdDev    |
+|-------------------------- |------- |-------------------- |---------:|----------:|----------:|
+| ToImmutableArrayBenchmark | v9.0.0 | /p:SciVersion=9.0.0 | 1.173 μs | 0.0057 μs | 0.0086 μs |
+| ToImmutableArrayBenchmark | v9.0.3 | /p:SciVersion=9.0.3 | 1.173 μs | 0.0038 μs | 0.0058 μs |
+| ToImmutableArrayBenchmark | v9.0.5 | /p:SciVersion=9.0.5 | 1.172 μs | 0.0107 μs | 0.0157 μs |
 
 ### Links
 

samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj

@@ -19,9 +19,12 @@
   <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
     <Reference Include="System.Reflection" />
   </ItemGroup>
+  <PropertyGroup>
+    <!-- Use 9.0.0 as baseline package for IntroNuGet -->
+    <SciVersion Condition="'$(SciVersion)' == ''">9.0.0</SciVersion>
+  </PropertyGroup>
   <ItemGroup>
-    <!-- Use v9.0.0 as baseline package for WithNuGet tests -->
-    <PackageReference Include="System.Collections.Immutable" Version="9.0.0" />
+    <PackageReference Include="System.Collections.Immutable" Version="$(SciVersion)" />
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="System.Drawing.Common" Version="9.0.5" />

samples/BenchmarkDotNet.Samples/IntroNuGet.cs

@@ -11,21 +11,26 @@ namespace BenchmarkDotNet.Samples
     /// Benchmarks between various versions of a NuGet package
     /// </summary>
     /// <remarks>
-    /// Only supported with the CsProjCoreToolchain toolchain
+    /// Only supported with CsProj toolchains.
     /// </remarks>
     [Config(typeof(Config))]
     public class IntroNuGet
     {
-        // Specify jobs with different versions of the same NuGet package to benchmark.
-        // The NuGet versions referenced on these jobs must be greater or equal to the
-        // same NuGet version referenced in this benchmark project.
-        // Example: This benchmark project references Newtonsoft.Json 13.0.1
+        // Setup your csproj like this:
+        /*
+        <PropertyGroup>
+          <!-- Use 9.0.0 as default package version if not specified -->
+          <SciVersion Condition="'$(SciVersion)' == ''">9.0.0</SciVersion>
+        </PropertyGroup>
+        <ItemGroup>
+          <PackageReference Include="System.Collections.Immutable" Version="$(SciVersion)" />
+        </ItemGroup>
+        */
+        // All versions of the package must be source-compatible with your benchmark code.
         private class Config : ManualConfig
         {
             public Config()
             {
-                var baseJob = Job.MediumRun;
-
                 string[] targetVersions = [
                     "9.0.0",
                     "9.0.3",
@@ -34,8 +39,10 @@ public Config()
 
                 foreach (var version in targetVersions)
                 {
-                    AddJob(baseJob.WithNuGet("System.Collections.Immutable", version)
-                                  .WithId($"v{version}"));
+                    AddJob(Job.MediumRun
+                        .WithMsBuildArguments($"/p:SciVersion={version}")
+                        .WithId($"v{version}")
+                    );
                 }
             }
         }

src/BenchmarkDotNet/Environments/InfrastructureResolver.cs

@@ -17,7 +17,10 @@ private InfrastructureResolver()
             Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName);
 
             Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty<Argument>);
+
+#pragma warning disable CS0618 // Type or member is obsolete
             Register(InfrastructureMode.NuGetReferencesCharacteristic, Array.Empty<NuGetReference>);
+#pragma warning restore CS0618 // Type or member is obsolete
         }
     }
 }

src/BenchmarkDotNet/Jobs/InfrastructureMode.cs

@@ -1,4 +1,6 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
 using System.Diagnostics.CodeAnalysis;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Engines;
@@ -18,6 +20,9 @@ public sealed class InfrastructureMode : JobMode<InfrastructureMode>
         public static readonly Characteristic<IEngineFactory> EngineFactoryCharacteristic = CreateCharacteristic<IEngineFactory>(nameof(EngineFactory));
         public static readonly Characteristic<string> BuildConfigurationCharacteristic = CreateCharacteristic<string>(nameof(BuildConfiguration));
         public static readonly Characteristic<IReadOnlyList<Argument>> ArgumentsCharacteristic = CreateCharacteristic<IReadOnlyList<Argument>>(nameof(Arguments));
+
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("This will soon be removed")]
         public static readonly Characteristic<IReadOnlyCollection<NuGetReference>> NuGetReferencesCharacteristic = CreateCharacteristic<IReadOnlyCollection<NuGetReference>>(nameof(NuGetReferences));
 
         public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessEmitToolchain.Instance);
@@ -64,6 +69,8 @@ public IReadOnlyList<Argument> Arguments
             set => ArgumentsCharacteristic[this] = value;
         }
 
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("This will soon be removed")]
         public IReadOnlyCollection<NuGetReference> NuGetReferences
         {
             get => NuGetReferencesCharacteristic[this];

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -339,6 +339,8 @@ public static Job WithEnvironmentVariable(this Job job, string key, string value
         /// <param name="source">(optional)Indicate the URI of the NuGet package source to use during the restore operation.</param>
         /// <param name="prerelease">(optional)Allows prerelease packages to be installed.</param>
         /// <returns></returns>
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("This method will soon be removed, please start using .WithMsBuildArguments() instead.")]
         public static Job WithNuGet(this Job job, string packageName, string? packageVersion = null, Uri? source = null, bool prerelease = false) =>
             job.WithCore(j => j.Infrastructure.NuGetReferences =
                 new NuGetReferenceList(j.Infrastructure.NuGetReferences ?? Array.Empty<NuGetReference>())
@@ -352,9 +354,14 @@ public static Job WithNuGet(this Job job, string packageName, string? packageVer
         /// <param name="job"></param>
         /// <param name="nuGetReferences">A collection of NuGet dependencies</param>
         /// <returns></returns>
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("This method will soon be removed, please start using .WithMsBuildArguments() instead.")]
         public static Job WithNuGet(this Job job, NuGetReferenceList nuGetReferences) =>
             job.WithCore(j => j.Infrastructure.NuGetReferences = nuGetReferences);
 
+        public static Job WithMsBuildArguments(this Job job, params string[] msBuildArguments)
+            => job.WithArguments([.. msBuildArguments.Select(a => new MsBuildArgument(a))]);
+
         // Accuracy
         /// <summary>
         /// Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error).
@@ -437,8 +444,10 @@ private static Job WithCore(this Job job, Action<Job> updateCallback)
             return newJob;
         }
 
-        internal static bool HasDynamicBuildCharacteristic(this Job job) =>
-            job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)
+        internal static bool HasDynamicBuildCharacteristic(this Job job)
+#pragma warning disable CS0618 // Type or member is obsolete
+            => job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)
+#pragma warning restore CS0618 // Type or member is obsolete
             || job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic)
             || job.HasValue(InfrastructureMode.ArgumentsCharacteristic);
     }

src/BenchmarkDotNet/Jobs/NugetReference.cs

@@ -1,8 +1,11 @@
 using System;
+using System.ComponentModel;
 using System.Text.RegularExpressions;
 
 namespace BenchmarkDotNet.Jobs
 {
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    [Obsolete("This type will soon be removed")]
     public class NuGetReference : IEquatable<NuGetReference>
     {
         public NuGetReference(string packageName, string packageVersion, Uri? source = null, bool prerelease = false)

src/BenchmarkDotNet/Jobs/NugetReferenceList.cs

@@ -1,12 +1,15 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.ComponentModel;
 
 namespace BenchmarkDotNet.Jobs
 {
     /// <summary>
     /// An ordered list of NuGet references. Does not allow duplicate references with the same PackageName.
     /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    [Obsolete("This type will soon be removed")]
     public class NuGetReferenceList : IReadOnlyCollection<NuGetReference>
     {
         private readonly List<NuGetReference> references = new List<NuGetReference>();

src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs

@@ -47,8 +47,10 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y)
                     return false;
                 if (AreDifferent(jobX.Infrastructure.Arguments, jobY.Infrastructure.Arguments)) // arguments can be anything (Mono runtime settings or MsBuild parameters)
                     return false;
+#pragma warning disable CS0618 // Type or member is obsolete
                 if (AreDifferent(jobX.Infrastructure.NuGetReferences, jobY.Infrastructure.NuGetReferences))
                     return false;
+#pragma warning restore CS0618 // Type or member is obsolete
                 if (!jobX.Environment.Gc.Equals(jobY.Environment.Gc)) // GC settings are per .config/.csproj
                     return false;
 
@@ -81,8 +83,10 @@ public int GetHashCode(BenchmarkCase obj)
                 hashCode.Add(job.Infrastructure.BuildConfiguration);
                 foreach (var arg in job.Infrastructure.Arguments ?? Array.Empty<Argument>())
                     hashCode.Add(arg);
+#pragma warning disable CS0618 // Type or member is obsolete
                 foreach (var reference in job.Infrastructure.NuGetReferences ?? Array.Empty<NuGetReference>())
                     hashCode.Add(reference);
+#pragma warning restore CS0618 // Type or member is obsolete
                 return hashCode.ToHashCode();
             }
 

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -57,7 +58,9 @@ public BuildResult RestoreThenBuild()
         {
             DotNetCliCommandExecutor.LogEnvVars(WithArguments(null));
 
+#pragma warning disable CS0618 // Type or member is obsolete
             var packagesResult = AddPackages();
+#pragma warning restore CS0618 // Type or member is obsolete
             if (!packagesResult.IsSuccess)
                 return BuildResult.Failure(GenerateResult, packagesResult.AllInformation);
 
@@ -97,7 +100,9 @@ public BuildResult RestoreThenBuildThenPublish()
         {
             DotNetCliCommandExecutor.LogEnvVars(WithArguments(null));
 
+#pragma warning disable CS0618 // Type or member is obsolete
             var packagesResult = AddPackages();
+#pragma warning restore CS0618 // Type or member is obsolete
             if (!packagesResult.IsSuccess)
                 return BuildResult.Failure(GenerateResult, packagesResult.AllInformation);
 
@@ -115,6 +120,8 @@ public BuildResult RestoreThenBuildThenPublish()
             return PublishNoRestore().ToBuildResult(GenerateResult);
         }
 
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("This method will soon be removed")]
         public DotNetCliCommandResult AddPackages()
         {
             var executionTime = new TimeSpan(0);
@@ -150,6 +157,7 @@ public DotNetCliCommandResult PublishNoRestore()
             => DotNetCliCommandExecutor.Execute(WithArguments(
                 GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore", "publish-no-restore")));
 
+        [Obsolete]
         internal static IEnumerable<string> GetAddPackagesCommands(BuildPartition buildPartition)
             => GetNuGetAddPackageCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver);
 
@@ -204,6 +212,7 @@ private static string GetCustomMsBuildArguments(BenchmarkCase benchmarkCase, IRe
             return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation));
         }
 
+        [Obsolete]
         private static IEnumerable<string> GetNuGetAddPackageCommands(BenchmarkCase benchmarkCase, IResolver resolver)
         {
             if (!benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic))
@@ -229,6 +238,7 @@ private static string GetMandatoryMsBuildSettings(string buildConfiguration)
             return $"{NoMsBuildZombieProcesses} {EnforceOptimizations}";
         }
 
+        [Obsolete]
         private static string BuildAddPackageCommand(NuGetReference reference)
         {
             var commandBuilder = new StringBuilder();