feat: Change default job order to numeric order (#2933)

* feat: change default job order to numeric order

* chore: remove JobOrderPolicy.Default and modify JobComparer constructor

* chore: modify helper class access modifier

* Add .NET10 tfm support and add modify unit tests

Author: filzrev

samples/BenchmarkDotNet.Samples/IntroOrderAttr.cs

@@ -4,7 +4,7 @@
 
 namespace BenchmarkDotNet.Samples
 {
-    [Orderer(SummaryOrderPolicy.FastestToSlowest, MethodOrderPolicy.Declared, jobOrderPolicy: JobOrderPolicy.Default)]
+    [Orderer(SummaryOrderPolicy.FastestToSlowest, MethodOrderPolicy.Declared, jobOrderPolicy: JobOrderPolicy.Numeric)]
     [DryJob]
     public class IntroOrderAttr
     {

src/BenchmarkDotNet/Attributes/OrdererAttribute.cs

@@ -10,7 +10,7 @@ public class OrdererAttribute : Attribute, IConfigSource
         public OrdererAttribute(
             SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default,
             MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared,
-            JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default)
+            JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Numeric)
         {
             Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy, jobOrderPolicy));
         }

src/BenchmarkDotNet/BenchmarkDotNet.csproj

@@ -2,7 +2,7 @@
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
     <AssemblyTitle>BenchmarkDotNet</AssemblyTitle>
-    <TargetFrameworks>netstandard2.0;net6.0;net8.0;net9.0</TargetFrameworks>
+    <TargetFrameworks>netstandard2.0;net6.0;net8.0;net9.0;net10.0</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <NoWarn>$(NoWarn);1701;1702;1705;1591;3005;NU1702;CS3001;CS3003</NoWarn>
     <AssemblyName>BenchmarkDotNet</AssemblyName>

src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs

@@ -221,7 +221,7 @@ private static ImmutableHashSet<IValidator> GetValidators(IEnumerable<IValidator
         /// </summary>
         private static IReadOnlyList<Job> GetRunnableJobs(IEnumerable<Job> jobs)
         {
-            var unique = jobs.Distinct(JobComparer.Instance).ToArray();
+            var unique = jobs.Distinct(JobComparer.Default).ToArray();
             var result = new List<Job>();
 
             foreach (var standardJob in unique.Where(job => !job.Meta.IsMutator && !job.Meta.IsDefault))

src/BenchmarkDotNet/Jobs/JobComparer.cs

@@ -10,15 +10,28 @@ namespace BenchmarkDotNet.Jobs
     internal class JobComparer : IComparer<Job>, IEqualityComparer<Job>
     {
         private readonly IComparer<string> Comparer;
+        private static IComparer<string> NumericComparer =>
+#if NET10_0_OR_GREATER
+            StringComparer.Create(System.Globalization.CultureInfo.InvariantCulture, System.Globalization.CompareOptions.NumericOrdering);
+#else
+            new NumericStringComparer();
+#endif
 
-        public static readonly JobComparer Instance = new JobComparer(JobOrderPolicy.Default);
-        public static readonly JobComparer Numeric = new JobComparer(JobOrderPolicy.Numeric);
+        public static readonly JobComparer Default = new(JobOrderPolicy.Numeric);
+        public static readonly JobComparer Ordinal = new(JobOrderPolicy.Ordinal);
 
-        public JobComparer(JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default)
+        private JobComparer(JobOrderPolicy jobOrderPolicy)
         {
-            Comparer = jobOrderPolicy == JobOrderPolicy.Default
-                ? StringComparer.Ordinal
-                : new NumericStringComparer();  // TODO: Use `StringComparer.Create(CultureInfo.InvariantCulture, CompareOptions.NumericOrdering)` for .NET10 or greater.
+            switch (jobOrderPolicy)
+            {
+                case JobOrderPolicy.Ordinal:
+                    Comparer = StringComparer.Ordinal;
+                    break;
+
+                default:
+                    Comparer = NumericComparer;
+                    break;
+            }
         }
 
         public int Compare(Job? x, Job? y)
@@ -66,7 +79,8 @@ public int Compare(Job? x, Job? y)
 
         public int GetHashCode(Job obj) => obj.Id.GetHashCode();
 
-        internal class NumericStringComparer : IComparer<string>
+#if !NET10_0_OR_GREATER
+        private class NumericStringComparer : IComparer<string>
         {
             public int Compare(string? x, string? y)
             {
@@ -139,5 +153,6 @@ public int Compare(string? x, string? y)
                 return (spanX.Length - i).CompareTo(spanY.Length - j);
             }
         }
+#endif
     }
 }

src/BenchmarkDotNet/Order/DefaultOrderer.cs

@@ -28,13 +28,13 @@ public class DefaultOrderer : IOrderer
         public DefaultOrderer(
             SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default,
             MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared,
-            JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default)
+            JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Numeric)
         {
             SummaryOrderPolicy = summaryOrderPolicy;
             MethodOrderPolicy = methodOrderPolicy;
-            jobComparer = jobOrderPolicy == JobOrderPolicy.Default
-                ? JobComparer.Instance
-                : JobComparer.Numeric;
+            jobComparer = jobOrderPolicy == JobOrderPolicy.Ordinal
+                ? JobComparer.Ordinal
+                : JobComparer.Default;
             targetComparer = new DescriptorComparer(methodOrderPolicy);
         }
 

src/BenchmarkDotNet/Order/JobOrderPolicy.cs

@@ -3,12 +3,12 @@
 public enum JobOrderPolicy
 {
     /// <summary>
-    /// Compare job characteristics in ordinal order.
+    /// Compare job characteristics in numeric order.
     /// </summary>
-    Default,
+    Numeric,
 
     /// <summary>
-    /// Compare job characteristics in numeric order.
+    /// Compare job characteristics in ordinal order.
     /// </summary>
-    Numeric,
+    Ordinal,
 }

src/BenchmarkDotNet/Validators/BaselineValidator.cs

@@ -30,7 +30,7 @@ public IEnumerable<ValidationError> Validate(ValidationParameters input)
             {
                 var benchmarks = allBenchmarks.Where((benchmark, index) => benchmarkLogicalGroups[index] == logicalGroup).ToArray();
                 int methodBaselineCount = benchmarks.Select(b => b.Descriptor).Distinct().Count(it => it.Baseline);
-                int jobBaselineCount = benchmarks.Select(b => b.Job).Distinct(JobComparer.Instance).Count(it => it.Meta.Baseline);
+                int jobBaselineCount = benchmarks.Select(b => b.Job).Distinct(JobComparer.Default).Count(it => it.Meta.Baseline);
                 string className = benchmarks.First().Descriptor.Type.Name;
 
                 if (methodBaselineCount > 1)

tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj

@@ -2,7 +2,7 @@
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
     <AssemblyTitle>BenchmarkDotNet.Tests</AssemblyTitle>
-    <TargetFrameworks>net8.0;net462</TargetFrameworks>
+    <TargetFrameworks>net8.0;net10.0;net462</TargetFrameworks>
     <AssemblyName>BenchmarkDotNet.Tests</AssemblyName>
     <PackageId>BenchmarkDotNet.Tests</PackageId>
     <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>

tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs

@@ -26,27 +26,27 @@ public void TestJobOrders_ByJobId()
                    .WithId("v1.4.2"),
         ];
 
-        // Verify jobs are sorted by JobId's ordinal order.
+        // Verify jobs are sorted by JobId's default order (numeric order).
         {
             // Act
-            var comparer = JobComparer.Instance;
+            var comparer = JobComparer.Default;
             var results = jobs.OrderBy(x => x, comparer)
                               .Select(x => x.Job.Id)
                               .ToArray();
 
             // Assert
-            Assert.Equal(["v1.4.1", "v1.4.10", "v1.4.2"], results);
+            Assert.Equal(["v1.4.1", "v1.4.2", "v1.4.10"], results);
         }
 
-        // Verify jobs are sorted by JobId's numeric order.
+        // Verify jobs are sorted by JobId's ordinal order.
         {
             // Act
-            var comparer = JobComparer.Numeric;
+            var comparer = JobComparer.Ordinal;
             var results = jobs.OrderBy(d => d, comparer)
                               .Select(x => x.Job.Id)
                               .ToArray();
             // Assert
-            Assert.Equal(["v1.4.1", "v1.4.2", "v1.4.10"], results);
+            Assert.Equal(["v1.4.1", "v1.4.10", "v1.4.2"], results);
         }
     }
 
@@ -65,8 +65,8 @@ public void TestJobOrders_ByRuntime()
         ];
 
         // Act
-        // Verify jobs are sorted by Runtime's numeric order.
-        var results = jobs.OrderBy(d => d, JobComparer.Numeric)
+        // Verify jobs are sorted by Runtime name order.
+        var results = jobs.OrderBy(d => d, JobComparer.Default)
                           .Select(x => x.Job.Environment.GetRuntime().Name)
                           .ToArray();
 
@@ -92,8 +92,8 @@ public void TestJobOrders_ByToolchain()
         ];
 
         // Act
-        // Verify jobs are sorted by Toolchain's numeric order.
-        var results = jobs.OrderBy(d => d, JobComparer.Numeric)
+        // Verify jobs are sorted by Toolchain name order.
+        var results = jobs.OrderBy(d => d, JobComparer.Default)
                           .Select(x => x.Job.GetToolchain().Name)
                           .ToArray();
 
@@ -132,12 +132,15 @@ public void TestJobOrders_ByToolchain()
     [InlineData(".NET 08", ".NET 10", -1)]
     [InlineData(".NET 10", ".NET 08", 1)]
     // Arguments that contains null
-    [InlineData(null, "a", -1)]
-    [InlineData("a", null, 1)]
-    [InlineData(null, null, 0)]
-    public void TestNumericComparer(string? a, string? b, int expectedSign)
+    [InlineData("", "a", -1)]
+    [InlineData("a", "", 1)]
+    [InlineData("", "", 0)]
+    public void TestNumericComparer(string a, string b, int expectedSign)
     {
-        int result = new JobComparer.NumericStringComparer().Compare(a, b);
+        var jobA = new Job(a);
+        var jobB = new Job(b);
+
+        int result = JobComparer.Default.Compare(jobA, jobB);
         Assert.Equal(expectedSign, NormalizeSign(result));
 
         static int NormalizeSign(int value)