Categories should have the highest priority in logical group order, fixes #1864

Author: AndreyAkinshin

src/BenchmarkDotNet/Order/CategoryComparer.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace BenchmarkDotNet.Order
+{
+    internal class CategoryComparer : IComparer<string[]>
+    {
+        private const string Separator = "§";
+        public static readonly CategoryComparer Instance = new();
+
+        public int Compare(string[] x, string[] y)
+        {
+            return string.Compare(GetUniqueId(x), GetUniqueId(y), StringComparison.Ordinal);
+        }
+
+        private static string GetUniqueId(string[] categories)
+        {
+            var list = categories.ToList();
+            list.Sort();
+            return string.Join(Separator, categories);
+        }
+    }
+}

src/BenchmarkDotNet/Order/DefaultOrderer.cs

@@ -16,6 +16,7 @@ public class DefaultOrderer : IOrderer
     {
         public static readonly IOrderer Instance = new DefaultOrderer();
 
+        private readonly IComparer<string[]> categoryComparer = CategoryComparer.Instance;
         private readonly IComparer<ParameterInstances> paramsComparer = ParameterComparer.Instance;
         private readonly IComparer<Job> jobComparer = JobComparer.Instance;
         private readonly IComparer<BenchmarkCase> benchmarkComparer;
@@ -31,7 +32,7 @@ public DefaultOrderer(
             SummaryOrderPolicy = summaryOrderPolicy;
             MethodOrderPolicy = methodOrderPolicy;
             IComparer<Descriptor> targetComparer = new DescriptorComparer(methodOrderPolicy);
-            benchmarkComparer = new BenchmarkComparer(paramsComparer, jobComparer, targetComparer);
+            benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer);
             logicalGroupComparer = new LogicalGroupComparer(benchmarkComparer);
         }
 
@@ -103,14 +104,14 @@ public string GetLogicalGroupKey(ImmutableArray<BenchmarkCase> allBenchmarksCase
             }
 
             var keys = new List<string>();
+            if (rules.Contains(BenchmarkLogicalGroupRule.ByCategory))
+                keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories));
             if (rules.Contains(BenchmarkLogicalGroupRule.ByMethod))
                 keys.Add(benchmarkCase.Descriptor.DisplayInfo);
             if (rules.Contains(BenchmarkLogicalGroupRule.ByJob))
                 keys.Add(benchmarkCase.Job.DisplayInfo);
             if (rules.Contains(BenchmarkLogicalGroupRule.ByParams))
                 keys.Add(benchmarkCase.Parameters.ValueInfo);
-            if (rules.Contains(BenchmarkLogicalGroupRule.ByCategory))
-                keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories));
 
             string logicalGroupKey = string.Join("-", keys.Where(key => key != string.Empty));
             return logicalGroupKey == string.Empty ? "*" : logicalGroupKey;
@@ -127,12 +128,18 @@ public virtual IEnumerable<IGrouping<string, BenchmarkCase>> GetLogicalGroupOrde
 
         private class BenchmarkComparer : IComparer<BenchmarkCase>
         {
+            private readonly IComparer<string[]> categoryComparer;
             private readonly IComparer<ParameterInstances> paramsComparer;
             private readonly IComparer<Job> jobComparer;
             private readonly IComparer<Descriptor> targetComparer;
 
-            public BenchmarkComparer(IComparer<ParameterInstances> paramsComparer, IComparer<Job> jobComparer, IComparer<Descriptor> targetComparer)
+            public BenchmarkComparer(
+                IComparer<string[]> categoryComparer,
+                IComparer<ParameterInstances> paramsComparer,
+                IComparer<Job> jobComparer,
+                IComparer<Descriptor> targetComparer)
             {
+                this.categoryComparer = categoryComparer;
                 this.targetComparer = targetComparer;
                 this.jobComparer = jobComparer;
                 this.paramsComparer = paramsComparer;
@@ -145,6 +152,7 @@ public int Compare(BenchmarkCase x, BenchmarkCase y)
                 if (x == null) return -1;
                 return new[]
                 {
+                    categoryComparer?.Compare(x.Descriptor.Categories, y.Descriptor.Categories) ?? 0,
                     paramsComparer?.Compare(x.Parameters, y.Parameters) ?? 0,
                     jobComparer?.Compare(x.Job, y.Job) ?? 0,
                     targetComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0,

src/BenchmarkDotNet/Running/BenchmarkCase.cs

@@ -20,7 +20,7 @@ public class BenchmarkCase : IComparable<BenchmarkCase>, IDisposable
 
         public override string ToString() => DisplayInfo;
 
-        private BenchmarkCase(Descriptor descriptor, Job job, ParameterInstances parameters, ImmutableConfig config)
+        internal BenchmarkCase(Descriptor descriptor, Job job, ParameterInstances parameters, ImmutableConfig config)
         {
             Descriptor = descriptor;
             Job = job;

tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Globalization;
+using System.Linq;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Order;
+using BenchmarkDotNet.Parameters;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Validators;
+using Xunit;
+
+namespace BenchmarkDotNet.Tests.Order
+{
+    public class DefaultOrdererTests
+    {
+        [Fact]
+        public void CategoriesHasHigherPriorityThanParameters()
+        {
+            var summary = new Summary("", ImmutableArray<BenchmarkReport>.Empty, HostEnvironmentInfo.GetCurrent(),
+                "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray<ValidationError>.Empty);
+
+            BenchmarkCase CreateBenchmarkCase(string category, int parameter) => new(
+                new Descriptor(null, null, categories: new[] { category }),
+                new Job(),
+                new ParameterInstances(new[]
+                {
+                    new ParameterInstance(new ParameterDefinition("P", false, null, false, null, 0), parameter, SummaryStyle.Default)
+                }),
+                DefaultConfig.Instance.CreateImmutableConfig()
+            );
+
+            string GetId(BenchmarkCase benchmarkCase) => benchmarkCase.Descriptor.Categories.First() + benchmarkCase.Parameters.Items.First().Value;
+
+            var benchmarkCases = new List<BenchmarkCase>
+            {
+                CreateBenchmarkCase("A", 1),
+                CreateBenchmarkCase("B", 1),
+                CreateBenchmarkCase("A", 2),
+                CreateBenchmarkCase("B", 2)
+            }.ToImmutableArray();
+
+            string[] sortedBenchmarkCases = DefaultOrderer.Instance.GetSummaryOrder(benchmarkCases, summary).Select(GetId).ToArray();
+            Assert.Equal(new[] {"A1", "A2", "B1", "B2"}, sortedBenchmarkCases);
+        }
+    }
+}