Sorting parameter columns with parameter priority (#1612)

JohannesDeml · web-flow · commit aef9cbecb7f5 · 2021-01-15T10:09:10.000+01:00
fixes #1150
diff --git a/docs/articles/features/parameterization.md b/docs/articles/features/parameterization.md
@@ -13,8 +13,12 @@ name: Benchmark Parameterization
 
 [!include[IntroParamsAllValues](../samples/IntroParamsAllValues.md)]
 
+[!include[IntroParamsPriority](../samples/IntroParamsPriority.md)]
+
 [!include[IntroArguments](../samples/IntroArguments.md)]
 
 [!include[IntroArgumentsSource](../samples/IntroArgumentsSource.md)]
 
 [!include[IntroArrayParam](../samples/IntroArrayParam.md)]
+
+[!include[IntroArguments](../samples/IntroArgumentsPriority.md)]
diff --git a/docs/articles/samples/IntroArgumentsPriority.md b/docs/articles/samples/IntroArgumentsPriority.md
@@ -0,0 +1,30 @@
+---
+uid: BenchmarkDotNet.Samples.IntroArgumentsPriority
+---
+
+## Sample: IntroArgumentsPriority
+
+Like Params also Argument columns can be sorted in the table result through their `Priority`. The priority should be defined only once for multiple Arguments and will keep their inner order as they are defined in the method.
+
+### Source code
+
+[!code-csharp[IntroArgumentsPriority.cs](../../../samples/BenchmarkDotNet.Samples/IntroArgumentsPriority.cs)]
+
+### Output
+
+```markdown
+|        Method |  b |   A | c | d |     Mean |   Error |  StdDev |
+|-------------- |--- |---- |-- |-- |---------:|--------:|--------:|
+| ManyArguments |  ? | 100 | 1 | 2 | 103.4 ms | 0.09 ms | 0.08 ms |
+|     Benchmark |  5 | 100 | ? | ? | 105.5 ms | 0.21 ms | 0.19 ms |
+|     Benchmark | 10 | 100 | ? | ? | 110.5 ms | 0.14 ms | 0.14 ms |
+|     Benchmark | 20 | 100 | ? | ? | 120.4 ms | 0.16 ms | 0.15 ms |
+```
+
+### Links
+
+* Priority BaseClass [`PriorityAttribute.cs`](xref:BenchmarkDotNet.Attributes.PriorityAttribute)
+* @docs.parameterization
+* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroArgumentsPriority
+
+---
diff --git a/docs/articles/samples/IntroParams.md b/docs/articles/samples/IntroParams.md
@@ -17,12 +17,12 @@ As a result, you will get results for each combination of params values.
 ### Output
 
 ```markdown
-   Method  |      Median |    StdDev |   A |  B
----------- |------------ |---------- |---- |---
- Benchmark | 115.3325 ms | 0.0242 ms | 100 | 10
- Benchmark | 125.3282 ms | 0.0245 ms | 100 | 20
- Benchmark | 215.3024 ms | 0.0375 ms | 200 | 10
- Benchmark | 225.2710 ms | 0.0434 ms | 200 | 20
+|    Method |   A |  B |     Mean |   Error |  StdDev |
+|---------- |---- |--- |---------:|--------:|--------:|
+| Benchmark | 100 | 10 | 115.3 ms | 0.13 ms | 0.12 ms |
+| Benchmark | 100 | 20 | 125.4 ms | 0.14 ms | 0.12 ms |
+| Benchmark | 200 | 10 | 215.5 ms | 0.19 ms | 0.18 ms |
+| Benchmark | 200 | 20 | 225.4 ms | 0.17 ms | 0.16 ms |
 ```
 
 ### Links
diff --git a/docs/articles/samples/IntroParamsAllValues.md b/docs/articles/samples/IntroParamsAllValues.md
@@ -28,10 +28,6 @@ If you want to use all possible values of an `enum` or another type with a small
  Benchmark | Three |     ? | 301.4 ms |    NA |
  Benchmark | Three | False | 311.5 ms |    NA |
  Benchmark | Three |  True | 320.8 ms |    NA |
-
-// * Legends *
-  E     : Value of the 'E' parameter
-  B     : Value of the 'B' parameter
 ```
 
 ### Links
diff --git a/docs/articles/samples/IntroParamsPriority.md b/docs/articles/samples/IntroParamsPriority.md
@@ -0,0 +1,27 @@
+---
+uid: BenchmarkDotNet.Samples.IntroParamsPriority
+---
+
+## Sample: IntroParamsPriority
+
+In order to sort columns of parameters in the results table you can use the Property `Priority` inside the params attribute. The priority range is `[Int32.MinValue;Int32.MaxValue]`, lower priorites will appear earlier in the column order. The default priority is set to `0`.
+
+### Source code
+
+[!code-csharp[IntroParamsPriority.cs](../../../samples/BenchmarkDotNet.Samples/IntroParamsPriority.cs)]
+
+### Output
+
+```markdown
+|    Method |  B |   A |     Mean |   Error |  StdDev |
+|---------- |--- |---- |---------:|--------:|--------:|
+| Benchmark | 10 | 100 | 115.4 ms | 0.12 ms | 0.11 ms |
+```
+
+### Links
+
+* Priority BaseClass [`PriorityAttribute.cs`](xref:BenchmarkDotNet.Attributes.PriorityAttribute)
+* @docs.parameterization
+* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroParamsPriority
+
+---
diff --git a/docs/articles/samples/IntroParamsSource.md b/docs/articles/samples/IntroParamsSource.md
@@ -19,12 +19,12 @@ The source must be within benchmarked type!
 ### Output
 
 ```markdown
-   Method  |      Median |    StdDev |   A |  B
----------- |------------ |---------- |---- |---
- Benchmark | 115.3325 ms | 0.0242 ms | 100 | 10
- Benchmark | 125.3282 ms | 0.0245 ms | 100 | 20
- Benchmark | 215.3024 ms | 0.0375 ms | 200 | 10
- Benchmark | 225.2710 ms | 0.0434 ms | 200 | 20
+|    Method |  B |   A |     Mean |   Error |  StdDev |
+|---------- |--- |---- |---------:|--------:|--------:|
+| Benchmark | 10 | 100 | 115.5 ms | 0.17 ms | 0.16 ms |
+| Benchmark | 10 | 200 | 215.6 ms | 0.15 ms | 0.14 ms |
+| Benchmark | 20 | 100 | 125.5 ms | 0.19 ms | 0.18 ms |
+| Benchmark | 20 | 200 | 225.5 ms | 0.23 ms | 0.22 ms |
 ```
 
 ### Remarks
diff --git a/samples/BenchmarkDotNet.Samples/IntroArgumentsPriority.cs b/samples/BenchmarkDotNet.Samples/IntroArgumentsPriority.cs
@@ -0,0 +1,27 @@
+﻿using System.Collections.Generic;
+using System.Threading;
+using BenchmarkDotNet.Attributes;
+
+namespace BenchmarkDotNet.Samples
+{
+    public class IntroArgumentsPriority
+    {
+        [Params(100, Priority = 0)] // Argument priority can be combined with Params priority
+        public int A { get; set; }
+
+        [Arguments(5, Priority = -10)] // Define priority just once for multiple argument attributes
+        [Arguments(10)]
+        [Arguments(20)]
+        [Benchmark]
+        public void Benchmark(int b) => Thread.Sleep(A + b);
+
+        [Benchmark]
+        [ArgumentsSource(nameof(Numbers), Priority = 10)]
+        public void ManyArguments(int c, int d) => Thread.Sleep(A + c + d);
+
+        public IEnumerable<object[]> Numbers()
+        {
+            yield return new object[] { 1, 2 };
+        }
+    }
+}
diff --git a/samples/BenchmarkDotNet.Samples/IntroParamsPriority.cs b/samples/BenchmarkDotNet.Samples/IntroParamsPriority.cs
@@ -0,0 +1,17 @@
+﻿using System.Threading;
+using BenchmarkDotNet.Attributes;
+
+namespace BenchmarkDotNet.Samples
+{
+    public class IntroParamsPriority
+    {
+        [Params(100)]
+        public int A { get; set; }
+
+        [Params(10, Priority = -100)]
+        public int B { get; set; }
+
+        [Benchmark]
+        public void Benchmark() => Thread.Sleep(A + B + 5);
+    }
+}
diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsAttribute.cs
@@ -4,7 +4,7 @@
 namespace BenchmarkDotNet.Attributes
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
-    public class ArgumentsAttribute : Attribute
+    public class ArgumentsAttribute : PriorityAttribute
     {
         public object[] Values { get; }
 
diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs
@@ -3,7 +3,7 @@
 namespace BenchmarkDotNet.Attributes
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class ArgumentsSourceAttribute : Attribute
+    public class ArgumentsSourceAttribute : PriorityAttribute
     {
         public string Name { get; }
 
diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ParamsAllValuesAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ParamsAllValuesAttribute.cs
@@ -2,7 +2,7 @@
 namespace BenchmarkDotNet.Attributes
 {
     [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)]
-    public class ParamsAllValuesAttribute : Attribute
+    public class ParamsAllValuesAttribute : PriorityAttribute
     {
     }
 }
diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs
@@ -3,7 +3,7 @@
 namespace BenchmarkDotNet.Attributes
 {
     [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)]
-    public class ParamsAttribute : Attribute
+    public class ParamsAttribute : PriorityAttribute
     {
         public object[] Values { get; }
 
diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs
@@ -3,7 +3,7 @@
 namespace BenchmarkDotNet.Attributes
 {
     [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)]
-    public class ParamsSourceAttribute : Attribute
+    public class ParamsSourceAttribute : PriorityAttribute
     {
         public string Name { get; }
 
diff --git a/src/BenchmarkDotNet.Annotations/Attributes/PriorityAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/PriorityAttribute.cs
@@ -0,0 +1,12 @@
+﻿using System;
+
+namespace BenchmarkDotNet.Attributes
+{
+    public abstract class PriorityAttribute : Attribute
+    {
+        /// <summary>
+        /// Defines display order of column in the same category.
+        /// </summary>
+        public int Priority { get; set; } = 0;
+    }
+}
diff --git a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs
@@ -76,9 +76,9 @@ private class ParamsColumnProvider : IColumnProvider
         {
             public IEnumerable<IColumn> GetColumns(Summary summary) => summary
                 .BenchmarksCases
-                .SelectMany(b => b.Parameters.Items.Select(item => item.Name))
+                .SelectMany(b => b.Parameters.Items.Select(item => item.Definition))
                 .Distinct()
-                .Select(name => new ParamColumn(name));
+                .Select(definition => new ParamColumn(definition.Name, definition.PriorityInCategory));
         }
 
         private class MetricsColumnProvider : IColumnProvider
diff --git a/src/BenchmarkDotNet/Columns/ParamColumn.cs b/src/BenchmarkDotNet/Columns/ParamColumn.cs
@@ -10,9 +10,10 @@ public class ParamColumn : IColumn
         public string Id => nameof(ParamColumn) + "." + ColumnName;
         public string ColumnName { get; }
 
-        public ParamColumn(string columnName)
+        public ParamColumn(string columnName, int priorityInCategory = 0)
         {
             ColumnName = columnName;
+            PriorityInCategory = priorityInCategory;
         }
 
         public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false;
@@ -23,7 +24,7 @@ public string GetValue(Summary summary, BenchmarkCase benchmarkCase) =>
         public bool IsAvailable(Summary summary) => true;
         public bool AlwaysShow => true;
         public ColumnCategory Category => ColumnCategory.Params;
-        public int PriorityInCategory => 0;
+        public int PriorityInCategory { get; private set; }
         public override string ToString() => ColumnName;
         public bool IsNumeric => false;
         public UnitType UnitType => UnitType.Dimensionless;
diff --git a/src/BenchmarkDotNet/Parameters/ParameterDefinition.cs b/src/BenchmarkDotNet/Parameters/ParameterDefinition.cs
@@ -9,14 +9,16 @@ public class ParameterDefinition
         public object[] Values { get; }
         public bool IsArgument { get; }
         public Type ParameterType { get; }
+        public int PriorityInCategory { get; }
 
-        public ParameterDefinition(string name, bool isStatic, object[] values, bool isArgument, Type parameterType)
+        public ParameterDefinition(string name, bool isStatic, object[] values, bool isArgument, Type parameterType, int priorityInCategory)
         {
             Name = name;
             IsStatic = isStatic;
             Values = values;
             IsArgument = isArgument;
             ParameterType = parameterType;
+            PriorityInCategory = priorityInCategory;
         }
     }
 }
diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs
@@ -165,7 +165,7 @@ private static Descriptor CreateDescriptor(
 
         private static ParameterDefinitions GetParameterDefinitions(Type type)
         {
-            IEnumerable<ParameterDefinition> GetDefinitions<TAttribute>(Func<TAttribute, Type, object[]> getValidValues) where TAttribute : Attribute
+            IEnumerable<ParameterDefinition> GetDefinitions<TAttribute>(Func<TAttribute, Type, object[]> getValidValues) where TAttribute : PriorityAttribute
             {
                 const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
 
@@ -176,7 +176,8 @@ IEnumerable<ParameterDefinition> GetDefinitions<TAttribute>(Func<TAttribute, Typ
                         member.IsStatic,
                         getValidValues(member.Attribute, member.ParameterType),
                         false,
-                        member.ParameterType));
+                        member.ParameterType,
+                        member.Attribute.Priority));
             }
 
             var paramsDefinitions = GetDefinitions<ParamsAttribute>((attribute, parameterType) => GetValidValues(attribute.Values, parameterType));
@@ -190,14 +191,16 @@ IEnumerable<ParameterDefinition> GetDefinitions<TAttribute>(Func<TAttribute, Typ
             var paramsAllValuesDefinitions = GetDefinitions<ParamsAllValuesAttribute>((_, parameterType) => GetAllValidValues(parameterType));
 
             var definitions = paramsDefinitions.Concat(paramsSourceDefinitions).Concat(paramsAllValuesDefinitions).ToArray();
-
             return new ParameterDefinitions(definitions);
         }
 
         private static IEnumerable<ParameterInstances> GetArgumentsDefinitions(MethodInfo benchmark, Type target, SummaryStyle summaryStyle)
         {
+            var argumentsAttributes = benchmark.GetCustomAttributes<PriorityAttribute>();
+            int priority = argumentsAttributes.Select(attribute => attribute.Priority).Sum();
+
             var parameterDefinitions = benchmark.GetParameters()
-                .Select(parameter => new ParameterDefinition(parameter.Name, false, Array.Empty<object>(), true, parameter.ParameterType))
+                .Select(parameter => new ParameterDefinition(parameter.Name, false, Array.Empty<object>(), true, parameter.ParameterType, priority))
                 .ToArray();
 
             if (parameterDefinitions.IsEmpty())
diff --git a/tests/BenchmarkDotNet.IntegrationTests/PriorityTests.cs b/tests/BenchmarkDotNet.IntegrationTests/PriorityTests.cs
diff --git a/tests/BenchmarkDotNet.Tests/ParameterComparerTests.cs b/tests/BenchmarkDotNet.Tests/ParameterComparerTests.cs
diff --git a/tests/BenchmarkDotNet.Tests/ParameterInstanceTests.cs b/tests/BenchmarkDotNet.Tests/ParameterInstanceTests.cs

Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@`
`4`	`4`	`namespace BenchmarkDotNet.Attributes`
`5`	`5`	`{`
`6`	`6`	`[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]`
`7`		`- public class ArgumentsAttribute : Attribute`
	`7`	`+ public class ArgumentsAttribute : PriorityAttribute`
`8`	`8`	`{`
`9`	`9`	`public object[] Values { get; }`
`10`	`10`
Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@`
`3`	`3`	`namespace BenchmarkDotNet.Attributes`
`4`	`4`	`{`
`5`	`5`	`[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]`
`6`		`- public class ArgumentsSourceAttribute : Attribute`
	`6`	`+ public class ArgumentsSourceAttribute : PriorityAttribute`
`7`	`7`	`{`
`8`	`8`	`public string Name { get; }`
`9`	`9`