Fix duplicated test name in Perf.TypeName and update validation (#4898)

Author: caaavik-msft

src/benchmarks/micro/libraries/System.Reflection.Metadata/Perf.TypeName.cs

@@ -11,40 +11,53 @@ namespace System.Reflection.Metadata
     [BenchmarkCategory(Categories.Libraries)]
     public class Perf_TypeName
     {
+        public class ArgumentTypeWrapper
+        {
+            private readonly string displayName;
+
+            public ArgumentTypeWrapper(Type type, string displayName)
+            {
+                Type = type;
+                this.displayName = displayName;
+            }
+
+            public Type Type { get; }
+            public override string ToString() => displayName;
+        }
+
         public IEnumerable<object> TypeArguments()
         {
-            // We don't return strings here, as they change over the time when assembly versions get increased.
-            // This would change benchmark ID and loose historical data tracking.
-            yield return typeof(int); // elemental type
-            yield return typeof(int).MakePointerType(); // a pointer to an elemental type
-            yield return typeof(int).MakeByRefType(); // a reference to an elemental type
-            yield return typeof(int[]); // SZArray
-            yield return typeof(int).MakeArrayType(1); // single-dimensional array, but not indexed from 0
-            yield return typeof(int).MakeArrayType(2); // multi-dimensional array
-            yield return typeof(Dictionary<string, bool>); // generic type
-            yield return typeof(Dictionary<string, bool>[]); // an array of generic types
-            yield return typeof(Nested); // nested type
-            yield return typeof(Nested.NestedGeneric<string, bool>); // nested generic type
-            yield return typeof(Dictionary<List<int[]>[,], List<int?[][][,]>>[]); // complex generic type (node count = 16)
+            // We use a wrapper type for each argument to ensure that the test name remains the same into the future
+            yield return new ArgumentTypeWrapper(typeof(int), "typeof(int)"); // elemental type
+            yield return new ArgumentTypeWrapper(typeof(int).MakePointerType(), "typeof(System.Int32*)"); // a pointer to an elemental type
+            yield return new ArgumentTypeWrapper(typeof(int).MakeByRefType(), "typeof(System.Int32&)"); // a reference to an elemental type
+            yield return new ArgumentTypeWrapper(typeof(int[]), "typeof(System.Int32[])"); // SZArray
+            yield return new ArgumentTypeWrapper(typeof(int).MakeArrayType(1), "typeof(System.Int32[*])"); // single-dimensional array, but not indexed from 0
+            yield return new ArgumentTypeWrapper(typeof(int).MakeArrayType(2), "typeof(System.Int32[,])"); // multi-dimensional array
+            yield return new ArgumentTypeWrapper(typeof(Dictionary<string, bool>), "typeof(System.Collections.Generic.Dictionary<String, Boolean>)"); // generic type
+            yield return new ArgumentTypeWrapper(typeof(Dictionary<string, bool>[]), "typeof(System.Collections.Generic.Dictionary`2[])"); // an array of generic types
+            yield return new ArgumentTypeWrapper(typeof(Nested), "typeof(System.Reflection.Metadata.Nested)"); // nested type
+            yield return new ArgumentTypeWrapper(typeof(Nested.NestedGeneric<string, bool>), "typeof(System.Reflection.Metadata.NestedGeneric<String, Boolean>)"); // nested generic type
+            yield return new ArgumentTypeWrapper(typeof(Dictionary<List<int[]>[,], List<int?[][][,]>>[]), "typeof(System.Collections.Generic.Dictionary`2[]) (COMPLEX)"); // complex generic type (node count = 16)
         }
 
         [Benchmark]
         [ArgumentsSource(nameof(TypeArguments))]
-        public TypeName Parse_FullNames(Type input) => TypeName.Parse(input.FullName);
+        public TypeName Parse_FullNames(ArgumentTypeWrapper input) => TypeName.Parse(input.Type.FullName);
 
         [Benchmark]
         [ArgumentsSource(nameof(TypeArguments))]
-        public TypeName Parse_AssemblyQualifiedName(Type input) => TypeName.Parse(input.AssemblyQualifiedName);
+        public TypeName Parse_AssemblyQualifiedName(ArgumentTypeWrapper input) => TypeName.Parse(input.Type.AssemblyQualifiedName);
 
         // The Name, FullName and AssemblyQualifiedName properties are lazy and cached,
-        // so we need to parse a new TypName instance in order to get these properties calculated.
+        // so we need to parse a new TypeName instance in order to get these properties calculated.
         [Benchmark]
         [ArgumentsSource(nameof(TypeArguments))]
-        public string ParseAndGetFullName(Type input) => TypeName.Parse(input.FullName).FullName;
+        public string ParseAndGetFullName(ArgumentTypeWrapper input) => TypeName.Parse(input.Type.FullName).FullName;
 
         [Benchmark]
         [ArgumentsSource(nameof(TypeArguments))]
-        public string ParseAndGetAssemblyQualifiedName(Type input) => TypeName.Parse(input.AssemblyQualifiedName).AssemblyQualifiedName;
+        public string ParseAndGetAssemblyQualifiedName(ArgumentTypeWrapper input) => TypeName.Parse(input.Type.AssemblyQualifiedName).AssemblyQualifiedName;
 
         public IEnumerable<string> InvalidArguments()
         {

src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Exporters;
 
 namespace BenchmarkDotNet.Extensions
 {
@@ -29,14 +30,17 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
         private class BenchmarkArgumentsComparer : IEqualityComparer<BenchmarkCase>
         {
             public bool Equals(BenchmarkCase x, BenchmarkCase y)
-                => Enumerable.SequenceEqual(
+            {
+                if (FullNameProvider.GetBenchmarkName(x).Equals(FullNameProvider.GetBenchmarkName(y), System.StringComparison.Ordinal))
+                    return true;
+
+                return Enumerable.SequenceEqual(
                     x.Parameters.Items.Select(argument => argument.Value), 
                     y.Parameters.Items.Select(argument => argument.Value));
+            }
 
             public int GetHashCode(BenchmarkCase obj)
-                => obj.Parameters.Items
-                    .Where(item => item.Value != null)
-                    .Aggregate(seed: 0, (hashCode, argument) => hashCode ^= argument.Value.GetHashCode());
+                => FullNameProvider.GetBenchmarkName(obj).GetHashCode();
         }
     }
 }

src/tools/Reporting/Reporting/Reporter.cs

@@ -59,9 +59,7 @@ public void AddTest(Test test)
     {
         if (Tests.Any(t => t.Name.Equals(test.Name)))
         {
-            // TODO: Make this surface as a failure so it gets fixed, this is just temporary to fix the pipeline
-            Console.Error.WriteLine($"Duplicate test name: {test.Name}, skipping from results");
-            return;
+            throw new Exception($"Duplicate test name: {test.Name} found");
         }
 
         Tests.Add(test);