Add support for user-supplied project file detection

BenchmarkDotNet.sln

@@ -59,6 +59,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.P
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.IntegrationTests.FileLocators", "tests\BenchmarkDotNet.IntegrationTests.FileLocators\BenchmarkDotNet.IntegrationTests.FileLocators.csproj", "{7AD9FCF9-69B5-4984-93AC-D6E30344DADC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -161,6 +163,10 @@ Global
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7AD9FCF9-69B5-4984-93AC-D6E30344DADC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7AD9FCF9-69B5-4984-93AC-D6E30344DADC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7AD9FCF9-69B5-4984-93AC-D6E30344DADC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7AD9FCF9-69B5-4984-93AC-D6E30344DADC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -190,6 +196,7 @@ Global
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9}
+		{7AD9FCF9-69B5-4984-93AC-D6E30344DADC} = {14195214-591A-45B7-851A-19D3BA2413F9}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

src/BenchmarkDotNet/Configs/DebugConfig.cs

@@ -8,6 +8,7 @@
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Locators;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
@@ -58,6 +59,7 @@ public abstract class DebugConfig : IConfig
         public IEnumerable<IValidator> GetValidators() => Array.Empty<IValidator>();
         public IEnumerable<IColumnProvider> GetColumnProviders() => DefaultColumnProviders.Instance;
         public IEnumerable<IExporter> GetExporters() => Array.Empty<IExporter>();
+        public IEnumerable<IFileLocator> GetFileLocators() => Array.Empty<IFileLocator>();
         public IEnumerable<ILogger> GetLoggers() => new[] { ConsoleLogger.Default };
         public IEnumerable<IDiagnoser> GetDiagnosers() => Array.Empty<IDiagnoser>();
         public IEnumerable<IAnalyser> GetAnalysers() => Array.Empty<IAnalyser>();

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -11,6 +11,7 @@
 using BenchmarkDotNet.Exporters.Csv;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Locators;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Portability;
@@ -40,6 +41,8 @@ public IEnumerable<IExporter> GetExporters()
             yield return HtmlExporter.Default;
         }
 
+        public IEnumerable<IFileLocator> GetFileLocators() => Array.Empty<IFileLocator>();
+
         public IEnumerable<ILogger> GetLoggers()
         {
             if (LinqPadLogger.IsAvailable)

src/BenchmarkDotNet/Configs/IConfig.cs

@@ -8,6 +8,7 @@
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Locators;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
@@ -20,6 +21,7 @@ public interface IConfig
     {
         IEnumerable<IColumnProvider> GetColumnProviders();
         IEnumerable<IExporter> GetExporters();
+        IEnumerable<IFileLocator> GetFileLocators();
         IEnumerable<ILogger> GetLoggers();
         IEnumerable<IDiagnoser> GetDiagnosers();
         IEnumerable<IAnalyser> GetAnalysers();

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Globalization;
@@ -10,6 +10,7 @@
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Locators;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
@@ -24,6 +25,7 @@ public sealed class ImmutableConfig : IConfig
         // if something is an array here instead of hashset it means it must have a guaranteed order of elements
         private readonly ImmutableArray<IColumnProvider> columnProviders;
         private readonly ImmutableArray<IExporter> exporters;
+        private readonly ImmutableArray<IFileLocator> fileLocators;
         private readonly ImmutableHashSet<ILogger> loggers;
         private readonly ImmutableHashSet<IDiagnoser> diagnosers;
         private readonly ImmutableHashSet<IAnalyser> analysers;
@@ -41,6 +43,7 @@ internal ImmutableConfig(
             ImmutableHashSet<HardwareCounter> uniqueHardwareCounters,
             ImmutableHashSet<IDiagnoser> uniqueDiagnosers,
             ImmutableArray<IExporter> uniqueExporters,
+            ImmutableArray<IFileLocator> uniqueFileLocators,
             ImmutableHashSet<IAnalyser> uniqueAnalyzers,
             ImmutableHashSet<IValidator> uniqueValidators,
             ImmutableHashSet<IFilter> uniqueFilters,
@@ -63,6 +66,7 @@ internal ImmutableConfig(
             hardwareCounters = uniqueHardwareCounters;
             diagnosers = uniqueDiagnosers;
             exporters = uniqueExporters;
+            fileLocators = uniqueFileLocators;
             analysers = uniqueAnalyzers;
             validators = uniqueValidators;
             filters = uniqueFilters;
@@ -92,6 +96,7 @@ internal ImmutableConfig(
 
         public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
         public IEnumerable<IExporter> GetExporters() => exporters;
+        public IEnumerable<IFileLocator> GetFileLocators() => fileLocators;
         public IEnumerable<ILogger> GetLoggers() => loggers;
         public IEnumerable<IDiagnoser> GetDiagnosers() => diagnosers;
         public IEnumerable<IAnalyser> GetAnalysers() => analysers;

src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
 using BenchmarkDotNet.Analysers;
@@ -44,6 +44,7 @@ public static ImmutableConfig Create(IConfig source)
             var uniqueHardwareCounters = source.GetHardwareCounters().Where(counter => counter != HardwareCounter.NotSet).ToImmutableHashSet();
             var uniqueDiagnosers = GetDiagnosers(source.GetDiagnosers(), uniqueHardwareCounters);
             var uniqueExporters = GetExporters(source.GetExporters(), uniqueDiagnosers, configAnalyse);
+            var uniqueFileLocators = source.GetFileLocators().ToImmutableArray();
             var uniqueAnalyzers = GetAnalysers(source.GetAnalysers(), uniqueDiagnosers);
 
             var uniqueValidators = GetValidators(source.GetValidators(), MandatoryValidators, source.Options);
@@ -61,6 +62,7 @@ public static ImmutableConfig Create(IConfig source)
                 uniqueHardwareCounters,
                 uniqueDiagnosers,
                 uniqueExporters,
+                uniqueFileLocators,
                 uniqueAnalyzers,
                 uniqueValidators,
                 uniqueFilters,

src/BenchmarkDotNet/Configs/ManualConfig.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Globalization;
@@ -11,6 +11,7 @@
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Locators;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
@@ -26,6 +27,7 @@ public class ManualConfig : IConfig
 
         private readonly List<IColumnProvider> columnProviders = new List<IColumnProvider>();
         private readonly List<IExporter> exporters = new List<IExporter>();
+        private readonly List<IFileLocator> locators = new List<IFileLocator>();
         private readonly List<ILogger> loggers = new List<ILogger>();
         private readonly List<IDiagnoser> diagnosers = new List<IDiagnoser>();
         private readonly List<IAnalyser> analysers = new List<IAnalyser>();
@@ -39,6 +41,7 @@ public class ManualConfig : IConfig
 
         public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
         public IEnumerable<IExporter> GetExporters() => exporters;
+        public IEnumerable<IFileLocator> GetFileLocators() => locators;
         public IEnumerable<ILogger> GetLoggers() => loggers;
         public IEnumerable<IDiagnoser> GetDiagnosers() => diagnosers;
         public IEnumerable<IAnalyser> GetAnalysers() => analysers;
@@ -139,6 +142,12 @@ public ManualConfig AddExporter(params IExporter[] newExporters)
             return this;
         }
 
+        public ManualConfig AddFileLocator(params IFileLocator[] newLocators)
+        {
+            locators.AddRange(newLocators);
+            return this;
+        }
+
         [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete("This method will soon be removed, please start using .AddLogger() instead.")]
         public void Add(params ILogger[] newLoggers) => AddLogger(newLoggers);
@@ -256,6 +265,7 @@ public void Add(IConfig config)
         {
             columnProviders.AddRange(config.GetColumnProviders());
             exporters.AddRange(config.GetExporters());
+            locators.AddRange(config.GetFileLocators());
             loggers.AddRange(config.GetLoggers());
             diagnosers.AddRange(config.GetDiagnosers());
             analysers.AddRange(config.GetAnalysers());

src/BenchmarkDotNet/Locators/FileLocatorArgs.cs

@@ -0,0 +1,16 @@
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
+
+namespace BenchmarkDotNet.Locators;
+
+public class FileLocatorArgs
+{
+    public FileLocatorArgs(BenchmarkCase benchmarkCase, ILogger logger)
+    {
+        BenchmarkCase = benchmarkCase;
+        Logger = logger;
+    }
+
+    public BenchmarkCase BenchmarkCase { get; }
+    public ILogger Logger { get; }
+}

src/BenchmarkDotNet/Locators/FileLocatorType.cs

@@ -0,0 +1,6 @@
+namespace BenchmarkDotNet.Locators;
+
+public enum FileLocatorType
+{
+    Project
+}

src/BenchmarkDotNet/Locators/IFileLocator.cs

@@ -0,0 +1,22 @@
+using System.IO;
+
+namespace BenchmarkDotNet.Locators;
+
+/// <summary>
+/// Locators can be used to extend the default behavior of finding files
+/// </summary>
+public interface IFileLocator
+{
+    /// <summary>
+    /// The type of locator
+    /// </summary>
+    FileLocatorType LocatorType { get; }
+
+    /// <summary>
+    /// Tries to locate a file
+    /// </summary>
+    /// <param name="fileLocatorArgs">The arguments such as benchmark and logger</param>
+    /// <param name="fileInfo">The file is provided by the implementation</param>
+    /// <returns>True when a file was successfully found, False otherwise.</returns>
+    bool TryLocate(FileLocatorArgs fileLocatorArgs, out FileInfo fileInfo);
+}

src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics.CodeAnalysis;
@@ -11,6 +11,7 @@
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Locators;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.DotNetCli;
@@ -71,7 +72,7 @@ protected override string GetIntermediateDirectoryPath(string buildArtifactsDire
         protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger)
         {
             var benchmark = buildPartition.RepresentativeBenchmarkCase;
-            var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger);
+            var projectFile = GetProjectFilePath(benchmark, logger);
 
             var xmlDoc = new XmlDocument();
             xmlDoc.Load(projectFile.FullName);
@@ -246,8 +247,29 @@ private static string GetIndentedXmlString(XmlDocument doc)
         /// returns a path to the project file which defines the benchmarks
         /// </summary>
         [PublicAPI]
-        protected virtual FileInfo GetProjectFilePath(Type benchmarkTarget, ILogger logger)
+        protected virtual FileInfo GetProjectFilePath(BenchmarkCase benchmark, ILogger logger)
         {
+            var args = new FileLocatorArgs(benchmark, logger);
+
+            // Try locators first. Logic is provided by the user for uses-cases such as they have set AssemblyName to a custom value.
+            var notFound = new List<string>();
+            foreach (var locator in benchmark.Config.GetFileLocators())
+            {
+                if (locator.LocatorType != FileLocatorType.Project)
+                {
+                    continue;
+                }
+
+                if (locator.TryLocate(args, out var fileInfo))
+                {
+                    if (fileInfo.Exists)
+                        return fileInfo;
+
+                    notFound.Add(fileInfo.FullName);
+                }
+            }
+
+            // Fall back to default project detection logic
             if (!GetSolutionRootDirectory(out var rootDirectory) && !GetProjectRootDirectory(out rootDirectory))
             {
                 logger.WriteLineError(
@@ -256,7 +278,7 @@ protected virtual FileInfo GetProjectFilePath(Type benchmarkTarget, ILogger logg
             }
 
             // important assumption! project's file name === output dll name
-            string projectName = benchmarkTarget.GetTypeInfo().Assembly.GetName().Name;
+            string projectName = benchmark.Descriptor.Type.GetTypeInfo().Assembly.GetName().Name;
 
             var possibleNames = new HashSet<string> { $"{projectName}.csproj", $"{projectName}.fsproj", $"{projectName}.vbproj" };
             var projectFiles = rootDirectory
@@ -266,12 +288,18 @@ protected virtual FileInfo GetProjectFilePath(Type benchmarkTarget, ILogger logg
 
             if (projectFiles.Length == 0)
             {
-                throw new NotSupportedException(
-                    $"Unable to find {projectName} in {rootDirectory.FullName} and its subfolders. Most probably the name of output exe is different than the name of the .(c/f)sproj");
+                string message;
+
+                if (notFound.Count > 0)
+                    message = $"Unable to find {projectName} in any of the paths: {string.Join(", ", notFound)} or in {rootDirectory.FullName} and its subfolders";
+                else
+                    message = $"Unable to find {projectName} in {rootDirectory.FullName} and its subfolders. Most probably the name of output exe is different than the name of the .(c/f)sproj. You can add an IFileLocator to the config if this is on purpose.";
+
+                throw new FileNotFoundException(message);
             }
             else if (projectFiles.Length > 1)
             {
-                throw new NotSupportedException(
+                throw new InvalidOperationException(
                     $"Found more than one matching project file for {projectName} in {rootDirectory.FullName} and its subfolders: {string.Join(",", projectFiles.Select(pf => $"'{pf.FullName}'"))}. Benchmark project names needs to be unique.");
             }
 

src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs

@@ -28,7 +28,7 @@ public MonoAotLLVMGenerator(string targetFrameworkMoniker, string cliPath, strin
         protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger)
         {
             BenchmarkCase benchmark = buildPartition.RepresentativeBenchmarkCase;
-            var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger);
+            var projectFile = GetProjectFilePath(benchmark, logger);
 
             string useLLVM = AotCompilerMode == MonoAotCompilerMode.llvm ? "true" : "false";
 

src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs

@@ -41,7 +41,7 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts
         protected void GenerateProjectFile(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, bool aot, ILogger logger)
         {
             BenchmarkCase benchmark = buildPartition.RepresentativeBenchmarkCase;
-            var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger);
+            var projectFile = GetProjectFilePath(benchmark, logger);
 
             WasmRuntime runtime = (WasmRuntime) buildPartition.Runtime;
 

src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs

@@ -151,17 +151,17 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif
   </ItemGroup>
   <ItemGroup>
     {GetILCompilerPackageReference()}
-    <ProjectReference Include=""{GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).FullName}"" />
+    <ProjectReference Include=""{GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase, logger).FullName}"" />
   </ItemGroup>
   <ItemGroup>
-    {string.Join(Environment.NewLine, GetRdXmlFiles(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).Select(file => $"<RdXmlFile Include=\"{file}\" />"))}
+    {string.Join(Environment.NewLine, GetRdXmlFiles(buildPartition.RepresentativeBenchmarkCase, logger).Select(file => $"<RdXmlFile Include=\"{file}\" />"))}
   </ItemGroup>
 {GetCustomProperties(buildPartition, logger)}
 </Project>";
 
         private string GetCustomProperties(BuildPartition buildPartition, ILogger logger)
         {
-            var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger);
+            var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase, logger);
             var xmlDoc = new XmlDocument();
             xmlDoc.Load(projectFile.FullName);
 
@@ -186,11 +186,11 @@ private string GetInstructionSetSettings(BuildPartition buildPartition)
             return !string.IsNullOrEmpty(instructionSet) ? $"<IlcInstructionSet>{instructionSet}</IlcInstructionSet>" : "";
         }
 
-        public IEnumerable<string> GetRdXmlFiles(Type benchmarkTarget, ILogger logger)
+        public IEnumerable<string> GetRdXmlFiles(BenchmarkCase benchmark, ILogger logger)
         {
             yield return GeneratedRdXmlFileName;
 
-            var projectFile = GetProjectFilePath(benchmarkTarget, logger);
+            var projectFile = GetProjectFilePath(benchmark, logger);
             var projectFileFolder = projectFile.DirectoryName;
             var rdXml = Path.Combine(projectFileFolder, "rd.xml");
             if (File.Exists(rdXml))

tests/BenchmarkDotNet.IntegrationTests.FileLocators/AssemblyNameIsSetBenchmarks.cs

@@ -0,0 +1,13 @@
+using BenchmarkDotNet.Attributes;
+
+namespace BenchmarkDotNet.IntegrationTests.FileLocators
+{
+    public class AssemblyNameIsSetBenchmarks
+    {
+        [Benchmark]
+        public string Benchmark()
+        {
+            return "This will only run when a FileLocator is set due to <AssemblyName> in the csproj";
+        }
+    }
+}