Add the args parameter to RunAll and RunAllJoined (#2339)

* Add args parameter to RunAll and RunAllJoined

* Fix test

* Fix NRE

Author: YegorStepanov

src/BenchmarkDotNet/BenchmarkDotNet.csproj

@@ -17,7 +17,7 @@
     <EmbeddedResource Include="Environments\microarchitectures.txt" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.4.3" />
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
     <PackageReference Include="Gee.External.Capstone" Version="2.3.0" />
     <PackageReference Include="Iced" Version="1.17.0" />
     <PackageReference Include="Microsoft.Diagnostics.Runtime" Version="2.2.332302" />

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -86,6 +86,31 @@ public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse
             return result;
         }
 
+        internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action<CommandLineOptions> updater)
+        {
+            (bool isSuccess, CommandLineOptions options) result = default;
+
+            ILogger logger = NullLogger.Instance;
+            using (var parser = CreateParser(logger))
+            {
+                parser
+                    .ParseArguments<CommandLineOptions>(args)
+                    .WithParsed(options => result = Validate(options, logger) ? (true, options) : (false, default))
+                    .WithNotParsed(errors => result = (false,  default));
+
+                if (!result.isSuccess)
+                {
+                    updatedArgs = null;
+                    return false;
+                }
+
+                updater(result.options);
+
+                updatedArgs = parser.FormatCommandLine(result.options, settings => settings.SkipDefault = true).Split();
+                return true;
+            }
+        }
+
         private static Parser CreateParser(ILogger logger)
             => new Parser(settings =>
             {

src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs

@@ -53,12 +53,26 @@ public class BenchmarkSwitcher
         /// <summary>
         /// Run all available benchmarks.
         /// </summary>
-        [PublicAPI] public IEnumerable<Summary> RunAll(IConfig? config = null) => Run(new[] { "--filter", "*" }, config);
+        [PublicAPI] public IEnumerable<Summary> RunAll(IConfig? config = null, string[]? args = null)
+        {
+            args ??= Array.Empty<string>();
+            if (ConfigParser.TryUpdateArgs(args, out var updatedArgs, options => options.Filters = new[] { "*" }))
+                args = updatedArgs;
+
+            return Run(args, config);
+        }
 
         /// <summary>
         /// Run all available benchmarks and join them to a single summary
         /// </summary>
-        [PublicAPI] public Summary RunAllJoined(IConfig? config = null) => Run(new[] { "--filter", "*", "--join" }, config).Single();
+        [PublicAPI] public Summary RunAllJoined(IConfig? config = null, string[]? args = null)
+        {
+            args ??= Array.Empty<string>();
+            if (ConfigParser.TryUpdateArgs(args, out var updatedArgs, options => (options.Join, options.Filters) = (true, new[] { "*" })))
+                args = updatedArgs;
+
+            return Run(args, config).Single();
+        }
 
         [PublicAPI]
         public IEnumerable<Summary> Run(string[]? args = null, IConfig? config = null)

tests/BenchmarkDotNet.Tests/ConfigParserTests.cs

@@ -586,5 +586,31 @@ public void UsersCanSpecifyWithoutOverheadEvalution()
                 Assert.False(job.Accuracy.EvaluateOverhead);
             }
         }
+
+        [Theory]
+        [InlineData("--filter abc", "--filter *")]
+        [InlineData("-f abc", "--filter *")]
+        [InlineData("-f *", "--filter *")]
+        [InlineData("--runtimes net7.0 --join", "--filter * --join --runtimes net7.0")]
+        [InlineData("--join abc", "--filter * --join")]
+        public void CheckUpdateValidArgs(string strArgs, string expected)
+        {
+            var args = strArgs.Split();
+            _ = ConfigParser.TryUpdateArgs(args, out var updatedArgs, options => options.Filters = new[] { "*" });
+
+            Assert.Equal(expected.Split(), updatedArgs);
+        }
+
+        [Theory]
+        [InlineData("--filter abc -f abc")]
+        [InlineData("--runtimes net")]
+        public void CheckUpdateInvalidArgs(string strArgs)
+        {
+            var args = strArgs.Split();
+            bool isSuccess = ConfigParser.TryUpdateArgs(args, out var updatedArgs, options => options.Filters = new[] { "*" });
+
+            Assert.Null(updatedArgs);
+            Assert.False(isSuccess);
+        }
     }
 }