when user specifies both --runtimes and --corerun, multiple independent jobs should be created (#2002)

* update tests

* when user specifies both --runtimes and --corerun, multiple independent jobs should be created

Author: adamsitnik

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -78,7 +78,7 @@ public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse
             {
                 parser
                     .ParseArguments<CommandLineOptions>(args)
-                    .WithParsed(options => result = Validate(options, logger) ? (true, CreateConfig(options, globalConfig), options) : (false, default, default))
+                    .WithParsed(options => result = Validate(options, logger) ? (true, CreateConfig(options, globalConfig, args), options) : (false, default, default))
                     .WithNotParsed(errors => result = (false, default, default));
             }
 
@@ -163,9 +163,9 @@ private static bool Validate(CommandLineOptions options, ILogger logger)
                 return false;
             }
 
-            if (options.Runtimes.Count() > 1 && !options.CoreRunPaths.IsNullOrEmpty())
+            if (!options.CoreRunPaths.IsNullOrEmpty() && !RuntimeInformation.IsNetCore)
             {
-                logger.WriteLineError("CoreRun path can't be combined with multiple .NET Runtimes");
+                logger.WriteLineError("CoreRun path can be used only for .NET Core host processes.");
                 return false;
             }
 
@@ -197,12 +197,12 @@ private static bool Validate(CommandLineOptions options, ILogger logger)
             return true;
         }
 
-        private static IConfig CreateConfig(CommandLineOptions options, IConfig globalConfig)
+        private static IConfig CreateConfig(CommandLineOptions options, IConfig globalConfig, string[] args)
         {
             var config = new ManualConfig();
 
             var baseJob = GetBaseJob(options, globalConfig);
-            var expanded = Expand(baseJob.UnfreezeCopy(), options).ToArray(); // UnfreezeCopy ensures that each of the expanded jobs will have it's own ID
+            var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); // UnfreezeCopy ensures that each of the expanded jobs will have it's own ID
             if (expanded.Length > 1)
                 expanded[0] = expanded[0].AsBaseline(); // if the user provides multiple jobs, then the first one should be a baseline
             config.AddJob(expanded);
@@ -313,20 +313,46 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig)
                 .AsMutator(); // we mark it as mutator so it will be applied to other jobs defined via attributes and merged later in GetRunnableJobs method
         }
 
-        private static IEnumerable<Job> Expand(Job baseJob, CommandLineOptions options)
+        private static IEnumerable<Job> Expand(Job baseJob, CommandLineOptions options, string[] args)
         {
             if (options.RunInProcess)
+            {
                 yield return baseJob.WithToolchain(InProcessEmitToolchain.Instance);
+            }
             else if (!string.IsNullOrEmpty(options.ClrVersion))
+            {
                 yield return baseJob.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(options.ClrVersion)); // local builds of .NET Runtime
-            else if (options.CoreRunPaths.Any())
-                foreach (var coreRunPath in options.CoreRunPaths)
-                    yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local CoreFX and CoreCLR builds
-            else if (options.CliPath != null && options.Runtimes.IsEmpty())
+            }
+            else if (options.CliPath != null && options.Runtimes.IsEmpty() && options.CoreRunPaths.IsEmpty())
+            {
                 yield return CreateCoreJobWithCli(baseJob, options);
+            }
             else
-                foreach (string runtime in options.Runtimes) // known runtimes
-                    yield return CreateJobForGivenRuntime(baseJob, runtime, options);
+            {
+                // in case both --runtimes and --corerun are specified, the first one is returned first and becomes a baseline job
+                string first = args.FirstOrDefault(arg =>
+                    arg.Equals("--runtimes", StringComparison.OrdinalIgnoreCase)
+                    || arg.Equals("-r", StringComparison.OrdinalIgnoreCase)
+
+                    || arg.Equals("--corerun", StringComparison.OrdinalIgnoreCase));
+
+                if (first is null || first.Equals("--corerun", StringComparison.OrdinalIgnoreCase))
+                {
+                    foreach (var coreRunPath in options.CoreRunPaths)
+                        yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds
+
+                    foreach (string runtime in options.Runtimes) // known runtimes
+                        yield return CreateJobForGivenRuntime(baseJob, runtime, options);
+                }
+                else
+                {
+                    foreach (string runtime in options.Runtimes) // known runtimes
+                        yield return CreateJobForGivenRuntime(baseJob, runtime, options);
+
+                    foreach (var coreRunPath in options.CoreRunPaths)
+                        yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds
+                }
+            }
         }
 
         private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options)
@@ -479,7 +505,7 @@ private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, Fil
                 .WithToolchain(new CoreRunToolchain(
                     coreRunPath,
                     createCopy: true,
-                    targetFrameworkMoniker: options.Runtimes.SingleOrDefault() ?? RuntimeInformation.GetCurrentRuntime().MsBuildMoniker,
+                    targetFrameworkMoniker: RuntimeInformation.GetCurrentRuntime().MsBuildMoniker,
                     customDotNetCliPath: options.CliPath,
                     restorePath: options.RestorePath,
                     displayName: GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath)));

tests/BenchmarkDotNet.Tests/ConfigParserTests.cs

@@ -150,25 +150,70 @@ public void CoreRunConfigParsedCorrectlyWhenRuntimeNotSpecified()
             Assert.Equal(fakeRestorePackages, toolchain.RestorePath.FullName);
         }
 
-        [Fact]
-        public void CoreRunConfigParsedCorrectlyWhenRuntimeSpecified()
+        [FactClassicDotNetOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
+        public void SpecifyingCoreRunWithFullFrameworkHostGivesError()
+        {
+            var fakePath = typeof(object).Assembly.Location;
+            Assert.False(ConfigParser.Parse(new[] { "--corerun", fakePath }, new OutputLogger(Output)).isSuccess);
+        }
+
+        [FactDotNetCoreOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
+        public void SpecifyingCoreRunAndRuntimeCreatesTwoJobs()
         {
-            const string runtime = "netcoreapp3.0";
+            const string runtime = "net7.0";
             var fakeDotnetCliPath = typeof(object).Assembly.Location;
             var fakeCoreRunPath = typeof(ConfigParserTests).Assembly.Location;
             var fakeRestorePackages = Path.GetTempPath();
             var config = ConfigParser.Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages, "-r", runtime }, new OutputLogger(Output)).config;
 
-            Assert.Single(config.GetJobs());
-            CoreRunToolchain toolchain = config.GetJobs().Single().GetToolchain() as CoreRunToolchain;
-            Assert.NotNull(toolchain);
-            Assert.Equal(runtime, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); // runtime was provided and used
-            Assert.Equal(fakeCoreRunPath, toolchain.SourceCoreRun.FullName);
-            Assert.Equal(fakeDotnetCliPath, toolchain.CustomDotNetCliPath.FullName);
-            Assert.Equal(fakeRestorePackages, toolchain.RestorePath.FullName);
+            Assert.Equal(2, config.GetJobs().Count());
+
+            Job coreRunJob = config.GetJobs().Single(job => job.GetToolchain() is CoreRunToolchain);
+            Job runtimeJob = config.GetJobs().Single(job => job.GetToolchain() is CsProjCoreToolchain);
+
+            CoreRunToolchain coreRunToolchain = (CoreRunToolchain)coreRunJob.GetToolchain();
+            DotNetCliGenerator generator = (DotNetCliGenerator)coreRunToolchain.Generator;
+            Assert.Equal(RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, generator.TargetFrameworkMoniker);
+            Assert.Equal(fakeCoreRunPath, coreRunToolchain.SourceCoreRun.FullName);
+            Assert.Equal(fakeDotnetCliPath, coreRunToolchain.CustomDotNetCliPath.FullName);
+            Assert.Equal(fakeRestorePackages, coreRunToolchain.RestorePath.FullName);
+
+            CsProjCoreToolchain coreToolchain = (CsProjCoreToolchain)runtimeJob.GetToolchain();
+            generator = (DotNetCliGenerator)coreToolchain.Generator;
+            Assert.Equal(runtime, ((DotNetCliGenerator)coreToolchain.Generator).TargetFrameworkMoniker);
+            Assert.Equal(fakeDotnetCliPath, coreToolchain.CustomDotNetCliPath);
+            Assert.Equal(fakeRestorePackages, generator.PackagesPath);
         }
 
-        [Fact]
+        [FactDotNetCoreOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
+        public void FirstJobIsBaseline_RuntimesCoreRun()
+        {
+            const string runtime1 = "net5.0";
+            const string runtime2 = "net6.0";
+            string fakePath = typeof(object).Assembly.Location;
+            var config = ConfigParser.Parse(new[] { "--runtimes", runtime1, runtime2, "--coreRun", fakePath }, new OutputLogger(Output)).config;
+
+            Assert.Equal(3, config.GetJobs().Count());
+            Job baselineJob = config.GetJobs().Single(job => job.Meta.Baseline == true);
+            Assert.False(baselineJob.GetToolchain() is CoreRunToolchain);
+            Assert.Equal(runtime1, ((DotNetCliGenerator)baselineJob.GetToolchain().Generator).TargetFrameworkMoniker);
+        }
+
+        [FactDotNetCoreOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
+        public void FirstJobIsBaseline_CoreRunsRuntimes()
+        {
+            const string runtime1 = "net5.0";
+            const string runtime2 = "net6.0";
+            string fakePath1 = typeof(object).Assembly.Location;
+            string fakePath2 = typeof(FactAttribute).Assembly.Location;
+            var config = ConfigParser.Parse(new[] { "--coreRun", fakePath1, fakePath2, "--runtimes", runtime1, runtime2 }, new OutputLogger(Output)).config;
+
+            Assert.Equal(4, config.GetJobs().Count());
+            Job baselineJob = config.GetJobs().Single(job => job.Meta.Baseline == true);
+            Assert.Equal(fakePath1, ((CoreRunToolchain)baselineJob.GetToolchain()).SourceCoreRun.FullName);
+        }
+
+        [FactDotNetCoreOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
         public void UserCanSpecifyMultipleCoreRunPaths()
         {
             var fakeCoreRunPath_1 = typeof(object).Assembly.Location;