Add mix test --dry-run flag (elixir-lang#14499)

Author: Nezteb

lib/ex_unit/lib/ex_unit/cli_formatter.ex

@@ -18,6 +18,7 @@ defmodule ExUnit.CLIFormatter do
     IO.puts("")
 
     config = %{
+      dry_run: opts[:dry_run],
       trace: opts[:trace],
       colors: colors(opts),
       width: get_terminal_width(),
@@ -35,6 +36,8 @@ defmodule ExUnit.CLIFormatter do
   end
 
   def handle_cast({:suite_started, _opts}, config) do
+    if config.dry_run, do: IO.puts(extra_info("Tests that would be executed:", config))
+
     {:noreply, config}
   end
 
@@ -154,7 +157,15 @@ defmodule ExUnit.CLIFormatter do
     {:noreply, config}
   end
 
-  def handle_cast({:module_finished, %ExUnit.TestModule{state: nil}}, config) do
+  def handle_cast({:module_finished, %ExUnit.TestModule{state: nil} = module}, config) do
+    if config.dry_run do
+      file_path = Path.relative_to_cwd(module.file)
+
+      Enum.each(module.tests, fn test ->
+        IO.puts("#{file_path}:#{test.tags.line}")
+      end)
+    end
+
     {:noreply, config}
   end
 
@@ -453,6 +464,10 @@ defmodule ExUnit.CLIFormatter do
     colorize(:failure, msg, config)
   end
 
+  defp extra_info(msg, config) do
+    colorize(:extra_info, msg, config)
+  end
+
   # Diff formatting
 
   defp formatter(:diff_enabled?, _, %{colors: colors}),

lib/ex_unit/lib/ex_unit/runner.ex

@@ -88,7 +88,8 @@ defmodule ExUnit.Runner do
       seed: opts[:seed],
       stats_pid: stats_pid,
       timeout: opts[:timeout],
-      trace: opts[:trace]
+      trace: opts[:trace],
+      dry_run: opts[:dry_run]
     }
   end
 
@@ -306,6 +307,10 @@ defmodule ExUnit.Runner do
     {test_module, [], []}
   end
 
+  defp run_module_tests(%{dry_run: true}, test_module, _async?, tests) do
+    {test_module, [], tests}
+  end
+
   defp run_module_tests(config, test_module, async?, tests) do
     Process.put(@current_key, test_module)
     %ExUnit.TestModule{name: module, tags: tags, parameters: params} = test_module

lib/ex_unit/mix.exs

@@ -39,7 +39,8 @@ defmodule ExUnit.MixProject do
         timeout: 60000,
         trace: false,
         after_suite: [],
-        repeat_until_failure: 0
+        repeat_until_failure: 0,
+        dry_run: false
       ]
     ]
   end

lib/mix/lib/mix/compilers/test.ex

@@ -39,11 +39,12 @@ defmodule Mix.Compilers.Test do
   end
 
   defp require_and_run(matched_test_files, test_paths, opts) do
-    stale = opts[:stale]
+    dry_run = Keyword.get(opts, :dry_run, false)
+    stale = Keyword.get(opts, :stale, false)
     max_requires = opts[:max_requires]
 
     {test_files, stale_manifest_pid, parallel_require_callbacks} =
-      if stale do
+      if stale and not dry_run do
         set_up_stale(matched_test_files, test_paths, opts)
       else
         {matched_test_files, nil, []}

lib/mix/lib/mix/tasks/test.ex

@@ -121,6 +121,10 @@ defmodule Mix.Tasks.Test do
 
     * `--cover` - runs coverage tool. See "Coverage" section below
 
+    * `--dry-run` *(since v1.19.0)* - prints which tests would be run based on current options,
+      but does not actually run any tests. This combines with all other options
+      like `--stale`, `--only`, `--exclude`, and so on.
+
     * `--exclude` - excludes tests that match the filter. This option may be given
       several times to apply different filters, such as `--exclude ci --exclude slow`
 
@@ -494,7 +498,8 @@ defmodule Mix.Tasks.Test do
     warnings_as_errors: :boolean,
     profile_require: :string,
     exit_status: :integer,
-    repeat_until_failure: :integer
+    repeat_until_failure: :integer,
+    dry_run: :boolean
   ]
 
   @cover [output: "cover", tool: Mix.Tasks.Test.Coverage]
@@ -847,7 +852,8 @@ defmodule Mix.Tasks.Test do
     :only_test_ids,
     :test_location_relative_path,
     :exit_status,
-    :repeat_until_failure
+    :repeat_until_failure,
+    :dry_run
   ]
 
   @doc false

lib/mix/test/mix/tasks/test_test.exs

@@ -716,6 +716,65 @@ defmodule Mix.Tasks.TestTest do
     end
   end
 
+  describe "--dry-run" do
+    test "works with --stale" do
+      in_fixture("test_stale", fn ->
+        File.write!("test/dry_run_one_test_stale.exs", """
+        defmodule DryRunOneTest do
+          use ExUnit.Case
+
+          test "new test" do
+            assert true
+          end
+        end
+        """)
+
+        File.write!("test/dry_run_two_test_stale.exs", """
+        defmodule DryRunTwoTest do
+          use ExUnit.Case
+
+          @tag :skip
+          test "skipped test" do
+            assert true
+          end
+        end
+        """)
+
+        output = mix(["test", "--dry-run", "--stale"])
+        assert output =~ "Tests that would be executed:"
+        assert output =~ "test/a_test_stale.exs:4"
+        assert output =~ "test/b_test_stale.exs:4"
+        assert output =~ "test/dry_run_one_test_stale.exs:4"
+        refute output =~ "test/dry_run_two_test_stale.exs:5"
+        assert output =~ "1 test, 0 failures, 1 skipped"
+
+        # Tests should still be marked as stale
+        output = mix(["test", "--dry-run", "--stale"])
+        assert output =~ "1 test, 0 failures, 1 skipped"
+      end)
+    end
+
+    test "works with --failed" do
+      in_fixture("test_failed", fn ->
+        output = mix(["test"])
+        assert output =~ "4 tests, 2 failures"
+
+        output = mix(["test", "--dry-run", "--failed"])
+        assert output =~ "Tests that would be executed:"
+        assert output =~ "test/only_failing_test_failed.exs:4"
+        assert output =~ "test/passing_and_failing_test_failed.exs:5"
+        assert output =~ "0 tests, 0 failures"
+
+        # Force the tests to pass, verify dry-run doesn't actually run them
+        System.put_env("PASS_FAILING_TESTS", "true")
+        output = mix(["test", "--dry-run", "--failed"])
+        assert output =~ "0 tests, 0 failures"
+      end)
+    after
+      System.delete_env("PASS_FAILING_TESTS")
+    end
+  end
+
   defp receive_until_match(port, expected, acc) do
     receive do
       {^port, {:data, output}} ->