Skip to content

[ex_unit] Omit excluded tests from test count in CLI summary #14078

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Dec 23, 2024
Merged

[ex_unit] Omit excluded tests from test count in CLI summary #14078

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
76bff37
[ex_unit] omit excluded tests from test count in CLI summary
kieraneglin Dec 17, 2024
4e7a5b1
Refactored based on feedback
kieraneglin Dec 19, 2024
c64c54f
Removed the need for a default 'test_counter' value
kieraneglin Dec 20, 2024
190afac
Changed output order so excluded is at the end of the summary
kieraneglin Dec 20, 2024
0d75496
Placed excluded count in parens to improve clarity
kieraneglin Dec 22, 2024
ce56313
Update test_test.exs
josevalim Dec 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions lib/ex_unit/lib/ex_unit/cli_formatter.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ defmodule ExUnit.CLIFormatter do
def handle_cast({:suite_finished, times_us}, config) do
test_type_counts = collect_test_type_counts(config)

if test_type_counts > 0 && config.excluded_counter == test_type_counts do
if test_type_counts == 0 and config.excluded_counter > 0 do
IO.puts(invalid("All tests have been excluded.", config))
end

Expand Down Expand Up @@ -256,6 +256,10 @@ defmodule ExUnit.CLIFormatter do
end
end

defp update_test_counter(test_counter, %{state: {:excluded, _reason}}) do
test_counter
end

defp update_test_counter(test_counter, %{tags: %{test_type: test_type}}) do
Map.update(test_counter, test_type, 1, &(&1 + 1))
end
Expand Down Expand Up @@ -331,8 +335,9 @@ defmodule ExUnit.CLIFormatter do
## Printing

defp print_summary(config, force_failures?) do
formatted_test_type_counts = format_test_type_counts(config)
test_type_counts = collect_test_type_counts(config)
test_counter = test_counter_or_default(config, test_type_counts)
formatted_test_type_counts = format_test_type_counts(test_counter)
failure_pl = pluralize(config.failure_counter, "failure", "failures")

message =
Expand Down Expand Up @@ -384,15 +389,24 @@ defmodule ExUnit.CLIFormatter do
IO.puts(formatted)
end

defp format_test_type_counts(%{test_counter: test_counter} = _config) do
defp format_test_type_counts(test_counter) do
test_counter
|> Enum.sort()
|> Enum.map(fn {test_type, count} ->
type_pluralized = pluralize(count, test_type, ExUnit.plural_rule(test_type |> to_string()))

"#{count} #{type_pluralized}, "
end)
end

defp test_counter_or_default(_config, 0) do
%{test: 0}
end

defp test_counter_or_default(%{test_counter: test_counter} = _config, _test_type_counts) do
test_counter
end

defp collect_test_type_counts(%{test_counter: test_counter} = _config) do
Enum.reduce(test_counter, 0, fn {_, count}, acc ->
acc + count
Expand Down
29 changes: 15 additions & 14 deletions lib/ex_unit/test/ex_unit_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ defmodule ExUnitTest do
assert capture_io(fn ->
assert ExUnit.async_run() |> ExUnit.await_run() ==
%{failures: 0, skipped: 0, total: 0, excluded: 0}
end) =~ "\n0 failures\n"
end) =~ "\n0 tests, 0 failures\n"
end

test "supports rerunning given modules" do
Expand Down Expand Up @@ -137,7 +137,7 @@ defmodule ExUnitTest do
assert result =~ """
Showing results so far...

0 failures
0 tests, 0 failures
"""
end

Expand Down Expand Up @@ -331,19 +331,19 @@ defmodule ExUnitTest do

{result, output} = run_with_filter([exclude: [even: true]], [ParityTest])
assert result == %{failures: 0, skipped: 0, excluded: 1, total: 4}
assert output =~ "\n4 tests, 0 failures, 1 excluded\n"
assert output =~ "\n3 tests, 0 failures, 1 excluded\n"

{result, output} = run_with_filter([exclude: :even], [ParityTest])
assert result == %{failures: 0, skipped: 0, excluded: 3, total: 4}
assert output =~ "\n4 tests, 0 failures, 3 excluded\n"
assert output =~ "\n1 test, 0 failures, 3 excluded\n"

{result, output} = run_with_filter([exclude: :even, include: [even: true]], [ParityTest])
assert result == %{failures: 1, skipped: 0, excluded: 2, total: 4}
assert output =~ "\n4 tests, 1 failure, 2 excluded\n"
assert output =~ "\n2 tests, 1 failure, 2 excluded\n"

{result, output} = run_with_filter([exclude: :test, include: [even: true]], [ParityTest])
assert result == %{failures: 1, skipped: 0, excluded: 3, total: 4}
assert output =~ "\n4 tests, 1 failure, 3 excluded\n"
assert output =~ "\n1 test, 1 failure, 3 excluded\n"
end

test "log capturing" do
Expand Down Expand Up @@ -508,16 +508,17 @@ defmodule ExUnitTest do

# Empty because it is already loaded
{result, output} = run_with_filter([exclude: :module], [])

assert result == %{failures: 0, skipped: 0, excluded: 2, total: 2}
assert output =~ "\n2 tests, 0 failures, 2 excluded\n"
assert output =~ "\n0 tests, 0 failures, 2 excluded\n"

{result, output} =
[exclude: :test, include: [module: "ExUnitTest.SecondTestModule"]]
|> run_with_filter([FirstTestModule, SecondTestModule])

assert result == %{failures: 1, skipped: 0, excluded: 1, total: 2}
assert output =~ "\n 1) test false (ExUnitTest.SecondTestModule)\n"
assert output =~ "\n2 tests, 1 failure, 1 excluded\n"
assert output =~ "\n1 test, 1 failure, 1 excluded\n"
end

test "raises on reserved tag :file in module" do
Expand Down Expand Up @@ -680,7 +681,7 @@ defmodule ExUnitTest do
end)

refute output =~ max_failures_reached_msg()
assert output =~ "\n6 tests, 0 failures, 1 excluded, 4 invalid, 1 skipped\n"
assert output =~ "\n5 tests, 0 failures, 1 excluded, 4 invalid, 1 skipped\n"
end

test "parameterized tests" do
Expand Down Expand Up @@ -787,7 +788,7 @@ defmodule ExUnitTest do
end)

assert output =~ max_failures_reached_msg()
assert output =~ "\n6 tests, 2 failures, 1 excluded, 1 skipped\n"
assert output =~ "\n5 tests, 2 failures, 1 excluded, 1 skipped\n"
end

test ":max_failures is not reached" do
Expand Down Expand Up @@ -818,7 +819,7 @@ defmodule ExUnitTest do
end)

refute output =~ max_failures_reached_msg()
assert output =~ "\n8 tests, 2 failures, 2 excluded, 1 skipped\n"
assert output =~ "\n6 tests, 2 failures, 2 excluded, 1 skipped\n"
end

test ":max_failures has been reached" do
Expand Down Expand Up @@ -852,7 +853,7 @@ defmodule ExUnitTest do
end)

assert output =~ max_failures_reached_msg()
assert output =~ "\n7 tests, 2 failures, 2 excluded, 2 skipped\n"
assert output =~ "\n5 tests, 2 failures, 2 excluded, 2 skipped\n"
end

# Excluded and skipped tests are detected before setup_all
Expand Down Expand Up @@ -886,7 +887,7 @@ defmodule ExUnitTest do
end)

assert output =~ max_failures_reached_msg()
assert output =~ "\n4 tests, 0 failures, 1 excluded, 2 invalid, 1 skipped\n"
assert output =~ "\n3 tests, 0 failures, 1 excluded, 2 invalid, 1 skipped\n"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I have one last nitpick. In this large list, we have excluded, invalid, and skipped, and it is hard to know which ones are counted as tests. I think we should either move "excluded" as the last one or perhaps show it in parens, such as "1 skipped (1 excluded)". WDYT?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't be sorry! I appreciate the patience (:

I tried both locally and I think moving it to the last of the list reads the best. I'll push that up shortly and see what you think!

end

test ":max_failures flushes all async/sync cases" do
Expand Down Expand Up @@ -1059,7 +1060,7 @@ defmodule ExUnitTest do
end)

assert output =~ "All tests have been excluded.\n"
assert output =~ "2 tests, 0 failures, 2 excluded\n"
assert output =~ "0 tests, 0 failures, 2 excluded\n"
end

test "tests are run in compile order (FIFO)" do
Expand Down