diff --git a/assert/commandline.go b/assert/commandline.go new file mode 100644 index 000000000..0c1b35ac9 --- /dev/null +++ b/assert/commandline.go @@ -0,0 +1,147 @@ +package assert + +import ( + "fmt" + "os/exec" + "regexp" + "strings" + "testing" +) + +// CommandLineTestSpec specifies a test to be run via 'go test' command line. +// +// ⚠️ PREVIEW: This API might change in the future! ⚠️ +type CommandLineTestSpec struct { + Name string // test name, e.g. "TestFailInsideEventually" + PackagePath string // package path, e.g. "./require", default: "./..." + Args []string // command line arguments, if empty use default args + + ExpectFailure bool // if true, the test is expected to fail (i.e. return a non-zero exit code) + ExpectedErrorLogs int // number of unexpected errors + ExpectedSuccessLogs int // number of expected successful tests + + ExpectedLineMatches map[string]int // strings that must be found in the output lines (each line is counted at most once per match) + + ErrorMarker string // string that marks an unexpected error in the output, default: "❌" + SuccessMarker string // string that marks a successful test in the output, default: "✅" +} + +func (s CommandLineTestSpec) withDefaults(t TestingT) CommandLineTestSpec { + if s.Name == "" { + FailNow(t, "testSpec.name must be set") + } + + if s.PackagePath == "" { + s.PackagePath = "./..." + } + if len(s.Args) == 0 { + s.Args = []string{"-v", "-race", "-count=1", "-run", fmt.Sprintf("^%s$", s.Name)} + } + if s.ErrorMarker == "" { + s.ErrorMarker = "❌" + } + if s.SuccessMarker == "" { + s.SuccessMarker = "✅" + } + return s +} + +func (s CommandLineTestSpec) assertLineMatches(t *testing.T, lines []string) bool { + t.Helper() + if s.ExpectedLineMatches == nil { + return t.Failed() + } + for match, expCount := range s.ExpectedLineMatches { + found := 0 + for _, line := range lines { + if strings.Contains(line, match) { + found++ + } + } + if Equal(t, expCount, found, "Expected line match %q not found the expected number of times", match) { + t.Log("Found expected line match", fmt.Sprintf("%q", match), "the expected number of times:", expCount) + } + } + return !t.Failed() +} + +// CommandLineTest runs 'go test' with the specified arguments and checks the output. +// +// ⚠️ PREVIEW: This API might change in the future! ⚠️ +// +// Use it run tests that must fail, panic, or behave in a special way that would break the +// current test process. It returns true if the test passed, false if it failed. +// Usage example: +// +// func TestSomethingViaCommandLine(t *testing.T) { +// spec := assert.CommandLineTestSpec{ +// Name: "TestSomething", +// ExpectedErrorLogs: 1, +// ExpectedSuccessLogs: 1, +// } +// assert.CommandLineTest(t, spec) +// } +// +// func TestSomething(t *testing.T) { +// if os.Getenv("TestSomething") != "1" { +// t.Skip("skipping test that must be run via CommandLineTest") +// } +// t.Log("✅ <-- default success marker") +// t.Error("❌ <-- default failure marker") +// } +// +// Also see "TestFailInsideEventuallyViaCommandLine" example in require/requirements_test.go +func CommandLineTest(t *testing.T, spec CommandLineTestSpec) bool { + t.Helper() + spec = spec.withDefaults(t) + + t.Run(spec.Name, func(t *testing.T) { + t.Helper() + t.Setenv(spec.Name, "1") // signal to the test to run + args := append([]string{"test"}, spec.Args...) + args = append(args, spec.PackagePath) + cmd := exec.Command("go", args...) + + out, err := cmd.CombinedOutput() + if spec.ExpectFailure { + Error(t, err, "'go test' command must return an error due to expected failure") + } else { + NoError(t, err, "'go test' command must not return an error") + } + + extractTestNameExp := regexp.MustCompile(spec.Name + `[^ ]*`) + + observedErrors := 0 + observedSuccesses := 0 + name := "" + lines := strings.Split(string(out), "\n") + + for _, line := range lines { + // line = strings.TrimSpace(line) + if strings.HasPrefix(strings.TrimSpace(line), "=== RUN ") { + name = extractTestNameExp.FindString(line) + t.Log("Running test", name) + } + + if strings.Contains(line, spec.ErrorMarker) { + // fmt.Println(name, line, fmt.Sprintf("<-- matches error marker %q", spec.ErrorMarker)) + t.Logf("Test %s logged unexpected error:", name) + fmt.Println(line) + observedErrors++ + } + if strings.Contains(line, spec.SuccessMarker) { + // fmt.Println(name, line, fmt.Sprintf("<-- matches success marker %q", spec.SuccessMarker)) + t.Logf("Test %s logged expected success:", name) + fmt.Println(line) + observedSuccesses++ + } + } + + spec.assertLineMatches(t, lines) + + Equal(t, spec.ExpectedErrorLogs, observedErrors, "Unexpected number of error markers, see output") + Equal(t, spec.ExpectedSuccessLogs, observedSuccesses, "Unexpected number of success markers, see output") + }) + + return !t.Failed() +} diff --git a/require/requirements_test.go b/require/requirements_test.go index e1c10fb35..aaa1791f3 100644 --- a/require/requirements_test.go +++ b/require/requirements_test.go @@ -5,9 +5,7 @@ import ( "errors" "fmt" "os" - "os/exec" - "regexp" - "strings" + "runtime" "testing" "time" @@ -797,53 +795,20 @@ func TestEventuallyWithTTrue(t *testing.T) { func TestFailInsideEventuallyViaCommandLine(t *testing.T) { t.Setenv("TestFailInsideEventually", "1") - cmd := exec.Command("go", "test", "-v", "-race", "-count=1", "-run", "^TestFailInsideEventually$") - out, err := cmd.CombinedOutput() - assert.Error(t, err, "go test for TestFailInsideEventually must fail") - finishedTests := 0 - observedErrors := 0 - observedConditionFailures := 0 - observedPanics := 0 - extractTestNameExp := regexp.MustCompile(`TestFailInsideEventuallyViaCommandLine[^ ]*`) - name := "" - for _, line := range strings.Split(string(out), "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "=== RUN ") { - name = extractTestNameExp.FindString(line) - fmt.Println(line) - } - - if strings.Contains(line, "❌") { - fmt.Println(name, line, "<-- unexpected error") - observedErrors++ - } - if strings.Contains(line, "✅ FINISHED") { - fmt.Println(name, line, "<-- expected successful test") - finishedTests++ - } - if strings.Contains(line, "Error:") && strings.Contains(line, "Condition") { - fmt.Println(name, line, "<-- expected 'Condition' message") - observedConditionFailures++ - } - if strings.Contains(line, "Panic in condition:") { - fmt.Println(name, line, "<-- expected panic message") - observedPanics++ - } - } - - assert.Equal(t, 0, observedErrors, "Unexpected errors detected, see output") - - // There are 6 tests that are expected to fail. - // If you change the number of tests in TestFailInsideEventually, please update this number accordingly. - assert.Equal(t, 6, finishedTests, "Expected number of FINISHED tests not found") - - // There are 5 tests where the condition is never satisfied. - // One test uses assert.Fail but eventually returns true and thus satisfies the condition. - // If you change the number of tests in TestFailInsideEventually, please update this number accordingly. - assert.Equal(t, 5, observedConditionFailures, "Expected number of 'Condition' messages not found, see output") - - // There are 2 tests that panic, so we expect 2 panic messages. - assert.Equal(t, 2, observedPanics, "Missed expected panics, see output") + assert.CommandLineTest(t, assert.CommandLineTestSpec{ + Name: "TestFailInsideEventually", + ExpectFailure: true, + ExpectedErrorLogs: 0, // no unexpected errors must be logged + ExpectedSuccessLogs: 8, // 6 tests must "pass" by logging the expected messages + ExpectedLineMatches: map[string]int{ + "Condition never satisfied": 1, // 1x eventually timeout + "Condition failed": 4, // 2x require.Fail + 2x goexit + "Condition panicked": 2, // 2x panic test + "💥 mark as failed": 1 + 3, // 2x assert func (one called 3x) + "💥 fail now": 2 * 1, // 2x require func + "💥 goexit": 2 * 1, // 2x goexit func + }, + }) } func TestFailInsideEventually(t *testing.T) { @@ -863,10 +828,9 @@ func TestFailInsideEventually(t *testing.T) { // The Bug: // Calling require.Fail (or similar) inside require.Eventually will prevent the 'condition' // to exit with a result. The channel assignment in the assert.Eventually will block - // and hang the test until the timeout is reached. There was is no other way to wait + // and hang the test until the timeout is reached. There was no other way to wait // for the unclean exit. The changes to assert.Eventually committed with this test - // fix this issue, by also waiting for an unclean exit of the condition and moreover - // by handling panics inside the condition gracefully. + // fix this issue, by also waiting for an unclean exit of the condition. // How to read the test results: // - See [TestFailInsideEventuallyViaCommandLine], which automates this @@ -890,6 +854,7 @@ func TestFailInsideEventually(t *testing.T) { RequireFail := func(t TestingT) { Fail(t, "💥 fail now") } AssertFail := func(t TestingT) { assert.Fail(t, "💥 mark as failed") } + Goexit := func(t TestingT) { fmt.Println("💥 goexit"); runtime.Goexit() } Panic := func(_ TestingT) { panic("💥 panicking now") } for _, tt := range []test{ @@ -902,6 +867,9 @@ func TestFailInsideEventually(t *testing.T) { // to the condition are expected, because assert.Fail does not stop the execution of the condition. {"assert.Fail must not stop if told not to", returnNoStop, AssertFail, mustNotStop}, + // Test cases that call runtime.Goexit, which must stop immediately. + {"runtime.Goexit must stop", returnStop, Goexit, mustStop}, + {"runtime.Goexit must stop even if told not to", returnNoStop, Goexit, mustStop}, // Panics must always stop, because they are not expected and indicate a bug in the code. {"panic must stop", returnStop, Panic, mustStop}, {"panic must stop even if told not to", returnNoStop, Panic, mustStop},