From bcbb0306fbb372d92e5f149ac4eb2b05b79ea247 Mon Sep 17 00:00:00 2001 From: Uwe Jugel Date: Tue, 14 Oct 2025 23:22:47 +0200 Subject: [PATCH 1/4] add indirect test for different condition failures --- assert/assertions.go | 6 +- require/requirements_test.go | 147 +++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 3 deletions(-) diff --git a/assert/assertions.go b/assert/assertions.go index 558201ff5..f8a54afde 100644 --- a/assert/assertions.go +++ b/assert/assertions.go @@ -2099,12 +2099,12 @@ func (*CollectT) Copy(TestingT) { } func (c *CollectT) fail() { - if !c.failed() { + if !c.Failed() { c.errors = []error{} // Make it non-nil to mark a failure. } } -func (c *CollectT) failed() bool { +func (c *CollectT) Failed() bool { return c.errors != nil } @@ -2164,7 +2164,7 @@ func EventuallyWithT(t TestingT, condition func(collect *CollectT), waitFor time tickC = nil go checkCond() case collect := <-ch: - if !collect.failed() { + if !collect.Failed() { return true } // Keep the errors from the last ended condition, so that they can be copied to t if timeout is reached. diff --git a/require/requirements_test.go b/require/requirements_test.go index 7cb63a554..71cc62ba8 100644 --- a/require/requirements_test.go +++ b/require/requirements_test.go @@ -3,6 +3,11 @@ package require import ( "encoding/json" "errors" + "fmt" + "os" + "os/exec" + "regexp" + "strings" "testing" "time" @@ -26,6 +31,7 @@ type AssertionTesterNonConformingObject struct { } type MockT struct { + // Failed marks the test as failed. Failed bool } @@ -788,3 +794,144 @@ func TestEventuallyWithTTrue(t *testing.T) { False(t, mockT.Failed, "Check should pass") Equal(t, 2, counter, "Condition is expected to be called 2 times") } + +func TestFailInsideEventuallyViaCommandLine(t *testing.T) { + t.Setenv("TestFailInsideEventually", "1") + cmd := exec.Command("go", "test", "-v", "-race", "-count=1", "-run", "^TestFailInsideEventually$") + out, err := cmd.CombinedOutput() + assert.Error(t, err, "go test for TestFailInsideEventually must fail") + finishedTests := 0 + observedErrors := 0 + observedConditionFailures := 0 + extractTestNameExp := regexp.MustCompile(`TestFailInsideEventuallyViaCommandLine[^ ]*`) + name := "" + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "=== RUN ") { + name = extractTestNameExp.FindString(line) + fmt.Println(line) + } + + if strings.Contains(line, "❌") { + fmt.Println(name, line, "<-- unexpected error") + observedErrors++ + } + if strings.Contains(line, "✅ FINISHED") { + fmt.Println(name, line, "<-- expected successful test") + finishedTests++ + } + if strings.Contains(line, "Error:") && strings.Contains(line, "Condition") { + fmt.Println(name, line, "<-- expected 'Condition' message") + observedConditionFailures++ + } + } + + assert.Equal(t, 0, observedErrors, "Unexpected errors detected, see output") + + // There are 4 tests that are expected to fail. + // If you change the number of tests in TestFailInsideEventually, please update this number accordingly. + assert.Equal(t, 4, finishedTests, "Expected number of FINISHED tests not found") + + // There are 3 tests where the condition is never satisfied. + // One test uses assert.Fail but eventually returns true and thus satisfies the condition. + // If you change the number of tests in TestFailInsideEventually, please update this number accordingly. + assert.Equal(t, 3, observedConditionFailures, "Expected number of 'Condition' messages not found, see output") +} + +func TestFailInsideEventually(t *testing.T) { + if os.Getenv("TestFailInsideEventually") == "" { + t.Skip("Skipping test, run via TestFailInsideEventuallyViaCommandLine") + } + + // Using testing.T: + // Enable this test temporarily to manually check that the issue is fixed. + // Note that MockT does not reproduce the issue, so we have to use the real *testing.T. + // TODO: Enhance MockT to reproduce the issue, then we can remove the t.Skip above. + // UPDATE: I tried to enhance MockT, but it still does not reproduce the issue or fails + // in a different way. Therefore I keep using *testing.T for now. + // In general, require.MockT does not play well when assert.Fail is called. + // The test will not be marked as failed, even though Fail is called. + + // The Bug: + // Calling require.Fail (or similar) inside require.Eventually will prevent the 'condition' + // to exit with a result. The channel assignment in the assert.Eventually will block + // and hang the test until the timeout is reached. There was no other way to wait + // for the unclean exit. The changes to assert.Eventually committed with this test + // fix this issue, by also waiting for an unclean exit of the condition. + + // How to read the test results: + // - See [TestFailInsideEventuallyViaCommandLine], which automates this + // - The test will always fail, because it calls Fail or assert.Fail or panics + // - The test is "successful" if it fails quickly and cleanly, i.e. without + // multiple calls to the eventually function, except if expected. + // - The test logs should contain specific messages, see below. + // - The test must not log any UNCLEAN EXIT or MISSED ASSERTIONS messages or any errors ❌. + + type test struct { + Name string + Return bool + FailFunc func(t TestingT) + MustStop bool // after the FailFunc is called + } + + const returnStop = true + const returnNoStop = false + const mustStop = true + const mustNotStop = false + + RequireFail := func(t TestingT) { Fail(t, "💥 fail now") } + AssertFail := func(t TestingT) { assert.Fail(t, "💥 mark as failed") } + + for _, tt := range []test{ + // Test cases that must exit immediately after the first call to the condition. + {"require.Fail must stop", returnStop, RequireFail, mustStop}, + {"require.Fail must stop even if told not to", returnNoStop, RequireFail, mustStop}, + {"assert.Fail must stop if told to", returnStop, AssertFail, mustStop}, + + // The following test case is the only one that must not stop and where multiple calls + // to the condition are expected, because assert.Fail does not stop the execution of the condition. + {"assert.Fail must not stop if told not to", returnNoStop, AssertFail, mustNotStop}, + + // Make sure to update the assertions in TestFailInsideEventuallyViaCommandLine + // accordingly if you change the number of tests here. + } { + count := 0 + start := time.Now() + timeout := time.Second * 1 + tick := time.Second / 3 + ok := false + + ok = t.Run(tt.Name, func(t *testing.T) { + // Cannot use a MockT here, because it does reproduce the issue. + Eventually(t, func() bool { + count++ + t.Log("🪲 eventually call number:", count, "calling FailNow!") + tt.FailFunc(t) // any case that calls Fail or assert.Fail should stop retrying + return tt.Return // indicate whether to stop retrying + }, timeout, tick) + }) + + dur := time.Since(start) + t.Log("🪲 test duration:", dur) + + // TODO: Replace with plain t with MockT once it can reproduce the issue. + // Until can only indicate that the test should have failed using stdout. + + c := new(assert.CollectT) + assert.True(c, !ok, "❌ UNCLEAN EXIT: test was expected to fail, but passed") + + if tt.MustStop { + assert.Equal(c, 1, count, "❌ UNCLEAN EXIT: eventually func should be called exactly once") + assert.Less(c, dur, tick, "❌ UNCLEAN EXIT: eventually func should be called only once, but took too long") + } else { + assert.Greater(c, count, 1, "❌ MISSED ASSERTIONS: eventually func should be called multiple times, but was called only once") + assert.Greater(c, dur, tick, "❌ MISSED ASSERTIONS: eventually func should be called multiple times over time, but total duration was too short") + } + + if c.Failed() { + t.Log("❌ TEST FAILED") + } else { + t.Log("✅ FINISHED", tt.Name) + } + } +} From 75b2a13444bcc7cbd5972d323c504a287bc28f19 Mon Sep 17 00:00:00 2001 From: Uwe Jugel Date: Thu, 16 Oct 2025 12:15:29 +0200 Subject: [PATCH 2/4] add commandline tool based test capability --- assert/commandline.go | 145 +++++++++++++++++++++++++++++++++++ require/requirements_test.go | 60 +++++---------- 2 files changed, 163 insertions(+), 42 deletions(-) create mode 100644 assert/commandline.go diff --git a/assert/commandline.go b/assert/commandline.go new file mode 100644 index 000000000..369190429 --- /dev/null +++ b/assert/commandline.go @@ -0,0 +1,145 @@ +package assert + +import ( + "fmt" + "os/exec" + "regexp" + "strings" + "testing" +) + +// CommandLineTestSpec specifies a test to be run via 'go test' command line. +// +// ⚠️ PREVIEW: This API might change in the future! ⚠️ +type CommandLineTestSpec struct { + Name string // test name, e.g. "TestFailInsideEventually" + PackagePath string // package path, e.g. "./require", default: "./..." + Args []string // command line arguments, if empty use default args + + ExpectFailure bool // if true, the test is expected to fail (i.e. return a non-zero exit code) + ExpectedErrorLogs int // number of unexpected errors + ExpectedSuccessLogs int // number of expected successful tests + + ExpectedLineMatches map[string]int // strings that must be found in the output lines (each line is counted at most once per match) + + ErrorMarker string // string that marks an unexpected error in the output, default: "❌" + SuccessMarker string // string that marks a successful test in the output, default: "✅" +} + +func (s CommandLineTestSpec) withDefaults(t TestingT) CommandLineTestSpec { + if s.Name == "" { + FailNow(t, "testSpec.name must be set") + } + + if s.PackagePath == "" { + s.PackagePath = "./..." + } + if len(s.Args) == 0 { + s.Args = []string{"-v", "-race", "-count=1", "-run", fmt.Sprintf("^%s$", s.Name)} + } + if s.ErrorMarker == "" { + s.ErrorMarker = "❌" + } + if s.SuccessMarker == "" { + s.SuccessMarker = "✅" + } + return s +} + +func (s CommandLineTestSpec) assertLineMatches(t *testing.T, lines []string) bool { + t.Helper() + if s.ExpectedLineMatches == nil { + return t.Failed() + } + for match, expCount := range s.ExpectedLineMatches { + found := 0 + for _, line := range lines { + if strings.Contains(line, match) { + found++ + } + } + if Equal(t, expCount, found, "Expected line match %q not found the expected number of times", match) { + t.Log("Found expected line match", fmt.Sprintf("%q", match), "the expected number of times:", expCount) + } + } + return !t.Failed() +} + +// CommandLineTest runs 'go test' with the specified arguments and checks the output. +// +// ⚠️ PREVIEW: This API might change in the future! ⚠️ +// +// Use it run tests that must fail, panic, or behave in a special way that would break the +// current test process. It returns true if the test passed, false if it failed. +// Usage example: +// +// func TestSomethingViaCommandLine(t *testing.T) { +// spec := assert.CommandLineTestSpec{ +// Name: "TestSomething", +// ExpectedErrorLogs: 1, +// ExpectedSuccessLogs: 1, +// } +// assert.CommandLineTest(t, spec) +// } +// +// func TestSomething(t *testing.T) { +// if os.Getenv("TestSomething") != "1" { +// t.Skip("skipping test that must be run via CommandLineTest") +// } +// t.Log("✅ <-- default success marker") +// t.Error("❌ <-- default failure marker") +// } +// +// Also see "TestFailInsideEventuallyViaCommandLine" example in require/requirements_test.go +func CommandLineTest(t *testing.T, spec CommandLineTestSpec) bool { + spec = spec.withDefaults(t) + + t.Run(spec.Name, func(t *testing.T) { + t.Setenv(spec.Name, "1") // signal to the test to run + args := append([]string{"test"}, spec.Args...) + args = append(args, spec.PackagePath) + cmd := exec.Command("go", args...) + + out, err := cmd.CombinedOutput() + if spec.ExpectFailure { + Error(t, err, "'go test' command must return an error due to expected failure") + } else { + NoError(t, err, "'go test' command must not return an error") + } + + extractTestNameExp := regexp.MustCompile(spec.Name + `[^ ]*`) + + observedErrors := 0 + observedSuccesses := 0 + name := "" + lines := strings.Split(string(out), "\n") + + for _, line := range lines { + // line = strings.TrimSpace(line) + if strings.HasPrefix(strings.TrimSpace(line), "=== RUN ") { + name = extractTestNameExp.FindString(line) + t.Log("Running test", name) + } + + if strings.Contains(line, spec.ErrorMarker) { + // fmt.Println(name, line, fmt.Sprintf("<-- matches error marker %q", spec.ErrorMarker)) + t.Logf("Test %s logged unexpected error:", name) + fmt.Println(line) + observedErrors++ + } + if strings.Contains(line, spec.SuccessMarker) { + // fmt.Println(name, line, fmt.Sprintf("<-- matches success marker %q", spec.SuccessMarker)) + t.Logf("Test %s logged expected success:", name) + fmt.Println(line) + observedSuccesses++ + } + } + + spec.assertLineMatches(t, lines) + + Equal(t, spec.ExpectedErrorLogs, observedErrors, "Unexpected number of error markers, see output") + Equal(t, spec.ExpectedSuccessLogs, observedSuccesses, "Unexpected number of success markers, see output") + }) + + return !t.Failed() +} diff --git a/require/requirements_test.go b/require/requirements_test.go index 71cc62ba8..377d28216 100644 --- a/require/requirements_test.go +++ b/require/requirements_test.go @@ -5,9 +5,7 @@ import ( "errors" "fmt" "os" - "os/exec" - "regexp" - "strings" + "runtime" "testing" "time" @@ -797,45 +795,18 @@ func TestEventuallyWithTTrue(t *testing.T) { func TestFailInsideEventuallyViaCommandLine(t *testing.T) { t.Setenv("TestFailInsideEventually", "1") - cmd := exec.Command("go", "test", "-v", "-race", "-count=1", "-run", "^TestFailInsideEventually$") - out, err := cmd.CombinedOutput() - assert.Error(t, err, "go test for TestFailInsideEventually must fail") - finishedTests := 0 - observedErrors := 0 - observedConditionFailures := 0 - extractTestNameExp := regexp.MustCompile(`TestFailInsideEventuallyViaCommandLine[^ ]*`) - name := "" - for _, line := range strings.Split(string(out), "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "=== RUN ") { - name = extractTestNameExp.FindString(line) - fmt.Println(line) - } - - if strings.Contains(line, "❌") { - fmt.Println(name, line, "<-- unexpected error") - observedErrors++ - } - if strings.Contains(line, "✅ FINISHED") { - fmt.Println(name, line, "<-- expected successful test") - finishedTests++ - } - if strings.Contains(line, "Error:") && strings.Contains(line, "Condition") { - fmt.Println(name, line, "<-- expected 'Condition' message") - observedConditionFailures++ - } - } - - assert.Equal(t, 0, observedErrors, "Unexpected errors detected, see output") - - // There are 4 tests that are expected to fail. - // If you change the number of tests in TestFailInsideEventually, please update this number accordingly. - assert.Equal(t, 4, finishedTests, "Expected number of FINISHED tests not found") - - // There are 3 tests where the condition is never satisfied. - // One test uses assert.Fail but eventually returns true and thus satisfies the condition. - // If you change the number of tests in TestFailInsideEventually, please update this number accordingly. - assert.Equal(t, 3, observedConditionFailures, "Expected number of 'Condition' messages not found, see output") + assert.CommandLineTest(t, assert.CommandLineTestSpec{ + Name: "TestFailInsideEventually", + ExpectFailure: true, + ExpectedErrorLogs: 0, // no unexpected errors must be logged + ExpectedSuccessLogs: 6, // 6 tests must "pass" by logging the expected messages + ExpectedLineMatches: map[string]int{ + "Condition never satisfied": 5 * 1, // 5x early exit, 1x assert + return true + "💥 mark as failed": 1 + 3, // 2x assert func (one called 3x) + "💥 fail now": 2 * 1, // 2x require func + "💥 goexit": 2 * 1, // 2x goexit func + }, + }) } func TestFailInsideEventually(t *testing.T) { @@ -881,6 +852,7 @@ func TestFailInsideEventually(t *testing.T) { RequireFail := func(t TestingT) { Fail(t, "💥 fail now") } AssertFail := func(t TestingT) { assert.Fail(t, "💥 mark as failed") } + Goexit := func(t TestingT) { fmt.Println("💥 goexit"); runtime.Goexit() } for _, tt := range []test{ // Test cases that must exit immediately after the first call to the condition. @@ -892,6 +864,10 @@ func TestFailInsideEventually(t *testing.T) { // to the condition are expected, because assert.Fail does not stop the execution of the condition. {"assert.Fail must not stop if told not to", returnNoStop, AssertFail, mustNotStop}, + // Test cases that call runtime.Goexit, which must stop immediately. + {"runtime.Goexit must stop", returnStop, Goexit, mustStop}, + {"runtime.Goexit must stop even if told not to", returnNoStop, Goexit, mustStop}, + // Make sure to update the assertions in TestFailInsideEventuallyViaCommandLine // accordingly if you change the number of tests here. } { From 8501b5280dfabb54a5e6ddb98097a1df131142e0 Mon Sep 17 00:00:00 2001 From: Uwe Jugel Date: Thu, 16 Oct 2025 12:39:07 +0200 Subject: [PATCH 3/4] extend test --- require/requirements_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/require/requirements_test.go b/require/requirements_test.go index 9893a35de..aaa1791f3 100644 --- a/require/requirements_test.go +++ b/require/requirements_test.go @@ -799,9 +799,11 @@ func TestFailInsideEventuallyViaCommandLine(t *testing.T) { Name: "TestFailInsideEventually", ExpectFailure: true, ExpectedErrorLogs: 0, // no unexpected errors must be logged - ExpectedSuccessLogs: 6, // 6 tests must "pass" by logging the expected messages + ExpectedSuccessLogs: 8, // 6 tests must "pass" by logging the expected messages ExpectedLineMatches: map[string]int{ - "Condition never satisfied": 5 * 1, // 5x early exit, 1x assert + return true + "Condition never satisfied": 1, // 1x eventually timeout + "Condition failed": 4, // 2x require.Fail + 2x goexit + "Condition panicked": 2, // 2x panic test "💥 mark as failed": 1 + 3, // 2x assert func (one called 3x) "💥 fail now": 2 * 1, // 2x require func "💥 goexit": 2 * 1, // 2x goexit func From 2435a43ffadc056ea85fb308b0430c6325ab4335 Mon Sep 17 00:00:00 2001 From: Uwe Jugel Date: Thu, 16 Oct 2025 13:25:06 +0200 Subject: [PATCH 4/4] make helpers helpers --- assert/commandline.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assert/commandline.go b/assert/commandline.go index 369190429..0c1b35ac9 100644 --- a/assert/commandline.go +++ b/assert/commandline.go @@ -92,9 +92,11 @@ func (s CommandLineTestSpec) assertLineMatches(t *testing.T, lines []string) boo // // Also see "TestFailInsideEventuallyViaCommandLine" example in require/requirements_test.go func CommandLineTest(t *testing.T, spec CommandLineTestSpec) bool { + t.Helper() spec = spec.withDefaults(t) t.Run(spec.Name, func(t *testing.T) { + t.Helper() t.Setenv(spec.Name, "1") // signal to the test to run args := append([]string{"test"}, spec.Args...) args = append(args, spec.PackagePath)