TEST: add indirect test for different condition failures

assert/commandline.go

@@ -0,0 +1,147 @@
+package assert
+
+import (
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+// CommandLineTestSpec specifies a test to be run via 'go test' command line.
+//
+// ⚠️ PREVIEW: This API might change in the future! ⚠️
+type CommandLineTestSpec struct {
+	Name        string   // test name, e.g. "TestFailInsideEventually"
+	PackagePath string   // package path, e.g. "./require", default: "./..."
+	Args        []string // command line arguments, if empty use default args
+
+	ExpectFailure       bool // if true, the test is expected to fail (i.e. return a non-zero exit code)
+	ExpectedErrorLogs   int  // number of unexpected errors
+	ExpectedSuccessLogs int  // number of expected successful tests
+
+	ExpectedLineMatches map[string]int // strings that must be found in the output lines (each line is counted at most once per match)
+
+	ErrorMarker   string // string that marks an unexpected error in the output, default: "❌"
+	SuccessMarker string // string that marks a successful test in the output, default: "✅"
+}
+
+func (s CommandLineTestSpec) withDefaults(t TestingT) CommandLineTestSpec {
+	if s.Name == "" {
+		FailNow(t, "testSpec.name must be set")
+	}
+
+	if s.PackagePath == "" {
+		s.PackagePath = "./..."
+	}
+	if len(s.Args) == 0 {
+		s.Args = []string{"-v", "-race", "-count=1", "-run", fmt.Sprintf("^%s$", s.Name)}
+	}
+	if s.ErrorMarker == "" {
+		s.ErrorMarker = "❌"
+	}
+	if s.SuccessMarker == "" {
+		s.SuccessMarker = "✅"
+	}
+	return s
+}
+
+func (s CommandLineTestSpec) assertLineMatches(t *testing.T, lines []string) bool {
+	t.Helper()
+	if s.ExpectedLineMatches == nil {
+		return t.Failed()
+	}
+	for match, expCount := range s.ExpectedLineMatches {
+		found := 0
+		for _, line := range lines {
+			if strings.Contains(line, match) {
+				found++
+			}
+		}
+		if Equal(t, expCount, found, "Expected line match %q not found the expected number of times", match) {
+			t.Log("Found expected line match", fmt.Sprintf("%q", match), "the expected number of times:", expCount)
+		}
+	}
+	return !t.Failed()
+}
+
+// CommandLineTest runs 'go test' with the specified arguments and checks the output.
+//
+// ⚠️ PREVIEW: This API might change in the future! ⚠️
+//
+// Use it run tests that must fail, panic, or behave in a special way that would break the
+// current test process. It returns true if the test passed, false if it failed.
+// Usage example:
+//
+//	func TestSomethingViaCommandLine(t *testing.T) {
+//		spec := assert.CommandLineTestSpec{
+//			Name:                "TestSomething",
+//			ExpectedErrorLogs:   1,
+//			ExpectedSuccessLogs: 1,
+//		}
+//		assert.CommandLineTest(t, spec)
+//	}
+//
+//	func TestSomething(t *testing.T) {
+//		if os.Getenv("TestSomething") != "1" {
+//			t.Skip("skipping test that must be run via CommandLineTest")
+//		}
+//		t.Log("✅ <-- default success marker")
+//		t.Error("❌ <-- default failure marker")
+//	}
+//
+// Also see "TestFailInsideEventuallyViaCommandLine" example in require/requirements_test.go
+func CommandLineTest(t *testing.T, spec CommandLineTestSpec) bool {
+	t.Helper()
+	spec = spec.withDefaults(t)
+
+	t.Run(spec.Name, func(t *testing.T) {
+		t.Helper()
+		t.Setenv(spec.Name, "1") // signal to the test to run
+		args := append([]string{"test"}, spec.Args...)
+		args = append(args, spec.PackagePath)
+		cmd := exec.Command("go", args...)
+
+		out, err := cmd.CombinedOutput()
+		if spec.ExpectFailure {
+			Error(t, err, "'go test' command must return an error due to expected failure")
+		} else {
+			NoError(t, err, "'go test' command must not return an error")
+		}
+
+		extractTestNameExp := regexp.MustCompile(spec.Name + `[^ ]*`)
+
+		observedErrors := 0
+		observedSuccesses := 0
+		name := ""
+		lines := strings.Split(string(out), "\n")
+
+		for _, line := range lines {
+			// line = strings.TrimSpace(line)
+			if strings.HasPrefix(strings.TrimSpace(line), "=== RUN   ") {
+				name = extractTestNameExp.FindString(line)
+				t.Log("Running test", name)
+			}
+
+			if strings.Contains(line, spec.ErrorMarker) {
+				// fmt.Println(name, line, fmt.Sprintf("<-- matches error marker %q", spec.ErrorMarker))
+				t.Logf("Test %s logged unexpected error:", name)
+				fmt.Println(line)
+				observedErrors++
+			}
+			if strings.Contains(line, spec.SuccessMarker) {
+				// fmt.Println(name, line, fmt.Sprintf("<-- matches success marker %q", spec.SuccessMarker))
+				t.Logf("Test %s logged expected success:", name)
+				fmt.Println(line)
+				observedSuccesses++
+			}
+		}
+
+		spec.assertLineMatches(t, lines)
+
+		Equal(t, spec.ExpectedErrorLogs, observedErrors, "Unexpected number of error markers, see output")
+		Equal(t, spec.ExpectedSuccessLogs, observedSuccesses, "Unexpected number of success markers, see output")
+	})
+
+	return !t.Failed()
+}

require/requirements_test.go

@@ -5,9 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
-	"regexp"
-	"strings"
+	"runtime"
 	"testing"
 	"time"
 
@@ -797,53 +795,20 @@ func TestEventuallyWithTTrue(t *testing.T) {
 
 func TestFailInsideEventuallyViaCommandLine(t *testing.T) {
 	t.Setenv("TestFailInsideEventually", "1")
-	cmd := exec.Command("go", "test", "-v", "-race", "-count=1", "-run", "^TestFailInsideEventually$")
-	out, err := cmd.CombinedOutput()
-	assert.Error(t, err, "go test for TestFailInsideEventually must fail")
-	finishedTests := 0
-	observedErrors := 0
-	observedConditionFailures := 0
-	observedPanics := 0
-	extractTestNameExp := regexp.MustCompile(`TestFailInsideEventuallyViaCommandLine[^ ]*`)
-	name := ""
-	for _, line := range strings.Split(string(out), "\n") {
-		line = strings.TrimSpace(line)
-		if strings.HasPrefix(line, "=== RUN   ") {
-			name = extractTestNameExp.FindString(line)
-			fmt.Println(line)
-		}
-
-		if strings.Contains(line, "❌") {
-			fmt.Println(name, line, "<-- unexpected error")
-			observedErrors++
-		}
-		if strings.Contains(line, "✅ FINISHED") {
-			fmt.Println(name, line, "<-- expected successful test")
-			finishedTests++
-		}
-		if strings.Contains(line, "Error:") && strings.Contains(line, "Condition") {
-			fmt.Println(name, line, "<-- expected 'Condition' message")
-			observedConditionFailures++
-		}
-		if strings.Contains(line, "Panic in condition:") {
-			fmt.Println(name, line, "<-- expected panic message")
-			observedPanics++
-		}
-	}
-
-	assert.Equal(t, 0, observedErrors, "Unexpected errors detected, see output")
-
-	// There are 6 tests that are expected to fail.
-	// If you change the number of tests in TestFailInsideEventually, please update this number accordingly.
-	assert.Equal(t, 6, finishedTests, "Expected number of FINISHED tests not found")
-
-	// There are 5 tests where the condition is never satisfied.
-	// One test uses assert.Fail but eventually returns true and thus satisfies the condition.
-	// If you change the number of tests in TestFailInsideEventually, please update this number accordingly.
-	assert.Equal(t, 5, observedConditionFailures, "Expected number of 'Condition' messages not found, see output")
-
-	// There are 2 tests that panic, so we expect 2 panic messages.
-	assert.Equal(t, 2, observedPanics, "Missed expected panics, see output")
+	assert.CommandLineTest(t, assert.CommandLineTestSpec{
+		Name:                "TestFailInsideEventually",
+		ExpectFailure:       true,
+		ExpectedErrorLogs:   0, // no unexpected errors must be logged
+		ExpectedSuccessLogs: 8, // 6 tests must "pass" by logging the expected messages
+		ExpectedLineMatches: map[string]int{
+			"Condition never satisfied": 1,     // 1x eventually timeout
+			"Condition failed":          4,     // 2x require.Fail  + 2x goexit
+			"Condition panicked":        2,     // 2x panic test
+			"💥 mark as failed":          1 + 3, // 2x assert func (one called 3x)
+			"💥 fail now":                2 * 1, // 2x require func
+			"💥 goexit":                  2 * 1, // 2x goexit func
+		},
+	})
 }
 
 func TestFailInsideEventually(t *testing.T) {
@@ -863,10 +828,9 @@ func TestFailInsideEventually(t *testing.T) {
 	// The Bug:
 	// Calling require.Fail (or similar) inside require.Eventually will prevent the 'condition'
 	// to exit with a result. The channel assignment in the assert.Eventually will block
-	// and hang the test until the timeout is reached. There was is no other way to wait
+	// and hang the test until the timeout is reached. There was no other way to wait
 	// for the unclean exit. The changes to assert.Eventually committed with this test
-	// fix this issue, by also waiting for an unclean exit of the condition and moreover
-	// by handling panics inside the condition gracefully.
+	// fix this issue, by also waiting for an unclean exit of the condition.
 
 	// How to read the test results:
 	// - See [TestFailInsideEventuallyViaCommandLine], which automates this
@@ -890,6 +854,7 @@ func TestFailInsideEventually(t *testing.T) {
 
 	RequireFail := func(t TestingT) { Fail(t, "💥 fail now") }
 	AssertFail := func(t TestingT) { assert.Fail(t, "💥 mark as failed") }
+	Goexit := func(t TestingT) { fmt.Println("💥 goexit"); runtime.Goexit() }
 	Panic := func(_ TestingT) { panic("💥 panicking now") }
 
 	for _, tt := range []test{
@@ -902,6 +867,9 @@ func TestFailInsideEventually(t *testing.T) {
 		// to the condition are expected, because assert.Fail does not stop the execution of the condition.
 		{"assert.Fail must not stop if told not to", returnNoStop, AssertFail, mustNotStop},
 
+		// Test cases that call runtime.Goexit, which must stop immediately.
+		{"runtime.Goexit must stop", returnStop, Goexit, mustStop},
+		{"runtime.Goexit must stop even if told not to", returnNoStop, Goexit, mustStop},
 		// Panics must always stop, because they are not expected and indicate a bug in the code.
 		{"panic must stop", returnStop, Panic, mustStop},
 		{"panic must stop even if told not to", returnNoStop, Panic, mustStop},