Fixes flakeguard bug that silently absorbs build errors (#1739)

Author: kalverra

tools/flakeguard/runner/runner.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -25,8 +26,10 @@ const (
 )
 
 var (
-	startPanicRe = regexp.MustCompile(`^panic:`)
-	startRaceRe  = regexp.MustCompile(`^WARNING: DATA RACE`)
+	startPanicRe         = regexp.MustCompile(`^panic:`)
+	startRaceRe          = regexp.MustCompile(`^WARNING: DATA RACE`)
+	buildErr             = errors.New("failed to build test code")
+	failedToShowBuildErr = errors.New("flakeguard failed to show build errors")
 )
 
 // Runner describes the test run parameters and raw test outputs
@@ -297,6 +300,19 @@ func (r *Runner) parseTestResults(runPrefix string, runCount int) ([]reports.Tes
 				context := append(precedingLines, followingLines...)
 				return nil, fmt.Errorf("failed to parse json test output near lines:\n%s\nerror: %w", strings.Join(context, "\n"), err)
 			}
+			if entryLine.Action == "build-fail" {
+				_, err := file.Seek(0, 0)
+				if err != nil {
+					return nil, fmt.Errorf("%w: %w", failedToShowBuildErr, buildErr)
+				}
+				// Print all build errors
+				buildErrs, err := io.ReadAll(file)
+				if err != nil {
+					return nil, fmt.Errorf("%w: %w", failedToShowBuildErr, buildErr)
+				}
+				fmt.Println(string(buildErrs))
+				return nil, buildErr
+			}
 
 			var result *reports.TestResult
 			if entryLine.Test != "" {

tools/flakeguard/runner/runner_test.go

@@ -41,6 +41,24 @@ type expectedTestResult struct {
 	seen bool
 }
 
+func TestNoCompileTests(t *testing.T) {
+	// Test that we are not swallowing test compilation errors
+	t.Parallel()
+
+	runner := Runner{
+		ProjectPath:    "./",
+		Verbose:        true,
+		RunCount:       1,
+		GoTestRaceFlag: false,
+		FailFast:       false,
+	}
+
+	_, err := runner.RunTestPackages([]string{"./example_bad_test_package"})
+	require.Error(t, err)
+	require.ErrorIs(t, err, buildErr, "expected a compile error")
+	require.NotErrorIs(t, err, failedToShowBuildErr, "should be able to print out build errors")
+}
+
 func TestPrettyProjectPath(t *testing.T) {
 	t.Parallel()