Collect PackageOutputs

Author: lukaszcl

tools/flakeguard/reports/reports.go

@@ -21,6 +21,7 @@ type TestResult struct {
 	Runs                int
 	Outputs             []string  // Stores outputs for a test
 	Durations           []float64 // Stores elapsed time in seconds for each run of the test
+	PackageOutputs      []string  // Stores package-level outputs
 }
 
 // FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.

tools/flakeguard/runner/runner.go

@@ -119,64 +119,83 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
 				return nil, fmt.Errorf("failed to parse json test output: %s, err: %w", scanner.Text(), err)
 			}
 
-			// Determine the key based on whether Test is empty
-			var key string
+			// Only create TestResult for test-level entries
+			var result *reports.TestResult
 			if entry.Test != "" {
-				key = entry.Package + "/" + entry.Test
-			} else {
-				key = entry.Package // Package-level key
-			}
-
-			if _, exists := testDetails[key]; !exists {
-				testName := entry.Test
-				if testName == "" {
-					testName = "Package Level"
-				}
-				testDetails[key] = &reports.TestResult{
-					TestName:    testName,
-					TestPackage: entry.Package,
-					Runs:        0,
-					PassRatio:   0,
-					Outputs:     []string{},
+				// Determine the key
+				key := entry.Package + "/" + entry.Test
+
+				if _, exists := testDetails[key]; !exists {
+					testDetails[key] = &reports.TestResult{
+						TestName:       entry.Test,
+						TestPackage:    entry.Package,
+						Runs:           0,
+						PassRatio:      0,
+						Outputs:        []string{},
+						PackageOutputs: []string{},
+					}
 				}
+				result = testDetails[key]
 			}
 
-			result := testDetails[key]
-
-			// Collect test outputs
+			// Collect outputs
 			if entry.Output != "" {
-				result.Outputs = append(result.Outputs, entry.Output)
+				if entry.Test != "" {
+					// Test-level output
+					result.Outputs = append(result.Outputs, entry.Output)
+				} else {
+					// Package-level output
+					// Append to PackageOutputs of all TestResults in the same package
+					for _, res := range testDetails {
+						if res.TestPackage == entry.Package {
+							res.PackageOutputs = append(res.PackageOutputs, entry.Output)
+						}
+					}
+				}
 			}
 
 			switch entry.Action {
 			case "run":
-				result.Runs++
+				if entry.Test != "" {
+					result.Runs++
+				}
 			case "pass":
-				result.PassRatio = (result.PassRatio*float64(result.Runs-1) + 1) / float64(result.Runs)
-				result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
-				result.Durations = append(result.Durations, entry.Elapsed)
+				if entry.Test != "" {
+					result.PassRatio = (result.PassRatio*float64(result.Runs-1) + 1) / float64(result.Runs)
+					result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
+					result.Durations = append(result.Durations, entry.Elapsed)
+				}
 			case "fail":
-				result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
-				result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
-				result.Durations = append(result.Durations, entry.Elapsed)
+				if entry.Test != "" {
+					result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
+					result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
+					result.Durations = append(result.Durations, entry.Elapsed)
+				}
 			case "output":
 				// Output already handled above
 				if panicRe.MatchString(entry.Output) {
 					if entry.Test != "" {
 						// Test-level panic
 						result.Panicked = true
+						result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
+						result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
+						result.Durations = append(result.Durations, entry.Elapsed)
 					} else {
 						// Package-level panic
-						result.PackagePanicked = true
+						// Mark PackagePanicked for all TestResults in the package
+						for _, res := range testDetails {
+							if res.TestPackage == entry.Package {
+								res.PackagePanicked = true
+							}
+						}
 					}
-					result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
-					result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
-					result.Durations = append(result.Durations, entry.Elapsed)
 				}
 			case "skip":
-				result.Skipped = true
-				result.Runs++
-				result.Durations = append(result.Durations, entry.Elapsed)
+				if entry.Test != "" {
+					result.Skipped = true
+					result.Runs++
+					result.Durations = append(result.Durations, entry.Elapsed)
+				}
 			}
 		}