Save test outputs and logs to separate file

Author: lukaszcl

tools/flakeguard/cmd/aggregate.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"fmt"
 	"log"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
@@ -13,27 +12,24 @@ var AggregateAllCmd = &cobra.Command{
 	Short: "Aggregate all test results and output them to a file",
 	Run: func(cmd *cobra.Command, args []string) {
 		resultsFolderPath, _ := cmd.Flags().GetString("results-path")
-		outputPath, _ := cmd.Flags().GetString("output-json")
+		outputResultsPath, _ := cmd.Flags().GetString("output-results")
+		outputLogsPath, _ := cmd.Flags().GetString("output-logs")
 
 		// Aggregate all test results
 		allResults, err := reports.AggregateTestResults(resultsFolderPath)
 		if err != nil {
 			log.Fatalf("Error aggregating results: %v", err)
 		}
 
-		// Output all results to JSON file
-		if outputPath != "" && len(allResults) > 0 {
-			if err := saveResults(outputPath, allResults); err != nil {
-				log.Fatalf("Error writing aggregated results to file: %v", err)
-			}
-			fmt.Printf("Aggregated test results saved to %s\n", outputPath)
-		} else {
-			fmt.Println("No test results found.")
+		// Output all results to JSON files
+		if len(allResults) > 0 {
+			reports.SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath, allResults)
 		}
 	},
 }
 
 func init() {
 	AggregateAllCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
-	AggregateAllCmd.Flags().String("output-json", "all_tests.json", "Path to output the aggregated test results in JSON format")
+	AggregateFailedCmd.Flags().String("output-results", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
+	AggregateFailedCmd.Flags().String("output-logs", "failed_logs.json", "Path to output the filtered failed test logs in JSON format")
 }

tools/flakeguard/cmd/aggregate_failed.go

@@ -1,10 +1,7 @@
 package cmd
 
 import (
-	"encoding/json"
-	"fmt"
 	"log"
-	"os"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 	"github.com/spf13/cobra"
@@ -15,7 +12,8 @@ var AggregateFailedCmd = &cobra.Command{
 	Short: "Aggregate all test results, then filter and output only failed tests based on a threshold",
 	Run: func(cmd *cobra.Command, args []string) {
 		resultsFolderPath, _ := cmd.Flags().GetString("results-path")
-		outputPath, _ := cmd.Flags().GetString("output-json")
+		outputResultsPath, _ := cmd.Flags().GetString("output-results")
+		outputLogsPath, _ := cmd.Flags().GetString("output-logs")
 		threshold, _ := cmd.Flags().GetFloat64("threshold")
 		minPassRatio, _ := cmd.Flags().GetFloat64("min-pass-ratio")
 
@@ -33,30 +31,17 @@ var AggregateFailedCmd = &cobra.Command{
 			}
 		}
 
-		// Output failed results to JSON file
-		if outputPath != "" && len(failedResults) > 0 {
-			if err := saveResults(outputPath, failedResults); err != nil {
-				log.Fatalf("Error writing failed results to file: %v", err)
-			}
-			fmt.Printf("Filtered failed test results saved to %s\n", outputPath)
-		} else {
-			fmt.Println("No failed tests found based on the specified threshold and min pass ratio.")
+		// Output results to JSON files
+		if len(failedResults) > 0 {
+			reports.SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath, failedResults)
 		}
 	},
 }
 
 func init() {
 	AggregateFailedCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
-	AggregateFailedCmd.Flags().String("output-json", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
+	AggregateFailedCmd.Flags().String("output-results", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
+	AggregateFailedCmd.Flags().String("output-logs", "failed_logs.json", "Path to output the filtered failed test logs in JSON format")
 	AggregateFailedCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed")
 	AggregateFailedCmd.Flags().Float64("min-pass-ratio", 0.001, "Minimum pass ratio for considering a test as flaky. Used to distinguish between tests that are truly flaky (with inconsistent results) and those that are consistently failing.")
 }
-
-// Helper function to save results to JSON file
-func saveResults(filePath string, results []reports.TestResult) error {
-	data, err := json.MarshalIndent(results, "", "  ")
-	if err != nil {
-		return fmt.Errorf("error marshaling results: %v", err)
-	}
-	return os.WriteFile(filePath, data, 0644)
-}

tools/flakeguard/reports/reports.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"path/filepath"
 	"sort"
@@ -139,3 +140,87 @@ func PrintTests(tests []TestResult, w io.Writer) {
 		fmt.Fprintf(w, "Outputs:\n%s\n", strings.Join(test.Outputs, ""))
 	}
 }
+
+// Helper function to save filtered results and logs to specified paths
+func SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath string, failedResults []TestResult) {
+	if outputResultsPath != "" {
+		if err := saveResults(outputResultsPath, failedResults); err != nil {
+			log.Fatalf("Error writing failed results to file: %v", err)
+		}
+		fmt.Printf("Filtered failed test results saved to %s\n", outputResultsPath)
+	} else {
+		fmt.Println("No failed tests found based on the specified threshold and min pass ratio.")
+	}
+
+	if outputLogsPath != "" {
+		if err := saveTestOutputs(outputLogsPath, failedResults); err != nil {
+			log.Fatalf("Error writing failed logs to file: %v", err)
+		}
+		fmt.Printf("Filtered failed test logs saved to %s\n", outputLogsPath)
+	}
+}
+
+// Helper function to save results to JSON file
+func saveResults(filePath string, results []TestResult) error {
+	// Define a struct type without Outputs and PackageOutputs
+	type filteredTestResult struct {
+		TestName            string
+		TestPackage         string
+		Panicked            bool
+		PackagePanicked     bool
+		PassRatio           float64
+		PassRatioPercentage string
+		Skipped             bool
+		Runs                int
+		Durations           []float64
+	}
+
+	var filteredResults []filteredTestResult
+	for _, r := range results {
+		filteredResults = append(filteredResults, filteredTestResult{
+			TestName:            r.TestName,
+			TestPackage:         r.TestPackage,
+			Panicked:            r.Panicked,
+			PackagePanicked:     r.PackagePanicked,
+			PassRatio:           r.PassRatio,
+			PassRatioPercentage: r.PassRatioPercentage,
+			Skipped:             r.Skipped,
+			Runs:                r.Runs,
+			Durations:           r.Durations,
+		})
+	}
+
+	data, err := json.MarshalIndent(filteredResults, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling results: %v", err)
+	}
+	return os.WriteFile(filePath, data, 0644)
+}
+
+// Helper function to save test names, packages, and outputs to JSON file
+func saveTestOutputs(filePath string, results []TestResult) error {
+	// Define a struct type with only the required fields
+	type outputOnlyResult struct {
+		TestName       string
+		TestPackage    string
+		Outputs        []string
+		PackageOutputs []string
+	}
+
+	// Convert results to the filtered struct
+	var outputResults []outputOnlyResult
+	for _, r := range results {
+		outputResults = append(outputResults, outputOnlyResult{
+			TestName:       r.TestName,
+			TestPackage:    r.TestPackage,
+			Outputs:        r.Outputs,
+			PackageOutputs: r.PackageOutputs,
+		})
+	}
+
+	data, err := json.MarshalIndent(outputResults, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling outputs: %v", err)
+	}
+	return os.WriteFile(filePath, data, 0644)
+}