Add cmd to aggregate all test results

Author: lukaszcl

tools/flakeguard/cmd/aggregate.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
+	"github.com/spf13/cobra"
+)
+
+var AggregateAllCmd = &cobra.Command{
+	Use:   "aggregate-all",
+	Short: "Aggregate all test results and output them to a file",
+	Run: func(cmd *cobra.Command, args []string) {
+		resultsFolderPath, _ := cmd.Flags().GetString("results-path")
+		outputPath, _ := cmd.Flags().GetString("output-json")
+
+		// Aggregate all test results
+		allResults, err := reports.AggregateTestResults(resultsFolderPath)
+		if err != nil {
+			log.Fatalf("Error aggregating results: %v", err)
+		}
+
+		// Output all results to JSON file
+		if outputPath != "" && len(allResults) > 0 {
+			if err := saveResults(outputPath, allResults); err != nil {
+				log.Fatalf("Error writing aggregated results to file: %v", err)
+			}
+			fmt.Printf("Aggregated test results saved to %s\n", outputPath)
+		} else {
+			fmt.Println("No test results found.")
+		}
+	},
+}
+
+func init() {
+	AggregateAllCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
+	AggregateAllCmd.Flags().String("output-json", "all_tests.json", "Path to output the aggregated test results in JSON format")
+}

tools/flakeguard/cmd/aggregate_failed.go

@@ -10,44 +10,44 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var AggregateFailedTestsCmd = &cobra.Command{
+var AggregateFailedCmd = &cobra.Command{
 	Use:   "aggregate-failed",
-	Short: "Aggregate test results and output only failed tests based on a threshold",
+	Short: "Aggregate all test results, then filter and output only failed tests based on a threshold",
 	Run: func(cmd *cobra.Command, args []string) {
-		folderPath, _ := cmd.Flags().GetString("folder-path")
+		resultsFolderPath, _ := cmd.Flags().GetString("results-path")
 		outputPath, _ := cmd.Flags().GetString("output-json")
 		threshold, _ := cmd.Flags().GetFloat64("threshold")
 
-		// Aggregate and merge results from the specified folder
-		allResults, err := reports.AggregateTestResults(folderPath)
+		// Aggregate all test results
+		allResults, err := reports.AggregateTestResults(resultsFolderPath)
 		if err != nil {
 			log.Fatalf("Error aggregating results: %v", err)
 		}
 
-		// Filter failed tests based on threshold
-		failedTests := reports.FilterFailedTests(allResults, threshold)
-
-		// Format PassRatio as a percentage for display
-		for i := range failedTests {
-			failedTests[i].PassRatioPercentage = fmt.Sprintf("%.0f%%", failedTests[i].PassRatio*100)
+		// Filter to only include failed tests based on threshold
+		var failedResults []reports.TestResult
+		for _, result := range allResults {
+			if result.PassRatio < threshold && !result.Skipped {
+				failedResults = append(failedResults, result)
+			}
 		}
 
-		// Output failed tests to JSON file
-		if outputPath != "" && len(failedTests) > 0 {
-			if err := saveResults(outputPath, failedTests); err != nil {
-				log.Fatalf("Error writing failed tests to file: %v", err)
+		// Output failed results to JSON file
+		if outputPath != "" && len(failedResults) > 0 {
+			if err := saveResults(outputPath, failedResults); err != nil {
+				log.Fatalf("Error writing failed results to file: %v", err)
 			}
-			fmt.Printf("Aggregated failed test results saved to %s\n", outputPath)
+			fmt.Printf("Filtered failed test results saved to %s\n", outputPath)
 		} else {
 			fmt.Println("No failed tests found based on the specified threshold.")
 		}
 	},
 }
 
 func init() {
-	AggregateFailedTestsCmd.Flags().String("folder-path", "testresult/", "Path to the folder containing JSON test result files")
-	AggregateFailedTestsCmd.Flags().String("output-json", "failed_tests.json", "Path to output the aggregated failed test results in JSON format")
-	AggregateFailedTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed")
+	AggregateFailedCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
+	AggregateFailedCmd.Flags().String("output-json", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
+	AggregateFailedCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed")
 }
 
 // Helper function to save results to JSON file

tools/flakeguard/main.go

@@ -28,7 +28,8 @@ func init() {
 
 	rootCmd.AddCommand(cmd.FindTestsCmd)
 	rootCmd.AddCommand(cmd.RunTestsCmd)
-	rootCmd.AddCommand(cmd.AggregateFailedTestsCmd)
+	rootCmd.AddCommand(cmd.AggregateAllCmd)
+	rootCmd.AddCommand(cmd.AggregateFailedCmd)
 }
 
 func main() {

tools/flakeguard/reports/reports.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 )
 
@@ -53,7 +54,7 @@ func FilterSkippedTests(results []TestResult) []TestResult {
 	return skippedTests
 }
 
-// Helper function to aggregate all JSON test results from a folder
+// AggregateTestResults aggregates all JSON test results.
 func AggregateTestResults(folderPath string) ([]TestResult, error) {
 	// Map to hold unique tests based on their TestName and TestPackage
 	testMap := make(map[string]TestResult)
@@ -83,15 +84,14 @@ func AggregateTestResults(folderPath string) ([]TestResult, error) {
 					existingResult.Durations = append(existingResult.Durations, result.Durations...)
 					existingResult.Outputs = append(existingResult.Outputs, result.Outputs...)
 
-					// Calculate total successful runs and aggregate pass ratio
+					// Calculate total successful runs for correct pass ratio calculation
 					successfulRuns := existingResult.PassRatio*float64(existingResult.Runs) + result.PassRatio*float64(result.Runs)
 					existingResult.Runs = totalRuns
 					existingResult.PassRatio = successfulRuns / float64(totalRuns)
 					existingResult.Skipped = existingResult.Skipped && result.Skipped // Mark as skipped only if all occurrences are skipped
 
 					// Update the map with the aggregated result
 					testMap[key] = existingResult
-
 				} else {
 					// Add new entry to the map
 					testMap[key] = result
@@ -110,6 +110,12 @@ func AggregateTestResults(folderPath string) ([]TestResult, error) {
 		result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
 		aggregatedResults = append(aggregatedResults, result)
 	}
+
+	// Sort by PassRatio in ascending order
+	sort.Slice(aggregatedResults, func(i, j int) bool {
+		return aggregatedResults[i].PassRatio < aggregatedResults[j].PassRatio
+	})
+
 	return aggregatedResults, nil
 }
 

tools/flakeguard/reports/reports_test.go

@@ -4,9 +4,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 )
@@ -115,14 +115,24 @@ func TestPrintTests(t *testing.T) {
 	}
 }
 
-// Helper function to write temporary JSON files for testing
+// Sorts TestResult slice by TestName and TestPackage for consistent comparison
+func sortTestResults(results []TestResult) {
+	sort.Slice(results, func(i, j int) bool {
+		if results[i].TestName == results[j].TestName {
+			return results[i].TestPackage < results[j].TestPackage
+		}
+		return results[i].TestName < results[j].TestName
+	})
+}
+
+// Helper function to write a JSON file for testing
 func writeTempJSONFile(t *testing.T, dir string, filename string, data interface{}) string {
 	filePath := filepath.Join(dir, filename)
 	fileData, err := json.Marshal(data)
 	if err != nil {
 		t.Fatalf("Failed to marshal JSON: %v", err)
 	}
-	if err := ioutil.WriteFile(filePath, fileData, 0644); err != nil {
+	if err := os.WriteFile(filePath, fileData, 0644); err != nil {
 		t.Fatalf("Failed to write JSON file: %v", err)
 	}
 	return filePath
@@ -143,7 +153,7 @@ func TestAggregateTestResults(t *testing.T) {
 		expectedOutput []TestResult
 	}{
 		{
-			description: "Unique test results, no aggregation",
+			description: "Unique test results without aggregation",
 			inputFiles: []interface{}{
 				[]TestResult{
 					{
@@ -154,6 +164,7 @@ func TestAggregateTestResults(t *testing.T) {
 						Skipped:             false,
 						Runs:                2,
 						Durations:           []float64{0.01, 0.02},
+						Outputs:             []string{"Output1", "Output2"},
 					},
 				},
 				[]TestResult{
@@ -165,6 +176,7 @@ func TestAggregateTestResults(t *testing.T) {
 						Skipped:             false,
 						Runs:                4,
 						Durations:           []float64{0.05, 0.05, 0.05, 0.05},
+						Outputs:             []string{"Output3", "Output4", "Output5", "Output6"},
 					},
 				},
 			},
@@ -177,6 +189,7 @@ func TestAggregateTestResults(t *testing.T) {
 					Skipped:             false,
 					Runs:                2,
 					Durations:           []float64{0.01, 0.02},
+					Outputs:             []string{"Output1", "Output2"},
 				},
 				{
 					TestName:            "TestB",
@@ -186,11 +199,12 @@ func TestAggregateTestResults(t *testing.T) {
 					Skipped:             false,
 					Runs:                4,
 					Durations:           []float64{0.05, 0.05, 0.05, 0.05},
+					Outputs:             []string{"Output3", "Output4", "Output5", "Output6"},
 				},
 			},
 		},
 		{
-			description: "Duplicate test results, aggregation of PassRatio and Durations",
+			description: "Duplicate test results with aggregation",
 			inputFiles: []interface{}{
 				[]TestResult{
 					{
@@ -201,6 +215,7 @@ func TestAggregateTestResults(t *testing.T) {
 						Skipped:             false,
 						Runs:                2,
 						Durations:           []float64{0.1, 0.1},
+						Outputs:             []string{"Output7", "Output8"},
 					},
 				},
 				[]TestResult{
@@ -212,6 +227,7 @@ func TestAggregateTestResults(t *testing.T) {
 						Skipped:             false,
 						Runs:                2,
 						Durations:           []float64{0.2, 0.2},
+						Outputs:             []string{"Output9", "Output10"},
 					},
 				},
 			},
@@ -224,6 +240,7 @@ func TestAggregateTestResults(t *testing.T) {
 					Skipped:             false,
 					Runs:                4,
 					Durations:           []float64{0.1, 0.1, 0.2, 0.2},
+					Outputs:             []string{"Output7", "Output8", "Output9", "Output10"},
 				},
 			},
 		},
@@ -239,6 +256,7 @@ func TestAggregateTestResults(t *testing.T) {
 						Skipped:             true,
 						Runs:                3,
 						Durations:           []float64{0.1, 0.2, 0.1},
+						Outputs:             []string{"Output11", "Output12", "Output13"},
 					},
 				},
 				[]TestResult{
@@ -250,6 +268,7 @@ func TestAggregateTestResults(t *testing.T) {
 						Skipped:             true,
 						Runs:                2,
 						Durations:           []float64{0.15, 0.15},
+						Outputs:             []string{"Output14", "Output15"},
 					},
 				},
 			},
@@ -262,6 +281,7 @@ func TestAggregateTestResults(t *testing.T) {
 					Skipped:             true, // Should remain true as all runs are skipped
 					Runs:                5,
 					Durations:           []float64{0.1, 0.2, 0.1, 0.15, 0.15},
+					Outputs:             []string{"Output11", "Output12", "Output13", "Output14", "Output15"},
 				},
 			},
 		},
@@ -280,6 +300,10 @@ func TestAggregateTestResults(t *testing.T) {
 				t.Fatalf("AggregateTestResults failed: %v", err)
 			}
 
+			// Sort both result and expectedOutput for consistent comparison
+			sortTestResults(result)
+			sortTestResults(tc.expectedOutput)
+
 			// Compare the result with the expected output
 			if len(result) != len(tc.expectedOutput) {
 				t.Fatalf("Expected %d results, got %d", len(tc.expectedOutput), len(result))
@@ -299,6 +323,9 @@ func TestAggregateTestResults(t *testing.T) {
 				if len(got.Durations) != len(expected.Durations) {
 					t.Errorf("Result %d - expected %d durations, got %d", i, len(expected.Durations), len(got.Durations))
 				}
+				if len(got.Outputs) != len(expected.Outputs) {
+					t.Errorf("Result %d - expected %d outputs, got %d", i, len(expected.Outputs), len(got.Outputs))
+				}
 			}
 		})
 	}