Makes TestReport Aggregation More Flexible (#1429)

* Makes TestReport aggregation more flexible

* Better name

* Better printing

* Remove markdown only formatting

* Markdown optional

Author: kalverra

framework/components/clnode/clnode.go

@@ -5,17 +5,18 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"text/template"
+	"time"
+
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/go-connections/nat"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/postgres"
 	tc "github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
-	"os"
-	"path/filepath"
-	"sync"
-	"text/template"
-	"time"
 )
 
 const (
@@ -114,10 +115,10 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 	ctx := context.Background()
 
 	passwordPath, err := WriteTmpFile(DefaultPasswordTxt, "password.txt")
-	apiCredentialsPath, err := WriteTmpFile(DefaultAPICredentials, "apicredentials")
 	if err != nil {
 		return nil, err
 	}
+	apiCredentialsPath, err := WriteTmpFile(DefaultAPICredentials, "apicredentials")
 	if err != nil {
 		return nil, err
 	}

framework/components/clnode/clnode_test.go

@@ -1,12 +1,13 @@
 package clnode_test
 
 import (
+	"sync"
+	"testing"
+
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/clnode"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/postgres"
 	"github.com/stretchr/testify/require"
-	"sync"
-	"testing"
 )
 
 type testCase struct {

framework/components/clnode/connect_network.go

@@ -2,6 +2,7 @@ package clnode
 
 import (
 	"fmt"
+
 	"github.com/google/uuid"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"

framework/components/clnode/default.go

@@ -8,11 +8,11 @@ Dynamic settings are usually used to connect Docker components together
 
 const (
 	DefaultTestKeystorePassword = "thispasswordislongenough"
-	DefaultPasswordTxt          = `T.tLHkcmwePT/p,]sYuntjwHKAsrhm#4eRs4LuKHwvHejWYAC2JP4M8HimwgmbaZ`
+	DefaultPasswordTxt          = `T.tLHkcmwePT/p,]sYuntjwHKAsrhm#4eRs4LuKHwvHejWYAC2JP4M8HimwgmbaZ` //nolint:gosec
 	DefaultAPICredentials       = `[email protected]
 fj293fbBnlQ!f9vNs`
 	DefaultAPIUser     = `[email protected]`
-	DefaultAPIPassword = `fj293fbBnlQ!f9vNs`
+	DefaultAPIPassword = `fj293fbBnlQ!f9vNs` //nolint:gosec
 )
 
 const defaultConfigTmpl = `

tools/flakeguard/cmd/aggregate_results.go

@@ -1,7 +1,10 @@
 package cmd
 
 import (
+	"encoding/json"
 	"log"
+	"os"
+	"path/filepath"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 	"github.com/spf13/cobra"
@@ -21,7 +24,31 @@ var AggregateResultsCmd = &cobra.Command{
 	Use:   "aggregate-results",
 	Short: "Aggregate test results and optionally filter failed tests based on a threshold",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		allReport, err := reports.AggregateTestResults(resultsFolderPath)
+		// Read test reports from files
+		var testReports []*reports.TestReport
+		err := filepath.Walk(resultsFolderPath, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if !info.IsDir() && filepath.Ext(path) == ".json" {
+				// Read file content
+				data, readErr := os.ReadFile(path)
+				if readErr != nil {
+					return readErr
+				}
+				var report *reports.TestReport
+				if jsonErr := json.Unmarshal(data, &report); jsonErr != nil {
+					return jsonErr
+				}
+				testReports = append(testReports, report)
+			}
+			return nil
+		})
+		if err != nil {
+			log.Fatalf("Error reading test reports: %v", err)
+		}
+
+		allReport, err := reports.Aggregate(testReports...)
 		if err != nil {
 			log.Fatalf("Error aggregating results: %v", err)
 		}

tools/flakeguard/cmd/run.go

@@ -71,7 +71,7 @@ var RunTestsCmd = &cobra.Command{
 		// Print all failed tests including flaky tests
 		if printFailedTests {
 			fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", maxPassRatio)
-			reports.PrintTests(os.Stdout, testReport.Results, maxPassRatio, false)
+			reports.PrintResults(os.Stdout, testReport.Results, maxPassRatio, false, false)
 		}
 
 		// Save the test results in JSON format