Skip to content

Commit fbe0544

Browse files
kalverrakalverra
committed
Makes TestReport aggregation more flexible
1 parent f03577d commit fbe0544

File tree

7 files changed

+84
-77
lines changed

7 files changed

+84
-77
lines changed

‎framework/components/clnode/clnode.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,18 @@ import (
55
"context"
66
"errors"
77
"fmt"
8+
"os"
9+
"path/filepath"
10+
"sync"
11+
"text/template"
12+
"time"
13+
814
"github.com/docker/docker/api/types/container"
915
"github.com/docker/go-connections/nat"
1016
"github.com/smartcontractkit/chainlink-testing-framework/framework"
1117
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/postgres"
1218
tc "github.com/testcontainers/testcontainers-go"
1319
"github.com/testcontainers/testcontainers-go/wait"
14-
"os"
15-
"path/filepath"
16-
"sync"
17-
"text/template"
18-
"time"
1920
)
2021

2122
const (
@@ -114,10 +115,10 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
114115
ctx := context.Background()
115116

116117
passwordPath, err := WriteTmpFile(DefaultPasswordTxt, "password.txt")
117-
apiCredentialsPath, err := WriteTmpFile(DefaultAPICredentials, "apicredentials")
118118
if err != nil {
119119
return nil, err
120120
}
121+
apiCredentialsPath, err := WriteTmpFile(DefaultAPICredentials, "apicredentials")
121122
if err != nil {
122123
return nil, err
123124
}

‎framework/components/clnode/clnode_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
package clnode_test
22

33
import (
4+
"sync"
5+
"testing"
6+
47
"github.com/smartcontractkit/chainlink-testing-framework/framework"
58
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/clnode"
69
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/postgres"
710
"github.com/stretchr/testify/require"
8-
"sync"
9-
"testing"
1011
)
1112

1213
type testCase struct {

‎framework/components/clnode/connect_network.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package clnode
22

33
import (
44
"fmt"
5+
56
"github.com/google/uuid"
67
"github.com/smartcontractkit/chainlink-testing-framework/framework"
78
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"

‎framework/components/clnode/default.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@ Dynamic settings are usually used to connect Docker components together
88

99
const (
1010
DefaultTestKeystorePassword = "thispasswordislongenough"
11-
DefaultPasswordTxt = `T.tLHkcmwePT/p,]sYuntjwHKAsrhm#4eRs4LuKHwvHejWYAC2JP4M8HimwgmbaZ`
11+
DefaultPasswordTxt = `T.tLHkcmwePT/p,]sYuntjwHKAsrhm#4eRs4LuKHwvHejWYAC2JP4M8HimwgmbaZ` //nolint:gosec
1212
DefaultAPICredentials = `[email protected]
1313
fj293fbBnlQ!f9vNs`
1414
DefaultAPIUser = `[email protected]`
15-
DefaultAPIPassword = `fj293fbBnlQ!f9vNs`
15+
DefaultAPIPassword = `fj293fbBnlQ!f9vNs` //nolint:gosec
1616
)
1717

1818
const defaultConfigTmpl = `

‎tools/flakeguard/cmd/aggregate_results.go‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package cmd
22

33
import (
4+
"encoding/json"
45
"log"
6+
"os"
7+
"path/filepath"
58

69
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
710
"github.com/spf13/cobra"
@@ -21,7 +24,31 @@ var AggregateResultsCmd = &cobra.Command{
2124
Use: "aggregate-results",
2225
Short: "Aggregate test results and optionally filter failed tests based on a threshold",
2326
RunE: func(cmd *cobra.Command, args []string) error {
24-
allReport, err := reports.AggregateTestResults(resultsFolderPath)
27+
// Read test reports from files
28+
var testReports []*reports.TestReport
29+
err := filepath.Walk(resultsFolderPath, func(path string, info os.FileInfo, err error) error {
30+
if err != nil {
31+
return err
32+
}
33+
if !info.IsDir() && filepath.Ext(path) == ".json" {
34+
// Read file content
35+
data, readErr := os.ReadFile(path)
36+
if readErr != nil {
37+
return readErr
38+
}
39+
var report *reports.TestReport
40+
if jsonErr := json.Unmarshal(data, &report); jsonErr != nil {
41+
return jsonErr
42+
}
43+
testReports = append(testReports, report)
44+
}
45+
return nil
46+
})
47+
if err != nil {
48+
log.Fatalf("Error reading test reports: %v", err)
49+
}
50+
51+
allReport, err := reports.AggregateTestReports(testReports...)
2552
if err != nil {
2653
log.Fatalf("Error aggregating results: %v", err)
2754
}

‎tools/flakeguard/reports/reports.go‎

Lines changed: 41 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,8 @@ func FilterSkippedTests(results []TestResult) []TestResult {
9191
return skippedTests
9292
}
9393

94-
// AggregateTestResults aggregates all JSON test results.
95-
func AggregateTestResults(folderPath string) (*TestReport, error) {
94+
// AggregateTestReports aggregates multiple test reports into a single report.
95+
func AggregateTestReports(reportsToAggregate ...*TestReport) (*TestReport, error) {
9696
var (
9797
// Map to hold unique tests based on their TestName and TestPackage
9898
// Key: TestName|TestPackage, Value: TestResult
@@ -103,67 +103,49 @@ func AggregateTestResults(folderPath string) (*TestReport, error) {
103103
)
104104

105105
// Read all JSON files in the folder
106-
err := filepath.Walk(folderPath, func(path string, info os.FileInfo, err error) error {
107-
if err != nil {
108-
return err
106+
for _, report := range reportsToAggregate {
107+
if fullReport.GoProject == "" {
108+
fullReport.GoProject = report.GoProject
109+
} else if fullReport.GoProject != report.GoProject {
110+
return nil, fmt.Errorf("reports with different Go projects found, expected %s, got %s", fullReport.GoProject, report.GoProject)
109111
}
110-
if !info.IsDir() && filepath.Ext(path) == ".json" {
111-
// Read file content
112-
data, readErr := os.ReadFile(path)
113-
if readErr != nil {
114-
return readErr
115-
}
116-
var report TestReport
117-
if jsonErr := json.Unmarshal(data, &report); jsonErr != nil {
118-
return jsonErr
119-
}
120-
if fullReport.GoProject == "" {
121-
fullReport.GoProject = report.GoProject
122-
} else if fullReport.GoProject != report.GoProject {
123-
return fmt.Errorf("multiple projects found in the results folder, expected %s, got %s", fullReport.GoProject, report.GoProject)
124-
}
125-
fullReport.TestRunCount += report.TestRunCount
126-
fullReport.RaceDetection = report.RaceDetection && fullReport.RaceDetection
127-
for _, test := range report.ExcludedTests {
128-
excludedTests[test] = struct{}{}
129-
}
130-
for _, test := range report.SelectedTests {
131-
selectedTests[test] = struct{}{}
132-
}
133-
// Process each test results
134-
for _, result := range report.Results {
135-
// Unique key for each test based on TestName and TestPackage
136-
key := result.TestName + "|" + result.TestPackage
137-
if existingResult, found := testMap[key]; found {
138-
// Aggregate runs, durations, and outputs
139-
existingResult.Runs = existingResult.Runs + result.Runs
140-
existingResult.Durations = append(existingResult.Durations, result.Durations...)
141-
existingResult.Outputs = append(existingResult.Outputs, result.Outputs...)
142-
existingResult.PackageOutputs = append(existingResult.PackageOutputs, result.PackageOutputs...)
143-
existingResult.Successes += result.Successes
144-
existingResult.Failures += result.Failures
145-
existingResult.Panic = existingResult.Panic || result.Panic
146-
existingResult.Race = existingResult.Race || result.Race
147-
existingResult.Skips += result.Skips
148-
existingResult.PassRatio = 1.0
149-
if existingResult.Runs > 0 {
150-
existingResult.PassRatio = float64(existingResult.Successes) / float64(existingResult.Runs)
151-
}
152-
153-
existingResult.Skipped = existingResult.Skipped && result.Skipped // Mark as skipped only if all occurrences are skipped
154-
155-
// Update the map with the aggregated result
156-
testMap[key] = existingResult
157-
} else {
158-
// Add new entry to the map
159-
testMap[key] = result
112+
fullReport.TestRunCount += report.TestRunCount
113+
fullReport.RaceDetection = report.RaceDetection && fullReport.RaceDetection
114+
for _, test := range report.ExcludedTests {
115+
excludedTests[test] = struct{}{}
116+
}
117+
for _, test := range report.SelectedTests {
118+
selectedTests[test] = struct{}{}
119+
}
120+
// Process each test results
121+
for _, result := range report.Results {
122+
// Unique key for each test based on TestName and TestPackage
123+
key := result.TestName + "|" + result.TestPackage
124+
if existingResult, found := testMap[key]; found {
125+
// Aggregate runs, durations, and outputs
126+
existingResult.Runs = existingResult.Runs + result.Runs
127+
existingResult.Durations = append(existingResult.Durations, result.Durations...)
128+
existingResult.Outputs = append(existingResult.Outputs, result.Outputs...)
129+
existingResult.PackageOutputs = append(existingResult.PackageOutputs, result.PackageOutputs...)
130+
existingResult.Successes += result.Successes
131+
existingResult.Failures += result.Failures
132+
existingResult.Panic = existingResult.Panic || result.Panic
133+
existingResult.Race = existingResult.Race || result.Race
134+
existingResult.Skips += result.Skips
135+
existingResult.PassRatio = 1.0
136+
if existingResult.Runs > 0 {
137+
existingResult.PassRatio = float64(existingResult.Successes) / float64(existingResult.Runs)
160138
}
139+
140+
existingResult.Skipped = existingResult.Skipped && result.Skipped // Mark as skipped only if all occurrences are skipped
141+
142+
// Update the map with the aggregated result
143+
testMap[key] = existingResult
144+
} else {
145+
// Add new entry to the map
146+
testMap[key] = result
161147
}
162148
}
163-
return nil
164-
})
165-
if err != nil {
166-
return nil, fmt.Errorf("error reading files: %v", err)
167149
}
168150
// Aggregate
169151
for test := range excludedTests {

‎tools/flakeguard/reports/reports_test.go‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package reports
33
import (
44
"bytes"
55
"encoding/json"
6-
"fmt"
76
"os"
87
"path/filepath"
98
"testing"
@@ -145,7 +144,7 @@ func TestPrintTests(t *testing.T) {
145144
t.Parallel()
146145
var buf bytes.Buffer
147146

148-
runs, passes, fails, skips, panickedTests, racedTests, flakyTests := PrintTests(&buf, tc.testResults, tc.maxPassRatio)
147+
runs, passes, fails, skips, panickedTests, racedTests, flakyTests := PrintTests(&buf, tc.testResults, tc.maxPassRatio, false)
149148
assert.Equal(t, tc.expectedRuns, runs, "wrong number of runs")
150149
assert.Equal(t, tc.expectedPasses, passes, "wrong number of passes")
151150
assert.Equal(t, tc.expectedFails, fails, "wrong number of failures")
@@ -371,11 +370,7 @@ func TestAggregateTestResults(t *testing.T) {
371370

372371
for _, tc := range testCases {
373372
t.Run(tc.description, func(t *testing.T) {
374-
for i, inputData := range tc.inputReports {
375-
writeTempJSONFile(t, tempDir, fmt.Sprintf("input%d.json", i), inputData)
376-
}
377-
378-
finalReport, err := AggregateTestResults(tempDir)
373+
finalReport, err := AggregateTestReports(tc.inputReports...)
379374
if err != nil {
380375
t.Fatalf("AggregateTestResults failed: %v", err)
381376
}

0 commit comments

Comments
 (0)