Skip to content

Commit 8fcbb59

Browse files
lukaszcllukaszcl
committed
Add cmd to aggregate failed test results
1 parent aa22cd4 commit 8fcbb59

File tree

4 files changed

+326
-7
lines changed

4 files changed

+326
-7
lines changed

‎tools/flakeguard/cmd/aggregate_failed.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package cmd
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"log"
7+
"os"
8+
9+
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
10+
"github.com/spf13/cobra"
11+
)
12+
13+
var AggregateFailedTestsCmd = &cobra.Command{
14+
Use: "aggregate-failed",
15+
Short: "Aggregate test results and output only failed tests based on a threshold",
16+
Run: func(cmd *cobra.Command, args []string) {
17+
folderPath, _ := cmd.Flags().GetString("folder-path")
18+
outputPath, _ := cmd.Flags().GetString("output-json")
19+
threshold, _ := cmd.Flags().GetFloat64("threshold")
20+
21+
// Aggregate and merge results from the specified folder
22+
allResults, err := reports.AggregateTestResults(folderPath)
23+
if err != nil {
24+
log.Fatalf("Error aggregating results: %v", err)
25+
}
26+
27+
// Filter failed tests based on threshold
28+
failedTests := reports.FilterFailedTests(allResults, threshold)
29+
30+
// Format PassRatio as a percentage for display
31+
for i := range failedTests {
32+
failedTests[i].PassRatioPercentage = fmt.Sprintf("%.0f%%", failedTests[i].PassRatio*100)
33+
}
34+
35+
// Output failed tests to JSON file
36+
if outputPath != "" && len(failedTests) > 0 {
37+
if err := saveResults(outputPath, failedTests); err != nil {
38+
log.Fatalf("Error writing failed tests to file: %v", err)
39+
}
40+
fmt.Printf("Aggregated failed test results saved to %s\n", outputPath)
41+
} else {
42+
fmt.Println("No failed tests found based on the specified threshold.")
43+
}
44+
},
45+
}
46+
47+
func init() {
48+
AggregateFailedTestsCmd.Flags().String("folder-path", "testresult/", "Path to the folder containing JSON test result files")
49+
AggregateFailedTestsCmd.Flags().String("output-json", "failed_tests.json", "Path to output the aggregated failed test results in JSON format")
50+
AggregateFailedTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed")
51+
}
52+
53+
// Helper function to save results to JSON file
54+
func saveResults(filePath string, results []reports.TestResult) error {
55+
data, err := json.MarshalIndent(results, "", " ")
56+
if err != nil {
57+
return fmt.Errorf("error marshaling results: %v", err)
58+
}
59+
return os.WriteFile(filePath, data, 0644)
60+
}

‎tools/flakeguard/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ func init() {
2828

2929
rootCmd.AddCommand(cmd.FindTestsCmd)
3030
rootCmd.AddCommand(cmd.RunTestsCmd)
31+
rootCmd.AddCommand(cmd.AggregateFailedTestsCmd)
3132
}
3233

3334
func main() {

‎tools/flakeguard/reports/reports.go‎

Lines changed: 71 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,23 @@
11
package reports
22

33
import (
4+
"encoding/json"
45
"fmt"
56
"io"
7+
"os"
8+
"path/filepath"
69
"strings"
710
)
811

912
type TestResult struct {
10-
TestName string
11-
TestPackage string
12-
PassRatio float64
13-
Skipped bool // Indicates if the test was skipped
14-
Runs int
15-
Outputs []string // Stores outputs for a test
16-
Durations []float64 // Stores elapsed time in seconds for each run of the test
13+
TestName string
14+
TestPackage string
15+
PassRatio float64 // Pass ratio in decimal format like 0.5
16+
PassRatioPercentage string // Pass ratio in percentage format like "50%"
17+
Skipped bool // Indicates if the test was skipped
18+
Runs int
19+
Outputs []string // Stores outputs for a test
20+
Durations []float64 // Stores elapsed time in seconds for each run of the test
1721
}
1822

1923
// FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.
@@ -49,6 +53,66 @@ func FilterSkippedTests(results []TestResult) []TestResult {
4953
return skippedTests
5054
}
5155

56+
// Helper function to aggregate all JSON test results from a folder
57+
func AggregateTestResults(folderPath string) ([]TestResult, error) {
58+
// Map to hold unique tests based on their TestName and TestPackage
59+
testMap := make(map[string]TestResult)
60+
61+
err := filepath.Walk(folderPath, func(path string, info os.FileInfo, err error) error {
62+
if err != nil {
63+
return err
64+
}
65+
if !info.IsDir() && filepath.Ext(path) == ".json" {
66+
// Read file content
67+
data, readErr := os.ReadFile(path)
68+
if readErr != nil {
69+
return readErr
70+
}
71+
// Parse JSON data into TestResult slice
72+
var results []TestResult
73+
if jsonErr := json.Unmarshal(data, &results); jsonErr != nil {
74+
return jsonErr
75+
}
76+
// Process each result
77+
for _, result := range results {
78+
// Unique key for each test based on TestName and TestPackage
79+
key := result.TestName + "|" + result.TestPackage
80+
if existingResult, found := testMap[key]; found {
81+
// Aggregate runs, durations, and outputs
82+
totalRuns := existingResult.Runs + result.Runs
83+
existingResult.Durations = append(existingResult.Durations, result.Durations...)
84+
existingResult.Outputs = append(existingResult.Outputs, result.Outputs...)
85+
86+
// Calculate total successful runs and aggregate pass ratio
87+
successfulRuns := existingResult.PassRatio*float64(existingResult.Runs) + result.PassRatio*float64(result.Runs)
88+
existingResult.Runs = totalRuns
89+
existingResult.PassRatio = successfulRuns / float64(totalRuns)
90+
existingResult.Skipped = existingResult.Skipped && result.Skipped // Mark as skipped only if all occurrences are skipped
91+
92+
// Update the map with the aggregated result
93+
testMap[key] = existingResult
94+
95+
} else {
96+
// Add new entry to the map
97+
testMap[key] = result
98+
}
99+
}
100+
}
101+
return nil
102+
})
103+
if err != nil {
104+
return nil, fmt.Errorf("error reading files: %v", err)
105+
}
106+
107+
// Convert map to slice of TestResult and set PassRatioPercentage
108+
aggregatedResults := make([]TestResult, 0, len(testMap))
109+
for _, result := range testMap {
110+
result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
111+
aggregatedResults = append(aggregatedResults, result)
112+
}
113+
return aggregatedResults, nil
114+
}
115+
52116
// PrintTests prints tests in a pretty format
53117
func PrintTests(tests []TestResult, w io.Writer) {
54118
for i, test := range tests {

‎tools/flakeguard/reports/reports_test.go‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ package reports
22

33
import (
44
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"io/ioutil"
8+
"os"
9+
"path/filepath"
510
"strings"
611
"testing"
712
)
@@ -109,3 +114,192 @@ func TestPrintTests(t *testing.T) {
109114
}
110115
}
111116
}
117+
118+
// Helper function to write temporary JSON files for testing
119+
func writeTempJSONFile(t *testing.T, dir string, filename string, data interface{}) string {
120+
filePath := filepath.Join(dir, filename)
121+
fileData, err := json.Marshal(data)
122+
if err != nil {
123+
t.Fatalf("Failed to marshal JSON: %v", err)
124+
}
125+
if err := ioutil.WriteFile(filePath, fileData, 0644); err != nil {
126+
t.Fatalf("Failed to write JSON file: %v", err)
127+
}
128+
return filePath
129+
}
130+
131+
func TestAggregateTestResults(t *testing.T) {
132+
// Create a temporary directory for test JSON files
133+
tempDir, err := os.MkdirTemp("", "aggregatetestresults")
134+
if err != nil {
135+
t.Fatalf("Failed to create temp directory: %v", err)
136+
}
137+
defer os.RemoveAll(tempDir)
138+
139+
// Test cases
140+
testCases := []struct {
141+
description string
142+
inputFiles []interface{}
143+
expectedOutput []TestResult
144+
}{
145+
{
146+
description: "Unique test results, no aggregation",
147+
inputFiles: []interface{}{
148+
[]TestResult{
149+
{
150+
TestName: "TestA",
151+
TestPackage: "pkgA",
152+
PassRatio: 1,
153+
PassRatioPercentage: "100%",
154+
Skipped: false,
155+
Runs: 2,
156+
Durations: []float64{0.01, 0.02},
157+
},
158+
},
159+
[]TestResult{
160+
{
161+
TestName: "TestB",
162+
TestPackage: "pkgB",
163+
PassRatio: 0.5,
164+
PassRatioPercentage: "50%",
165+
Skipped: false,
166+
Runs: 4,
167+
Durations: []float64{0.05, 0.05, 0.05, 0.05},
168+
},
169+
},
170+
},
171+
expectedOutput: []TestResult{
172+
{
173+
TestName: "TestA",
174+
TestPackage: "pkgA",
175+
PassRatio: 1,
176+
PassRatioPercentage: "100%",
177+
Skipped: false,
178+
Runs: 2,
179+
Durations: []float64{0.01, 0.02},
180+
},
181+
{
182+
TestName: "TestB",
183+
TestPackage: "pkgB",
184+
PassRatio: 0.5,
185+
PassRatioPercentage: "50%",
186+
Skipped: false,
187+
Runs: 4,
188+
Durations: []float64{0.05, 0.05, 0.05, 0.05},
189+
},
190+
},
191+
},
192+
{
193+
description: "Duplicate test results, aggregation of PassRatio and Durations",
194+
inputFiles: []interface{}{
195+
[]TestResult{
196+
{
197+
TestName: "TestC",
198+
TestPackage: "pkgC",
199+
PassRatio: 1,
200+
PassRatioPercentage: "100%",
201+
Skipped: false,
202+
Runs: 2,
203+
Durations: []float64{0.1, 0.1},
204+
},
205+
},
206+
[]TestResult{
207+
{
208+
TestName: "TestC",
209+
TestPackage: "pkgC",
210+
PassRatio: 0.5,
211+
PassRatioPercentage: "50%",
212+
Skipped: false,
213+
Runs: 2,
214+
Durations: []float64{0.2, 0.2},
215+
},
216+
},
217+
},
218+
expectedOutput: []TestResult{
219+
{
220+
TestName: "TestC",
221+
TestPackage: "pkgC",
222+
PassRatio: 0.75, // Calculated as (2*1 + 2*0.5) / 4
223+
PassRatioPercentage: "75%",
224+
Skipped: false,
225+
Runs: 4,
226+
Durations: []float64{0.1, 0.1, 0.2, 0.2},
227+
},
228+
},
229+
},
230+
{
231+
description: "All Skipped test results",
232+
inputFiles: []interface{}{
233+
[]TestResult{
234+
{
235+
TestName: "TestD",
236+
TestPackage: "pkgD",
237+
PassRatio: 1,
238+
PassRatioPercentage: "100%",
239+
Skipped: true,
240+
Runs: 3,
241+
Durations: []float64{0.1, 0.2, 0.1},
242+
},
243+
},
244+
[]TestResult{
245+
{
246+
TestName: "TestD",
247+
TestPackage: "pkgD",
248+
PassRatio: 1,
249+
PassRatioPercentage: "100%",
250+
Skipped: true,
251+
Runs: 2,
252+
Durations: []float64{0.15, 0.15},
253+
},
254+
},
255+
},
256+
expectedOutput: []TestResult{
257+
{
258+
TestName: "TestD",
259+
TestPackage: "pkgD",
260+
PassRatio: 1,
261+
PassRatioPercentage: "100%",
262+
Skipped: true, // Should remain true as all runs are skipped
263+
Runs: 5,
264+
Durations: []float64{0.1, 0.2, 0.1, 0.15, 0.15},
265+
},
266+
},
267+
},
268+
}
269+
270+
for _, tc := range testCases {
271+
t.Run(tc.description, func(t *testing.T) {
272+
// Write input files to the temporary directory
273+
for i, inputData := range tc.inputFiles {
274+
writeTempJSONFile(t, tempDir, fmt.Sprintf("input%d.json", i), inputData)
275+
}
276+
277+
// Run AggregateTestResults
278+
result, err := AggregateTestResults(tempDir)
279+
if err != nil {
280+
t.Fatalf("AggregateTestResults failed: %v", err)
281+
}
282+
283+
// Compare the result with the expected output
284+
if len(result) != len(tc.expectedOutput) {
285+
t.Fatalf("Expected %d results, got %d", len(tc.expectedOutput), len(result))
286+
}
287+
288+
for i, expected := range tc.expectedOutput {
289+
got := result[i]
290+
if got.TestName != expected.TestName || got.TestPackage != expected.TestPackage || got.Runs != expected.Runs || got.Skipped != expected.Skipped {
291+
t.Errorf("Result %d - expected %+v, got %+v", i, expected, got)
292+
}
293+
if got.PassRatio != expected.PassRatio {
294+
t.Errorf("Result %d - expected PassRatio %f, got %f", i, expected.PassRatio, got.PassRatio)
295+
}
296+
if got.PassRatioPercentage != expected.PassRatioPercentage {
297+
t.Errorf("Result %d - expected PassRatioPercentage %s, got %s", i, expected.PassRatioPercentage, got.PassRatioPercentage)
298+
}
299+
if len(got.Durations) != len(expected.Durations) {
300+
t.Errorf("Result %d - expected %d durations, got %d", i, len(expected.Durations), len(got.Durations))
301+
}
302+
}
303+
})
304+
}
305+
}

0 commit comments

Comments
 (0)