Skip to content

Commit af77c7b

Browse files
committed
Even Prettier printing
1 parent ebc0969 commit af77c7b

File tree

6 files changed

+83
-44
lines changed

6 files changed

+83
-44
lines changed

tools/flakeguard/Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ test:
1212
example:
1313
rm -rf example_results
1414
mkdir -p example_results
15-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic,TestFlakyPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
16-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic,TestFlakyPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
17-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic,TestFlakyPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
15+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
16+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
17+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
1818
go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
1919

2020
.PHONY: example_panic

tools/flakeguard/reports/reports.go

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -167,10 +167,7 @@ func AggregateTestResults(folderPath string) (*TestReport, error) {
167167
allSuccesses += result.Successes
168168
}
169169

170-
// Sort by PassRatio in ascending order
171-
sort.Slice(aggregatedResults, func(i, j int) bool {
172-
return aggregatedResults[i].PassRatio < aggregatedResults[j].PassRatio
173-
})
170+
sortTestResults(aggregatedResults)
174171
fullReport.Results = aggregatedResults
175172

176173
return fullReport, nil
@@ -182,6 +179,7 @@ func PrintTests(
182179
tests []TestResult,
183180
maxPassRatio float64,
184181
) (runs, passes, fails, skips, panickedTests, racedTests, flakyTests int) {
182+
sortTestResults(tests)
185183
headers := []string{
186184
"**Test**",
187185
"**Pass Ratio**",
@@ -318,7 +316,7 @@ func MarkdownSummary(w io.Writer, testReport *TestReport, maxPassRatio float64)
318316

319317
rows := [][]string{
320318
{"**Setting**", "**Value**"},
321-
{"Go Project", testReport.GoProject},
319+
{"Project", testReport.GoProject},
322320
{"Max Pass Ratio", fmt.Sprintf("%.2f%%", maxPassRatio*100)},
323321
{"Test Run Count", fmt.Sprintf("%d", testReport.TestRunCount)},
324322
{"Race Detection", fmt.Sprintf("%t", testReport.RaceDetection)},
@@ -443,3 +441,16 @@ func avgDuration(durations []time.Duration) time.Duration {
443441
}
444442
return total / time.Duration(len(durations))
445443
}
444+
445+
// sortTestResults sorts results by PassRatio, TestPackage, and TestName for consistent comparison and pretty printing
446+
func sortTestResults(results []TestResult) {
447+
sort.Slice(results, func(i, j int) bool {
448+
if results[i].PassRatio != results[j].PassRatio {
449+
return results[i].PassRatio < results[j].PassRatio
450+
}
451+
if results[i].TestName == results[j].TestName {
452+
return results[i].TestPackage < results[j].TestPackage
453+
}
454+
return results[i].TestName < results[j].TestName
455+
})
456+
}

tools/flakeguard/reports/reports_test.go

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"fmt"
77
"os"
88
"path/filepath"
9-
"sort"
109
"testing"
1110
"time"
1211

@@ -165,16 +164,6 @@ func TestPrintTests(t *testing.T) {
165164

166165
}
167166

168-
// Sorts TestResult slice by TestName and TestPackage for consistent comparison
169-
func sortTestResults(results []TestResult) {
170-
sort.Slice(results, func(i, j int) bool {
171-
if results[i].TestName == results[j].TestName {
172-
return results[i].TestPackage < results[j].TestPackage
173-
}
174-
return results[i].TestName < results[j].TestName
175-
})
176-
}
177-
178167
// Helper function to write a JSON file for testing
179168
func writeTempJSONFile(t *testing.T, dir string, filename string, data interface{}) string {
180169
t.Helper()

tools/flakeguard/runner/runner.go

Lines changed: 56 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ var (
2626

2727
type Runner struct {
2828
ProjectPath string // Path to the Go project directory.
29+
prettyProjectPath string // Go project package path, formatted for pretty printing.
2930
Verbose bool // If true, provides detailed logging.
3031
RunCount int // Number of times to run the tests.
3132
UseRace bool // Enable race detector.
@@ -66,21 +67,12 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
6667
}
6768
}
6869

69-
results, err := parseTestResults(jsonFilePaths)
70+
results, err := parseTestResults(r.RunCount, jsonFilePaths)
7071
if err != nil {
7172
return nil, fmt.Errorf("failed to parse test results: %w", err)
7273
}
73-
for index, report := range results {
74-
if report.Runs > r.RunCount { // Panics can introduce double-counting test failures, this is a hacky correction for it
75-
if report.Panic && report.Failures > report.Runs {
76-
results[index].Failures = report.Runs
77-
} else {
78-
fmt.Printf("WARN: '%s' has %d test runs, exceeding expected amount in an unexpected way, this may be due to oddly presented panics\n", report.TestName, report.Runs)
79-
}
80-
}
81-
}
8274
return &reports.TestReport{
83-
GoProject: r.ProjectPath,
75+
GoProject: r.prettyProjectPath,
8476
TestRunCount: r.RunCount,
8577
RaceDetection: r.UseRace,
8678
ExcludedTests: r.SkipTests,
@@ -132,6 +124,10 @@ func (r *Runner) runTests(packageName string) (string, bool, error) {
132124
}
133125
defer tmpFile.Close()
134126

127+
r.prettyProjectPath, err = prettyProjectPath(r.ProjectPath)
128+
if err != nil {
129+
return "", false, fmt.Errorf("failed to get pretty project path: %w", err)
130+
}
135131
// Run the command with output directed to the file
136132
cmd := exec.Command("go", args...)
137133
cmd.Dir = r.ProjectPath
@@ -179,7 +175,7 @@ func (e entry) String() string {
179175
// panics and failures at that point.
180176
// Subtests add more complexity, as panics in subtests are only reported in their parent's output,
181177
// and cannot be accurately attributed to the subtest that caused them.
182-
func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
178+
func parseTestResults(expectedRuns int, filePaths []string) ([]reports.TestResult, error) {
183179
var (
184180
testDetails = make(map[string]*reports.TestResult) // Holds run, pass counts, and other details for each test
185181
panickedPackages = map[string]struct{}{} // Packages with tests that panicked
@@ -385,15 +381,23 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
385381
testDetails[subTestKey].Outputs = append(testDetails[subTestKey].Outputs, "Panic in parent test")
386382
}
387383
} else {
388-
fmt.Printf("WARN: subtest %s has no results\n", subTestKey)
384+
fmt.Printf("WARN: expected to fine subtest '%s' inside parent test '%s', but not found\n", parentTestKey, subTestKey)
389385
}
390386
}
391387
}
392388
} else {
393-
fmt.Printf("WARN: parent test %s has no results\n", parentTestKey)
389+
fmt.Printf("WARN: expected to find parent test '%s' for sub tests, but not found\n", parentTestKey)
394390
}
395391
}
396392
for _, result := range testDetails {
393+
if result.Runs > expectedRuns { // Panics can introduce double-counting test failures, this is a hacky correction for it
394+
if result.Panic {
395+
result.Failures = expectedRuns
396+
result.Runs = expectedRuns
397+
} else {
398+
fmt.Printf("WARN: '%s' has %d test runs, exceeding expected amount of %d in an unexpected way, this may be due to oddly presented panics\n", result.TestName, result.Runs, expectedRuns)
399+
}
400+
}
397401
// If a package panicked, all tests in that package will be marked as panicking
398402
if _, panicked := panickedPackages[result.TestPackage]; panicked {
399403
result.PackagePanic = true
@@ -445,3 +449,41 @@ func parseSubTest(testName string) (parentTestName, subTestName string) {
445449
}
446450
return parts[0], parts[1]
447451
}
452+
453+
// prettyProjectPath returns the project path formatted for pretty printing in results
454+
func prettyProjectPath(projectPath string) (string, error) {
455+
// Walk up the directory structure to find go.mod
456+
absPath, err := filepath.Abs(projectPath)
457+
if err != nil {
458+
return "", fmt.Errorf("failed to resolve absolute path: %w", err)
459+
}
460+
dir := absPath
461+
for {
462+
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
463+
break
464+
}
465+
466+
parent := filepath.Dir(dir)
467+
if parent == dir { // Reached the root without finding go.mod
468+
return "", fmt.Errorf("go.mod not found in project path, started at %s, ended at %s", projectPath, dir)
469+
}
470+
dir = parent
471+
}
472+
473+
// Read go.mod to extract the module path
474+
goModPath := filepath.Join(dir, "go.mod")
475+
goModData, err := os.ReadFile(goModPath)
476+
if err != nil {
477+
return "", fmt.Errorf("failed to read go.mod: %w", err)
478+
}
479+
480+
for _, line := range strings.Split(string(goModData), "\n") {
481+
if strings.HasPrefix(line, "module ") {
482+
goProject := strings.TrimSpace(strings.TrimPrefix(line, "module "))
483+
projectPath = strings.TrimPrefix(projectPath, goProject)
484+
return filepath.Join(goProject, projectPath), nil
485+
}
486+
}
487+
488+
return "", fmt.Errorf("module path not found in go.mod")
489+
}

tools/flakeguard/runner/runner_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@ type expectedTestResult struct {
3939
seen bool
4040
}
4141

42+
func TestPrettyProjectPath(t *testing.T) {
43+
t.Parallel()
44+
45+
prettyPath, err := prettyProjectPath("./")
46+
require.NoError(t, err)
47+
assert.Equal(t, "github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard", prettyPath)
48+
}
49+
4250
func TestRun(t *testing.T) {
4351
var (
4452
zeroRuns = 0

tools/flakeguard/x.md

Lines changed: 0 additions & 11 deletions
This file was deleted.

0 commit comments

Comments
 (0)