Skip to content

Commit 9dbae89

Browse files
spilchenspilchen
committed
roachtest: add Mocha test parser for node-postgres tests
This commit introduces Mocha test output parsing for the node-postgres roachtest. The parser handles hierarchical test suites (suite => subsuite => test) and properly categorizes test results against blocklists and ignorelists. With this ignorelist in place, this also ignores a test flake as outlined here: #152728 (comment) Fixes #152728 Release note: None Epic: None
1 parent e3e757c commit 9dbae89

File tree

5 files changed

+351
-22
lines changed

5 files changed

+351
-22
lines changed

‎pkg/cmd/roachtest/tests/BUILD.bazel‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,9 @@ go_library(
141141
"multitenant_upgrade.go",
142142
"mvcc_gc.go",
143143
"network.go",
144+
"nodejs_helpers.go",
144145
"nodejs_postgres.go",
146+
"nodejs_postgres_blocklist.go",
145147
"npgsql.go",
146148
"npgsql_blocklist.go",
147149
"online_restore.go",
@@ -366,6 +368,7 @@ go_test(
366368
"blocklist_test.go",
367369
"cdc_helper_test.go",
368370
"drt_test.go",
371+
"nodejs_helpers_test.go",
369372
"query_comparison_util_test.go",
370373
"restore_test.go",
371374
"sysbench_test.go",

‎pkg/cmd/roachtest/tests/nodejs_helpers.go‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
// Copyright 2025 The Cockroach Authors.
2+
//
3+
// Use of this software is governed by the CockroachDB Software License
4+
// included in the /LICENSE file.
5+
6+
package tests
7+
8+
import (
9+
"context"
10+
"fmt"
11+
"regexp"
12+
"strings"
13+
14+
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
15+
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
16+
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
17+
)
18+
19+
const (
20+
// mochaDetailedFailureIndent is the indentation string used by Mocha for
21+
// lines in the detailed failure section (after "X failing").
22+
// This distinguishes them from the test output section which uses 2-4 spaces.
23+
mochaDetailedFailureIndent = " " // 7 spaces
24+
)
25+
26+
// mochaTestResult represents the result of a single test from Mocha output
27+
type mochaTestResult struct {
28+
name string
29+
status status
30+
}
31+
32+
// parseMochaOutput parses Mocha test output and extracts test results.
33+
// Mocha output format:
34+
//
35+
// ✓ passing test name
36+
// 1) failing test name
37+
// - skipped test name
38+
//
39+
// Returns a slice of test results.
40+
func parseMochaOutput(output string) []mochaTestResult {
41+
var results []mochaTestResult
42+
seen := make(map[string]bool) // Track seen test names to avoid duplicates.
43+
lines := strings.Split(output, "\n")
44+
45+
// Regex patterns for different test result types.
46+
passingPattern := regexp.MustCompile(`^\s*✓\s+(.+)$`)
47+
failingPattern := regexp.MustCompile(`^\s*\d+\)\s+(.+?)(?::.*)?$`)
48+
skippedPattern := regexp.MustCompile(`^\s*-\s+(.+)$`)
49+
50+
for i, line := range lines {
51+
line = strings.TrimSpace(line)
52+
if line == "" {
53+
continue
54+
}
55+
56+
if match := passingPattern.FindStringSubmatch(line); match != nil {
57+
testName := strings.TrimSpace(match[1])
58+
if !seen[testName] {
59+
seen[testName] = true
60+
results = append(results, mochaTestResult{
61+
name: testName,
62+
status: statusPass,
63+
})
64+
}
65+
} else if match := failingPattern.FindStringSubmatch(line); match != nil {
66+
// Look ahead to see if this is a nested test failure.
67+
// Check next few lines to build the full test description.
68+
suiteName := strings.TrimSpace(match[1])
69+
var hierarchy []string
70+
foundDetailedFormat := false
71+
72+
// Look for indented continuation lines that describe the full test path.
73+
// Mocha can have multiple levels: suite -> subsuite -> test.
74+
for j := i + 1; j < len(lines) && j < i+10; j++ {
75+
nextLine := lines[j]
76+
trimmed := strings.TrimSpace(nextLine)
77+
78+
// Check if this is an indented line (part of the test hierarchy).
79+
// Use mochaDetailedFailureIndent to distinguish detailed failure section
80+
// from the test output section which has only 2-4 spaces.
81+
if strings.HasPrefix(nextLine, mochaDetailedFailureIndent) && len(trimmed) > 0 {
82+
// If it ends with :, it's the final test name.
83+
if strings.HasSuffix(trimmed, ":") {
84+
hierarchy = append(hierarchy, strings.TrimSuffix(trimmed, ":"))
85+
foundDetailedFormat = true
86+
break
87+
} else {
88+
// It's an intermediate level (subsuite).
89+
hierarchy = append(hierarchy, trimmed)
90+
}
91+
} else if trimmed != "" && !strings.HasPrefix(nextLine, " ") {
92+
// Non-indented non-empty line means we've left the failure description.
93+
break
94+
}
95+
}
96+
97+
// Only process entries from the detailed failure section (which have indented continuation).
98+
// Skip the summary section entries that appear inline with test output.
99+
if !foundDetailedFormat {
100+
continue
101+
}
102+
103+
// Build the full hierarchical test name.
104+
// Format: "suite => subsuite => ... => test"
105+
var combinedName string
106+
if len(hierarchy) > 0 {
107+
// Start with the suite name, then add all hierarchy levels.
108+
parts := []string{suiteName}
109+
parts = append(parts, hierarchy...)
110+
combinedName = strings.Join(parts, " => ")
111+
} else {
112+
// No hierarchy found, just use the suite name.
113+
combinedName = suiteName
114+
}
115+
116+
if !seen[combinedName] {
117+
seen[combinedName] = true
118+
results = append(results, mochaTestResult{
119+
name: combinedName,
120+
status: statusFail,
121+
})
122+
}
123+
} else if match := skippedPattern.FindStringSubmatch(line); match != nil {
124+
testName := strings.TrimSpace(match[1])
125+
if !seen[testName] {
126+
seen[testName] = true
127+
results = append(results, mochaTestResult{
128+
name: testName,
129+
status: statusSkip,
130+
})
131+
}
132+
}
133+
}
134+
135+
return results
136+
}
137+
138+
// parseAndSummarizeNodeJSTestResults parses Node.js/Mocha test output and
139+
// summarizes it. This is based off of parseAndSummarizeJavaORMTestsResults.
140+
func parseAndSummarizeNodeJSTestResults(
141+
ctx context.Context,
142+
t test.Test,
143+
c cluster.Cluster,
144+
node option.NodeListOption,
145+
testName string,
146+
testOutput string,
147+
blocklistName string,
148+
expectedFailures blocklist,
149+
ignorelist blocklist,
150+
version string,
151+
) {
152+
results := newORMTestsResults()
153+
154+
// Parse the Mocha output.
155+
mochaResults := parseMochaOutput(testOutput)
156+
157+
for _, testResult := range mochaResults {
158+
currentTestName := testResult.name
159+
160+
// Check if test is in ignore or block lists.
161+
ignoredIssue, expectedIgnored := ignorelist[currentTestName]
162+
issue, expectedFailure := expectedFailures[currentTestName]
163+
164+
if len(issue) == 0 || issue == "unknown" {
165+
issue = "unknown"
166+
}
167+
168+
// Categorize the test result.
169+
switch {
170+
case expectedIgnored:
171+
results.results[currentTestName] = fmt.Sprintf("--- IGNORE: %s due to %s (expected)", currentTestName, ignoredIssue)
172+
results.ignoredCount++
173+
case testResult.status == statusSkip:
174+
results.results[currentTestName] = fmt.Sprintf("--- SKIP: %s", currentTestName)
175+
results.skipCount++
176+
case testResult.status == statusPass && !expectedFailure:
177+
results.results[currentTestName] = fmt.Sprintf("--- PASS: %s (expected)", currentTestName)
178+
results.passExpectedCount++
179+
case testResult.status == statusPass && expectedFailure:
180+
results.results[currentTestName] = fmt.Sprintf("--- PASS: %s - %s (unexpected)",
181+
currentTestName, maybeAddGithubLink(issue),
182+
)
183+
results.passUnexpectedCount++
184+
case testResult.status == statusFail && expectedFailure:
185+
results.results[currentTestName] = fmt.Sprintf("--- FAIL: %s - %s (expected)",
186+
currentTestName, maybeAddGithubLink(issue),
187+
)
188+
results.failExpectedCount++
189+
results.currentFailures = append(results.currentFailures, currentTestName)
190+
case testResult.status == statusFail && !expectedFailure:
191+
results.results[currentTestName] = fmt.Sprintf("--- FAIL: %s - %s (unexpected)",
192+
currentTestName, maybeAddGithubLink(issue))
193+
results.failUnexpectedCount++
194+
results.currentFailures = append(results.currentFailures, currentTestName)
195+
}
196+
197+
results.runTests[currentTestName] = struct{}{}
198+
results.allTests = append(results.allTests, currentTestName)
199+
}
200+
201+
results.summarizeAll(
202+
t, testName, blocklistName, expectedFailures, version, "N/A",
203+
)
204+
}

‎pkg/cmd/roachtest/tests/nodejs_helpers_test.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// Copyright 2025 The Cockroach Authors.
2+
//
3+
// Use of this software is governed by the CockroachDB Software License
4+
// included in the /LICENSE file.
5+
6+
package tests
7+
8+
import (
9+
"testing"
10+
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
// TestParseMochaOutput tests the Mocha output parser with real test output.
15+
func TestParseMochaOutput(t *testing.T) {
16+
testCases := []struct {
17+
name string
18+
input string
19+
expected []mochaTestResult
20+
}{
21+
{
22+
name: "single level hierarchy",
23+
input: `
24+
70 passing (17s)
25+
1 pending
26+
2 failing
27+
28+
1) events
29+
emits acquire every time a client is acquired:
30+
Uncaught Error: expected 0 to equal 20
31+
32+
2) pool size of 1
33+
can only send 1 query at a time:
34+
Error: expected values to match
35+
`,
36+
expected: []mochaTestResult{
37+
{name: "events => emits acquire every time a client is acquired", status: statusFail},
38+
{name: "pool size of 1 => can only send 1 query at a time", status: statusFail},
39+
},
40+
},
41+
{
42+
name: "multi-level hierarchy",
43+
input: `
44+
70 passing (17s)
45+
1 pending
46+
2 failing
47+
48+
1) pool
49+
with callbacks
50+
removes client if it errors in background:
51+
Error: Timeout of 2000ms exceeded
52+
53+
2) pool size of 1
54+
can only send 1 query at a time:
55+
Error: expected values to match
56+
`,
57+
expected: []mochaTestResult{
58+
{name: "pool => with callbacks => removes client if it errors in background", status: statusFail},
59+
{name: "pool size of 1 => can only send 1 query at a time", status: statusFail},
60+
},
61+
},
62+
{
63+
name: "passing tests",
64+
input: `
65+
✓ works totally unconfigured (44ms)
66+
✓ passes props to clients (43ms)
67+
`,
68+
expected: []mochaTestResult{
69+
{name: "works totally unconfigured (44ms)", status: statusPass},
70+
{name: "passes props to clients (43ms)", status: statusPass},
71+
},
72+
},
73+
{
74+
name: "skipped tests",
75+
input: `
76+
- is returned from the query method
77+
✓ verifies a client with a callback (44ms)
78+
`,
79+
expected: []mochaTestResult{
80+
{name: "is returned from the query method", status: statusSkip},
81+
{name: "verifies a client with a callback (44ms)", status: statusPass},
82+
},
83+
},
84+
}
85+
86+
for _, tc := range testCases {
87+
t.Run(tc.name, func(t *testing.T) {
88+
results := parseMochaOutput(tc.input)
89+
require.Equal(t, len(tc.expected), len(results), "unexpected number of results")
90+
for i, expected := range tc.expected {
91+
require.Equal(t, expected.name, results[i].name, "test name mismatch at index %d", i)
92+
require.Equal(t, expected.status, results[i].status, "test status mismatch at index %d", i)
93+
}
94+
})
95+
}
96+
}

‎pkg/cmd/roachtest/tests/nodejs_postgres.go‎

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ package tests
88
import (
99
"context"
1010
"fmt"
11-
"strings"
1211

1312
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
1413
"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
@@ -110,27 +109,33 @@ PGSSLCERT=$HOME/certs/client.%[1]s.crt PGSSLKEY=$HOME/certs/client.%[1]s.key PGS
110109

111110
rawResultsStr := result.Stdout + result.Stderr
112111
t.L().Printf("Test Results: %s", rawResultsStr)
113-
if err != nil {
114-
// Check for expected test failures. We allow:
115-
// 1. One failing test that is "pool size of 1"
116-
// 2. One failing test that is "events"
117-
// 3. Two failing tests that are exactly "events" and "pool size of 1"
118-
if strings.Contains(rawResultsStr, "1 failing") {
119-
// Single test failure case
120-
if strings.Contains(rawResultsStr, "1) pool size of 1") ||
121-
strings.Contains(rawResultsStr, "1) events") {
122-
err = nil
123-
}
124-
} else if strings.Contains(rawResultsStr, "2 failing") {
125-
// Two test failures case - must be exactly events and pool size of 1
126-
if strings.Contains(rawResultsStr, "1) events") &&
127-
strings.Contains(rawResultsStr, "2) pool size of 1") {
128-
err = nil
129-
}
130-
}
131-
if err != nil {
132-
t.Fatal(err)
133-
}
112+
113+
// Get version for reporting
114+
version, versionErr := fetchCockroachVersion(ctx, t.L(), c, node[0])
115+
if versionErr != nil {
116+
version = "unknown"
117+
}
118+
119+
// Use the new parsing system with blocklist and ignorelist
120+
const blocklistName = "nodePostgresBlockList"
121+
const ignorelistName = "nodePostgresIgnoreList"
122+
expectedFailures := nodePostgresBlockList
123+
ignorelist := nodePostgresIgnoreList
124+
125+
status := fmt.Sprintf("Running cockroach version %s, using blocklist %s, using ignorelist %s",
126+
version, blocklistName, ignorelistName)
127+
t.L().Printf("%s", status)
128+
129+
// Parse and summarize the test results
130+
// If there were no command errors, the test passed completely
131+
if err == nil {
132+
t.L().Printf("All tests passed successfully")
133+
} else {
134+
// Parse the output and check against blocklist/ignorelist
135+
parseAndSummarizeNodeJSTestResults(
136+
ctx, t, c, node, "node-postgres", rawResultsStr,
137+
blocklistName, expectedFailures, ignorelist, version,
138+
)
134139
}
135140
}
136141

0 commit comments

Comments
 (0)