Skip to content

Commit bdf8d55

Browse files
gagikgagik
authored
feat(tests): Add individual evergreen test results with XUnit (#2227)
Allows for individual testing results to be viewed and makes it easier to detect specific flaky tests with Foliage in the future.
1 parent 16b453e commit bdf8d55

File tree

28 files changed

+380
-25
lines changed

28 files changed

+380
-25
lines changed

‎.evergreen.yml‎

Lines changed: 309 additions & 1 deletion
Large diffs are not rendered by default.

‎.evergreen/evergreen.yml.in‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@ post:
130130
visibility: signed
131131
content_type: application/x-gzip
132132
optional: true
133+
- command: attach.xunit_results
134+
params:
135+
file: src/.logs/*.xml
133136

134137

135138
# Functions are any command that can be run.
@@ -264,6 +267,7 @@ functions:
264267
MONGOSH_RUN_ONLY_IN_PACKAGE: ${mongosh_run_only_in_package}
265268
AWS_AUTH_IAM_ACCESS_KEY_ID: ${devtools_ci_aws_key}
266269
AWS_AUTH_IAM_SECRET_ACCESS_KEY: ${devtools_ci_aws_secret}
270+
TASK_NAME: ${task_name}
267271
- command: s3.put
268272
params:
269273
aws_key: ${aws_key}
@@ -517,7 +521,7 @@ functions:
517521
AWS_AUTH_IAM_ACCESS_KEY_ID: ${devtools_ci_aws_key}
518522
AWS_AUTH_IAM_SECRET_ACCESS_KEY: ${devtools_ci_aws_secret}
519523
DISABLE_OPENSSL_SHARED_CONFIG_FOR_BUNDLED_OPENSSL: ${disable_openssl_shared_config_for_bundled_openssl}
520-
E2E_TASK_NAME: ${task_name}
524+
TASK_NAME: ${task_name}
521525

522526
###
523527
# PACKAGING AND UPLOADING
@@ -1124,6 +1128,7 @@ tasks:
11241128
mongosh_skip_node_version_check: "<% out(skipNodeVersionCheck) %>"
11251129
mongosh_test_id: "<% out(id) %>"
11261130
mongosh_run_only_in_package: "<% out(packageName) %>"
1131+
task_name: ${task_name}
11271132
<% } %>
11281133

11291134
###
@@ -1141,6 +1146,7 @@ tasks:
11411146
- func: test_vscode
11421147
vars:
11431148
node_js_version: "<% out(NODE_JS_VERSION_20) %>"
1149+
task_name: ${task_name}
11441150
- name: test_connectivity
11451151
tags: ["extra-integration-test"]
11461152
depends_on:
@@ -1154,6 +1160,7 @@ tasks:
11541160
- func: test_connectivity
11551161
vars:
11561162
node_js_version: "<% out(NODE_JS_VERSION_20) %>"
1163+
task_name: ${task_name}
11571164
- name: test_apistrict
11581165
tags: ["extra-integration-test"]
11591166
depends_on:
@@ -1169,6 +1176,7 @@ tasks:
11691176
node_js_version: "<% out(NODE_JS_VERSION_20) %>"
11701177
mongosh_server_test_version: "latest-alpha-enterprise"
11711178
mongosh_test_force_api_strict: "1"
1179+
task_name: ${task_name}
11721180
- name: compile_artifact
11731181
tags: ["compile-artifact"]
11741182
depends_on:
@@ -1273,6 +1281,7 @@ tasks:
12731281
node_js_version: "<% out(NODE_JS_VERSION_20) %>"
12741282
test_mongosh_executable: dist/mongosh
12751283
kerberos_jumphost_dockerfile: "Dockerfile.<% out(dockerFile) %>"
1284+
task_name: ${task_name}
12761285
<% } } %>
12771286

12781287
###
@@ -1406,6 +1415,7 @@ tasks:
14061415
vars:
14071416
node_js_version: "<% out(NODE_JS_VERSION_20) %>"
14081417
dockerfile: <% out(dockerfile) %>
1418+
task_name: ${task_name}
14091419
<% }; break;
14101420
// We don't have docker for platforms other than x64, so for those we just
14111421
// extract the archives locally.

‎.evergreen/run-e2e-tests.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
#!/usr/bin/env bash
22
set -e
33
export NODE_JS_VERSION=${NODE_JS_VERSION}
4+
export TASK_NAME=${TASK_NAME}
45

5-
if [[ "$DISABLE_OPENSSL_SHARED_CONFIG_FOR_BUNDLED_OPENSSL" == "true" ]] && [[ ! "$E2E_TASK_NAME" =~ openssl(3|11) ]]; then
6+
if [[ "$DISABLE_OPENSSL_SHARED_CONFIG_FOR_BUNDLED_OPENSSL" == "true" ]] && [[ ! "$TASK_NAME" =~ openssl(3|11) ]]; then
67
# On RHEL9 and based-distros, an additional configuration option
78
# `rh-allow-sha1-signatures` is present which is not recognizable to the
89
# OpenSSL version bundled with Node.js and hence the mongosh binary fails to

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,5 @@ mongocryptd.pid
2626
.sbom
2727
.nvm
2828
snapshot.blob
29+
.logs/*
30+
!.logs/empty.xml

‎.logs/empty.xml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
<!-- An empty test suite which is needed as not all tasks produce XUnit results and the current Evergreen setup always expects some file to be uploaded -->
2+
<testsuite name="empty" tests="0" failures="0" errors="0" skipped="0" timestamp="Fri, 18 Oct 2024 09:21:06 GMT" time="8.907">
3+
</testsuite>

‎configs/mocha-config-mongosh/reporter.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import { reporters } from 'mocha';
2+
import type { MochaOptions, Runner } from 'mocha';
3+
import path from 'path';
4+
5+
// Import the built-in reporters
6+
const Spec = reporters.Spec;
7+
const XUnit = reporters.XUnit;
8+
9+
export class MochaReporter extends reporters.Base {
10+
constructor(runner: Runner, options: MochaOptions) {
11+
super(runner, options);
12+
const suiteName = process.env.TASK_NAME ?? path.basename(process.cwd());
13+
14+
new Spec(runner);
15+
16+
runner.on('suite', (suite) => {
17+
if (suite.parent?.root) {
18+
suite.title = `${suiteName}__${suite.title}`;
19+
}
20+
});
21+
22+
new XUnit(runner, {
23+
reporterOptions: {
24+
suiteName,
25+
output: path.join(__dirname, '..', '..', '.logs', `${suiteName}.xml`),
26+
},
27+
});
28+
}
29+
}
30+
31+
module.exports = MochaReporter;

‎packages/arg-parser/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"unsafe-perm": true
1212
},
1313
"scripts": {
14-
"test": "mocha -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,lib}/**/*.spec.ts\"",
14+
"test": "mocha -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,lib}/**/*.spec.ts\" --reporter \"../../configs/mocha-config-mongosh/reporter.ts\"",
1515
"test-ci": "node ../../scripts/run-if-package-requested.js npm test",
1616
"test-coverage": "nyc --no-clean --cwd ../.. --reporter=none npm run test",
1717
"test-ci-coverage": "nyc --no-clean --cwd ../.. --reporter=none npm run test-ci",

‎packages/async-rewriter2/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"scripts": {
77
"pretest": "npm run compile",
88
"benchmark": "node -r ts-node/register benchmark/index.ts",
9-
"test": "mocha --experimental-vm-modules -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,lib}/**/*.spec.ts\"",
9+
"test": "mocha --experimental-vm-modules -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,lib}/**/*.spec.ts\" --reporter \"../../configs/mocha-config-mongosh/reporter.ts\"",
1010
"test-ci": "node ../../scripts/run-if-package-requested.js npm test",
1111
"test-coverage": "nyc --no-clean --cwd ../.. --reporter=none npm run test",
1212
"test-ci-coverage": "nyc --no-clean --cwd ../.. --reporter=none npm run test-ci",

‎packages/autocomplete/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
"node": ">=14.15.1"
1818
},
1919
"scripts": {
20-
"test": "mocha -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,lib}/**/*.spec.ts\"",
20+
"test": "mocha -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,lib}/**/*.spec.ts\" --reporter \"../../configs/mocha-config-mongosh/reporter.ts\"",
2121
"test-ci": "node ../../scripts/run-if-package-requested.js npm test",
2222
"test-coverage": "nyc --no-clean --cwd ../.. --reporter=none npm run test",
2323
"test-ci-coverage": "nyc --no-clean --cwd ../.. --reporter=none npm run test-ci",

‎packages/browser-runtime-core/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
"url": "git://github.com/mongodb-js/mongosh.git"
1616
},
1717
"scripts": {
18-
"test": "mocha -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,lib}/**/*.spec.ts\"",
18+
"test": "mocha -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,lib}/**/*.spec.ts\" --reporter \"../../configs/mocha-config-mongosh/reporter.ts\"",
1919
"test-ci": "node ../../scripts/run-if-package-requested.js npm test",
2020
"test-coverage": "nyc --no-clean --cwd ../.. --reporter=none npm run test",
2121
"test-ci-coverage": "nyc --no-clean --cwd ../.. --reporter=none npm run test-ci",

0 commit comments

Comments
 (0)