test(core): merge all package test reports (#6261)

## Problem
- CI test reporting only captures the last package's results
- Running `npm run testE2E` only preserves toolkit's report.xml, losing
other package results


## Solution
- Generate individual report.xml files per subproject
- Consolidate all existing package reports into root
.test-reports/report.xml
- This is done in the buildspecs themselves rather than directly in the
package.json (the original approach to solve this problem) because
mac/linux and windows have two different ways of getting the last error
code, resulting in a complicated package.json for something thats only
needed in our codebuilds


---

- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).

License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

Author: jpinkney-aws

buildspec/linuxE2ETests.yml

@@ -37,7 +37,7 @@ phases:
         commands:
             - export HOME=/home/codebuild-user
             # Ignore failure until throttling issues are fixed.
-            - xvfb-run npm run testE2E
+            - xvfb-run npm run testE2E; npm run mergeReports -- "$?"
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"

buildspec/linuxIntegrationTests.yml

@@ -92,7 +92,7 @@ phases:
     build:
         commands:
             - export HOME=/home/codebuild-user
-            - xvfb-run npm run testInteg
+            - xvfb-run npm run testInteg; npm run mergeReports -- "$?"
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"

buildspec/linuxTests.yml

@@ -41,7 +41,7 @@ phases:
                     # Ensure that "foo | run_and_report" fails correctly.
                     set -o pipefail
                     . buildspec/shared/common.sh
-                    2>&1 xvfb-run npm test --silent | run_and_report 2 \
+                    { 2>&1 xvfb-run npm test --silent; npm run mergeReports -- "$?"; } | run_and_report 2 \
                         'rejected promise not handled' \
                         'This typically indicates a bug. Read https://developer.mozilla.org/docs/Web/JavaScript/Guide/Using_promises#error_handling'
                 }

package.json

@@ -36,7 +36,8 @@
         "lintfix": "eslint -c .eslintrc.js --ignore-path .gitignore --ignore-pattern '**/*.json' --ignore-pattern '**/*.gen.ts' --ignore-pattern '**/types/*.d.ts' --ignore-pattern '**/src/testFixtures/**' --ignore-pattern '**/resources/js/graphStateMachine.js' --fix --ext .ts packages plugins",
         "clean": "npm run clean -w packages/ -w plugins/",
         "reset": "npm run clean && ts-node ./scripts/clean.ts node_modules && npm install",
-        "generateNonCodeFiles": "npm run generateNonCodeFiles -w packages/ --if-present"
+        "generateNonCodeFiles": "npm run generateNonCodeFiles -w packages/ --if-present",
+        "mergeReports": "ts-node ./scripts/mergeReports.ts"
     },
     "devDependencies": {
         "@aws-toolkits/telemetry": "^1.0.289",

packages/core/src/test/testRunner.ts

@@ -65,7 +65,8 @@ export async function runTests(
     }
 
     const root = getRoot()
-    const outputFile = path.resolve(root, '../../', '.test-reports', 'report.xml')
+    // output the report to the individual package
+    const outputFile = path.resolve(root, '.test-reports', 'report.xml')
     const colorOutput = !process.env['AWS_TOOLKIT_TEST_NO_COLOR']
 
     // Create the mocha test

scripts/mergeReports.ts

@@ -0,0 +1,71 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as fs from 'fs'
+import * as path from 'path'
+import * as xml2js from 'xml2js'
+
+/**
+ * Merge all of the packages/ test reports into a single directory
+ */
+async function mergeReports() {
+    console.log('Merging test reports')
+
+    const packagesDir = `${__dirname}/../packages`
+
+    // Get all packages/* directories
+    const packageDirs = fs.readdirSync(packagesDir).map((dir) => path.join(packagesDir, dir))
+
+    // Find report.xml files in .test-reports subdirectories
+    const testReports = packageDirs
+        .map((dir) => `${dir}/.test-reports/report.xml`)
+        .filter((file) => fs.existsSync(file))
+
+    const mergedReport = {
+        testsuites: {
+            testsuite: [],
+        },
+    }
+
+    // Collect all test reports into a single merged test report object
+    for (const file of testReports) {
+        const content = fs.readFileSync(file)
+        const result: { testsuites: { testsuite: [] } } = await xml2js.parseStringPromise(content)
+        if (result.testsuites && result.testsuites.testsuite) {
+            mergedReport.testsuites.testsuite.push(...result.testsuites.testsuite)
+        }
+    }
+
+    const builder = new xml2js.Builder()
+    const xml = builder.buildObject(mergedReport)
+
+    /**
+     * Create the new test reports directory and write the test report
+     */
+    const reportsDir = path.join(__dirname, '..', '.test-reports')
+
+    // Create reports directory if it doesn't exist
+    if (!fs.existsSync(reportsDir)) {
+        fs.mkdirSync(reportsDir, { recursive: true })
+    }
+
+    fs.writeFileSync(`${reportsDir}/report.xml`, xml)
+
+    const exitCodeArg = process.argv[2]
+    if (exitCodeArg) {
+        /**
+         * Retrieves the exit code from the previous test run execution.
+         *
+         * This allows us to:
+         * 1. Merge and upload test reports regardless of the test execution status
+         * 2. Preserve the original test run exit code
+         * 3. Report the test status back to CI
+         */
+        const exitCode = parseInt(exitCodeArg || '0', 10)
+        process.exit(exitCode)
+    }
+}
+
+mergeReports()