diff --git a/buildspec/linuxE2ETests.yml b/buildspec/linuxE2ETests.yml index def6dfd1cd0..37fc8377bb8 100644 --- a/buildspec/linuxE2ETests.yml +++ b/buildspec/linuxE2ETests.yml @@ -37,7 +37,7 @@ phases: commands: - export HOME=/home/codebuild-user # Ignore failure until throttling issues are fixed. - - xvfb-run npm run testE2E; npm run mergeReports -- "$?" + - xvfb-run npm run testE2E; npm run createTestReport -- "$?" - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}" - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g') - CI_BUILD_ID="${CODEBUILD_BUILD_ID}" diff --git a/buildspec/linuxIntegrationTests.yml b/buildspec/linuxIntegrationTests.yml index dacab125b89..6c44badeb87 100644 --- a/buildspec/linuxIntegrationTests.yml +++ b/buildspec/linuxIntegrationTests.yml @@ -92,7 +92,7 @@ phases: build: commands: - export HOME=/home/codebuild-user - - xvfb-run npm run testInteg; npm run mergeReports -- "$?" + - xvfb-run npm run testInteg; npm run createTestReport -- "$?" - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}" - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g') - CI_BUILD_ID="${CODEBUILD_BUILD_ID}" diff --git a/buildspec/linuxTests.yml b/buildspec/linuxTests.yml index 900b720e61a..5e8fc95ced0 100644 --- a/buildspec/linuxTests.yml +++ b/buildspec/linuxTests.yml @@ -41,7 +41,7 @@ phases: # Ensure that "foo | run_and_report" fails correctly. set -o pipefail . buildspec/shared/common.sh - { 2>&1 xvfb-run npm test --silent; npm run mergeReports -- "$?"; } | run_and_report 2 \ + { 2>&1 xvfb-run npm test --silent; npm run createTestReport -- "$?"; } | run_and_report 2 \ 'rejected promise not handled' \ 'This typically indicates a bug. Read https://developer.mozilla.org/docs/Web/JavaScript/Guide/Using_promises#error_handling' } diff --git a/package.json b/package.json index cb669449f9d..0819f9d78d3 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,10 @@ "buildCustomLintPlugin": "npm run build -w plugins/eslint-plugin-aws-toolkits", "compile": "npm run compile -w packages/", "testCompile": "npm run testCompile -w packages/ --if-present", - "test": "npm run test -w packages/ --if-present", - "testWeb": "npm run testWeb -w packages/ --if-present", - "testE2E": "npm run testE2E -w packages/ --if-present", - "testInteg": "npm run testInteg -w packages/ --if-present", + "test": "npm run test -w packages/ --if-present; npm run createTestReport", + "testWeb": "npm run testWeb -w packages/ --if-present; npm run createTestReport", + "testE2E": "npm run testE2E -w packages/ --if-present; npm run createTestReport", + "testInteg": "npm run testInteg -w packages/ --if-present; npm run createTestReport", "package": "npm run package -w packages/toolkit -w packages/amazonq", "newChange": "echo 'Must specify subproject/workspace with -w packages/' && false", "createRelease": "echo 'Must specify subproject/workspace with -w packages/' && false", @@ -37,7 +37,7 @@ "clean": "npm run clean -w packages/ -w plugins/", "reset": "npm run clean && ts-node ./scripts/clean.ts node_modules && npm install", "generateNonCodeFiles": "npm run generateNonCodeFiles -w packages/ --if-present", - "mergeReports": "ts-node ./scripts/mergeReports.ts" + "createTestReport": "ts-node ./scripts/createTestReport.ts" }, "devDependencies": { "@aws-toolkits/telemetry": "^1.0.289", diff --git a/packages/amazonq/test/unit/validation.test.ts b/packages/amazonq/test/unit/validation.test.ts index 4b19d94935a..13aafe13f6f 100644 --- a/packages/amazonq/test/unit/validation.test.ts +++ b/packages/amazonq/test/unit/validation.test.ts @@ -27,4 +27,32 @@ describe('package validations', function () { ) assert.deepStrictEqual(packageJson.contributes.icons, corePackageJson.contributes.icons) }) + + describe('foo', () => { + it('bar1', () => { + assert.ok(true) + }) + describe('fi', () => { + it('bar2', () => { + assert.ok(true) + }) + describe('fo234', () => { + it('bar3', () => { + throw new Error('foo') + }) + }) + }) + }) + + describe('package validations', () => { + it('wee', () => { + assert.ok(true) + }) + + describe('foo', () => { + it('wee 2', () => { + assert.ok(true) + }) + }) + }) }) diff --git a/packages/core/src/test/shared/applicationBuilder/explorer/nodes/appNode.test.ts b/packages/core/src/test/shared/applicationBuilder/explorer/nodes/appNode.test.ts index 80d83a8303c..281ed25dfb1 100644 --- a/packages/core/src/test/shared/applicationBuilder/explorer/nodes/appNode.test.ts +++ b/packages/core/src/test/shared/applicationBuilder/explorer/nodes/appNode.test.ts @@ -169,6 +169,7 @@ describe('AppNode', () => { describe('getTreeItem', () => { it('should return a TreeItem with the correct properties', () => { + assert.fail('failed') const treeItem = appNode.getTreeItem() const expextedLabel = path.join('VSCode Example Workspace', 'Project One Root Folder') diff --git a/scripts/createTestReport.ts b/scripts/createTestReport.ts new file mode 100644 index 00000000000..e3549a0ae61 --- /dev/null +++ b/scripts/createTestReport.ts @@ -0,0 +1,242 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as fs from 'fs' +import * as xml2js from 'xml2js' + +interface TestFailure { + $: { + message: string + } + _: string +} + +interface TestCase { + $: { + classname: string + name: string + time: string + } + failure?: TestFailure[] +} + +interface TestSuite { + $: { + name: string + tests: string + failures: string + errors: string + time: string + file: string + } + testcase: TestCase[] | undefined +} + +interface TestReport { + testsuites: { + testsuite: TestSuite[] + } +} + +interface TestSummary { + totalTests: number + totalFailures: number + totalTime: number + failedTests: FailedTest[] +} + +interface FailedTest { + suite: string + test: string + message: string + contents: string + path: string[] +} + +/** + * Merge all of the packages/ test reports into a single directory + */ +async function createTestReport() { + console.log('Merging test reports') + + const packagesDir = `${__dirname}/../packages` + + // Get all packages/* directories + const packageDirs = fs.readdirSync(packagesDir).map((dir) => `${packagesDir}/${dir}`) + + // Find report.xml files in .test-reports subdirectories + const testReports = packageDirs + .map((dir) => `${dir}/.test-reports/report.xml`) + .filter((file) => fs.existsSync(file)) + + const mergedReport: TestReport = { + testsuites: { + testsuite: [], + }, + } + + const failedTests: FailedTest[] = [] + let totalTests = 0 + let totalFailures = 0 + let totalTime = 0 + + let filePath = '' + let suites = new Set() + + /** + * Collect all test reports into a single merged test report object. + * Also keeps track of test count, test failures, and test run time + */ + for (const file of testReports) { + const content = fs.readFileSync(file) + const result: { testsuites: { testsuite: TestSuite[] } } = await xml2js.parseStringPromise(content) + if (result.testsuites && result.testsuites.testsuite) { + for (const suite of result.testsuites.testsuite) { + if (suite.$.file !== filePath) { + filePath = suite.$.file + suites = new Set() + } + + for (const testcase of suite.testcase ?? []) { + if (testcase.failure) { + const testPath = parseTestHierarchy(suites, testcase.$.classname, suite.$.name, testcase.$.name) + failedTests.push({ + suite: suite.$.name, + test: testcase.$.name, + message: testcase.failure[0].$.message, + contents: testcase.failure[0]._, + path: testPath, + }) + } + } + + totalTests += parseInt(suite.$.tests, 10) + totalFailures += parseInt(suite.$.failures, 10) + totalTime += parseFloat(suite.$.time) + + suites.add(suite.$.name) + } + + mergedReport.testsuites.testsuite.push(...result.testsuites.testsuite) + } + } + + printTestSummary({ + totalTests, + totalFailures, + totalTime, + failedTests, + }) + + writeReport(mergedReport) +} + +/** + * Extracts and constructs a hierarchical test path from a test case identifier + * + * @param suites - Set of known test suite names + * @param className - Name of the test class + * @param suiteName - Name of the test suite + * @param testcaseName - Full name of the test case + * @example + * parseTestHierarchy(new Set(["package validations"]), 'bar1', 'foo', 'package validations foo bar1') -> ["package validations", "bar1", "foo"] + * @returns An array of path components representing the test hierarchy + */ +function parseTestHierarchy(suites: Set, className: string, suiteName: string, testcaseName: string) { + let remainingPath = testcaseName + remainingPath = remainingPath.substring(0, remainingPath.lastIndexOf(className)) + remainingPath = remainingPath.substring(0, remainingPath.lastIndexOf(suiteName)) + + const pathComponents = remainingPath.trim().split(' ') + let index = 0 + let currentComponent = pathComponents[0] + const path = [] + while (remainingPath.length > 0) { + index++ + if (!suites.has(currentComponent)) { + currentComponent = currentComponent + ' ' + pathComponents[index] + } else { + path.push(currentComponent) + remainingPath = remainingPath.substring(currentComponent.length).trim() + currentComponent = pathComponents[index] + } + } + + path.push(suiteName) + path.push(className) + + return path +} + +function printTestSummary({ totalTests, totalFailures, totalTime, failedTests }: TestSummary) { + const passingTests = totalTests - totalFailures + const pendingTests = 0 + + console.log(`${passingTests} passing (${Math.round(totalTime)}s)`) + if (pendingTests > 0) { + console.log(`${pendingTests} pending`) + } + if (totalFailures > 0) { + console.log(`${totalFailures} failing`) + + failedTests.forEach((test, index) => { + let indent = ' ' + + for (let x = 0; x < test.path.length; x++) { + if (x == 0) { + console.log(`${indent}${index + 1}) ${test.path[x]}`) + indent += ' ' + } else { + console.log(`${indent}${test.path[x]}`) + } + indent += ' ' + } + + if (test.contents) { + // Indent the stack trace + console.log( + test.contents + .split('\n') + .map((line) => `${indent}${line}`) + .join('\n') + ) + } + console.log() // Add empty line between failures + }) + } +} + +function writeReport(mergedReport: TestReport) { + const builder = new xml2js.Builder() + const xml = builder.buildObject(mergedReport) + + /** + * Create the new test reports directory and write the test report + */ + const reportsDir = `${__dirname}/../.test-reports` + + // Create reports directory if it doesn't exist + if (!fs.existsSync(reportsDir)) { + fs.mkdirSync(reportsDir, { recursive: true }) + } + + fs.writeFileSync(`${reportsDir}/report.xml`, xml) + + const exitCodeArg = process.argv[2] + if (exitCodeArg) { + /** + * Retrieves the exit code from the previous test run execution. + * + * This allows us to: + * 1. Merge and upload test reports regardless of the test execution status + * 2. Preserve the original test run exit code + * 3. Report the test status back to CI + */ + const exitCode = parseInt(exitCodeArg || '0', 10) + process.exit(exitCode) + } +} + +createTestReport() diff --git a/scripts/mergeReports.ts b/scripts/mergeReports.ts deleted file mode 100644 index 08b9dd6cd23..00000000000 --- a/scripts/mergeReports.ts +++ /dev/null @@ -1,71 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import * as fs from 'fs' -import * as path from 'path' -import * as xml2js from 'xml2js' - -/** - * Merge all of the packages/ test reports into a single directory - */ -async function mergeReports() { - console.log('Merging test reports') - - const packagesDir = `${__dirname}/../packages` - - // Get all packages/* directories - const packageDirs = fs.readdirSync(packagesDir).map((dir) => path.join(packagesDir, dir)) - - // Find report.xml files in .test-reports subdirectories - const testReports = packageDirs - .map((dir) => `${dir}/.test-reports/report.xml`) - .filter((file) => fs.existsSync(file)) - - const mergedReport = { - testsuites: { - testsuite: [], - }, - } - - // Collect all test reports into a single merged test report object - for (const file of testReports) { - const content = fs.readFileSync(file) - const result: { testsuites: { testsuite: [] } } = await xml2js.parseStringPromise(content) - if (result.testsuites && result.testsuites.testsuite) { - mergedReport.testsuites.testsuite.push(...result.testsuites.testsuite) - } - } - - const builder = new xml2js.Builder() - const xml = builder.buildObject(mergedReport) - - /** - * Create the new test reports directory and write the test report - */ - const reportsDir = path.join(__dirname, '..', '.test-reports') - - // Create reports directory if it doesn't exist - if (!fs.existsSync(reportsDir)) { - fs.mkdirSync(reportsDir, { recursive: true }) - } - - fs.writeFileSync(`${reportsDir}/report.xml`, xml) - - const exitCodeArg = process.argv[2] - if (exitCodeArg) { - /** - * Retrieves the exit code from the previous test run execution. - * - * This allows us to: - * 1. Merge and upload test reports regardless of the test execution status - * 2. Preserve the original test run exit code - * 3. Report the test status back to CI - */ - const exitCode = parseInt(exitCodeArg || '0', 10) - process.exit(exitCode) - } -} - -mergeReports()