Merge pull request #6621 from dibarbet/jest_report_ci

Report jest test results to AzDo

Author: dibarbet

.vscode/launch.json

@@ -41,7 +41,8 @@
                 "${workspaceFolder}/**",
                 "!**/node_modules/**"
             ],
-            "preLaunchTask": "buildDev"
+            "preLaunchTask": "buildDev",
+            "internalConsoleOptions": "openOnSessionStart"
         },
         {
             "name": "Launch Current File BasicRazorApp2_1 Integration Tests",
@@ -69,7 +70,8 @@
                 "${workspaceFolder}/**",
                 "!**/node_modules/**"
             ],
-            "preLaunchTask": "buildDev"
+            "preLaunchTask": "buildDev",
+            "internalConsoleOptions": "openOnSessionStart"
         },
         {
             "name": "Omnisharp: Launch Current File Integration Tests",

azure-pipelines/test-omnisharp.yml

@@ -14,6 +14,17 @@ steps:
   env:
     DISPLAY: :99.0
 
+- task: PublishTestResults@2
+  condition: succeededOrFailed()
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFormat: 'JUnit'
+    testResultsFiles: '*junit.xml'
+    searchFolder: '$(Build.SourcesDirectory)/out'
+    publishRunAttachments: true
+    mergeTestResults: true
+    testRunTitle: OmniSharp $(Agent.JobName) (Attempt $(System.JobAttempt))
+
 - task: PublishPipelineArtifact@1
   condition: failed()
   displayName: 'Upload integration test logs'

azure-pipelines/test.yml

@@ -36,6 +36,17 @@ jobs:
     env:
       DISPLAY: :99.0
 
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    displayName: 'Publish Test Results'
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: '*junit.xml'
+      searchFolder: '$(Build.SourcesDirectory)/out'
+      publishRunAttachments: true
+      mergeTestResults: true
+      testRunTitle: $(Agent.JobName) (Attempt $(System.JobAttempt))
+
   - task: PublishPipelineArtifact@1
     condition: failed()
     displayName: 'Upload integration test logs'

jest.config.ts

@@ -14,6 +14,23 @@ const config: Config = {
         '<rootDir>/omnisharptest/omnisharpUnitTests/jest.config.ts',
         '<rootDir>/omnisharptest/omnisharpIntegrationTests/jest.config.ts',
     ],
+    // Reporters are a global jest configuration property and cannot be set in the project jest config.
+    // This configuration will create a 'junit.xml' file in the output directory, no matter which test project is running.
+    // In order to not overwrite test results in CI, we configure a unique output file name in the gulp testTasks.
+    reporters: [
+        'default',
+        [
+            'jest-junit',
+            {
+                outputDirectory: '<rootDir>/out/',
+                reportTestSuiteErrors: 'true',
+                // Azure DevOps does not display test suites (it ignores them entirely).
+                // So we have to put all the info in the test case name so the UI shows anything relatively useful.
+                // See https://github.com/microsoft/azure-pipelines-tasks/issues/7659
+                titleTemplate: '{filename} / {suitename} / {title}',
+            },
+        ],
+    ],
 };
 
 export default config;

package-lock.json

@@ -130,6 +130,7 @@
     "@typescript-eslint/eslint-plugin": "^5.61.0",
     "@typescript-eslint/parser": "^5.61.0",
     "@vscode/test-electron": "2.3.4",
+    "@vscode/vsce": "2.21.0",
     "archiver": "5.3.0",
     "del": "3.0.0",
     "eslint": "^8.43.0",
@@ -147,6 +148,7 @@
     "gulp": "4.0.2",
     "jest": "^29.6.2",
     "jest-cli": "^29.6.4",
+    "jest-junit": "^16.0.0",
     "js-yaml": ">=3.13.1",
     "minimatch": "3.0.5",
     "mock-http-server": "1.4.2",
@@ -159,7 +161,6 @@
     "ts-node": "9.1.1",
     "typescript": "^5.1.6",
     "unzipper": "0.10.11",
-    "@vscode/vsce": "2.21.0",
     "vscode-oniguruma": "^1.6.1",
     "vscode-textmate": "^6.0.0",
     "vscode-uri": "^3.0.7",

package.json

@@ -21,7 +21,9 @@ gulp.task('test:razor', async () => {
 
 const razorIntegrationTestProjects = ['BasicRazorApp2_1'];
 for (const projectName of razorIntegrationTestProjects) {
-    gulp.task(`test:razorintegration:${projectName}`, async () => runIntegrationTest(projectName, /* razor */ true));
+    gulp.task(`test:razorintegration:${projectName}`, async () =>
+        runIntegrationTest(projectName, 'razorIntegrationTests', `Razor Test Integration ${projectName}`)
+    );
 }
 
 gulp.task(
@@ -41,10 +43,10 @@ const omnisharpIntegrationTestProjects = ['singleCsproj', 'slnWithCsproj', 'slnF
 
 for (const projectName of omnisharpIntegrationTestProjects) {
     gulp.task(`omnisharptest:integration:${projectName}:stdio`, async () =>
-        runOmnisharpJestIntegrationTest(projectName, 'stdio')
+        runOmnisharpJestIntegrationTest(projectName, 'stdio', `OmniSharp Test Integration ${projectName} STDIO}`)
     );
     gulp.task(`omnisharptest:integration:${projectName}:lsp`, async () =>
-        runOmnisharpJestIntegrationTest(projectName, 'lsp')
+        runOmnisharpJestIntegrationTest(projectName, 'lsp', `OmniSharp Test Integration ${projectName} LSP}`)
     );
     gulp.task(
         `omnisharptest:integration:${projectName}`,
@@ -73,7 +75,9 @@ gulp.task('test:unit', async () => {
 
 const integrationTestProjects = ['slnWithCsproj'];
 for (const projectName of integrationTestProjects) {
-    gulp.task(`test:integration:${projectName}`, async () => runIntegrationTest(projectName));
+    gulp.task(`test:integration:${projectName}`, async () =>
+        runIntegrationTest(projectName, 'integrationTests', `Test Integration ${projectName}`)
+    );
 }
 
 gulp.task(
@@ -83,7 +87,7 @@ gulp.task(
 
 gulp.task('test', gulp.series('test:unit', 'test:integration', 'test:razor', 'test:razorintegration'));
 
-async function runOmnisharpJestIntegrationTest(testAssetName: string, engine: 'stdio' | 'lsp') {
+async function runOmnisharpJestIntegrationTest(testAssetName: string, engine: 'stdio' | 'lsp', suiteName: string) {
     const workspaceFile = `omnisharp${engine === 'lsp' ? '_lsp' : ''}_${testAssetName}.code-workspace`;
     const testFolder = path.join('omnisharptest', 'omnisharpIntegrationTests');
 
@@ -96,26 +100,28 @@ async function runOmnisharpJestIntegrationTest(testAssetName: string, engine: 's
         CODE_DISABLE_EXTENSIONS: 'true',
     };
 
-    await runJestIntegrationTest(testAssetName, testFolder, workspaceFile, env);
+    await runJestIntegrationTest(testAssetName, testFolder, workspaceFile, suiteName, env);
 }
 
-async function runIntegrationTest(testAssetName: string, razor = false) {
+async function runIntegrationTest(testAssetName: string, testFolderName: string, suiteName: string) {
     const vscodeWorkspaceFileName = `lsp_tools_host_${testAssetName}.code-workspace`;
-    const testFolder = path.join('test', razor ? 'razorIntegrationTests' : 'integrationTests');
-    return await runJestIntegrationTest(testAssetName, testFolder, vscodeWorkspaceFileName);
+    const testFolder = path.join('test', testFolderName);
+    return await runJestIntegrationTest(testAssetName, testFolder, vscodeWorkspaceFileName, suiteName);
 }
 
 /**
  * Runs jest based integration tests.
  * @param testAssetName the name of the test asset
  * @param testFolderName the relative path (from workspace root)
  * @param workspaceFileName the name of the vscode workspace file to use.
+ * @param suiteName a unique name for the test suite being run.
  * @param env any environment variables needed.
  */
 async function runJestIntegrationTest(
     testAssetName: string,
     testFolderName: string,
     workspaceFileName: string,
+    suiteName: string,
     env: NodeJS.ProcessEnv = {}
 ) {
     // Test assets are always in a testAssets folder inside the integration test folder.
@@ -143,6 +149,10 @@ async function runJestIntegrationTest(
     env.CODE_EXTENSIONS_PATH = rootPath;
     env.EXTENSIONS_TESTS_PATH = vscodeRunnerPath;
 
+    // Configure the file and suite name in CI to avoid having multiple test runs stomp on each other.
+    env.JEST_JUNIT_OUTPUT_NAME = getJUnitFileName(suiteName);
+    env.JEST_SUITE_NAME = suiteName;
+
     const result = await spawnNode([launcherPath, '--enable-source-maps'], { env, cwd: rootPath });
 
     if (result.code === null || result.code > 0) {
@@ -154,6 +164,8 @@ async function runJestIntegrationTest(
 }
 
 async function runJestTest(project: string) {
+    process.env.JEST_JUNIT_OUTPUT_NAME = getJUnitFileName(project);
+    process.env.JEST_SUITE_NAME = project;
     const configPath = path.join(rootPath, 'jest.config.ts');
     const { results } = await jest.runCLI(
         {
@@ -168,3 +180,7 @@ async function runJestTest(project: string) {
         throw new Error('Tests failed.');
     }
 }
+
+function getJUnitFileName(suiteName: string) {
+    return `${suiteName.replaceAll(' ', '_')}_junit.xml`;
+}

tasks/testTasks.ts

@@ -24,19 +24,18 @@ export async function runIntegrationTests(projectName: string) {
         verbose: true,
     } as Config.Argv;
 
-    let filter: string;
     if (process.env.TEST_FILE_FILTER) {
-        // If we have just a file, run that with runTestsByPath.
-        jestConfig.runTestsByPath = true;
+        // If we have just a file, run that with an explicit match.
         jestConfig.testMatch = [process.env.TEST_FILE_FILTER];
-        filter = process.env.TEST_FILE_FILTER;
-    } else {
-        filter = projectName;
     }
 
-    const { results } = await jest.runCLI(jestConfig, [filter]);
+    const { results } = await jest.runCLI(jestConfig, [projectName]);
 
     if (!results.success) {
-        throw new Error('Tests failed.');
+        console.log('Tests failed.');
     }
+
+    // Explicitly exit the process - VSCode likes to write a bunch of cancellation errors to the console after this
+    // which make it look like the tests always fail.  We're done with the tests at this point, so just exit.
+    process.exit(results.success ? 0 : 1);
 }