Add commands to debug tests multiple times (#1763)

* Add commands to debug tests multiple times

Adds two new commands, `swift.debugTestsMultipleTimes` and
`swift.debugTestsUntilFailure`. They behave the same as the existing run
multiple and run until failure commands, except they use the debugging
profile. When debugging multiple times a build is only performed for the
first iteration, after which the resulting artifacts are run multiple
times.

Also cleans up some of the test run console output.

Issue: #1747

* Update changelog

Author: plemarquand

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 - New `swift.createTasksForLibraryProducts` setting that when enabled causes the extension to automatically create and provide tasks for library products ([#1741](https://github.com/swiftlang/vscode-swift/pull/1741))
 - New `swift.outputChannelLogLevel` setting to control the verbosity of the `Swift` output channel ([#1746](https://github.com/swiftlang/vscode-swift/pull/1746))
+- New `swift.debugTestsMultipleTimes` and `swift.debugTestsUntilFailure` commands for debugging tests over multiple runs ([#1763](https://github.com/swiftlang/vscode-swift/pull/1763))
 - Optionally include LLDB DAP logs in the Swift diagnostics bundle ([#1768](https://github.com/swiftlang/vscode-swift/pull/1758))
 
 ### Changed

package.json

@@ -277,6 +277,16 @@
         "title": "Run Until Failure...",
         "category": "Swift"
       },
+      {
+        "command": "swift.debugTestsMultipleTimes",
+        "title": "Debug Multiple Times...",
+        "category": "Swift"
+      },
+      {
+        "command": "swift.debugTestsUntilFailure",
+        "title": "Debug Until Failure...",
+        "category": "Swift"
+      },
       {
         "command": "swift.pickProcess",
         "title": "Pick Process...",
@@ -976,6 +986,16 @@
           "command": "swift.runTestsUntilFailure",
           "group": "testExtras",
           "when": "testId in swift.tests"
+        },
+        {
+          "command": "swift.debugTestsMultipleTimes",
+          "group": "testExtras",
+          "when": "testId in swift.tests"
+        },
+        {
+          "command": "swift.debugTestsUntilFailure",
+          "group": "testExtras",
+          "when": "testId in swift.tests"
         }
       ],
       "testing/item/context": [
@@ -988,6 +1008,16 @@
           "command": "swift.runTestsUntilFailure",
           "group": "testExtras",
           "when": "testId in swift.tests"
+        },
+        {
+          "command": "swift.debugTestsMultipleTimes",
+          "group": "testExtras",
+          "when": "testId in swift.tests"
+        },
+        {
+          "command": "swift.debugTestsUntilFailure",
+          "group": "testExtras",
+          "when": "testId in swift.tests"
         }
       ],
       "file/newFile": [
@@ -1015,6 +1045,16 @@
           "group": "testExtras",
           "when": "false"
         },
+        {
+          "command": "swift.debugTestsMultipleTimes",
+          "group": "testExtras",
+          "when": "false"
+        },
+        {
+          "command": "swift.debugTestsUntilFailure",
+          "group": "testExtras",
+          "when": "false"
+        },
         {
           "command": "swift.generateLaunchConfigurations",
           "when": "swift.hasExecutableProduct"

src/TestExplorer/TestParsers/SwiftTestingOutputParser.ts

@@ -22,7 +22,7 @@ import {
 } from "./TestEventStreamReader";
 import { ITestRunState } from "./TestRunState";
 import { TestClass } from "../TestDiscovery";
-import { sourceLocationToVSCodeLocation } from "../../utilities/utilities";
+import { colorize, sourceLocationToVSCodeLocation } from "../../utilities/utilities";
 import { exec } from "child_process";
 import { lineBreakRegex } from "../../utilities/tasks";
 
@@ -649,15 +649,15 @@ export class SymbolRenderer {
             case TestSymbol.skip:
             case TestSymbol.difference:
             case TestSymbol.passWithKnownIssue:
-                return `${SymbolRenderer.ansiEscapeCodePrefix}90m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
+                return colorize(symbol, "grey");
             case TestSymbol.pass:
-                return `${SymbolRenderer.ansiEscapeCodePrefix}92m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
+                return colorize(symbol, "lightGreen");
             case TestSymbol.fail:
-                return `${SymbolRenderer.ansiEscapeCodePrefix}91m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
+                return colorize(symbol, "lightRed");
             case TestSymbol.warning:
-                return `${SymbolRenderer.ansiEscapeCodePrefix}93m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
+                return colorize(symbol, "lightYellow");
             case TestSymbol.attachment:
-                return `${SymbolRenderer.ansiEscapeCodePrefix}94m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
+                return colorize(symbol, "lightBlue");
             case TestSymbol.none:
             default:
                 return symbol;

src/TestExplorer/TestRunner.ts

@@ -337,6 +337,9 @@ export class TestRunProxy {
     public setIteration(iteration: number) {
         this.runState = TestRunProxy.initialTestRunState();
         this.iteration = iteration;
+        if (this.testRun) {
+            this.performAppendOutput(this.testRun, "\n\r");
+        }
     }
 
     public appendOutput(output: string) {
@@ -933,29 +936,34 @@ export class TestRunner {
     }
 
     /** Run test session inside debugger */
-    async debugSession(runState: TestRunnerTestRunState) {
-        // Perform a build all first to produce the binaries we'll run later.
-        let buildOutput = "";
-        try {
-            await this.runStandardSession(
-                // Capture the output to print it in case of a build error.
-                // We dont want to associate it with the test run.
-                new stream.Writable({
-                    write: (chunk, _encoding, next) => {
-                        buildOutput += chunk.toString();
-                        next();
-                    },
-                }),
-                await BuildConfigurationFactory.buildAll(
-                    this.folderContext,
-                    true,
-                    isRelease(this.testKind)
-                ),
-                this.testKind
-            );
-        } catch (buildExitCode) {
-            runState.recordOutput(undefined, buildOutput);
-            throw new Error(`Build failed with exit code ${buildExitCode}`);
+    async debugSession(
+        runState: TestRunnerTestRunState,
+        performBuild: boolean = true
+    ): Promise<TestRunState> {
+        if (performBuild) {
+            // Perform a build all first to produce the binaries we'll run later.
+            let buildOutput = "";
+            try {
+                await this.runStandardSession(
+                    // Capture the output to print it in case of a build error.
+                    // We dont want to associate it with the test run.
+                    new stream.Writable({
+                        write: (chunk, _encoding, next) => {
+                            buildOutput += chunk.toString();
+                            next();
+                        },
+                    }),
+                    await BuildConfigurationFactory.buildAll(
+                        this.folderContext,
+                        true,
+                        isRelease(this.testKind)
+                    ),
+                    this.testKind
+                );
+            } catch (buildExitCode) {
+                runState.recordOutput(undefined, buildOutput);
+                throw new Error(`Build failed with exit code ${buildExitCode}`);
+            }
         }
 
         const testRunTime = Date.now();
@@ -1136,6 +1144,8 @@ export class TestRunner {
                 this.workspaceContext.logger
             );
         });
+
+        return this.testRun.runState;
     }
 
     /** Returns a callback that handles a chunk of stdout output from a test run. */

src/commands.ts

@@ -91,6 +91,8 @@ export enum Commands {
     UPDATE_DEPENDENCIES = "swift.updateDependencies",
     RUN_TESTS_MULTIPLE_TIMES = "swift.runTestsMultipleTimes",
     RUN_TESTS_UNTIL_FAILURE = "swift.runTestsUntilFailure",
+    DEBUG_TESTS_MULTIPLE_TIMES = "swift.debugTestsMultipleTimes",
+    DEBUG_TESTS_UNTIL_FAILURE = "swift.debugTestsUntilFailure",
     RESET_PACKAGE = "swift.resetPackage",
     USE_LOCAL_DEPENDENCY = "swift.useLocalDependency",
     UNEDIT_DEPENDENCY = "swift.uneditDependency",
@@ -144,7 +146,13 @@ export function register(ctx: WorkspaceContext): vscode.Disposable[] {
             async (...args: (vscode.TestItem | number)[]) => {
                 const { testItems, count } = extractTestItemsAndCount(...args);
                 if (ctx.currentFolder) {
-                    return await runTestMultipleTimes(ctx.currentFolder, testItems, false, count);
+                    return await runTestMultipleTimes(
+                        ctx.currentFolder,
+                        testItems,
+                        false,
+                        TestKind.standard,
+                        count
+                    );
                 }
             }
         ),
@@ -153,7 +161,44 @@ export function register(ctx: WorkspaceContext): vscode.Disposable[] {
             async (...args: (vscode.TestItem | number)[]) => {
                 const { testItems, count } = extractTestItemsAndCount(...args);
                 if (ctx.currentFolder) {
-                    return await runTestMultipleTimes(ctx.currentFolder, testItems, true, count);
+                    return await runTestMultipleTimes(
+                        ctx.currentFolder,
+                        testItems,
+                        true,
+                        TestKind.standard,
+                        count
+                    );
+                }
+            }
+        ),
+
+        vscode.commands.registerCommand(
+            Commands.DEBUG_TESTS_MULTIPLE_TIMES,
+            async (...args: (vscode.TestItem | number)[]) => {
+                const { testItems, count } = extractTestItemsAndCount(...args);
+                if (ctx.currentFolder) {
+                    return await runTestMultipleTimes(
+                        ctx.currentFolder,
+                        testItems,
+                        false,
+                        TestKind.debug,
+                        count
+                    );
+                }
+            }
+        ),
+        vscode.commands.registerCommand(
+            Commands.DEBUG_TESTS_UNTIL_FAILURE,
+            async (...args: (vscode.TestItem | number)[]) => {
+                const { testItems, count } = extractTestItemsAndCount(...args);
+                if (ctx.currentFolder) {
+                    return await runTestMultipleTimes(
+                        ctx.currentFolder,
+                        testItems,
+                        true,
+                        TestKind.debug,
+                        count
+                    );
                 }
             }
         ),

src/commands/testMultipleTimes.ts

@@ -13,9 +13,10 @@
 //===----------------------------------------------------------------------===//
 
 import * as vscode from "vscode";
-import { TestKind } from "../TestExplorer/TestKind";
+import { isDebugging, TestKind } from "../TestExplorer/TestKind";
 import { TestRunner, TestRunnerTestRunState, TestRunState } from "../TestExplorer/TestRunner";
 import { FolderContext } from "../FolderContext";
+import { colorize } from "../utilities/utilities";
 
 /**
  * Runs the supplied TestItem a number of times. The user is prompted with a dialog
@@ -28,6 +29,7 @@ export async function runTestMultipleTimes(
     currentFolder: FolderContext,
     tests: vscode.TestItem[],
     untilFailure: boolean,
+    kind: TestKind,
     count: number | undefined = undefined,
     testRunner?: () => Promise<TestRunState>
 ) {
@@ -50,7 +52,7 @@ export async function runTestMultipleTimes(
     const token = new vscode.CancellationTokenSource();
     const testExplorer = currentFolder.testExplorer;
     const runner = new TestRunner(
-        TestKind.standard,
+        kind,
         new vscode.TestRunRequest(tests),
         currentFolder,
         testExplorer.controller,
@@ -66,11 +68,18 @@ export async function runTestMultipleTimes(
     const runStates: TestRunState[] = [];
     for (let i = 0; i < numExecutions; i++) {
         runner.setIteration(i);
-        runner.testRun.appendOutput(`\x1b[36mBeginning Test Iteration #${i + 1}\x1b[0m\n`);
+        runner.testRun.appendOutput(
+            colorize(`Beginning Test Iteration #${i + 1}`, "cyan") + "\n\r"
+        );
 
-        const runState = await (testRunner !== undefined
-            ? testRunner()
-            : runner.runSession(testRunState));
+        let runState: TestRunState;
+        if (testRunner !== undefined) {
+            runState = await testRunner();
+        } else if (isDebugging(kind)) {
+            runState = await runner.debugSession(testRunState, i === 0);
+        } else {
+            runState = await runner.runSession(testRunState);
+        }
 
         runStates.push(runState);
 

src/utilities/utilities.ts

@@ -154,6 +154,29 @@ export async function execFile(
     });
 }
 
+enum Color {
+    red = 31,
+    green = 32,
+    yellow = 33,
+    blue = 34,
+    magenta = 35,
+    cyan = 36,
+    white = 37,
+    grey = 90,
+    lightRed = 91,
+    lightGreen = 92,
+    lightYellow = 93,
+    lightBlue = 94,
+}
+
+export function colorize(text: string, color: keyof typeof Color): string {
+    const colorCode = Color[color];
+    if (colorCode !== undefined) {
+        return `\x1b[${colorCode}m${text}\x1b[0m`;
+    }
+    return text;
+}
+
 export async function execFileStreamOutput(
     executable: string,
     args: string[],

test/integration-tests/commands/runTestMultipleTimes.test.ts

@@ -18,6 +18,7 @@ import { runTestMultipleTimes } from "../../../src/commands/testMultipleTimes";
 import { FolderContext } from "../../../src/FolderContext";
 import { TestRunProxy } from "../../../src/TestExplorer/TestRunner";
 import { activateExtensionForSuite, folderInRootWorkspace } from "../utilities/testutilities";
+import { TestKind } from "../../../src/TestExplorer/TestKind";
 
 suite("Test Multiple Times Command Test Suite", () => {
     let folderContext: FolderContext;
@@ -39,13 +40,24 @@ suite("Test Multiple Times Command Test Suite", () => {
     });
 
     test("Runs successfully after testing 0 times", async () => {
-        const runState = await runTestMultipleTimes(folderContext, [testItem], false, 0);
+        const runState = await runTestMultipleTimes(
+            folderContext,
+            [testItem],
+            false,
+            TestKind.standard,
+            0
+        );
         expect(runState).to.be.an("array").that.is.empty;
     });
 
     test("Runs successfully after testing 3 times", async () => {
-        const runState = await runTestMultipleTimes(folderContext, [testItem], false, 3, () =>
-            Promise.resolve(TestRunProxy.initialTestRunState())
+        const runState = await runTestMultipleTimes(
+            folderContext,
+            [testItem],
+            false,
+            TestKind.standard,
+            3,
+            () => Promise.resolve(TestRunProxy.initialTestRunState())
         );
 
         expect(runState).to.deep.equal([
@@ -61,13 +73,20 @@ suite("Test Multiple Times Command Test Suite", () => {
             failed: [{ test: testItem, message: new vscode.TestMessage("oh no") }],
         };
         let ctr = 0;
-        const runState = await runTestMultipleTimes(folderContext, [testItem], true, 3, () => {
-            ctr += 1;
-            if (ctr === 2) {
-                return Promise.resolve(failure);
+        const runState = await runTestMultipleTimes(
+            folderContext,
+            [testItem],
+            true,
+            TestKind.standard,
+            3,
+            () => {
+                ctr += 1;
+                if (ctr === 2) {
+                    return Promise.resolve(failure);
+                }
+                return Promise.resolve(TestRunProxy.initialTestRunState());
             }
-            return Promise.resolve(TestRunProxy.initialTestRunState());
-        });
+        );
 
         expect(runState).to.deep.equal([TestRunProxy.initialTestRunState(), failure]);
     });