Swift Testing Attachments Support (#1187)

* Attachments Support

Support swift-testing attachments. Currently prints the attachments out as a report at the end of a test run. In order to leverage them we set the --experimental-attachment-path defined by a new swift.attachmentsPath setting. This is a directory path which can be global or scoped to the workspace. It can be relative or absolute. If relative, the absolute path is computed relative to the project workspace root. Defaults to .build/attachments.

If the attachmentsPath directory doesn't exist, it is created. Each test run that produces attachments creates a subfolder named with the date/time of the test run, and places the runs attachments within it.

Author: plemarquand

.gitignore

@@ -7,6 +7,7 @@ default.profraw
 *.vsix
 .vscode-test
 .build
+.index-build
 .DS_Store
 assets/documentation-webview
 assets/test/**/Package.resolved

assets/test/defaultPackage/Tests/PackageTests/PackageTests.swift

@@ -42,9 +42,13 @@ final class DebugReleaseTestSuite: XCTestCase {
   }
 }
 
-#if swift(>=6.0)
+#if swift(>=6.1)
+@_spi(Experimental) import Testing
+#elseif swift(>=6.0)
 import Testing
+#endif
 
+#if swift(>=6.0)
 @Test func topLevelTestPassing() {
   print("A print statement in a test.")
   #if !TEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING
@@ -98,7 +102,12 @@ struct MixedSwiftTestingSuite {
   }
   #expect(2 == 3)
 }
+#endif
 
+#if swift(>=6.1)
+@Test func testAttachment() throws {
+  Attachment("Hello, world!", named: "hello.txt").attach()
+}
 #endif
 
 final class DuplicateSuffixTests: XCTestCase {

package.json

@@ -434,6 +434,12 @@
                 }
               }
             }
+          },
+          "swift.attachmentsPath": {
+            "type": "string",
+            "default": ".build/attachments",
+            "markdownDescription": "The path to a directory that will be used to store attachments produced during a test run.\n\nA relative path resolves relative to the root directory of the workspace running the test(s)",
+            "scope": "machine-overridable"
           }
         }
       },

src/TestExplorer/TestParsers/SwiftTestingOutputParser.ts

@@ -52,7 +52,8 @@ export type EventRecordPayload =
     | TestCaseEnded
     | IssueRecorded
     | TestSkipped
-    | RunEnded;
+    | RunEnded
+    | ValueAttached;
 
 export interface EventRecord extends VersionedRecord {
     kind: "event";
@@ -95,6 +96,15 @@ interface RunEnded {
     messages: EventMessage[];
 }
 
+interface ValueAttached {
+    kind: "_valueAttached";
+    _attachment: {
+        path?: string;
+    };
+    testID: string;
+    messages: EventMessage[];
+}
+
 interface Instant {
     absolute: number;
     since1970: number;
@@ -148,6 +158,7 @@ export enum TestSymbol {
     difference = "difference",
     warning = "warning",
     details = "details",
+    attachment = "attachment",
     none = "none",
 }
 
@@ -169,7 +180,8 @@ export class SwiftTestingOutputParser {
 
     constructor(
         public testRunStarted: () => void,
-        public addParameterizedTestCase: (testClass: TestClass, parentIndex: number) => void
+        public addParameterizedTestCase: (testClass: TestClass, parentIndex: number) => void,
+        public onAttachment: (testIndex: number, path: string) => void
     ) {}
 
     /**
@@ -457,6 +469,12 @@ export class SwiftTestingOutputParser {
                 this.completionMap.set(testIndex, true);
                 runState.completed(testIndex, { timestamp: item.payload.instant.absolute });
                 return;
+            } else if (item.payload.kind === "_valueAttached" && item.payload._attachment.path) {
+                const testID = this.idFromOptionalTestCase(item.payload.testID);
+                const testIndex = this.getTestCaseIndex(runState, testID);
+
+                this.onAttachment(testIndex, item.payload._attachment.path);
+                return;
             }
         }
     }
@@ -495,7 +513,7 @@ export class SymbolRenderer {
      * @param message An event message, typically found on an `EventRecordPayload`.
      * @returns A string colorized with ANSI escape codes.
      */
-    static eventMessageSymbol(symbol: TestSymbol): string {
+    public static eventMessageSymbol(symbol: TestSymbol): string {
         return this.colorize(symbol, this.symbol(symbol));
     }
 
@@ -521,6 +539,8 @@ export class SymbolRenderer {
                     return "\u{25B2}"; // Unicode: BLACK UP-POINTING TRIANGLE
                 case TestSymbol.details:
                     return "\u{2192}"; // Unicode: RIGHTWARDS ARROW
+                case TestSymbol.attachment:
+                    return "\u{2399}"; // Unicode: PRINT SCREEN SYMBOL
                 case TestSymbol.none:
                     return "";
             }
@@ -540,6 +560,8 @@ export class SymbolRenderer {
                     return "\u{26A0}\u{FE0E}"; // Unicode: WARNING SIGN + VARIATION SELECTOR-15 (disable emoji)
                 case TestSymbol.details:
                     return "\u{21B3}"; // Unicode: DOWNWARDS ARROW WITH TIP RIGHTWARDS
+                case TestSymbol.attachment:
+                    return "\u{2399}"; // Unicode: PRINT SCREEN SYMBOL
                 case TestSymbol.none:
                     return " ";
             }
@@ -562,6 +584,8 @@ export class SymbolRenderer {
                 return `${SymbolRenderer.ansiEscapeCodePrefix}91m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
             case TestSymbol.warning:
                 return `${SymbolRenderer.ansiEscapeCodePrefix}93m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
+            case TestSymbol.attachment:
+                return `${SymbolRenderer.ansiEscapeCodePrefix}94m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
             case TestSymbol.none:
             default:
                 return symbol;

src/TestExplorer/TestRunner.ts

@@ -27,7 +27,11 @@ import {
     ParallelXCTestOutputParser,
     XCTestOutputParser,
 } from "./TestParsers/XCTestOutputParser";
-import { SwiftTestingOutputParser } from "./TestParsers/SwiftTestingOutputParser";
+import {
+    SwiftTestingOutputParser,
+    SymbolRenderer,
+    TestSymbol,
+} from "./TestParsers/SwiftTestingOutputParser";
 import { LoggingDebugAdapterTracker } from "../debugger/logTracker";
 import { TaskOperation } from "../tasks/TaskQueue";
 import { TestXUnitParser } from "./TestXUnitParser";
@@ -36,7 +40,12 @@ import { TestRunArguments } from "./TestRunArguments";
 import { TemporaryFolder } from "../utilities/tempFolder";
 import { TestClass, runnableTag, upsertTestItem } from "./TestDiscovery";
 import { TestCoverage } from "../coverage/LcovResults";
-import { BuildConfigurationFactory, TestingConfigurationFactory } from "../debugger/buildConfig";
+import {
+    BuildConfigurationFactory,
+    SwiftTestingBuildAguments,
+    SwiftTestingConfigurationSetup,
+    TestingConfigurationFactory,
+} from "../debugger/buildConfig";
 import { TestKind, isDebugging, isRelease } from "./TestKind";
 import { reduceTestItemChildren } from "./TestUtils";
 import { CompositeCancellationToken } from "../utilities/cancellation";
@@ -67,6 +76,7 @@ export class TestRunProxy {
     private queuedOutput: string[] = [];
     private _testItems: vscode.TestItem[];
     private iteration: number | undefined;
+    private attachments: { [key: string]: string[] } = {};
     public coverage: TestCoverage;
     public token: CompositeCancellationToken;
 
@@ -177,6 +187,12 @@ export class TestRunProxy {
         this.addedTestItems.push({ testClass, parentIndex });
     };
 
+    public addAttachment = (testIndex: number, attachment: string) => {
+        const attachments = this.attachments[testIndex] ?? [];
+        attachments.push(attachment);
+        this.attachments[testIndex] = attachments;
+    };
+
     public getTestIndex(id: string, filename?: string): number {
         return this.testItemFinder.getIndex(id, filename);
     }
@@ -231,6 +247,8 @@ export class TestRunProxy {
         if (!this.runStarted) {
             this.testRunStarted();
         }
+
+        this.reportAttachments();
         this.testRun?.end();
         this.testRunCompleteEmitter.fire();
         this.token.dispose();
@@ -259,6 +277,25 @@ export class TestRunProxy {
         }
     }
 
+    private reportAttachments() {
+        const attachmentKeys = Object.keys(this.attachments);
+        if (attachmentKeys.length > 0) {
+            let attachment = "";
+            const totalAttachments = attachmentKeys.reduce((acc, key) => {
+                const attachments = this.attachments[key];
+                attachment = attachments.length ? attachments[0] : attachment;
+                return acc + attachments.length;
+            }, 0);
+
+            if (attachment) {
+                attachment = path.dirname(attachment);
+                this.appendOutput(
+                    `${SymbolRenderer.eventMessageSymbol(TestSymbol.attachment)} ${SymbolRenderer.ansiEscapeCodePrefix}90mRecorded ${totalAttachments} attachment${totalAttachments === 1 ? "" : "s"} to ${attachment}${SymbolRenderer.resetANSIEscapeCode}`
+                );
+            }
+        }
+    }
+
     private performAppendOutput(
         testRun: vscode.TestRun,
         output: string,
@@ -321,7 +358,8 @@ export class TestRunner {
                 : new XCTestOutputParser();
         this.swiftTestOutputParser = new SwiftTestingOutputParser(
             this.testRun.testRunStarted,
-            this.testRun.addParameterizedTestCase
+            this.testRun.addParameterizedTestCase,
+            this.testRun.addAttachment
         );
     }
 
@@ -334,7 +372,8 @@ export class TestRunner {
         // The SwiftTestingOutputParser holds state and needs to be reset between iterations.
         this.swiftTestOutputParser = new SwiftTestingOutputParser(
             this.testRun.testRunStarted,
-            this.testRun.addParameterizedTestCase
+            this.testRun.addParameterizedTestCase,
+            this.testRun.addAttachment
         );
         this.testRun.setIteration(iteration);
     }
@@ -519,18 +558,28 @@ export class TestRunner {
         // Run swift-testing first, then XCTest.
         // swift-testing being parallel by default should help these run faster.
         if (this.testArgs.hasSwiftTestingTests) {
-            const fifoPipePath = this.generateFifoPipePath();
+            const testRunTime = Date.now();
+            const fifoPipePath = this.generateFifoPipePath(testRunTime);
 
-            await TemporaryFolder.withNamedTemporaryFile(fifoPipePath, async () => {
+            await TemporaryFolder.withNamedTemporaryFiles([fifoPipePath], async () => {
                 // macOS/Linux require us to create the named pipe before we use it.
                 // Windows just lets us communicate by specifying a pipe path without any ceremony.
                 if (process.platform !== "win32") {
                     await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
                 }
-
-                const testBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
+                // Create the swift-testing configuration JSON file, peparing any
+                // directories the configuration may require.
+                const attachmentFolder = await SwiftTestingConfigurationSetup.setupAttachmentFolder(
                     this.folderContext,
+                    testRunTime
+                );
+                const swiftTestingArgs = await SwiftTestingBuildAguments.build(
                     fifoPipePath,
+                    attachmentFolder
+                );
+                const testBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
+                    this.folderContext,
+                    swiftTestingArgs,
                     this.testKind,
                     this.testArgs.swiftTestArgs,
                     true
@@ -540,17 +589,25 @@ export class TestRunner {
                     return this.testRun.runState;
                 }
 
+                const outputStream = this.testOutputWritable(TestLibrary.swiftTesting, runState);
+
                 // Watch the pipe for JSONL output and parse the events into test explorer updates.
                 // The await simply waits for the watching to be configured.
                 await this.swiftTestOutputParser.watch(fifoPipePath, runState);
 
                 await this.launchTests(
                     runState,
                     this.testKind === TestKind.parallel ? TestKind.standard : this.testKind,
-                    this.testOutputWritable(TestLibrary.swiftTesting, runState),
+                    outputStream,
                     testBuildConfig,
                     TestLibrary.swiftTesting
                 );
+
+                await SwiftTestingConfigurationSetup.cleanupAttachmentFolder(
+                    this.folderContext,
+                    testRunTime,
+                    this.workspaceContext.outputChannel
+                );
             });
         }
 
@@ -774,21 +831,32 @@ export class TestRunner {
             throw new Error(`Build failed with exit code ${buildExitCode}`);
         }
 
+        const testRunTime = Date.now();
         const subscriptions: vscode.Disposable[] = [];
         const buildConfigs: Array<vscode.DebugConfiguration | undefined> = [];
-        const fifoPipePath = this.generateFifoPipePath();
+        const fifoPipePath = this.generateFifoPipePath(testRunTime);
 
-        await TemporaryFolder.withNamedTemporaryFile(fifoPipePath, async () => {
+        await TemporaryFolder.withNamedTemporaryFiles([fifoPipePath], async () => {
             if (this.testArgs.hasSwiftTestingTests) {
                 // macOS/Linux require us to create the named pipe before we use it.
                 // Windows just lets us communicate by specifying a pipe path without any ceremony.
                 if (process.platform !== "win32") {
                     await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
                 }
+                // Create the swift-testing configuration JSON file, peparing any
+                // directories the configuration may require.
+                const attachmentFolder = await SwiftTestingConfigurationSetup.setupAttachmentFolder(
+                    this.folderContext,
+                    testRunTime
+                );
+                const swiftTestingArgs = await SwiftTestingBuildAguments.build(
+                    fifoPipePath,
+                    attachmentFolder
+                );
 
                 const swiftTestBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
                     this.folderContext,
-                    fifoPipePath,
+                    swiftTestingArgs,
                     this.testKind,
                     this.testArgs.swiftTestArgs,
                     true
@@ -930,9 +998,6 @@ export class TestRunner {
                                     }
                                 },
                                 reason => {
-                                    this.workspaceContext.outputChannel.logDiagnostic(
-                                        `Failed to debug test: ${reason}`
-                                    );
                                     subscriptions.forEach(sub => sub.dispose());
                                     reject(reason);
                                 }
@@ -942,6 +1007,13 @@ export class TestRunner {
 
             // Run each debugging session sequentially
             await debugRuns.reduce((p, fn) => p.then(() => fn()), Promise.resolve());
+
+            // Clean up any leftover resources
+            await SwiftTestingConfigurationSetup.cleanupAttachmentFolder(
+                this.folderContext,
+                testRunTime,
+                this.workspaceContext.outputChannel
+            );
         });
     }
 
@@ -996,10 +1068,10 @@ export class TestRunner {
         }
     }
 
-    private generateFifoPipePath(): string {
+    private generateFifoPipePath(testRunDateNow: number): string {
         return process.platform === "win32"
-            ? `\\\\.\\pipe\\vscodemkfifo-${Date.now()}`
-            : path.join(os.tmpdir(), `vscodemkfifo-${Date.now()}`);
+            ? `\\\\.\\pipe\\vscodemkfifo-${testRunDateNow}`
+            : path.join(os.tmpdir(), `vscodemkfifo-${testRunDateNow}`);
     }
 }
 

src/configuration.ts

@@ -65,6 +65,8 @@ export interface FolderConfiguration {
     readonly autoGenerateLaunchConfigurations: boolean;
     /** disable automatic running of swift package resolve */
     readonly disableAutoResolve: boolean;
+    /** location to save swift-testing attachments */
+    readonly attachmentsPath: string;
     /** look up saved permissions for the supplied plugin */
     pluginPermissions(pluginId: string): PluginPermissionConfiguration;
 }
@@ -162,6 +164,11 @@ const configuration = {
                     .getConfiguration("swift", workspaceFolder)
                     .get<boolean>("searchSubfoldersForPackages", false);
             },
+            get attachmentsPath(): string {
+                return vscode.workspace
+                    .getConfiguration("swift", workspaceFolder)
+                    .get<string>("attachmentsPath", "./.build/attachments");
+            },
             pluginPermissions(pluginId: string): PluginPermissionConfiguration {
                 return (
                     vscode.workspace.getConfiguration("swift", workspaceFolder).get<{