add the concept of test tags to set timeouts

matthewbastien · matthewbastien · commit 7dbe8686f54d · 2025-09-03T10:23:41.000-04:00
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -68,7 +68,7 @@ jobs:
         VSCODE_SWIFT_VSIX_ID=${{needs.package.outputs.artifact-id}}
         GITHUB_REPOSITORY=${{github.repository}}
       windows_pre_build_command: . .github\workflows\scripts\windows\setup.ps1
-      windows_build_command: scripts\test_windows.ps1
+      windows_build_command: Invoke-Program scripts\test_windows.ps1
       enable_windows_docker: false
 
   soundness:
diff --git a/.vscode-test.js b/.vscode-test.js
@@ -17,7 +17,6 @@ const path = require("path");
 const { version, publisher, name } = require("./package.json");
 
 const isCIBuild = process.env["CI"] === "1";
-const isFastTestRun = process.env["FAST_TEST_RUN"] === "1";
 
 const dataDir = process.env["VSCODE_DATA_DIR"];
 
@@ -115,8 +114,6 @@ module.exports = defineConfig({
                 color: true,
                 timeout,
                 forbidOnly: isCIBuild,
-                grep: isFastTestRun ? "@slow" : undefined,
-                invert: isFastTestRun,
                 slow: 10000,
                 retries: 1,
                 reporter: path.join(__dirname, ".mocha-reporter.js"),
@@ -151,8 +148,6 @@ module.exports = defineConfig({
                 color: true,
                 timeout,
                 forbidOnly: isCIBuild,
-                grep: isFastTestRun ? "@slow" : undefined,
-                invert: isFastTestRun,
                 slow: 10000,
                 retries: 1,
                 reporter: path.join(__dirname, ".mocha-reporter.js"),
diff --git a/test/common.ts b/test/common.ts
@@ -19,7 +19,10 @@ import * as chai from "chai";
 import * as sinonChai from "sinon-chai";
 import * as chaiAsPromised from "chai-as-promised";
 import * as chaiSubset from "chai-subset";
+import { installTagSupport } from "./tags";
 
 chai.use(sinonChai);
 chai.use(chaiAsPromised);
 chai.use(chaiSubset);
+
+installTagSupport();
diff --git a/test/integration-tests/BackgroundCompilation.test.ts b/test/integration-tests/BackgroundCompilation.test.ts
@@ -18,8 +18,9 @@ import { WorkspaceContext } from "../../src/WorkspaceContext";
 import { testAssetUri } from "../fixtures";
 import { activateExtensionForTest, updateSettings } from "./utilities/testutilities";
 import { closeAllEditors } from "../utilities/commands";
+import { tag } from "../tags";
 
-suite("BackgroundCompilation Test Suite", () => {
+tag("large").suite("BackgroundCompilation Test Suite", () => {
     let workspaceContext: WorkspaceContext;
 
     activateExtensionForTest({
@@ -36,7 +37,7 @@ suite("BackgroundCompilation Test Suite", () => {
         await closeAllEditors();
     });
 
-    test("build all on save @slow", async () => {
+    test("build all on save", async () => {
         const taskPromise = new Promise<void>(res => {
             vscode.tasks.onDidStartTask(e => {
                 const task = e.execution.task;
@@ -56,5 +57,5 @@ suite("BackgroundCompilation Test Suite", () => {
         await vscode.workspace.save(uri);
 
         await taskPromise;
-    }).timeout(180000);
+    });
 });
diff --git a/test/integration-tests/DiagnosticsManager.test.ts b/test/integration-tests/DiagnosticsManager.test.ts
@@ -33,6 +33,7 @@ import {
 } from "./utilities/testutilities";
 import { DiagnosticStyle } from "../../src/configuration";
 import { expect } from "chai";
+import { tag } from "../tags";
 
 const isEqual = (d1: vscode.Diagnostic, d2: vscode.Diagnostic) => {
     return (
@@ -66,9 +67,7 @@ function assertWithoutDiagnostic(uri: vscode.Uri, expected: vscode.Diagnostic) {
     );
 }
 
-suite("DiagnosticsManager Test Suite", function () {
-    this.timeout(60 * 1000 * 5); // Allow up to 5 minutes for build
-
+tag("medium").suite("DiagnosticsManager Test Suite", function () {
     let workspaceContext: WorkspaceContext;
     let folderContext: FolderContext;
     let cFolderContext: FolderContext;
diff --git a/test/integration-tests/ExtensionActivation.test.ts b/test/integration-tests/ExtensionActivation.test.ts
@@ -24,6 +24,7 @@ import {
 import { WorkspaceContext } from "../../src/WorkspaceContext";
 import { testAssetUri } from "../fixtures";
 import { assertContains } from "./testexplorer/utilities";
+import { tag } from "../tags";
 
 suite("Extension Activation/Deactivation Tests", () => {
     suite("Extension Activation", () => {
@@ -101,9 +102,7 @@ suite("Extension Activation/Deactivation Tests", () => {
         });
     });
 
-    suite("Activates for cmake projects", function () {
-        this.timeout(60000);
-
+    tag("medium").suite("Activates for cmake projects", function () {
         let workspaceContext: WorkspaceContext;
 
         activateExtensionForTest({
diff --git a/test/integration-tests/SwiftPackage.test.ts b/test/integration-tests/SwiftPackage.test.ts
@@ -17,9 +17,9 @@ import { testAssetUri } from "../fixtures";
 import { SwiftPackage } from "../../src/SwiftPackage";
 import { SwiftToolchain } from "../../src/toolchain/toolchain";
 import { Version } from "../../src/utilities/version";
+import { tag } from "../tags";
 
-suite("SwiftPackage Test Suite", function () {
-    this.timeout(5 * 60 * 1000); // 5 minute timeout
+tag("medium").suite("SwiftPackage Test Suite", function () {
     let toolchain: SwiftToolchain;
 
     setup(async () => {
diff --git a/test/integration-tests/SwiftSnippet.test.ts b/test/integration-tests/SwiftSnippet.test.ts
@@ -29,10 +29,9 @@ import {
 import { WorkspaceContext } from "../../src/WorkspaceContext";
 import { closeAllEditors } from "../utilities/commands";
 import { Commands } from "../../src/commands";
+import { tag } from "../tags";
 
-suite("SwiftSnippet Test Suite @slow", function () {
-    this.timeout(180000);
-
+tag("large").suite("SwiftSnippet Test Suite", function () {
     const uri = testAssetUri("defaultPackage/Snippets/hello.swift");
     const breakpoints = [
         new vscode.SourceBreakpoint(new vscode.Location(uri, new vscode.Position(2, 0))),
diff --git a/test/integration-tests/WorkspaceContext.test.ts b/test/integration-tests/WorkspaceContext.test.ts
@@ -28,6 +28,7 @@ import {
 import { FolderContext } from "../../src/FolderContext";
 import { assertContains } from "./testexplorer/utilities";
 import { resolveScope } from "../../src/utilities/tasks";
+import { tag } from "../tags";
 
 function assertContainsArg(execution: SwiftExecution, arg: string) {
     assert(execution?.args.find(a => a === arg));
@@ -40,7 +41,7 @@ function assertNotContainsArg(execution: SwiftExecution, arg: string) {
     );
 }
 
-suite("WorkspaceContext Test Suite", () => {
+tag("medium").suite("WorkspaceContext Test Suite", () => {
     let workspaceContext: WorkspaceContext;
     const packageFolder: vscode.Uri = testAssetUri("defaultPackage");
 
@@ -88,7 +89,7 @@ suite("WorkspaceContext Test Suite", () => {
             } finally {
                 observer?.dispose();
             }
-        }).timeout(60000 * 2);
+        });
     });
 
     suite("Tasks", function () {
@@ -106,9 +107,6 @@ suite("WorkspaceContext Test Suite", () => {
             }
         });
 
-        // Was hitting a timeout in suiteSetup during CI build once in a while
-        this.timeout(15000);
-
         test("Default Task values", async () => {
             const folder = workspaceContext.folders.find(
                 f => f.folder.fsPath === packageFolder.fsPath
@@ -190,32 +188,16 @@ suite("WorkspaceContext Test Suite", () => {
             const execution = buildAllTask.execution as SwiftExecution;
             assertContainsArg(execution, "--replace-scm-with-registry");
         });
-
-        test("Swift Path", async () => {
-            /* Temporarily disabled (need swift path to update immediately for this to work)
-            const folder = workspaceContext.folders.find(
-                f => f.folder.fsPath === packageFolder.fsPath
-            );
-            assert(folder);
-            await swiftConfig.update("path", "/usr/bin/swift");
-            const buildAllTask = createBuildAllTask(folder);
-            const execution = buildAllTask.execution as SwiftExecution;
-            assert.strictEqual(execution?.command, "/usr/bin/swift");
-            await swiftConfig.update("path", "");*/
-        });
     });
 
     suite("Toolchain", function () {
-        // Increase the timeout as this takes several seconds longer for the codeWorkspaceTests variant
-        this.timeout(60000);
-
         activateExtensionForSuite({
             async setup(ctx) {
                 workspaceContext = ctx;
             },
         });
 
-        test("get project templates", async () => {
+        tag("small").test("get project templates", async () => {
             // This is only supported in swift versions >=5.8.0
             const swiftVersion = workspaceContext.globalToolchain.swiftVersion;
             if (swiftVersion.isLessThan(new Version(5, 8, 0))) {
@@ -248,6 +230,6 @@ suite("WorkspaceContext Test Suite", () => {
                 name: "Build Tool Plugin",
                 description: "A package that vends a build tool plugin.",
             });
-        }).timeout(1000);
+        });
     });
-}).timeout(15000);
+});
diff --git a/test/integration-tests/commands/build.test.ts b/test/integration-tests/commands/build.test.ts
@@ -23,11 +23,9 @@ import { Commands } from "../../../src/commands";
 import { continueSession, waitForDebugAdapterRequest } from "../../utilities/debug";
 import { activateExtensionForSuite, folderInRootWorkspace } from "../utilities/testutilities";
 import { Version } from "../../../src/utilities/version";
+import { tag } from "../../tags";
 
-suite("Build Commands @slow", function () {
-    // Default timeout is a bit too short, give it a little bit more time
-    this.timeout(3 * 60 * 1000);
-
+tag("large").suite("Build Commands", function () {
     let folderContext: FolderContext;
     let workspaceContext: WorkspaceContext;
     const uri = testAssetUri("defaultPackage/Sources/PackageExe/main.swift");
diff --git a/test/integration-tests/commands/captureDiagnostics.test.ts b/test/integration-tests/commands/captureDiagnostics.test.ts
@@ -27,6 +27,7 @@ import {
     updateSettings,
 } from "../utilities/testutilities";
 import { Version } from "../../../src/utilities/version";
+import { tag } from "../../tags";
 
 suite("captureDiagnostics Test Suite", () => {
     let workspaceContext: WorkspaceContext;
@@ -88,9 +89,7 @@ suite("captureDiagnostics Test Suite", () => {
         });
     });
 
-    suite("Full @slow", function () {
-        this.timeout(2 * 60 * 1000);
-
+    tag("large").suite("Full", function () {
         activateExtensionForSuite({
             async setup(ctx) {
                 workspaceContext = ctx;
diff --git a/test/integration-tests/commands/dependency.test.ts b/test/integration-tests/commands/dependency.test.ts
@@ -21,12 +21,9 @@ import { WorkspaceContext } from "../../../src/WorkspaceContext";
 import { Commands } from "../../../src/commands";
 import { activateExtensionForSuite, findWorkspaceFolder } from "../utilities/testutilities";
 import { waitForNoRunningTasks } from "../../utilities/tasks";
+import { tag } from "../../tags";
 
-suite("Dependency Commmands Test Suite @slow", function () {
-    // full workflow's interaction with spm is longer than the default timeout
-    // 3 minutes for each test should be more than enough
-    this.timeout(3 * 60 * 1000);
-
+tag("large").suite("Dependency Commmands Test Suite", function () {
     let depsContext: FolderContext;
     let workspaceContext: WorkspaceContext;
 
diff --git a/test/integration-tests/documentation/DocumentationLivePreview.test.ts b/test/integration-tests/documentation/DocumentationLivePreview.test.ts
@@ -27,11 +27,9 @@ import {
     WebviewContent,
 } from "../../../src/documentation/webview/WebviewMessage";
 import { PreviewEditorConstant } from "../../../src/documentation/DocumentationPreviewEditor";
+import { tag } from "../../tags";
 
-suite("Documentation Live Preview", function () {
-    // Tests are short, but rely on SourceKit-LSP: give 30 seconds for each one
-    this.timeout(30 * 1000);
-
+tag("medium").suite("Documentation Live Preview", function () {
     let folderContext: FolderContext;
     let workspaceContext: WorkspaceContext;
 
diff --git a/test/integration-tests/extension.test.ts b/test/integration-tests/extension.test.ts
@@ -21,7 +21,6 @@ import { activateExtensionForTest, findWorkspaceFolder } from "./utilities/testu
 import { expect } from "chai";
 
 suite("Extension Test Suite", function () {
-    this.timeout(60000);
     let workspaceContext: WorkspaceContext;
 
     activateExtensionForTest({
@@ -30,22 +29,7 @@ suite("Extension Test Suite", function () {
         },
     });
 
-    suite("Temporary Folder Test Suite", () => {
-        /*test("Create/Delete File", async () => {
-            const fileContents = "Test file";
-            //const tempFolder = await TemporaryFolder.create();
-            const fileName = workspaceContext.tempFolder.filename("test");
-            assert.doesNotThrow(async () => await fs.writeFile(fileName, fileContents));
-            assert.doesNotThrow(async () => {
-                const contents = await fs.readFile(fileName, "utf8");
-                assert.strictEqual(contents, fileContents);
-            });
-            assert.doesNotThrow(async () => await fs.rm(fileName));
-        }).timeout(5000);*/
-    });
-
     suite("Workspace", function () {
-        this.timeout(60000);
         /** Verify tasks.json is being loaded */
         test("Tasks.json", async () => {
             const folder = findWorkspaceFolder("defaultPackage", workspaceContext);
@@ -62,6 +46,6 @@ suite("Extension Test Suite", function () {
             ])) {
                 assert.ok(execution?.args.find(item => item === arg));
             }
-        }).timeout(60000);
+        });
     });
 });
diff --git a/test/integration-tests/language/LanguageClientIntegration.test.ts b/test/integration-tests/language/LanguageClientIntegration.test.ts
@@ -23,6 +23,7 @@ import { createBuildAllTask } from "../../../src/tasks/SwiftTaskProvider";
 import { activateExtensionForSuite, folderInRootWorkspace } from "../utilities/testutilities";
 import { waitForClientState } from "../utilities/lsputilities";
 import { FolderContext } from "../../../src/FolderContext";
+import { tag } from "../../tags";
 
 async function buildProject(ctx: WorkspaceContext, name: string) {
     await waitForNoRunningTasks();
@@ -33,9 +34,7 @@ async function buildProject(ctx: WorkspaceContext, name: string) {
     return folderContext;
 }
 
-suite("Language Client Integration Suite @slow", function () {
-    this.timeout(3 * 60 * 1000);
-
+tag("large").suite("Language Client Integration Suite", function () {
     let clientManager: LanguageClientManager;
     let folderContext: FolderContext;
 
diff --git a/test/integration-tests/tasks/SwiftPluginTaskProvider.test.ts b/test/integration-tests/tasks/SwiftPluginTaskProvider.test.ts
@@ -37,8 +37,6 @@ suite("SwiftPluginTaskProvider Test Suite", function () {
     let workspaceContext: WorkspaceContext;
     let folderContext: FolderContext;
 
-    this.timeout(120000); // Mostly only when running suite with .only
-
     activateExtensionForSuite({
         async setup(ctx) {
             workspaceContext = ctx;
diff --git a/test/integration-tests/tasks/SwiftTaskProvider.test.ts b/test/integration-tests/tasks/SwiftTaskProvider.test.ts
@@ -31,6 +31,7 @@ import {
     folderInRootWorkspace,
     updateSettings,
 } from "../utilities/testutilities";
+import { tag } from "../../tags";
 
 suite("SwiftTaskProvider Test Suite", () => {
     let workspaceContext: WorkspaceContext;
@@ -115,13 +116,13 @@ suite("SwiftTaskProvider Test Suite", () => {
                 expect(task?.detail).to.include("swift build --build-tests");
             });
 
-            test("executes @slow", async () => {
+            tag("large").test("executes", async () => {
                 assert(task);
                 const exitPromise = waitForEndTaskProcess(task);
                 await vscode.tasks.executeTask(task);
                 const exitCode = await exitPromise;
                 expect(exitCode).to.equal(0);
-            }).timeout(180000); // 3 minutes to build
+            });
         });
 
         suite("includes build all task from tasks.json", () => {
diff --git a/test/integration-tests/tasks/TaskQueue.test.ts b/test/integration-tests/tasks/TaskQueue.test.ts
@@ -18,6 +18,7 @@ import { testAssetPath } from "../../fixtures";
 import { WorkspaceContext } from "../../../src/WorkspaceContext";
 import { SwiftExecOperation, TaskOperation, TaskQueue } from "../../../src/tasks/TaskQueue";
 import { activateExtensionForSuite, findWorkspaceFolder } from "../utilities/testutilities";
+import { tag } from "../../tags";
 
 suite("TaskQueue Test Suite", () => {
     let workspaceContext: WorkspaceContext;
@@ -129,7 +130,7 @@ suite("TaskQueue Test Suite", () => {
 
     // Queue two tasks. The first one taking longer than the second. If they
     // are queued correctly the first will still finish before the second
-    test("Test execution order", async () => {
+    tag("medium").test("Test execution order", async () => {
         const sleepScript = testAssetPath("sleep.sh");
         const results: (number | undefined)[] = [];
         const task1 = new vscode.Task(
@@ -151,7 +152,7 @@ suite("TaskQueue Test Suite", () => {
             taskQueue.queueOperation(new TaskOperation(task2)).then(rt => results.push(rt)),
         ]);
         assert.notStrictEqual(results, [1, 2]);
-    }).timeout(15000);
+    });
 
     // check queuing task will return expected value
     test("swift exec", async () => {
diff --git a/test/integration-tests/testexplorer/TestExplorerIntegration.test.ts b/test/integration-tests/testexplorer/TestExplorerIntegration.test.ts
diff --git a/test/integration-tests/ui/ProjectPanelProvider.test.ts b/test/integration-tests/ui/ProjectPanelProvider.test.ts
diff --git a/test/sleep.test.ts b/test/sleep.test.ts
diff --git a/test/tags.ts b/test/tags.ts
diff --git a/test/tsconfig.json b/test/tsconfig.json
