Merge pull request RooCodeInc#926 from RooVetGit/feature/enhanceFullIntegrationTests

Modify Full Integration Test Structure to allow for multiple tests th…

Author: ColemanRoo

package-lock.json

@@ -278,7 +278,7 @@
 		"dev": "cd webview-ui && npm run dev",
 		"test": "jest && npm run test:webview",
 		"test:webview": "cd webview-ui && npm run test",
-		"test:integration": "npm run build && npm run compile:integration && npx dotenvx run -f .env.integration -- vscode-test",
+		"test:integration": "npm run build && npm run compile:integration && npx dotenvx run -f .env.integration -- node ./out-integration/test/runTest.js",
 		"prepare": "husky",
 		"publish:marketplace": "vsce publish && ovsx publish",
 		"publish": "npm run build && changeset publish && npm install --package-lock-only",
@@ -342,8 +342,9 @@
 		"@types/debug": "^4.1.12",
 		"@types/diff": "^5.2.1",
 		"@types/diff-match-patch": "^1.0.36",
+		"@types/glob": "^8.1.0",
 		"@types/jest": "^29.5.14",
-		"@types/mocha": "^10.0.7",
+		"@types/mocha": "^10.0.10",
 		"@types/node": "20.x",
 		"@types/string-similarity": "^4.0.2",
 		"@typescript-eslint/eslint-plugin": "^7.14.1",
@@ -354,10 +355,12 @@
 		"mkdirp": "^3.0.1",
 		"rimraf": "^6.0.1",
 		"eslint": "^8.57.0",
+		"glob": "^11.0.1",
 		"husky": "^9.1.7",
 		"jest": "^29.7.0",
 		"jest-simple-dot-reporter": "^1.0.5",
 		"lint-staged": "^15.2.11",
+		"mocha": "^11.1.0",
 		"npm-run-all": "^4.1.5",
 		"prettier": "^3.4.2",
 		"ts-jest": "^29.2.5",

package.json

@@ -0,0 +1,23 @@
+import * as path from "path"
+
+import { runTests } from "@vscode/test-electron"
+
+async function main() {
+	try {
+		// The folder containing the Extension Manifest package.json
+		// Passed to `--extensionDevelopmentPath`
+		const extensionDevelopmentPath = path.resolve(__dirname, "../../")
+
+		// The path to the extension test script
+		// Passed to --extensionTestsPath
+		const extensionTestsPath = path.resolve(__dirname, "./suite/index")
+
+		// Download VS Code, unzip it and run the integration test
+		await runTests({ extensionDevelopmentPath, extensionTestsPath })
+	} catch {
+		console.error("Failed to run tests")
+		process.exit(1)
+	}
+}
+
+main()

src/test/runTest.ts

@@ -48,16 +48,4 @@ suite("Roo Code Extension", () => {
 			assert.ok(commands.includes(cmd), `Command ${cmd} should be registered`)
 		}
 	})
-
-	test("Webview panel can be created", () => {
-		const view = vscode.window.createWebviewPanel(
-			"roo-cline.SidebarProvider",
-			"Roo Code",
-			vscode.ViewColumn.One,
-			{},
-		)
-
-		assert.ok(view, "Failed to create webview panel")
-		view.dispose()
-	})
 })

src/test/extension.test.ts renamed to src/test/suite/extension.test.ts

@@ -0,0 +1,94 @@
+import * as path from "path"
+import Mocha from "mocha"
+import { glob } from "glob"
+import { ClineAPI } from "../../exports/cline"
+import { ClineProvider } from "../../core/webview/ClineProvider"
+import * as vscode from "vscode"
+
+declare global {
+	var api: ClineAPI
+	var provider: ClineProvider
+	var extension: vscode.Extension<ClineAPI> | undefined
+	var panel: vscode.WebviewPanel | undefined
+}
+
+export async function run(): Promise<void> {
+	// Create the mocha test
+	const mocha = new Mocha({
+		ui: "tdd",
+		timeout: 600000, // 10 minutes to compensate for time communicating with LLM while running in GHA
+	})
+
+	const testsRoot = path.resolve(__dirname, "..")
+
+	try {
+		// Find all test files
+		const files = await glob("**/**.test.js", { cwd: testsRoot })
+
+		// Add files to the test suite
+		files.forEach((f: string) => mocha.addFile(path.resolve(testsRoot, f)))
+
+		//Set up global extension, api, provider, and panel
+		globalThis.extension = vscode.extensions.getExtension("RooVeterinaryInc.roo-cline")
+		if (!globalThis.extension) {
+			throw new Error("Extension not found")
+		}
+
+		globalThis.api = globalThis.extension.isActive
+			? globalThis.extension.exports
+			: await globalThis.extension.activate()
+		globalThis.provider = globalThis.api.sidebarProvider
+		await globalThis.provider.updateGlobalState("apiProvider", "openrouter")
+		await globalThis.provider.updateGlobalState("openRouterModelId", "anthropic/claude-3.5-sonnet")
+		await globalThis.provider.storeSecret(
+			"openRouterApiKey",
+			process.env.OPENROUTER_API_KEY || "sk-or-v1-fake-api-key",
+		)
+
+		globalThis.panel = vscode.window.createWebviewPanel(
+			"roo-cline.SidebarProvider",
+			"Roo Code",
+			vscode.ViewColumn.One,
+			{
+				enableScripts: true,
+				enableCommandUris: true,
+				retainContextWhenHidden: true,
+				localResourceRoots: [globalThis.extension?.extensionUri],
+			},
+		)
+
+		await globalThis.provider.resolveWebviewView(globalThis.panel)
+
+		let startTime = Date.now()
+		const timeout = 60000
+		const interval = 1000
+
+		while (Date.now() - startTime < timeout) {
+			if (globalThis.provider.viewLaunched) {
+				break
+			}
+
+			await new Promise((resolve) => setTimeout(resolve, interval))
+		}
+
+		// Run the mocha test
+		return new Promise((resolve, reject) => {
+			try {
+				mocha.run((failures: number) => {
+					if (failures > 0) {
+						reject(new Error(`${failures} tests failed.`))
+					} else {
+						resolve()
+					}
+				})
+			} catch (err) {
+				console.error(err)
+				reject(err)
+			}
+		})
+	} catch (err) {
+		console.error("Error while running tests:")
+		console.error(err)
+		throw err
+	}
+}

src/test/suite/index.ts

@@ -0,0 +1,99 @@
+import * as assert from "assert"
+import * as vscode from "vscode"
+
+suite("Roo Code Modes", () => {
+	test("Should handle switching modes correctly", async function () {
+		const timeout = 30000
+		const interval = 1000
+
+		if (!globalThis.extension) {
+			assert.fail("Extension not found")
+		}
+
+		try {
+			let startTime = Date.now()
+
+			// Ensure the webview is launched.
+			while (Date.now() - startTime < timeout) {
+				if (globalThis.provider.viewLaunched) {
+					break
+				}
+
+				await new Promise((resolve) => setTimeout(resolve, interval))
+			}
+
+			await globalThis.provider.updateGlobalState("mode", "Ask")
+			await globalThis.provider.updateGlobalState("alwaysAllowModeSwitch", true)
+			await globalThis.provider.updateGlobalState("autoApprovalEnabled", true)
+
+			// Start a new task.
+			await globalThis.api.startNewTask(
+				"For each mode (Code, Architect, Ask) respond with the mode name and what it specializes in after switching to that mode, do not start with the current mode, be sure to say 'I AM DONE' after the task is complete",
+			)
+
+			// Wait for task to appear in history with tokens.
+			startTime = Date.now()
+
+			while (Date.now() - startTime < timeout) {
+				const messages = globalThis.provider.messages
+
+				if (
+					messages.some(
+						({ type, text }) =>
+							type === "say" && text?.includes("I AM DONE") && !text?.includes("be sure to say"),
+					)
+				) {
+					break
+				}
+
+				await new Promise((resolve) => setTimeout(resolve, interval))
+			}
+			if (globalThis.provider.messages.length === 0) {
+				assert.fail("No messages received")
+			}
+
+			assert.ok(
+				globalThis.provider.messages.some(
+					({ type, text }) => type === "say" && text?.includes(`"request":"[switch_mode to 'code' because:`),
+				),
+				"Did not receive expected response containing 'Roo wants to switch to code mode'",
+			)
+			assert.ok(
+				globalThis.provider.messages.some(
+					({ type, text }) => type === "say" && text?.includes("software engineer"),
+				),
+				"Did not receive expected response containing 'I am Roo in Code mode, specializing in software engineering'",
+			)
+
+			assert.ok(
+				globalThis.provider.messages.some(
+					({ type, text }) =>
+						type === "say" && text?.includes(`"request":"[switch_mode to 'architect' because:`),
+				),
+				"Did not receive expected response containing 'Roo wants to switch to architect mode'",
+			)
+			assert.ok(
+				globalThis.provider.messages.some(
+					({ type, text }) =>
+						type === "say" && (text?.includes("technical planning") || text?.includes("technical leader")),
+				),
+				"Did not receive expected response containing 'I am Roo in Architect mode, specializing in analyzing codebases'",
+			)
+
+			assert.ok(
+				globalThis.provider.messages.some(
+					({ type, text }) => type === "say" && text?.includes(`"request":"[switch_mode to 'ask' because:`),
+				),
+				"Did not receive expected response containing 'Roo wants to switch to ask mode'",
+			)
+			assert.ok(
+				globalThis.provider.messages.some(
+					({ type, text }) =>
+						type === "say" && (text?.includes("technical knowledge") || text?.includes("technical assist")),
+				),
+				"Did not receive expected response containing 'I am Roo in Ask mode, specializing in answering questions'",
+			)
+		} finally {
+		}
+	})
+})

src/test/suite/modes.test.ts

@@ -0,0 +1,54 @@
+import * as assert from "assert"
+import * as vscode from "vscode"
+
+suite("Roo Code Task", () => {
+	test("Should handle prompt and response correctly", async function () {
+		const timeout = 30000
+		const interval = 1000
+
+		if (!globalThis.extension) {
+			assert.fail("Extension not found")
+		}
+
+		try {
+			// Ensure the webview is launched.
+			let startTime = Date.now()
+
+			while (Date.now() - startTime < timeout) {
+				if (globalThis.provider.viewLaunched) {
+					break
+				}
+
+				await new Promise((resolve) => setTimeout(resolve, interval))
+			}
+
+			await globalThis.api.startNewTask("Hello world, what is your name? Respond with 'My name is ...'")
+
+			// Wait for task to appear in history with tokens.
+			startTime = Date.now()
+
+			while (Date.now() - startTime < timeout) {
+				const state = await globalThis.provider.getState()
+				const task = state.taskHistory?.[0]
+
+				if (task && task.tokensOut > 0) {
+					break
+				}
+
+				await new Promise((resolve) => setTimeout(resolve, interval))
+			}
+
+			if (globalThis.provider.messages.length === 0) {
+				assert.fail("No messages received")
+			}
+
+			assert.ok(
+				globalThis.provider.messages.some(
+					({ type, text }) => type === "say" && text?.includes("My name is Roo"),
+				),
+				"Did not receive expected response containing 'My name is Roo'",
+			)
+		} finally {
+		}
+	})
+})