From d7b44a2828bd7df85b837c7c8970cb1660f0430c Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Tue, 11 Feb 2025 10:55:04 -0600 Subject: [PATCH 1/8] Converting modes.test.ts to use grading to measure if the test passes Add documentation for VSCode Integration Tests Use explicit list of tests instead of searching ot make it easier to run 1 test at time locally. --- src/test/VSCODE_INTEGRATION_TESTS.md | 160 +++++++++++++++++++++++++++ src/test/suite/index.ts | 3 +- src/test/suite/modes.test.ts | 79 +++++++------ 3 files changed, 201 insertions(+), 41 deletions(-) create mode 100644 src/test/VSCODE_INTEGRATION_TESTS.md diff --git a/src/test/VSCODE_INTEGRATION_TESTS.md b/src/test/VSCODE_INTEGRATION_TESTS.md new file mode 100644 index 00000000000..6192c4488fa --- /dev/null +++ b/src/test/VSCODE_INTEGRATION_TESTS.md @@ -0,0 +1,160 @@ +# VSCode Integration Tests + +This document describes the integration test setup for the Roo Code VSCode extension. + +## Overview + +The integration tests use the `@vscode/test-electron` package to run tests in a real VSCode environment. These tests verify that the extension works correctly within VSCode, including features like mode switching, webview interactions, and API communication. + +## Test Setup + +### Directory Structure + +``` +src/test/ +├── runTest.ts # Main test runner +├── suite/ +│ ├── index.ts # Test suite configuration +│ ├── modes.test.ts # Mode switching tests +│ ├── tasks.test.ts # Task execution tests +│ └── extension.test.ts # Extension activation tests +``` + +### Test Runner Configuration + +The test runner (`runTest.ts`) is responsible for: + +- Setting up the extension development path +- Configuring the test environment +- Running the integration tests using `@vscode/test-electron` + +### Environment Setup + +1. Create a `.env.integration` file in the root directory with required environment variables: + +``` +OPENROUTER_API_KEY=sk-or-v1-... +``` + +2. The test suite (`suite/index.ts`) configures: + +- Mocha test framework with TDD interface +- 10-minute timeout for LLM communication +- Global extension API access +- WebView panel setup +- OpenRouter API configuration + +## Test Suite Structure + +Tests are organized using Mocha's TDD interface (`suite` and `test` functions). The main test files are: + +- `modes.test.ts`: Tests mode switching functionality +- `tasks.test.ts`: Tests task execution +- `extension.test.ts`: Tests extension activation + +### Global Objects + +The following global objects are available in tests: + +```typescript +declare global { + var api: ClineAPI + var provider: ClineProvider + var extension: vscode.Extension + var panel: vscode.WebviewPanel +} +``` + +## Running Tests + +1. Ensure you have the required environment variables set in `.env.integration` + +2. Run the integration tests: + +```bash +npm run test:integration +``` + +The tests will: + +- Download and launch a clean VSCode instance +- Install the extension +- Execute the test suite +- Report results + +## Writing New Tests + +When writing new integration tests: + +1. Create a new test file in `src/test/suite/` with the `.test.ts` extension + +2. Add the test file to the `files` array in `suite/index.ts` (you can temporarily comment out the other tests to run just the new test): + +```typescript +const files = ["suite/modes.test.js", "suite/tasks.test.js", "suite/extension.test.js", "suite/your-new-test.test.js"] +``` + +3. Structure your tests using the TDD interface: + +```typescript +import * as assert from "assert" +import * as vscode from "vscode" + +suite("Your Test Suite Name", () => { + test("Should do something specific", async function () { + // Your test code here + }) +}) +``` + +4. Use the global objects (`api`, `provider`, `extension`, `panel`) to interact with the extension + +### Best Practices + +1. **Timeouts**: Use appropriate timeouts for async operations: + +```typescript +const timeout = 30000 +const interval = 1000 +``` + +2. **State Management**: Reset extension state before/after tests: + +```typescript +await globalThis.provider.updateGlobalState("mode", "Ask") +await globalThis.provider.updateGlobalState("alwaysAllowModeSwitch", true) +``` + +3. **Assertions**: Use clear assertions with meaningful messages: + +```typescript +assert.ok(condition, "Descriptive message about what failed") +``` + +4. **Error Handling**: Wrap test code in try/catch blocks and clean up resources: + +```typescript +try { + // Test code +} finally { + // Cleanup code +} +``` + +5. **Wait for Operations**: Use polling when waiting for async operations: + +```typescript +let startTime = Date.now() +while (Date.now() - startTime < timeout) { + if (condition) break + await new Promise((resolve) => setTimeout(resolve, interval)) +} +``` + +6. **Grading**: When grading tests, use the `Grade:` format to ensure the test is graded correctly (See modes.test.ts for an example). + +```typescript +await globalThis.api.startNewTask( + `Given this prompt: ${testPrompt} grade the response from 1 to 10 in the format of "Grade: (1-10)": ${output} \n Be sure to say 'I AM DONE GRADING' after the task is complete`, +) +``` diff --git a/src/test/suite/index.ts b/src/test/suite/index.ts index ffb8de7473e..c538541a4bd 100644 --- a/src/test/suite/index.ts +++ b/src/test/suite/index.ts @@ -23,7 +23,8 @@ export async function run(): Promise { try { // Find all test files - const files = await glob("**/**.test.js", { cwd: testsRoot }) + //const files = await glob("**/**.test.js", { cwd: testsRoot } leaving this commented out for now since we only have three tests + const files = ["suite/modes.test.js", "suite/tasks.test.js", "suite/extension.test.js"] // Add files to the test suite files.forEach((f: string) => mocha.addFile(path.resolve(testsRoot, f))) diff --git a/src/test/suite/modes.test.ts b/src/test/suite/modes.test.ts index edd1a6a4ae2..6f20ee3b9e0 100644 --- a/src/test/suite/modes.test.ts +++ b/src/test/suite/modes.test.ts @@ -5,7 +5,8 @@ suite("Roo Code Modes", () => { test("Should handle switching modes correctly", async function () { const timeout = 30000 const interval = 1000 - + const testPrompt = + "For each mode (Code, Architect, Ask) respond with the mode name and what it specializes in after switching to that mode, do not start with the current mode, be sure to say 'I AM DONE' after the task is complete" if (!globalThis.extension) { assert.fail("Extension not found") } @@ -27,9 +28,7 @@ suite("Roo Code Modes", () => { await globalThis.provider.updateGlobalState("autoApprovalEnabled", true) // Start a new task. - await globalThis.api.startNewTask( - "For each mode (Code, Architect, Ask) respond with the mode name and what it specializes in after switching to that mode, do not start with the current mode, be sure to say 'I AM DONE' after the task is complete", - ) + await globalThis.api.startNewTask(testPrompt) // Wait for task to appear in history with tokens. startTime = Date.now() @@ -52,46 +51,46 @@ suite("Roo Code Modes", () => { assert.fail("No messages received") } - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => type === "say" && text?.includes(`"request":"[switch_mode to 'code' because:`), - ), - "Did not receive expected response containing 'Roo wants to switch to code mode'", - ) - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => type === "say" && text?.includes("software engineer"), - ), - "Did not receive expected response containing 'I am Roo in Code mode, specializing in software engineering'", + await globalThis.provider.updateGlobalState("mode", "Ask") + let output = globalThis.provider.messages.map(({ type, text }) => (type === "say" ? text : "")).join("\n") + await globalThis.api.startNewTask( + `Given this prompt: ${testPrompt} grade the response from 1 to 10 in the format of "Grade: (1-10)": ${output} \n Be sure to say 'I AM DONE GRADING' after the task is complete`, ) - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => - type === "say" && text?.includes(`"request":"[switch_mode to 'architect' because:`), - ), - "Did not receive expected response containing 'Roo wants to switch to architect mode'", - ) - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => - type === "say" && (text?.includes("technical planning") || text?.includes("technical leader")), - ), - "Did not receive expected response containing 'I am Roo in Architect mode, specializing in analyzing codebases'", - ) + startTime = Date.now() + while (Date.now() - startTime < timeout) { + const messages = globalThis.provider.messages + + if ( + messages.some( + ({ type, text }) => + type === "say" && text?.includes("I AM DONE GRADING") && !text?.includes("be sure to say"), + ) + ) { + break + } + + await new Promise((resolve) => setTimeout(resolve, interval)) + } + if (globalThis.provider.messages.length === 0) { + assert.fail("No messages received") + } + globalThis.provider.messages.forEach(({ type, text }) => { + if (type === "say" && text?.includes("Grade:")) { + console.log(text) + } + }) + const grade = globalThis.provider.messages.find( + ({ type, text }) => type === "say" && !text?.includes("Grade: (1-10)") && text?.includes("Grade:"), + )?.text + console.log("THIS IS THE GRADE", grade) assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => type === "say" && text?.includes(`"request":"[switch_mode to 'ask' because:`), - ), - "Did not receive expected response containing 'Roo wants to switch to ask mode'", - ) - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => - type === "say" && (text?.includes("technical knowledge") || text?.includes("technical assist")), - ), - "Did not receive expected response containing 'I am Roo in Ask mode, specializing in answering questions'", + grade?.includes("Grade: 10") || + grade?.includes("Grade: 9") || + grade?.includes("Grade: 8") || + grade?.includes("Grade: 7"), + "Did not receive expected response containing 'Grade: 10' or 'Grade: 9' or 'Grade: 8' or 'Grade: 7'", ) } finally { } From 21f04f580ad9d04ca44fff4ddb0527417d8ab1ad Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Tue, 11 Feb 2025 11:06:11 -0600 Subject: [PATCH 2/8] Fix typo for test list Modify where we log messages to the console for the test --- src/test/suite/index.ts | 2 +- src/test/suite/modes.test.ts | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/test/suite/index.ts b/src/test/suite/index.ts index c538541a4bd..0d0e68015f3 100644 --- a/src/test/suite/index.ts +++ b/src/test/suite/index.ts @@ -24,7 +24,7 @@ export async function run(): Promise { try { // Find all test files //const files = await glob("**/**.test.js", { cwd: testsRoot } leaving this commented out for now since we only have three tests - const files = ["suite/modes.test.js", "suite/tasks.test.js", "suite/extension.test.js"] + const files = ["suite/modes.test.js", "suite/task.test.js", "suite/extension.test.js"] // Add files to the test suite files.forEach((f: string) => mocha.addFile(path.resolve(testsRoot, f))) diff --git a/src/test/suite/modes.test.ts b/src/test/suite/modes.test.ts index 6f20ee3b9e0..56eda6e4174 100644 --- a/src/test/suite/modes.test.ts +++ b/src/test/suite/modes.test.ts @@ -51,6 +51,14 @@ suite("Roo Code Modes", () => { assert.fail("No messages received") } + //Log the messages to the console + globalThis.provider.messages.forEach(({ type, text }) => { + if (type === "say") { + console.log(text) + } + }) + + //Start Grading Portion of test to grade the response from 1 to 10 await globalThis.provider.updateGlobalState("mode", "Ask") let output = globalThis.provider.messages.map(({ type, text }) => (type === "say" ? text : "")).join("\n") await globalThis.api.startNewTask( @@ -84,7 +92,6 @@ suite("Roo Code Modes", () => { const grade = globalThis.provider.messages.find( ({ type, text }) => type === "say" && !text?.includes("Grade: (1-10)") && text?.includes("Grade:"), )?.text - console.log("THIS IS THE GRADE", grade) assert.ok( grade?.includes("Grade: 10") || grade?.includes("Grade: 9") || From 63346fbc6e6249399e42de5add614fb2666d0269 Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Tue, 11 Feb 2025 14:21:33 -0600 Subject: [PATCH 3/8] update task.test.ts to run in any order with other tests --- src/test/suite/task.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/suite/task.test.ts b/src/test/suite/task.test.ts index 5bea3754a56..71bf9da5ceb 100644 --- a/src/test/suite/task.test.ts +++ b/src/test/suite/task.test.ts @@ -22,6 +22,10 @@ suite("Roo Code Task", () => { await new Promise((resolve) => setTimeout(resolve, interval)) } + await globalThis.provider.updateGlobalState("mode", "Code") + await globalThis.provider.updateGlobalState("alwaysAllowModeSwitch", true) + await globalThis.provider.updateGlobalState("autoApprovalEnabled", true) + await globalThis.api.startNewTask("Hello world, what is your name? Respond with 'My name is ...'") // Wait for task to appear in history with tokens. From a41fb07adc36b33eec11fefecd6601e961427c44 Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Tue, 11 Feb 2025 14:34:03 -0600 Subject: [PATCH 4/8] task.test.ts fix to check for messages before assersion instead of tokens used --- src/test/suite/task.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/suite/task.test.ts b/src/test/suite/task.test.ts index 71bf9da5ceb..2d34bc78ff3 100644 --- a/src/test/suite/task.test.ts +++ b/src/test/suite/task.test.ts @@ -32,10 +32,9 @@ suite("Roo Code Task", () => { startTime = Date.now() while (Date.now() - startTime < timeout) { - const state = await globalThis.provider.getState() - const task = state.taskHistory?.[0] + const messages = globalThis.provider.messages - if (task && task.tokensOut > 0) { + if (messages.some(({ type, text }) => type === "say" && text?.includes("My name is Roo"))) { break } From eb826b1c90e5d393bd0a8dc190a0f0157fd77341 Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Wed, 12 Feb 2025 09:05:29 -0600 Subject: [PATCH 5/8] Change index.ts to always run all tests Add regex look for grade in in modes.test.ts --- src/test/suite/index.ts | 6 ++++-- src/test/suite/modes.test.ts | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/suite/index.ts b/src/test/suite/index.ts index 0d0e68015f3..5fe18f1ed0e 100644 --- a/src/test/suite/index.ts +++ b/src/test/suite/index.ts @@ -23,8 +23,10 @@ export async function run(): Promise { try { // Find all test files - //const files = await glob("**/**.test.js", { cwd: testsRoot } leaving this commented out for now since we only have three tests - const files = ["suite/modes.test.js", "suite/task.test.js", "suite/extension.test.js"] + const files = await glob("**/**.test.js", { cwd: testsRoot }) + + //If you want to run a specific test, comment out the above line and uncomment the following line and add the test file to the array + //const files = ["suite/modes.test.js"] // Add files to the test suite files.forEach((f: string) => mocha.addFile(path.resolve(testsRoot, f))) diff --git a/src/test/suite/modes.test.ts b/src/test/suite/modes.test.ts index 56eda6e4174..3f162499e73 100644 --- a/src/test/suite/modes.test.ts +++ b/src/test/suite/modes.test.ts @@ -89,9 +89,10 @@ suite("Roo Code Modes", () => { console.log(text) } }) - const grade = globalThis.provider.messages.find( + const gradeMessage = globalThis.provider.messages.find( ({ type, text }) => type === "say" && !text?.includes("Grade: (1-10)") && text?.includes("Grade:"), )?.text + const grade = gradeMessage?.match(`Grade: (10|[1-9])`) assert.ok( grade?.includes("Grade: 10") || grade?.includes("Grade: 9") || From 1e279b4728c76d1d615a5d6865ad4e77eca5ae97 Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Wed, 12 Feb 2025 09:09:03 -0600 Subject: [PATCH 6/8] ellipsis comment on regex format --- src/test/suite/modes.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/suite/modes.test.ts b/src/test/suite/modes.test.ts index 3f162499e73..367e545cdd5 100644 --- a/src/test/suite/modes.test.ts +++ b/src/test/suite/modes.test.ts @@ -92,7 +92,7 @@ suite("Roo Code Modes", () => { const gradeMessage = globalThis.provider.messages.find( ({ type, text }) => type === "say" && !text?.includes("Grade: (1-10)") && text?.includes("Grade:"), )?.text - const grade = gradeMessage?.match(`Grade: (10|[1-9])`) + const grade = gradeMessage?.match(/Grade: (10|[1-9])/) assert.ok( grade?.includes("Grade: 10") || grade?.includes("Grade: 9") || From 937c3b6c498f438f2898ea31ee74a1d08af60246 Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Thu, 13 Feb 2025 16:08:52 -0600 Subject: [PATCH 7/8] Remove test files list from index.ts Update documentation Update modes.test.ts to use PR comment suggestion for pulling out reponse grade --- src/test/VSCODE_INTEGRATION_TESTS.md | 12 ++++-------- src/test/suite/index.ts | 3 --- src/test/suite/modes.test.ts | 11 +++-------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/test/VSCODE_INTEGRATION_TESTS.md b/src/test/VSCODE_INTEGRATION_TESTS.md index 6192c4488fa..96307996cee 100644 --- a/src/test/VSCODE_INTEGRATION_TESTS.md +++ b/src/test/VSCODE_INTEGRATION_TESTS.md @@ -75,6 +75,8 @@ declare global { npm run test:integration ``` +3. If you want to run a specific test, you can use the `test.only` function in the test file. This will run only the test you specify and ignore the others. + The tests will: - Download and launch a clean VSCode instance @@ -88,13 +90,7 @@ When writing new integration tests: 1. Create a new test file in `src/test/suite/` with the `.test.ts` extension -2. Add the test file to the `files` array in `suite/index.ts` (you can temporarily comment out the other tests to run just the new test): - -```typescript -const files = ["suite/modes.test.js", "suite/tasks.test.js", "suite/extension.test.js", "suite/your-new-test.test.js"] -``` - -3. Structure your tests using the TDD interface: +2. Structure your tests using the TDD interface: ```typescript import * as assert from "assert" @@ -107,7 +103,7 @@ suite("Your Test Suite Name", () => { }) ``` -4. Use the global objects (`api`, `provider`, `extension`, `panel`) to interact with the extension +3. Use the global objects (`api`, `provider`, `extension`, `panel`) to interact with the extension ### Best Practices diff --git a/src/test/suite/index.ts b/src/test/suite/index.ts index 5fe18f1ed0e..ffb8de7473e 100644 --- a/src/test/suite/index.ts +++ b/src/test/suite/index.ts @@ -25,9 +25,6 @@ export async function run(): Promise { // Find all test files const files = await glob("**/**.test.js", { cwd: testsRoot }) - //If you want to run a specific test, comment out the above line and uncomment the following line and add the test file to the array - //const files = ["suite/modes.test.js"] - // Add files to the test suite files.forEach((f: string) => mocha.addFile(path.resolve(testsRoot, f))) diff --git a/src/test/suite/modes.test.ts b/src/test/suite/modes.test.ts index 367e545cdd5..2fe0eaa597f 100644 --- a/src/test/suite/modes.test.ts +++ b/src/test/suite/modes.test.ts @@ -92,14 +92,9 @@ suite("Roo Code Modes", () => { const gradeMessage = globalThis.provider.messages.find( ({ type, text }) => type === "say" && !text?.includes("Grade: (1-10)") && text?.includes("Grade:"), )?.text - const grade = gradeMessage?.match(/Grade: (10|[1-9])/) - assert.ok( - grade?.includes("Grade: 10") || - grade?.includes("Grade: 9") || - grade?.includes("Grade: 8") || - grade?.includes("Grade: 7"), - "Did not receive expected response containing 'Grade: 10' or 'Grade: 9' or 'Grade: 8' or 'Grade: 7'", - ) + const gradeMatch = gradeMessage?.match(/Grade: (\d+)/) + const gradeNum = gradeMatch ? parseInt(gradeMatch[1]) : undefined + assert.ok(gradeNum !== undefined && gradeNum >= 7 && gradeNum <= 10, "Grade must be between 7 and 10") } finally { } }) From 4886cb7e7d023523d4cfcb195e1095a3f00e84a1 Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Thu, 13 Feb 2025 16:21:34 -0600 Subject: [PATCH 8/8] ellipsis comment update to test document --- src/test/VSCODE_INTEGRATION_TESTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/VSCODE_INTEGRATION_TESTS.md b/src/test/VSCODE_INTEGRATION_TESTS.md index 96307996cee..f5882fea1ea 100644 --- a/src/test/VSCODE_INTEGRATION_TESTS.md +++ b/src/test/VSCODE_INTEGRATION_TESTS.md @@ -75,7 +75,7 @@ declare global { npm run test:integration ``` -3. If you want to run a specific test, you can use the `test.only` function in the test file. This will run only the test you specify and ignore the others. +3. If you want to run a specific test, you can use the `test.only` function in the test file. This will run only the test you specify and ignore the others. Be sure to remove the `test.only` function before committing your changes. The tests will: