Setup Playwright E2E Testing

.github/workflows/code-qa.yml

@@ -90,9 +90,28 @@ jobs:
               working-directory: apps/vscode-e2e
               run: xvfb-run -a pnpm test:ci
 
+    playwright-e2e-test:
+        runs-on: ubuntu-latest
+        needs: [check-openrouter-api-key]
+        if: needs.check-openrouter-api-key.outputs.exists == 'true'
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v4
+            - name: Setup Node.js and pnpm
+              uses: ./.github/actions/setup-node-pnpm
+            - name: Install Playwright browsers
+              working-directory: apps/playwright-e2e
+              run: npx playwright install --with-deps
+            - name: Create .env.local file
+              working-directory: apps/playwright-e2e
+              run: echo "OPENROUTER_API_KEY=${{ secrets.OPENROUTER_API_KEY }}" > .env.local
+            - name: Run Playwright E2E tests
+              working-directory: apps/playwright-e2e
+              run: xvfb-run -a pnpm test:ci
+
     notify-slack-on-failure:
         runs-on: ubuntu-latest
-        needs: [check-translations, knip, compile, unit-test, integration-test]
+        needs: [check-translations, knip, compile, unit-test, integration-test, playwright-e2e-test]
         if: ${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/main' && contains(needs.*.result, 'failure') }}
         steps:
             - name: Checkout code

apps/playwright-e2e/.env.local.sample

@@ -0,0 +1,3 @@
+# Copy this file to .env.local and fill in your actual values
+
+OPENROUTER_API_KEY=your_openrouter_api_key_here

apps/playwright-e2e/.gitignore

@@ -0,0 +1,8 @@
+node_modules/
+test-results/
+playwright-report/
+playwright/.cache/
+screenshots/
+*.log
+.env
+.env.local

apps/playwright-e2e/eslint.config.mjs

@@ -0,0 +1,3 @@
+import { config } from "@roo-code/config-eslint/base"
+
+export default config

apps/playwright-e2e/helpers/webview-helpers.ts

@@ -0,0 +1,52 @@
+import { type Page, type FrameLocator, expect } from "@playwright/test"
+import type { WebviewMessage } from "../../../src/shared/WebviewMessage"
+import type { ProviderSettings } from "@roo-code/types"
+
+const defaultPlaywrightApiConfig = {
+	apiProvider: "openrouter" as const,
+	openRouterApiKey: process.env.OPENROUTER_API_KEY,
+	openRouterModelId: "openai/gpt-4o-mini",
+}
+
+export async function findWebview(workbox: Page): Promise<FrameLocator> {
+	const webviewFrameEl = workbox.frameLocator(
+		'iframe[src*="extensionId=RooVeterinaryInc.roo-cline"][src*="purpose=webviewView"]',
+	)
+	await webviewFrameEl.locator("#active-frame").waitFor()
+	return webviewFrameEl.frameLocator("#active-frame")
+}
+
+export async function waitForWebviewText(page: Page, text: string, timeout: number = 30000): Promise<void> {
+	const webviewFrame = await findWebview(page)
+	await expect(webviewFrame.locator("body")).toContainText(text, { timeout })
+}
+
+export async function postWebviewMessage(page: Page, message: WebviewMessage): Promise<void> {
+	const webviewFrame = await findWebview(page)
+	await webviewFrame.locator("body").evaluate((element, msg) => {
+		if (!window.vscode) {
+			throw new Error("Global vscode API not found")
+		}
+
+		window.vscode.postMessage(msg)
+	}, message)
+}
+
+export async function verifyExtensionInstalled(page: Page) {
+	try {
+		const activityBarIcon = page.locator('[aria-label*="Roo"]').first()
+		expect(await activityBarIcon).toBeDefined()
+		activityBarIcon.click()
+	} catch (_error) {
+		throw new Error("Failed to find the installed extension! Check if the build failed and try again.")
+	}
+}
+
+export async function upsertApiConfiguration(page: Page, apiConfiguration?: Partial<ProviderSettings>): Promise<void> {
+	await postWebviewMessage(page, {
+		type: "upsertApiConfiguration",
+		text: "default",
+		apiConfiguration: apiConfiguration ?? defaultPlaywrightApiConfig,
+	})
+	await postWebviewMessage(page, { type: "currentApiConfigName", text: "default" })
+}

apps/playwright-e2e/package.json

@@ -0,0 +1,25 @@
+{
+	"name": "@roo-code/playwright-e2e",
+	"private": true,
+	"scripts": {
+		"lint": "eslint tests --ext=ts --max-warnings=0",
+		"check-types": "tsc --noEmit",
+		"format": "prettier --write tests",
+		"test:run": "npx dotenvx run -f .env.local -- playwright test",
+		"test:ci": "pnpm -w bundle && pnpm --filter @roo-code/vscode-webview build && playwright test",
+		"playwright": "playwright",
+		"clean": "rimraf test-results .turbo"
+	},
+	"devDependencies": {
+		"@playwright/test": "^1.53.1",
+		"@roo-code/config-eslint": "workspace:^",
+		"@roo-code/config-typescript": "workspace:^",
+		"@roo-code/types": "workspace:^",
+		"@types/node": "^22.15.29",
+		"@vscode/test-electron": "^2.4.0",
+		"dotenv": "^16.4.5",
+		"rimraf": "^6.0.1",
+		"typescript": "5.8.3",
+		"vitest": "^3.2.3"
+	}
+}

apps/playwright-e2e/playwright.config.ts

@@ -0,0 +1,36 @@
+import { defineConfig } from "@playwright/test"
+import { TestOptions } from "./tests/playwright-base-test"
+import * as dotenv from "dotenv"
+import * as path from "path"
+
+const envPath = path.resolve(__dirname, ".env")
+dotenv.config({ path: envPath })
+
+export default defineConfig<void, TestOptions>({
+	reporter: process.env.CI ? "html" : "list",
+	timeout: 120_000,
+	workers: 1,
+	expect: {
+		timeout: 30_000,
+	},
+	globalSetup: "./playwright.globalSetup",
+	testDir: "./tests",
+	testIgnore: "**/helpers/__tests__/**",
+	outputDir: "./test-results",
+	projects: [
+		// { name: "VSCode insiders", use: { vscodeVersion: "insiders" } },
+		{ name: "VSCode stable", use: { vscodeVersion: "stable" } },
+	],
+	use: {
+		// Keep headless true for Playwright, but don't pass --headless to VS Code
+		headless: true,
+		trace: "on-first-retry",
+		screenshot: "only-on-failure",
+		video: "retain-on-failure",
+		// Add longer timeouts for CI environment
+		...(process.env.CI && {
+			actionTimeout: 60_000,
+			navigationTimeout: 60_000,
+		}),
+	},
+})

apps/playwright-e2e/playwright.globalSetup.ts

@@ -0,0 +1,11 @@
+import { downloadAndUnzipVSCode } from "@vscode/test-electron/out/download"
+
+export default async () => {
+	// console.log("Downloading VS Code insiders...")
+	// await downloadAndUnzipVSCode("insiders")
+
+	console.log("Downloading VS Code stable...")
+	await downloadAndUnzipVSCode("stable")
+
+	console.log("VS Code downloads completed!")
+}

apps/playwright-e2e/tests/chat-with-response.test.ts

@@ -0,0 +1,27 @@
+import { test } from "./playwright-base-test"
+import {
+	verifyExtensionInstalled,
+	upsertApiConfiguration,
+	waitForWebviewText,
+	findWebview as findWebview,
+} from "../helpers/webview-helpers"
+
+test.describe("Full E2E Test", () => {
+	test("should configure credentials and send a message", async ({ workbox: page }) => {
+		await verifyExtensionInstalled(page)
+
+		await waitForWebviewText(page, "Welcome to Roo Code!")
+
+		await upsertApiConfiguration(page)
+
+		await waitForWebviewText(page, "Generate, refactor, and debug code with AI assistance")
+
+		const webviewFrame = await findWebview(page)
+		const chatInput = webviewFrame.locator('textarea, input[type="text"]').first()
+		await chatInput.waitFor({ timeout: 5000 })
+
+		await chatInput.fill("Output only the result of '1+1'")
+		await chatInput.press("Enter")
+		await waitForWebviewText(page, "2", 30_000)
+	})
+})

apps/playwright-e2e/tests/playwright-base-test.ts

@@ -0,0 +1,160 @@
+import { test as base, type Page, _electron } from "@playwright/test"
+import { downloadAndUnzipVSCode } from "@vscode/test-electron/out/download"
+export { expect } from "@playwright/test"
+import * as path from "path"
+import * as os from "os"
+import * as fs from "fs"
+
+const __dirname = path.dirname(__filename)
+
+export type TestOptions = {
+	vscodeVersion: string
+}
+
+type TestFixtures = TestOptions & {
+	workbox: Page
+	createProject: () => Promise<string>
+	createTempDir: () => Promise<string>
+}
+
+export const test = base.extend<TestFixtures>({
+	vscodeVersion: ["stable", { option: true }],
+
+	workbox: async ({ vscodeVersion, createProject, createTempDir }, use) => {
+		const defaultCachePath = await createTempDir()
+		const vscodePath = await downloadAndUnzipVSCode(vscodeVersion)
+
+		const electronApp = await _electron.launch({
+			executablePath: vscodePath,
+			args: [
+				"--no-sandbox",
+				"--disable-gpu-sandbox",
+				"--disable-updates",
+				"--skip-welcome",
+				"--skip-release-notes",
+				"--disable-workspace-trust",
+				"--disable-telemetry",
+				"--disable-crash-reporter",
+				"--disable-gpu",
+				"--disable-dev-shm-usage",
+				// Add CI-specific args for better Docker compatibility
+				...(process.env.CI ? ["--no-sandbox", "--disable-setuid-sandbox"] : []),
+				`--extensionDevelopmentPath=${path.resolve(__dirname, "..", "..", "..", "src")}`,
+				`--extensions-dir=${path.join(defaultCachePath, "extensions")}`,
+				`--user-data-dir=${path.join(defaultCachePath, "user-data")}`,
+				"--enable-proposed-api=RooVeterinaryInc.roo-cline",
+				await createProject(),
+			],
+		})
+
+		// Add better error handling and logging for firstWindow()
+		console.log("🔄 Attempting to get VS Code first window...")
+		let workbox: Page
+		try {
+			workbox = await electronApp.firstWindow()
+			console.log("✅ Successfully got VS Code first window")
+		} catch (error) {
+			console.error("❌ Failed to get VS Code first window:", error)
+			console.log("📊 Electron app context info:")
+			console.log("  - Process count:", electronApp.context().pages().length)
+
+			// Try to get any available window
+			const pages = electronApp.context().pages()
+			if (pages.length > 0) {
+				console.log("🔄 Using first available page instead...")
+				workbox = pages[0]
+			} else {
+				throw new Error(`No VS Code window available. Original error: ${error}`)
+			}
+		}
+		await workbox.waitForLoadState("domcontentloaded")
+
+		try {
+			console.log("🔄 Waiting for VS Code workbench...")
+			await workbox.waitForSelector(".monaco-workbench", { timeout: 10000 })
+		} catch (_error) {
+			throw new Error("❌ .monaco-workbench not found!")
+		}
+
+		console.log("✅ VS Code workbox ready for testing")
+		await use(workbox)
+		await electronApp.close()
+
+		const logPath = path.join(defaultCachePath, "user-data")
+		if (fs.existsSync(logPath)) {
+			const logOutputPath = test.info().outputPath("vscode-logs")
+			await fs.promises.cp(logPath, logOutputPath, { recursive: true })
+		}
+	},
+
+	createProject: async ({ createTempDir }, use) => {
+		await use(async () => {
+			const projectPath = await createTempDir()
+			if (fs.existsSync(projectPath)) await fs.promises.rm(projectPath, { recursive: true })
+
+			console.log(`Creating test project in ${projectPath}`)
+			await fs.promises.mkdir(projectPath)
+
+			const packageJson = {
+				name: "test-project",
+				version: "1.0.0",
+				description: "Test project for ai agent extension",
+				main: "index.js",
+				scripts: {
+					test: 'echo "Error: no test specified" && exit 1',
+				},
+				keywords: [],
+				author: "",
+				license: "ISC",
+			}
+
+			await fs.promises.writeFile(path.join(projectPath, "package.json"), JSON.stringify(packageJson, null, 2))
+
+			const testFile = `// Test file for extension
+console.log('Hello from the test project!');
+
+function greet(name) {
+  return \`Hello, \${name}!\`;
+}
+
+module.exports = { greet };
+`
+
+			await fs.promises.writeFile(path.join(projectPath, "index.js"), testFile)
+
+			const readme = `# Test Project
+
+This is a test project created for testing the VS Code extension.
+
+## Features
+
+- Basic JavaScript file
+- Package.json configuration
+- Ready for AI assistant interaction
+`
+
+			await fs.promises.writeFile(path.join(projectPath, "README.md"), readme)
+
+			return projectPath
+		})
+	},
+
+	// eslint-disable-next-line no-empty-pattern
+	createTempDir: async ({}, use) => {
+		const tempDirs: string[] = []
+		await use(async () => {
+			const tempDirPath = await fs.promises.mkdtemp(path.join(os.tmpdir(), "e2e-test-"))
+			const tempDir = await fs.promises.realpath(tempDirPath)
+			tempDirs.push(tempDir)
+			return tempDir
+		})
+
+		for (const tempDir of tempDirs) {
+			try {
+				await fs.promises.rm(tempDir, { recursive: true })
+			} catch (error) {
+				console.warn(`Failed to cleanup temp dir ${tempDir}:`, error)
+			}
+		}
+	},
+})

apps/playwright-e2e/tests/sanity.test.ts

@@ -0,0 +1,26 @@
+import { test, expect } from "./playwright-base-test"
+import { verifyExtensionInstalled, findWebview } from "../helpers/webview-helpers"
+
+test.describe("Sanity Tests", () => {
+	test("should launch VS Code with extension installed", async ({ workbox: page }) => {
+		await expect(page.locator(".monaco-workbench")).toBeVisible()
+		console.log("✅ VS Code launched successfully")
+
+		await expect(page.locator(".activitybar")).toBeVisible()
+		console.log("✅ Activity bar visible")
+
+		await page.keyboard.press("Meta+Shift+P")
+		const commandPalette = page.locator(".quick-input-widget")
+		await expect(commandPalette).toBeVisible()
+
+		await page.keyboard.press("Escape")
+		await expect(commandPalette).not.toBeVisible()
+		console.log("✅ Command palette working")
+
+		await verifyExtensionInstalled(page)
+		await findWebview(page)
+
+		console.log("✅ Extension installed and webview loaded!")
+		await page.screenshot({ path: "screenshots/sanity.png" })
+	})
+})