Merge pull request #1498 from Kilo-Org/fix-settings-screenshot-flakes

Avoid flakey setting snapshot tests

Author: chrarnoldus

apps/playwright-e2e/tests/chat-with-response.test.ts

@@ -10,12 +10,13 @@ test.describe("E2E Chat Test", () => {
 
 		await configureApiKeyThroughUI(page)
 		await waitForWebviewText(page, "Generate, refactor, and debug code with AI assistance")
-		await takeScreenshot("ready-to-chat")
 
-		await sendMessage(page, "Fill in the blanks for this phrase: 'hello w_r_d'")
+		await page.waitForTimeout(1000) // Let the page settle to avoid flakey screenshots
+		await takeScreenshot("ready-to-chat")
 
 		// Don't take any more screenshots after the reponse starts-
 		// llm responses aren't deterministic any capturing the reponse would cause screenshot flakes
+		await sendMessage(page, "Fill in the blanks for this phrase: 'hello w_r_d'")
 		await waitForWebviewText(page, "hello world", 30_000)
 	})
 })

apps/playwright-e2e/tests/playwright-base-test.ts

@@ -54,6 +54,7 @@ export const test = base.extend<TestFixtures>({
 				"--disable-updates",
 				"--skip-welcome",
 				"--skip-release-notes",
+				"--skip-getting-started",
 				"--disable-workspace-trust",
 				"--disable-telemetry",
 				"--disable-crash-reporter",

apps/playwright-e2e/tests/settings-screenshots.test.ts

@@ -5,60 +5,38 @@ test.describe("Settings Screenshots", () => {
 	test("should take screenshots of all settings tabs", async ({ workbox: page, takeScreenshot }: TestFixtures) => {
 		await verifyExtensionInstalled(page)
 
-		// Configure API to ensure settings are accessible
 		await upsertApiConfiguration(page)
 
-		// Open the settings by clicking the settings button in the activity bar
-		const settingsButton = page.locator('[aria-label*="Settings"], [title*="Settings"]').first()
-		if (await settingsButton.isVisible()) {
-			await settingsButton.click()
-		} else {
-			// Alternative: Use keyboard shortcut to open command palette and run settings command
-			const modifier = process.platform === "darwin" ? "Meta" : "Control"
-			await page.keyboard.press(`${modifier}+Shift+P`)
-			await page.keyboard.type("Kilo Code: Settings")
-			await page.keyboard.press("Enter")
-		}
+		// Open the settings
+		page.locator('[aria-label*="Settings"], [title*="Settings"]').first().click()
+		await page.mouse.move(0, 0) // Move the mouse to (0, 0) to avoid triggering the tooltip!
 
-		// Wait for the webview to load
 		const webviewFrame = await findWebview(page)
-
-		// Wait for settings view to be visible by checking for the tablist role
 		await expect(webviewFrame.locator('[role="tablist"]')).toBeVisible({ timeout: 10000 })
 		console.log("✅ Settings view loaded")
 
-		// Wait for the tab list to be visible
 		await expect(webviewFrame.locator('[data-testid="settings-tab-list"]')).toBeVisible()
 		console.log("✅ Settings tab list visible")
 
-		// Find all tab buttons with role="tab"
 		const tabButtons = webviewFrame.locator('[role="tab"]')
 		const tabCount = await tabButtons.count()
 		console.log(`✅ Found ${tabCount} settings tabs`)
 
-		// Take screenshot of each tab
-		for (let i = 0; i < tabCount; i++) {
+		// Take screenshot of each tab (except for the last two)
+		// MCP settings page is flakey and the info page has the version which changes
+		for (let i = 0; i < tabCount - 2; i++) {
 			const tabButton = tabButtons.nth(i)
+			await tabButton.click()
+			await page.waitForTimeout(500)
 
-			// Get the tab name from the text content
 			const tabText = await tabButton.textContent()
 			const tabName = tabText?.trim() || `tab-${i}`
 
-			// Get the data-testid attribute to get the section ID
 			const testId = await tabButton.getAttribute("data-testid")
 			const sectionId = testId?.replace("tab-", "") || `section-${i}`
 
 			console.log(`📸 Taking screenshot of tab: ${tabName} (${sectionId})`)
-
-			// Click the tab to activate it
-			await tabButton.click()
-
-			// Wait a moment for the content to load
-			await page.waitForTimeout(500)
-
-			// Take screenshot with descriptive name
-			const screenshotName = `settings-${sectionId}-${tabName.toLowerCase().replace(/\s+/g, "-")}`
-			await takeScreenshot(screenshotName)
+			await takeScreenshot(`settings-${sectionId}-${tabName.toLowerCase().replace(/\s+/g, "-")}`)
 		}
 
 		console.log("✅ All settings tabs screenshots completed")