-
Notifications
You must be signed in to change notification settings - Fork 274
test(qdoc): Add ui tests for qdoc update readme flow #5435
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 24 commits
d142bd3
62d6bff
43b4617
a004a28
53ceb04
22809cc
c00d3cd
525d784
fa120b2
f1fa2c8
c4a34f5
e6b30c0
b730f6d
deb3d94
7264ff0
de4afb2
93ede19
50923b7
1d1b5a2
0ca4932
7f9f881
e99c056
cc99a6c
e828a95
28a1045
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| // Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package software.aws.toolkits.jetbrains.uitests.docTests | ||
|
|
||
| import java.nio.file.Paths | ||
|
|
||
| fun prepTestData(isCreate: Boolean) { | ||
| val process: Process | ||
| if (isCreate) { | ||
| val path = Paths.get("tstData", "qdoc", "createFlow", "README.md").toUri() | ||
| process = ProcessBuilder("rm", path.path).start() | ||
| } else { | ||
| val path = Paths.get("tstData", "qdoc", "updateFlow", "README.md").toUri() | ||
| process = ProcessBuilder("git", "restore", path.path).start() | ||
| } | ||
|
|
||
| val exitCode = process.waitFor() | ||
| if (exitCode != 0) { | ||
| println("Warning: git stash command failed with exit code $exitCode") | ||
| process.errorStream.bufferedReader().use { reader -> | ||
| println("Error: ${reader.readText()}") | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| // Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package software.aws.toolkits.jetbrains.uitests.docTests.scripts | ||
|
|
||
| // language=TS | ||
| val findAndClickButtonScript = """ | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is awesome! Maybe could be moved to a common utils file so other teams can also use it
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sure, doesn't seem blocking though, i can do that in a followup |
||
| const findAndClickButton = async ( | ||
| page, | ||
| buttonText, | ||
| clickButton = false, | ||
| timeout = 5000 | ||
| ) => { | ||
| try { | ||
| // Wait for any matching buttons to be present | ||
| await page.waitForSelector('button.mynah-button', { | ||
| visible: true, | ||
| timeout, | ||
| }); | ||
| // Find and verify the specific button | ||
| const buttonHandle = await page.evaluateHandle(text => { | ||
| const buttons = Array.from( | ||
| document.querySelectorAll('button.mynah-button') | ||
| ); | ||
| return buttons.find(button => { | ||
| const label = button.querySelector('.mynah-button-label'); | ||
| return label && label.textContent.trim() === text; | ||
| }); | ||
| }, buttonText); | ||
| // Check if button was found | ||
| const button = buttonHandle.asElement(); | ||
| if (!button) { | ||
| console.log(buttonText); | ||
| throw new Error(`Button with text not found`); | ||
| } | ||
| // Verify button is visible and enabled | ||
| const isVisible = await page.evaluate(el => { | ||
| const style = window.getComputedStyle(el); | ||
| return ( | ||
| style.display !== 'none' && | ||
| style.visibility !== 'hidden' && | ||
| style.opacity !== '0' | ||
| ); | ||
| }, button); | ||
| if (!isVisible) { | ||
| console.log(buttonText); | ||
| throw new Error(`Button with text is not visible`); | ||
| } | ||
| if (clickButton) { | ||
| // Click the button | ||
| await button.click(); | ||
| // Optional wait after click | ||
| await new Promise(resolve => setTimeout(resolve, 1000)); | ||
| console.log(`Successfully clicked button with text`); | ||
| console.log(buttonText); | ||
| } else { | ||
| return button; | ||
| } | ||
| } catch (error) { | ||
| console.error(`Error interacting with button:`, buttonText, error); | ||
| throw error; | ||
| } | ||
| }; | ||
| """.trimIndent() | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,172 @@ | ||
| // Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package software.aws.toolkits.jetbrains.uitests.docTests.scripts | ||
|
|
||
| // language=TS | ||
| val updateReadmeLatestChangesConfirmOptionsScript = """ | ||
| const puppeteer = require('puppeteer'); | ||
|
|
||
| async function testNavigation() { | ||
| const browser = await puppeteer.connect({ | ||
| browserURL: 'http://localhost:9222' | ||
| }) | ||
|
|
||
| try { | ||
|
|
||
| const pages = await browser.pages() | ||
|
|
||
| for(const page of pages) { | ||
| const contents = await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); | ||
|
|
||
| const element = await page.${'$'}('.mynah-chat-prompt-input') | ||
| if(element) { | ||
|
|
||
| console.log('Typing /doc in the chat window') | ||
|
|
||
| await page.type('.mynah-chat-prompt-input', '/doc') | ||
| await page.keyboard.press('Enter') | ||
|
|
||
| console.log('Attempting to find and click Update an existing README button') | ||
| await findAndClickButton(page, 'Update an existing README', true, 10000) | ||
| console.log('Attempting to find and click Update README to reflect code button') | ||
| await findAndClickButton(page, 'Update README to reflect code', true, 10000) | ||
| console.log('Attempting to find all available buttons') | ||
| const yesButton = await findAndClickButton(page, 'Yes', false, 10000) | ||
| const changeFolderButton = await findAndClickButton(page, 'Change folder', false, 10000) | ||
| const cancelButton = await findAndClickButton(page, 'Cancel', false, 10000) | ||
|
|
||
| if (!yesButton || !changeFolderButton || !cancelButton) { | ||
| console.log('Error: Test Failed') | ||
| console.log('Unable to find buttons for Yes/ChangeFolder/Cancel') | ||
| } else { | ||
| console.log('Found all expected buttons') | ||
| console.log('Test Successful') | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } finally { | ||
| await browser.close(); | ||
| } | ||
| } | ||
|
|
||
| testNavigation().catch((error) => { | ||
| console.log('Error: Test Failed'); | ||
| console.error(error); | ||
| }); | ||
| """.trimIndent() | ||
|
|
||
| // language=TS | ||
| val updateReadmeLatestChangesScript = """ | ||
|
|
||
| const puppeteer = require('puppeteer'); | ||
|
|
||
| async function testNavigation() { | ||
| const browser = await puppeteer.connect({ | ||
| browserURL: 'http://localhost:9222' | ||
| }) | ||
|
|
||
| try { | ||
|
|
||
| const pages = await browser.pages() | ||
|
|
||
| for(const page of pages) { | ||
| const contents = await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); | ||
|
|
||
| const element = await page.${'$'}('.mynah-chat-prompt-input') | ||
|
|
||
| if(element) { | ||
|
|
||
| console.log('Typing /doc in the chat window') | ||
|
|
||
| await page.type('.mynah-chat-prompt-input', '/doc') | ||
| await page.keyboard.press('Enter') | ||
|
|
||
| console.log('Attempting to find and click Update an existing README button') | ||
| await findAndClickButton(page, 'Update an existing README', true, 10000) | ||
| console.log('Attempting to find and click Update README to reflect code button') | ||
| await findAndClickButton(page, 'Update README to reflect code', true, 10000) | ||
| console.log('Attempting to find and click Yes button to confirm option') | ||
| await findAndClickButton(page, 'Yes', true, 10000) | ||
| console.log('Waiting for updated README to be generated') | ||
| await new Promise(resolve => setTimeout(resolve, 90000)); | ||
| console.log('Attempting to find and click Accept button') | ||
| await findAndClickButton(page, 'Accept', true, 10000) | ||
| } | ||
| } | ||
|
|
||
| } finally { | ||
| await browser.close(); | ||
| } | ||
| } | ||
|
|
||
| testNavigation().catch((error) => { | ||
| console.log('Error: Test Failed'); | ||
| console.error(error); | ||
| }); | ||
|
|
||
| """.trimIndent() | ||
|
|
||
| // language=TS | ||
| val updateReadmeLatestChangesMakeChangesFlowScript = """ | ||
|
|
||
| const puppeteer = require('puppeteer'); | ||
|
|
||
| async function testNavigation() { | ||
| const browser = await puppeteer.connect({ | ||
| browserURL: 'http://localhost:9222' | ||
| }); | ||
|
|
||
| try { | ||
|
|
||
| const pages = await browser.pages(); | ||
|
|
||
| for(const page of pages) { | ||
| const contents = await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); | ||
|
|
||
| const element = await page.${'$'}('.mynah-chat-prompt-input'); | ||
| if(element) { | ||
|
|
||
| console.log('Typing /doc in the chat window'); | ||
|
|
||
| await page.type('.mynah-chat-prompt-input', '/doc'); | ||
| await page.keyboard.press('Enter'); | ||
|
|
||
| console.log('Attempting to find and click Update an existing README button'); | ||
| await findAndClickButton(page, 'Update an existing README', true, 10000); | ||
| console.log('Attempting to find and click Update README to reflect code button'); | ||
| await findAndClickButton(page, 'Update README to reflect code', true, 10000); | ||
|
Comment on lines
+126
to
+139
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These few lines appear to be repeated in most tests, can they be abstracted?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. abstracting parts of the testing flow can cause issues while trying to read and debug code for someone unfamiliar. So we should keep testing scripts as is to understand what's happening in the test without needing to jump around. |
||
| console.log('Attempting to find and click Yes button to confirm option'); | ||
| await findAndClickButton(page, 'Yes', true, 10000); | ||
| console.log('Waiting for updated README to be generated'); | ||
| await new Promise(resolve => setTimeout(resolve, 90000)); | ||
| console.log('Attempting to find and click Make changes button'); | ||
| await findAndClickButton(page, 'Make changes', true, 10000); | ||
| const makeChangeText = await page.waitForSelector('[placeholder="Describe documentation changes"]'); | ||
| if (!makeChangeText) { | ||
| console.log('Error: Test Failed'); | ||
| console.log('Unable to find placeholder description test in Make Changes flow'); | ||
| } else { | ||
| console.log('Found expected placeholder text for Make Changes flow'); | ||
| console.log('Test Successful'); | ||
| } | ||
|
|
||
| } | ||
| } | ||
|
|
||
| } finally { | ||
| await browser.close(); | ||
| } | ||
| } | ||
|
|
||
| testNavigation().catch((error) => { | ||
| console.log('Error: Test Failed'); | ||
| console.error(error); | ||
| }); | ||
|
|
||
| """.trimIndent() | ||
|
|
||
| val updateReadmeLatestChangesConfirmOptionsTestScript = updateReadmeLatestChangesConfirmOptionsScript.plus(findAndClickButtonScript) | ||
| val updateReadmeLatestChangesTestScript = updateReadmeLatestChangesScript.plus(findAndClickButtonScript) | ||
| val updateReadmeLatestChangesMakeChangesFlowTestScript = updateReadmeLatestChangesMakeChangesFlowScript.plus(findAndClickButtonScript) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| // Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package software.aws.toolkits.jetbrains.uitests.docTests.scripts | ||
|
|
||
| // language=TS | ||
| val updateReadmeSpecificChangesMakeChangesFlowScript = """ | ||
|
|
||
| const puppeteer = require('puppeteer'); | ||
|
|
||
| async function testNavigation() { | ||
| const browser = await puppeteer.connect({ | ||
| browserURL: 'http://localhost:9222' | ||
| }); | ||
|
|
||
| try { | ||
|
|
||
| const pages = await browser.pages(); | ||
|
|
||
| for(const page of pages) { | ||
| const contents = await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); | ||
|
|
||
| const element = await page.${'$'}('.mynah-chat-prompt-input'); | ||
| if(element) { | ||
|
|
||
| console.log('Typing /doc in the chat window'); | ||
|
|
||
| await page.type('.mynah-chat-prompt-input', '/doc'); | ||
| await page.keyboard.press('Enter'); | ||
|
|
||
| console.log('Attempting to find and click Update an existing README button'); | ||
| await findAndClickButton(page, 'Update an existing README', true, 10000); | ||
| console.log('Attempting to find and click Make a specific change button'); | ||
| await findAndClickButton(page, 'Make a specific change', true, 10000); | ||
| console.log('Attempting to find and click Yes button to confirm option'); | ||
| await findAndClickButton(page, 'Yes', true, 10000); | ||
|
|
||
| console.log('Typing specific change instructions in the chat window'); | ||
| await page.type('.mynah-chat-prompt-input', 'Add a section with Installation instructions for this repository. Title this new section \"### Installation\"'); | ||
| await page.keyboard.press('Enter'); | ||
|
|
||
| console.log('Waiting for updated README to be generated'); | ||
| await new Promise(resolve => setTimeout(resolve, 90000)); | ||
| console.log('Attempting to find and click Make changes button'); | ||
| await findAndClickButton(page, 'Make changes', true, 10000); | ||
| const makeChangeText = await page.waitForSelector('[placeholder="Describe documentation changes"]'); | ||
| if (!makeChangeText) { | ||
| console.log('Error: Test Failed'); | ||
| console.log('Unable to find placeholder description text in Make Changes flow'); | ||
| } else { | ||
| console.log('Found expected placeholder text for Make Changes flow'); | ||
| console.log('Test Successful'); | ||
| } | ||
|
|
||
| } | ||
| } | ||
|
|
||
| } finally { | ||
| await browser.close(); | ||
| } | ||
| } | ||
|
|
||
| testNavigation().catch((error) => { | ||
| console.log('Error: Test Failed'); | ||
| console.error(error); | ||
| }); | ||
|
|
||
| """.trimIndent() | ||
|
|
||
| // language=TS | ||
| val updateReadmeSpecificChangesScript = """ | ||
|
|
||
| const puppeteer = require('puppeteer'); | ||
|
|
||
| async function testNavigation() { | ||
| const browser = await puppeteer.connect({ | ||
| browserURL: 'http://localhost:9222' | ||
| }); | ||
|
|
||
| try { | ||
|
|
||
| const pages = await browser.pages(); | ||
|
|
||
| for(const page of pages) { | ||
| const contents = await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); | ||
|
|
||
| const element = await page.${'$'}('.mynah-chat-prompt-input'); | ||
|
|
||
| if(element) { | ||
|
|
||
| console.log('Typing /doc in the chat window'); | ||
|
|
||
| await page.type('.mynah-chat-prompt-input', '/doc'); | ||
| await page.keyboard.press('Enter'); | ||
|
|
||
| console.log('Attempting to find and click Update an existing README button'); | ||
| await findAndClickButton(page, 'Update an existing README', true, 10000); | ||
| console.log('Attempting to find and click Make a specific change button'); | ||
| await findAndClickButton(page, 'Make a specific change', true, 10000); | ||
| console.log('Attempting to find and click Yes button to confirm option'); | ||
| await findAndClickButton(page, 'Yes', true, 10000); | ||
|
|
||
| console.log('Typing specific change instructions in the chat window'); | ||
| await page.type('.mynah-chat-prompt-input', 'Add a section with Installation instructions for this repository. Title this new section \"### Installation\"'); | ||
| await page.keyboard.press('Enter'); | ||
|
|
||
| console.log('Waiting for updated README to be generated'); | ||
| await new Promise(resolve => setTimeout(resolve, 90000)); | ||
|
|
||
| console.log('Attempting to find and click Accept button'); | ||
| await findAndClickButton(page, 'Accept', true, 10000); | ||
| } | ||
| } | ||
|
|
||
| } finally { | ||
| await browser.close(); | ||
| } | ||
| } | ||
|
|
||
| testNavigation().catch((error) => { | ||
| console.log('Error: Test Failed'); | ||
| console.error(error); | ||
| }); | ||
|
|
||
| """.trimIndent() | ||
|
|
||
| val updateReadmeSpecificChangesMakeChangesFlowTestScript = updateReadmeSpecificChangesMakeChangesFlowScript.plus(findAndClickButtonScript) | ||
gandhi-21 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| val updateReadmeSpecificChangesTestScript = updateReadmeSpecificChangesScript.plus(findAndClickButtonScript) | ||
Uh oh!
There was an error while loading. Please reload this page.