55
66import assert from 'assert'
77import { qTestingFramework } from './framework/framework'
8- import { FollowUpTypes } from '../../amazonqFeatureDev/types'
98import sinon from 'sinon'
109import { verifyTextOrder } from './framework/text'
11- import { examples } from '../../amazonqFeatureDev/userFacingText'
1210import { registerAuthHook , using } from '../../test/setupUtil'
1311import { loginToIdC } from './utils/setup'
12+ import { Messenger } from './framework/messenger'
13+ import { FollowUpTypes } from '../../amazonqFeatureDev/types'
14+ import { examples , newTaskChanges , sessionClosed } from '../../amazonqFeatureDev/userFacingText'
15+ import { ChatItem } from '@aws/mynah-ui'
1416
1517describe . skip ( 'Amazon Q Feature Dev' , function ( ) {
1618 let framework : qTestingFramework
19+ let tab : Messenger
20+
21+ const maxTestDuration = 600000
22+ const prompt = 'Implement twosum in typescript'
23+ const iterateApproachPrompt = prompt + ' and add tests'
24+ const codegenApproachPrompt = prompt + ' and add even more tests'
1725
1826 before ( async function ( ) {
1927 await using( registerAuthHook ( 'amazonq-test-account' ) , async ( ) => {
@@ -24,17 +32,18 @@ describe.skip('Amazon Q Feature Dev', function () {
2432 beforeEach ( ( ) => {
2533 registerAuthHook ( 'amazonq-test-account' )
2634 framework = new qTestingFramework ( 'featuredev' , true , true )
35+ tab = framework . createTab ( )
2736 } )
2837
2938 afterEach ( ( ) => {
39+ framework . removeTab ( tab . tabID )
3040 framework . dispose ( )
3141 sinon . restore ( )
3242 } )
3343
3444 describe ( 'quick action availability' , ( ) => {
35- it ( 'Shows /dev when feature dev is enabled' , ( ) => {
36- const q = framework . createTab ( )
37- const command = q . findCommand ( '/dev' )
45+ it ( 'Shows /dev when feature dev is enabled' , async ( ) => {
46+ const command = tab . findCommand ( '/dev' )
3847 if ( ! command ) {
3948 assert . fail ( 'Could not find command' )
4049 }
@@ -48,47 +57,184 @@ describe.skip('Amazon Q Feature Dev', function () {
4857 // The beforeEach registers a framework which accepts requests. If we don't dispose before building a new one we have duplicate messages
4958 framework . dispose ( )
5059 framework = new qTestingFramework ( 'featuredev' , false , true )
51- const q = framework . createTab ( )
52- const command = q . findCommand ( '/dev' )
60+ const tab = framework . createTab ( )
61+ const command = tab . findCommand ( '/dev' )
5362 if ( command . length > 0 ) {
5463 assert . fail ( 'Found command when it should not have been found' )
5564 }
5665 } )
5766 } )
5867
59- describe ( '/dev {msg} entry' , async ( ) => {
60- it ( 'Receives chat response' , async ( ) => {
61- this . timeout ( 60000 )
62- const q = framework . createTab ( )
63- const prompt = 'Implement twosum in typescript'
64- q . addChatMessage ( { command : '/dev' , prompt } )
65-
68+ function waitForButtons ( buttons : FollowUpTypes [ ] ) {
69+ return tab . waitForEvent ( ( ) => {
70+ return buttons . every ( value => tab . hasButton ( value ) )
71+ } )
72+ }
73+
74+ async function waitForText ( text : string ) {
75+ await tab . waitForEvent (
76+ ( ) => {
77+ return tab . getChatItems ( ) . some ( chatItem => chatItem . body === text )
78+ } ,
79+ {
80+ waitIntervalInMs : 250 ,
81+ waitTimeoutInMs : 2000 ,
82+ }
83+ )
84+ }
85+
86+ function verifyApproachState ( chatItems : ChatItem [ ] , expectedResponses : RegExp [ ] ) {
87+ // Verify that all the responses come back in the correct order
88+ verifyTextOrder ( chatItems , expectedResponses )
89+
90+ // Check that the UI has the two buttons
91+ assert . notStrictEqual ( chatItems . pop ( ) ?. followUp ?. options , [
92+ {
93+ type : FollowUpTypes . GenerateCode ,
94+ disabled : false ,
95+ } ,
96+ ] )
97+ }
98+
99+ async function iterate ( prompt : string ) {
100+ tab . addChatMessage ( { prompt } )
101+
102+ await retryIfRequired ( async ( ) => {
66103 // Wait for a backend response
67- await q . waitForChatFinishesLoading ( )
68-
69- const chatItems = q . getChatItems ( )
70-
71- /**
72- * Verify that all the responses come back in the correct order and that a response
73- * after the prompt is non empty (represents a response from the backend, since the same response isn't
74- * guarenteed we can't verify direct responses)
75- */
76- verifyTextOrder ( chatItems , [ / W e l c o m e t o \/ d e v / , new RegExp ( prompt ) , / .\S / ] )
77-
78- // Check that the last UI message has the two buttons
79- assert . notStrictEqual ( chatItems . pop ( ) ?. followUp ?. options , [
80- {
81- type : FollowUpTypes . NewTask ,
82- } ,
83- {
84- type : FollowUpTypes . GenerateCode ,
85- disabled : false ,
86- } ,
87- ] )
104+ await tab . waitForChatFinishesLoading ( )
105+ } )
106+ }
107+
108+ /**
109+ * Make the initial request and if the response has a retry button, click it until either
110+ * we can no longer retry or the tests recover.
111+ *
112+ * This allows the e2e tests to recover from potential one off backend problems
113+ */
114+ async function retryIfRequired ( request : ( ) = > Promise < void > ) {
115+ await request ( )
116+ while ( tab . hasButton ( FollowUpTypes . Retry ) ) {
117+ console . log ( 'Retrying request' )
118+ tab . clickButton ( FollowUpTypes . Retry )
119+ await request ( )
120+ }
121+
122+ // The backend never recovered
123+ if ( tab . hasButton ( FollowUpTypes . SendFeedback ) ) {
124+ assert. fail ( 'Encountered an error when attempting to call the feature dev backend. Could not continue' )
125+ }
126+ }
127+
128+ const functionalTests = ( ) = > {
129+ afterEach ( async function ( ) {
130+ // currentTest.state is undefined if a beforeEach fails
131+ if (
132+ this . currentTest ?. state === undefined ||
133+ this . currentTest ?. isFailed ( ) ||
134+ this . currentTest ?. isPending ( )
135+ ) {
136+ // Since the tests are long running this may help in diagnosing the issue
137+ console . log ( 'Current chat items at failure' )
138+ console . log ( JSON . stringify ( tab . getChatItems ( ) , undefined , 4 ) )
139+ }
140+ } )
141+
142+ it ( 'Should receive chat response' , async ( ) => {
143+ verifyApproachState ( tab . getChatItems ( ) , [ new RegExp ( prompt ) , / .\S / ] )
144+ } )
145+
146+ describe ( 'Moves directly from approach to codegen' , ( ) => {
147+ codegenTests ( )
88148 } )
149+
150+ describe ( 'Iterates on approach' , ( ) => {
151+ beforeEach ( async function ( ) {
152+ this . timeout ( maxTestDuration )
153+ await iterate ( iterateApproachPrompt )
154+ } )
155+
156+ it ( 'Should iterate successfully' , ( ) => {
157+ verifyApproachState ( tab . getChatItems ( ) , [ new RegExp ( prompt ) , / .\S / ] )
158+ } )
159+
160+ describe ( 'Moves to codegen after iteration' , ( ) => {
161+ codegenTests ( )
162+ } )
163+ } )
164+
165+ function codegenTests ( ) {
166+ beforeEach ( async function ( ) {
167+ this . timeout ( maxTestDuration )
168+ tab . clickButton ( FollowUpTypes . GenerateCode )
169+ await retryIfRequired ( async ( ) => {
170+ await Promise . any ( [
171+ waitForButtons ( [ FollowUpTypes . InsertCode , FollowUpTypes . ProvideFeedbackAndRegenerateCode ] ) ,
172+ waitForButtons ( [ FollowUpTypes . Retry ] ) ,
173+ ] )
174+ } )
175+ } )
176+
177+ describe ( 'Clicks accept code' , ( ) => {
178+ insertCodeTests ( )
179+ } )
180+
181+ describe ( 'Iterates on codegen' , ( ) => {
182+ beforeEach ( async function ( ) {
183+ this . timeout ( maxTestDuration )
184+ tab . clickButton ( FollowUpTypes . ProvideFeedbackAndRegenerateCode )
185+ await tab . waitForChatFinishesLoading ( )
186+ await iterate ( codegenApproachPrompt )
187+ } )
188+
189+ describe ( 'Clicks accept code' , ( ) => {
190+ insertCodeTests ( )
191+ } )
192+ } )
193+ }
194+
195+ function insertCodeTests ( ) {
196+ beforeEach ( async function ( ) {
197+ this . timeout ( maxTestDuration )
198+ tab . clickButton ( FollowUpTypes . InsertCode )
199+ await waitForButtons ( [ FollowUpTypes . NewTask , FollowUpTypes . CloseSession ] )
200+ } )
201+
202+ it ( 'clicks new task' , async ( ) => {
203+ tab . clickButton ( FollowUpTypes . NewTask )
204+ await waitForText ( newTaskChanges )
205+ assert . deepStrictEqual ( tab . getChatItems ( ) . pop ( ) ?. body , newTaskChanges )
206+ } )
207+
208+ it ( 'click close session' , async ( ) => {
209+ tab . clickButton ( FollowUpTypes . CloseSession )
210+ await waitForText ( sessionClosed )
211+ assert . deepStrictEqual ( tab . getPlaceholder ( ) , sessionClosed )
212+ } )
213+ }
214+ }
215+
216+ describe ( '/dev {msg} entry' , async ( ) => {
217+ beforeEach ( async function ( ) {
218+ this . timeout ( maxTestDuration )
219+ tab . addChatMessage ( { command : '/dev' , prompt } )
220+ await retryIfRequired ( async ( ) => {
221+ await tab . waitForChatFinishesLoading ( )
222+ } )
223+ } )
224+
225+ functionalTests ( )
89226 } )
90227
91228 describe ( '/dev entry' , ( ) => {
229+ beforeEach ( async function ( ) {
230+ this . timeout ( maxTestDuration )
231+ tab . addChatMessage ( { command : '/dev' } )
232+ tab . addChatMessage ( { prompt } )
233+ await retryIfRequired ( async ( ) => {
234+ await tab . waitForChatFinishesLoading ( )
235+ } )
236+ } )
237+
92238 it ( 'Clicks examples' , async ( ) => {
93239 const q = framework . createTab ( )
94240 q . addChatMessage ( { command : '/dev' } )
@@ -98,35 +244,6 @@ describe.skip('Amazon Q Feature Dev', function () {
98244 assert . deepStrictEqual ( lastChatItems ?. body , examples )
99245 } )
100246
101- it ( 'Receives chat response' , async ( ) => {
102- this . timeout ( 60000 )
103- const q = framework . createTab ( )
104- const prompt = 'Implement twosum in typescript'
105- q . addChatMessage ( { command : '/dev' } )
106- q . addChatMessage ( { prompt } )
107-
108- // Wait for a backend response
109- await q . waitForChatFinishesLoading ( )
110-
111- const chatItems = q . getChatItems ( )
112-
113- /**
114- * Verify that all the responses come back in the correct order and that a response
115- * after the prompt is non empty (represents a response from the backend, since the same response isn't
116- * guarenteed we can't verify direct responses)
117- */
118- verifyTextOrder ( chatItems , [ / W e l c o m e t o \/ d e v / , new RegExp ( prompt ) , / .\S / ] )
119-
120- // Check that the UI has the two buttons
121- assert . notStrictEqual ( chatItems . pop ( ) ?. followUp ?. options , [
122- {
123- type : FollowUpTypes . NewTask ,
124- } ,
125- {
126- type : FollowUpTypes . GenerateCode ,
127- disabled : false ,
128- } ,
129- ] )
130- } )
247+ functionalTests ( )
131248 } )
132249} )
0 commit comments