@@ -15,10 +15,13 @@ import {
1515 using ,
1616} from 'aws-core-vscode/test'
1717import { RecommendationHandler , RecommendationService , session } from 'aws-core-vscode/codewhisperer'
18- import { Commands , globals , sleep , waitUntil } from 'aws-core-vscode/shared'
18+ import { Commands , globals , sleep , waitUntil , collectionUtil } from 'aws-core-vscode/shared'
1919import { loginToIdC } from '../amazonq/utils/setup'
2020
2121describe ( 'Amazon Q Inline' , async function ( ) {
22+ const retries = 3
23+ this . retries ( retries )
24+
2225 let tempFolder : string
2326 const waitOptions = {
2427 interval : 500 ,
@@ -37,13 +40,24 @@ describe('Amazon Q Inline', async function () {
3740 const folder = await TestFolder . create ( )
3841 tempFolder = folder . path
3942 await closeAllEditors ( )
40- await resetCodeWhispererGlobalVariables ( false )
43+ await resetCodeWhispererGlobalVariables ( )
4144 } )
4245
4346 afterEach ( async function ( ) {
4447 await closeAllEditors ( )
48+ if ( this . currentTest ?. state === undefined || this . currentTest ?. isFailed ( ) || this . currentTest ?. isPending ( ) ) {
49+ logUserDecisionStatus ( )
50+ }
4551 } )
4652
53+ function logUserDecisionStatus ( ) {
54+ const events = getUserTriggerDecision ( )
55+ console . table ( {
56+ 'telemetry events' : JSON . stringify ( events ) ,
57+ 'recommendation service status' : RecommendationService . instance . isRunning ,
58+ } )
59+ }
60+
4761 async function setupEditor ( { name, contents } : { name ?: string; contents ?: string } = { } ) {
4862 const fileName = name ?? 'test.ts'
4963 const textContents =
@@ -58,16 +72,28 @@ describe('Amazon Q Inline', async function () {
5872 }
5973
6074 async function waitForRecommendations ( ) {
61- const ok = await waitUntil (
62- async ( ) =>
63- RecommendationHandler . instance . isSuggestionVisible ( ) || session . getSuggestionState ( 0 ) === 'Showed' ,
75+ const suggestionShown = await waitUntil ( async ( ) => session . getSuggestionState ( 0 ) === 'Showed' , waitOptions )
76+ if ( ! suggestionShown ) {
77+ throw new Error ( `Suggestion did not show. Suggestion States: ${ JSON . stringify ( session . suggestionStates ) } ` )
78+ }
79+ const suggestionVisible = await waitUntil (
80+ async ( ) => RecommendationHandler . instance . isSuggestionVisible ( ) ,
6481 waitOptions
6582 )
66- if ( ! ok ) {
67- assert . fail (
83+ if ( ! suggestionVisible ) {
84+ throw new Error (
6885 `Suggestions failed to become visible. Suggestion States: ${ JSON . stringify ( session . suggestionStates ) } `
6986 )
7087 }
88+ console . table ( {
89+ 'suggestions states' : JSON . stringify ( session . suggestionStates ) ,
90+ 'valid recommendation' : RecommendationHandler . instance . isValidResponse ( ) ,
91+ 'recommendation service status' : RecommendationService . instance . isRunning ,
92+ recommendations : session . recommendations ,
93+ } )
94+ if ( ! RecommendationHandler . instance . isValidResponse ( ) ) {
95+ throw new Error ( 'Did not find a valid response' )
96+ }
7197 }
7298
7399 /**
@@ -82,17 +108,23 @@ describe('Amazon Q Inline', async function () {
82108 } )
83109 return events . some ( ( event ) => event . codewhispererSuggestionState === suggestionState )
84110 } , waitOptions )
85- const events = globals . telemetry . logger . query ( {
86- metricName,
87- } )
88111 if ( ! ok ) {
89- assert. fail ( `Telemetry failed to be emitted. Current events: ${ JSON . stringify ( events ) } ` )
112+ assert. fail ( `Telemetry for ${ metricName } with suggestionState ${ suggestionState } was not emitted ` )
90113 }
114+ const events = getUserTriggerDecision ( )
91115 if ( events . length > 1 && events [ events . length - 1 ] . codewhispererSuggestionState !== suggestionState ) {
92- assert. fail ( `Telemetry events were emitted in the wrong order. Current events: ${ JSON . stringify ( events ) } ` )
116+ assert. fail ( `Telemetry events were emitted in the wrong order` )
93117 }
94118 }
95119
120+ function getUserTriggerDecision ( ) {
121+ return globals . telemetry . logger
122+ . query ( {
123+ metricName : 'codewhisperer_userTriggerDecision' ,
124+ } )
125+ . map ( ( e ) => collectionUtil . partialClone ( e , 3 , [ 'credentialStartUrl' ] , '[omitted]' ) )
126+ }
127+
96128 for ( const [ name , invokeCompletion ] of [
97129 [ 'automatic' , async ( ) => await vscode . commands . executeCommand ( 'type' , { text : '\n' } ) ] ,
98130 [ 'manual' , async ( ) => Commands . tryExecute ( 'aws.amazonq.invokeInlineCompletion' ) ] ,
@@ -101,7 +133,7 @@ describe('Amazon Q Inline', async function () {
101133 let originalEditorContents : string | undefined
102134
103135 describe ( 'supported filetypes' , ( ) => {
104- beforeEach ( async ( ) => {
136+ async function setup ( ) {
105137 await setupEditor ( )
106138
107139 /**
@@ -119,6 +151,31 @@ describe('Amazon Q Inline', async function () {
119151
120152 // wait until the ghost text appears
121153 await waitForRecommendations ( )
154+ }
155+
156+ beforeEach ( async ( ) => {
157+ /**
158+ * Every once and a while the backend won't respond with any recommendations.
159+ * In those cases, re-try the setup up-to ${retries} times
160+ */
161+ let attempt = 0
162+ while ( attempt < retries ) {
163+ try {
164+ await setup ( )
165+ console . log ( `test run ${ attempt } succeeded` )
166+ logUserDecisionStatus ( )
167+ break
168+ } catch ( e ) {
169+ console . log ( `test run ${ attempt } failed` )
170+ console . log ( e )
171+ logUserDecisionStatus ( )
172+ attempt ++
173+ await resetCodeWhispererGlobalVariables ( )
174+ }
175+ }
176+ if ( attempt === retries ) {
177+ assert . fail ( `Failed to invoke ${ name } tests after ${ attempt } attempts` )
178+ }
122179 } )
123180
124181 it ( `${ name } invoke accept` , async function ( ) {
0 commit comments