tests(amazonq): update inline tests for flare

Author: jpinkney-aws

packages/amazonq/src/app/inline/completion.ts

@@ -32,6 +32,7 @@ import {
     ReferenceLogViewProvider,
     ImportAdderProvider,
     CodeSuggestionsState,
+    vsCodeState,
 } from 'aws-core-vscode/codewhisperer'
 import { InlineGeneratingMessage } from './inlineGeneratingMessage'
 import { LineTracker } from './stateTracker/lineTracker'
@@ -203,6 +204,7 @@ export class AmazonQInlineCompletionItemProvider implements InlineCompletionItem
         context: InlineCompletionContext,
         token: CancellationToken
     ): Promise<InlineCompletionItem[] | InlineCompletionList> {
+        vsCodeState.isRecommendationsActive = true
         if (this.isNewSession) {
             const isAutoTrigger = context.triggerKind === InlineCompletionTriggerKind.Automatic
             if (isAutoTrigger && !CodeSuggestionsState.instance.isSuggestionsEnabled()) {
@@ -249,6 +251,7 @@ export class AmazonQInlineCompletionItemProvider implements InlineCompletionItem
             )
             ImportAdderProvider.instance.onShowRecommendation(document, position.line, item)
         }
+        vsCodeState.isRecommendationsActive = false
         return items as InlineCompletionItem[]
     }
 }

packages/amazonq/test/e2e/inline/inline.test.ts

@@ -5,18 +5,10 @@
 
 import * as vscode from 'vscode'
 import assert from 'assert'
-import {
-    closeAllEditors,
-    getTestWindow,
-    registerAuthHook,
-    resetCodeWhispererGlobalVariables,
-    TestFolder,
-    toTextEditor,
-    using,
-} from 'aws-core-vscode/test'
-import { session } from 'aws-core-vscode/codewhisperer'
+import { closeAllEditors, registerAuthHook, TestFolder, toTextEditor, using } from 'aws-core-vscode/test'
 import { Commands, globals, sleep, waitUntil, collectionUtil } from 'aws-core-vscode/shared'
 import { loginToIdC } from '../amazonq/utils/setup'
+import { vsCodeState } from 'aws-core-vscode/codewhisperer'
 
 describe('Amazon Q Inline', async function () {
     const retries = 3
@@ -40,7 +32,6 @@ describe('Amazon Q Inline', async function () {
         const folder = await TestFolder.create()
         tempFolder = folder.path
         await closeAllEditors()
-        await resetCodeWhispererGlobalVariables()
     })
 
     afterEach(async function () {
@@ -70,17 +61,6 @@ describe('Amazon Q Inline', async function () {
         })
     }
 
-    async function waitForRecommendations() {
-        const suggestionShown = await waitUntil(async () => session.getSuggestionState(0) === 'Showed', waitOptions)
-        if (!suggestionShown) {
-            throw new Error(`Suggestion did not show. Suggestion States: ${JSON.stringify(session.suggestionStates)}`)
-        }
-        console.table({
-            'suggestions states': JSON.stringify(session.suggestionStates),
-            recommendations: session.recommendations,
-        })
-    }
-
     /**
      * Waits for a specific telemetry event to be emitted with the expected suggestion state.
      * It looks like there might be a potential race condition in codewhisperer causing telemetry
@@ -134,8 +114,9 @@ describe('Amazon Q Inline', async function () {
                     await invokeCompletion()
                     originalEditorContents = vscode.window.activeTextEditor?.document.getText()
 
-                    // wait until the ghost text appears
-                    await waitForRecommendations()
+                    // wait until all the recommendations have finished
+                    await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === true), waitOptions)
+                    await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === false), waitOptions)
                 }
 
                 beforeEach(async () => {
@@ -148,14 +129,12 @@ describe('Amazon Q Inline', async function () {
                         try {
                             await setup()
                             console.log(`test run ${attempt} succeeded`)
-                            logUserDecisionStatus()
                             break
                         } catch (e) {
                             console.log(`test run ${attempt} failed`)
                             console.log(e)
                             logUserDecisionStatus()
                             attempt++
-                            await resetCodeWhispererGlobalVariables()
                         }
                     }
                     if (attempt === retries) {
@@ -201,29 +180,6 @@ describe('Amazon Q Inline', async function () {
                     assert.deepStrictEqual(vscode.window.activeTextEditor?.document.getText(), originalEditorContents)
                 })
             })
-
-            it(`${name} invoke on unsupported filetype`, async function () {
-                await setupEditor({
-                    name: 'test.zig',
-                    contents: `fn doSomething() void {
-        
-             }`,
-                })
-
-                /**
-                 * Add delay between editor loading and invoking completion
-                 * @see beforeEach in supported filetypes for more information
-                 */
-                await sleep(1000)
-                await invokeCompletion()
-
-                if (name === 'automatic') {
-                    // It should never get triggered since its not a supported file type
-                    // assert.deepStrictEqual(RecommendationService.instance.isRunning, false)
-                } else {
-                    await getTestWindow().waitForMessage('currently not supported by Amazon Q inline suggestions')
-                }
-            })
         })
     }
 })

packages/core/src/codewhisperer/models/model.ts

@@ -33,6 +33,10 @@ interface VsCodeState {
      * Flag indicates whether codewhisperer is doing vscode.TextEditor.edit
      */
     isCodeWhispererEditing: boolean
+    /**
+     * Keeps track of whether or not recommendations are currently running
+     */
+    isRecommendationsActive: boolean
     /**
      * Timestamp of previous user edit
      */
@@ -44,6 +48,9 @@ interface VsCodeState {
 export const vsCodeState: VsCodeState = {
     isIntelliSenseActive: false,
     isCodeWhispererEditing: false,
+    // hack to globally keep track of whether or not recommendations are currently running. This allows us to know
+    // when recommendations have ran during e2e tests
+    isRecommendationsActive: false,
     lastUserModificationTime: 0,
     isFreeTierLimitReached: false,
 }

packages/core/src/shared/telemetry/exemptMetrics.ts

@@ -29,6 +29,8 @@ const validationExemptMetrics: Set<string> = new Set([
     'codewhisperer_codePercentage',
     'codewhisperer_userModification',
     'codewhisperer_userTriggerDecision',
+    'codewhisperer_perceivedLatency', // flare doesn't currently set result property
+    'codewhisperer_serviceInvocation', // flare doesn't currently set result property
     'dynamicresource_selectResources',
     'dynamicresource_copyIdentifier',
     'dynamicresource_mutateResource',