tests(amazonq): update inline tests for flare

jpinkney-aws · jpinkney-aws · commit 964be29eb21e · 2025-05-09T11:48:34.000-04:00
diff --git a/packages/amazonq/src/app/inline/completion.ts b/packages/amazonq/src/app/inline/completion.ts
@@ -17,6 +17,7 @@ import {
     window,
     TextEditor,
     InlineCompletionTriggerKind,
+    Range,
 } from 'vscode'
 import { LanguageClient } from 'vscode-languageclient'
 import {
@@ -32,11 +33,13 @@ import {
     ReferenceLogViewProvider,
     ImportAdderProvider,
     CodeSuggestionsState,
+    vsCodeState,
 } from 'aws-core-vscode/codewhisperer'
 import { InlineGeneratingMessage } from './inlineGeneratingMessage'
 import { LineTracker } from './stateTracker/lineTracker'
 import { InlineTutorialAnnotation } from './tutorials/inlineTutorialAnnotation'
 import { TelemetryHelper } from './telemetryHelper'
+import { getLogger } from 'aws-core-vscode/shared'
 
 export class InlineCompletionManager implements Disposable {
     private disposable: Disposable
@@ -204,54 +207,68 @@ export class AmazonQInlineCompletionItemProvider implements InlineCompletionItem
         context: InlineCompletionContext,
         token: CancellationToken
     ): Promise<InlineCompletionItem[] | InlineCompletionList> {
-        if (this.isNewSession) {
-            const isAutoTrigger = context.triggerKind === InlineCompletionTriggerKind.Automatic
-            if (isAutoTrigger && !CodeSuggestionsState.instance.isSuggestionsEnabled()) {
-                // return early when suggestions are disabled with auto trigger
+        try {
+            // only exposed for testing
+            vsCodeState.isRecommendationsActive = true
+            if (this.isNewSession) {
+                const isAutoTrigger = context.triggerKind === InlineCompletionTriggerKind.Automatic
+                if (isAutoTrigger && !CodeSuggestionsState.instance.isSuggestionsEnabled()) {
+                    // return early when suggestions are disabled with auto trigger
+                    return []
+                }
+
+                // tell the tutorial that completions has been triggered
+                await this.inlineTutorialAnnotation.triggered(context.triggerKind)
+                TelemetryHelper.instance.setInvokeSuggestionStartTime()
+                TelemetryHelper.instance.setTriggerType(context.triggerKind)
+
+                // make service requests if it's a new session
+                await this.recommendationService.getAllRecommendations(
+                    this.languageClient,
+                    document,
+                    position,
+                    context,
+                    token
+                )
+            }
+            // get active item from session for displaying
+            const items = this.sessionManager.getActiveRecommendation()
+            const session = this.sessionManager.getActiveSession()
+            const editor = window.activeTextEditor
+            if (!session || !items.length || !editor) {
                 return []
             }
 
-            // tell the tutorial that completions has been triggered
-            await this.inlineTutorialAnnotation.triggered(context.triggerKind)
-            TelemetryHelper.instance.setInvokeSuggestionStartTime()
-            TelemetryHelper.instance.setTriggerType(context.triggerKind)
-
-            // make service requests if it's a new session
-            await this.recommendationService.getAllRecommendations(
-                this.languageClient,
-                document,
-                position,
-                context,
-                token
-            )
-        }
-        // get active item from session for displaying
-        const items = this.sessionManager.getActiveRecommendation()
-        const session = this.sessionManager.getActiveSession()
-        if (!session || !items.length) {
-            return []
-        }
-        const editor = window.activeTextEditor
-        for (const item of items) {
-            item.command = {
-                command: 'aws.amazonq.acceptInline',
-                title: 'On acceptance',
-                arguments: [
-                    session.sessionId,
-                    item,
-                    editor,
-                    session.requestStartTime,
+            const start = document.validatePosition(editor.selection.active)
+            const end = position
+            for (const item of items) {
+                item.command = {
+                    command: 'aws.amazonq.acceptInline',
+                    title: 'On acceptance',
+                    arguments: [
+                        session.sessionId,
+                        item,
+                        editor,
+                        session.requestStartTime,
+                        position.line,
+                        session.firstCompletionDisplayLatency,
+                    ],
+                }
+                item.range = new Range(start, end)
+                ReferenceInlineProvider.instance.setInlineReference(
                     position.line,
-                    session.firstCompletionDisplayLatency,
-                ],
+                    item.insertText as string,
+                    item.references
+                )
+                ImportAdderProvider.instance.onShowRecommendation(document, position.line, item)
             }
-            ReferenceInlineProvider.instance.setInlineReference(
-                position.line,
-                item.insertText as string,
-                item.references
-            )
-            ImportAdderProvider.instance.onShowRecommendation(document, position.line, item)
+            return items as InlineCompletionItem[]
+        } catch (e) {
+            getLogger('amazonqLsp').error('Completion request failed: %O', e)
+            return []
+        } finally {
+            // only exposed for testing
+            vsCodeState.isRecommendationsActive = false
         }
-        return items as InlineCompletionItem[]
     }
 }
diff --git a/packages/amazonq/test/e2e/inline/inline.test.ts b/packages/amazonq/test/e2e/inline/inline.test.ts
@@ -5,18 +5,10 @@
 
 import * as vscode from 'vscode'
 import assert from 'assert'
-import {
-    closeAllEditors,
-    getTestWindow,
-    registerAuthHook,
-    resetCodeWhispererGlobalVariables,
-    TestFolder,
-    toTextEditor,
-    using,
-} from 'aws-core-vscode/test'
-import { session } from 'aws-core-vscode/codewhisperer'
+import { closeAllEditors, registerAuthHook, TestFolder, toTextEditor, using } from 'aws-core-vscode/test'
 import { Commands, globals, sleep, waitUntil, collectionUtil } from 'aws-core-vscode/shared'
 import { loginToIdC } from '../amazonq/utils/setup'
+import { vsCodeState } from 'aws-core-vscode/codewhisperer'
 
 describe('Amazon Q Inline', async function () {
     const retries = 3
@@ -40,7 +32,6 @@ describe('Amazon Q Inline', async function () {
         const folder = await TestFolder.create()
         tempFolder = folder.path
         await closeAllEditors()
-        await resetCodeWhispererGlobalVariables()
     })
 
     afterEach(async function () {
@@ -70,17 +61,6 @@ describe('Amazon Q Inline', async function () {
         })
     }
 
-    async function waitForRecommendations() {
-        const suggestionShown = await waitUntil(async () => session.getSuggestionState(0) === 'Showed', waitOptions)
-        if (!suggestionShown) {
-            throw new Error(`Suggestion did not show. Suggestion States: ${JSON.stringify(session.suggestionStates)}`)
-        }
-        console.table({
-            'suggestions states': JSON.stringify(session.suggestionStates),
-            recommendations: session.recommendations,
-        })
-    }
-
     /**
      * Waits for a specific telemetry event to be emitted with the expected suggestion state.
      * It looks like there might be a potential race condition in codewhisperer causing telemetry
@@ -134,8 +114,9 @@ describe('Amazon Q Inline', async function () {
                     await invokeCompletion()
                     originalEditorContents = vscode.window.activeTextEditor?.document.getText()
 
-                    // wait until the ghost text appears
-                    await waitForRecommendations()
+                    // wait until all the recommendations have finished
+                    await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === true), waitOptions)
+                    await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === false), waitOptions)
                 }
 
                 beforeEach(async () => {
@@ -148,14 +129,12 @@ describe('Amazon Q Inline', async function () {
                         try {
                             await setup()
                             console.log(`test run ${attempt} succeeded`)
-                            logUserDecisionStatus()
                             break
                         } catch (e) {
                             console.log(`test run ${attempt} failed`)
                             console.log(e)
                             logUserDecisionStatus()
                             attempt++
-                            await resetCodeWhispererGlobalVariables()
                         }
                     }
                     if (attempt === retries) {
@@ -201,29 +180,6 @@ describe('Amazon Q Inline', async function () {
                     assert.deepStrictEqual(vscode.window.activeTextEditor?.document.getText(), originalEditorContents)
                 })
             })
-
-            it(`${name} invoke on unsupported filetype`, async function () {
-                await setupEditor({
-                    name: 'test.zig',
-                    contents: `fn doSomething() void {
-        
-             }`,
-                })
-
-                /**
-                 * Add delay between editor loading and invoking completion
-                 * @see beforeEach in supported filetypes for more information
-                 */
-                await sleep(1000)
-                await invokeCompletion()
-
-                if (name === 'automatic') {
-                    // It should never get triggered since its not a supported file type
-                    // assert.deepStrictEqual(RecommendationService.instance.isRunning, false)
-                } else {
-                    await getTestWindow().waitForMessage('currently not supported by Amazon Q inline suggestions')
-                }
-            })
         })
     }
 })
diff --git a/packages/core/src/codewhisperer/models/model.ts b/packages/core/src/codewhisperer/models/model.ts
@@ -33,6 +33,10 @@ interface VsCodeState {
      * Flag indicates whether codewhisperer is doing vscode.TextEditor.edit
      */
     isCodeWhispererEditing: boolean
+    /**
+     * Keeps track of whether or not recommendations are currently running
+     */
+    isRecommendationsActive: boolean
     /**
      * Timestamp of previous user edit
      */
@@ -44,6 +48,9 @@ interface VsCodeState {
 export const vsCodeState: VsCodeState = {
     isIntelliSenseActive: false,
     isCodeWhispererEditing: false,
+    // hack to globally keep track of whether or not recommendations are currently running. This allows us to know
+    // when recommendations have ran during e2e tests
+    isRecommendationsActive: false,
     lastUserModificationTime: 0,
     isFreeTierLimitReached: false,
 }
diff --git a/packages/core/src/shared/telemetry/exemptMetrics.ts b/packages/core/src/shared/telemetry/exemptMetrics.ts
@@ -29,6 +29,8 @@ const validationExemptMetrics: Set<string> = new Set([
     'codewhisperer_codePercentage',
     'codewhisperer_userModification',
     'codewhisperer_userTriggerDecision',
+    'codewhisperer_perceivedLatency', // flare doesn't currently set result property
+    'codewhisperer_serviceInvocation', // flare doesn't currently set result property
     'dynamicresource_selectResources',
     'dynamicresource_copyIdentifier',
     'dynamicresource_mutateResource',
