test(amazonq): adding back inline tests - part 2

packages/amazonq/test/e2e/inline/inline.test.ts

@@ -5,10 +5,18 @@
 
 import * as vscode from 'vscode'
 import assert from 'assert'
-import { closeAllEditors, registerAuthHook, TestFolder, toTextEditor, using } from 'aws-core-vscode/test'
+import {
+    closeAllEditors,
+    getTestWindow,
+    registerAuthHook,
+    resetCodeWhispererGlobalVariables,
+    TestFolder,
+    toTextEditor,
+    using,
+} from 'aws-core-vscode/test'
+import { RecommendationHandler, RecommendationService, session } from 'aws-core-vscode/codewhisperer'
 import { Commands, globals, sleep, waitUntil, collectionUtil } from 'aws-core-vscode/shared'
 import { loginToIdC } from '../amazonq/utils/setup'
-import { vsCodeState } from 'aws-core-vscode/codewhisperer'
 
 describe('Amazon Q Inline', async function () {
     const retries = 3
@@ -32,6 +40,7 @@ describe('Amazon Q Inline', async function () {
         const folder = await TestFolder.create()
         tempFolder = folder.path
         await closeAllEditors()
+        await resetCodeWhispererGlobalVariables()
     })
 
     afterEach(async function () {
@@ -45,6 +54,7 @@ describe('Amazon Q Inline', async function () {
         const events = getUserTriggerDecision()
         console.table({
             'telemetry events': JSON.stringify(events),
+            'recommendation service status': RecommendationService.instance.isRunning,
         })
     }
 
@@ -61,6 +71,31 @@ describe('Amazon Q Inline', async function () {
         })
     }
 
+    async function waitForRecommendations() {
+        const suggestionShown = await waitUntil(async () => session.getSuggestionState(0) === 'Showed', waitOptions)
+        if (!suggestionShown) {
+            throw new Error(`Suggestion did not show. Suggestion States: ${JSON.stringify(session.suggestionStates)}`)
+        }
+        const suggestionVisible = await waitUntil(
+            async () => RecommendationHandler.instance.isSuggestionVisible(),
+            waitOptions
+        )
+        if (!suggestionVisible) {
+            throw new Error(
+                `Suggestions failed to become visible. Suggestion States: ${JSON.stringify(session.suggestionStates)}`
+            )
+        }
+        console.table({
+            'suggestions states': JSON.stringify(session.suggestionStates),
+            'valid recommendation': RecommendationHandler.instance.isValidResponse(),
+            'recommendation service status': RecommendationService.instance.isRunning,
+            recommendations: session.recommendations,
+        })
+        if (!RecommendationHandler.instance.isValidResponse()) {
+            throw new Error('Did not find a valid response')
+        }
+    }
+
     /**
      * Waits for a specific telemetry event to be emitted with the expected suggestion state.
      * It looks like there might be a potential race condition in codewhisperer causing telemetry
@@ -114,9 +149,8 @@ describe('Amazon Q Inline', async function () {
                     await invokeCompletion()
                     originalEditorContents = vscode.window.activeTextEditor?.document.getText()
 
-                    // wait until all the recommendations have finished
-                    await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === true), waitOptions)
-                    await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === false), waitOptions)
+                    // wait until the ghost text appears
+                    await waitForRecommendations()
                 }
 
                 beforeEach(async () => {
@@ -129,12 +163,14 @@ describe('Amazon Q Inline', async function () {
                         try {
                             await setup()
                             console.log(`test run ${attempt} succeeded`)
+                            logUserDecisionStatus()
                             break
                         } catch (e) {
                             console.log(`test run ${attempt} failed`)
                             console.log(e)
                             logUserDecisionStatus()
                             attempt++
+                            await resetCodeWhispererGlobalVariables()
                         }
                     }
                     if (attempt === retries) {
@@ -180,6 +216,29 @@ describe('Amazon Q Inline', async function () {
                     assert.deepStrictEqual(vscode.window.activeTextEditor?.document.getText(), originalEditorContents)
                 })
             })
+
+            it(`${name} invoke on unsupported filetype`, async function () {
+                await setupEditor({
+                    name: 'test.zig',
+                    contents: `fn doSomething() void {
+
+             }`,
+                })
+
+                /**
+                 * Add delay between editor loading and invoking completion
+                 * @see beforeEach in supported filetypes for more information
+                 */
+                await sleep(1000)
+                await invokeCompletion()
+
+                if (name === 'automatic') {
+                    // It should never get triggered since its not a supported file type
+                    assert.deepStrictEqual(RecommendationService.instance.isRunning, false)
+                } else {
+                    await getTestWindow().waitForMessage('currently not supported by Amazon Q inline suggestions')
+                }
+            })
         })
     }
 })

packages/core/src/testE2E/codewhisperer/referenceTracker.test.ts

@@ -0,0 +1,124 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import assert from 'assert'
+import * as codewhispererClient from '../../codewhisperer/client/codewhisperer'
+import { ConfigurationEntry } from '../../codewhisperer/models/model'
+import { setValidConnection, skipTestIfNoValidConn } from '../util/connection'
+import { RecommendationHandler } from '../../codewhisperer/service/recommendationHandler'
+import { createMockTextEditor, resetCodeWhispererGlobalVariables } from '../../test/codewhisperer/testUtil'
+import { invokeRecommendation } from '../../codewhisperer/commands/invokeRecommendation'
+import { session } from '../../codewhisperer/util/codeWhispererSession'
+
+/*
+New model deployment may impact references returned. 
+These tests:
+    1) are not required for github approval flow 
+    2) will be auto-skipped until fix for manual runs is posted.
+*/
+
+const leftContext = `InAuto.GetContent(
+    InAuto.servers.auto, "vendors.json",
+    function (data) {
+        let block = '';
+        for(let i = 0; i < data.length; i++) {
+            block += '<a href="' + data`
+
+const rightContext = `[i].title + '">' + cars[i].title + '</a>';
+        }
+        $('#cars').html(block);
+    });`
+
+describe('CodeWhisperer service invocation', async function () {
+    let validConnection: boolean
+    const client = new codewhispererClient.DefaultCodeWhispererClient()
+    const configWithRefs: ConfigurationEntry = {
+        isShowMethodsEnabled: true,
+        isManualTriggerEnabled: true,
+        isAutomatedTriggerEnabled: true,
+        isSuggestionsWithCodeReferencesEnabled: true,
+    }
+    const configWithNoRefs: ConfigurationEntry = {
+        isShowMethodsEnabled: true,
+        isManualTriggerEnabled: true,
+        isAutomatedTriggerEnabled: true,
+        isSuggestionsWithCodeReferencesEnabled: false,
+    }
+
+    before(async function () {
+        validConnection = await setValidConnection()
+    })
+
+    beforeEach(function () {
+        void resetCodeWhispererGlobalVariables()
+        RecommendationHandler.instance.clearRecommendations()
+        // TODO: remove this line (this.skip()) when these tests no longer auto-skipped
+        this.skip()
+        // valid connection required to run tests
+        skipTestIfNoValidConn(validConnection, this)
+    })
+
+    it('trigger known to return recs with references returns rec with reference', async function () {
+        // check that handler is empty before invocation
+        const requestIdBefore = RecommendationHandler.instance.requestId
+        const sessionIdBefore = session.sessionId
+        const validRecsBefore = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestIdBefore.length === 0)
+        assert.ok(sessionIdBefore.length === 0)
+        assert.ok(!validRecsBefore)
+
+        const doc = leftContext + rightContext
+        const filename = 'test.js'
+        const language = 'javascript'
+        const line = 5
+        const character = 39
+        const mockEditor = createMockTextEditor(doc, filename, language, line, character)
+
+        await invokeRecommendation(mockEditor, client, configWithRefs)
+
+        const requestId = RecommendationHandler.instance.requestId
+        const sessionId = session.sessionId
+        const validRecs = RecommendationHandler.instance.isValidResponse()
+        const references = session.recommendations[0].references
+
+        assert.ok(requestId.length > 0)
+        assert.ok(sessionId.length > 0)
+        assert.ok(validRecs)
+        assert.ok(references !== undefined)
+        // TODO: uncomment this assert when this test is no longer auto-skipped
+        // assert.ok(references.length > 0)
+    })
+
+    // This test will fail if user is logged in with IAM identity center
+    it('trigger known to return rec with references does not return rec with references when reference tracker setting is off', async function () {
+        // check that handler is empty before invocation
+        const requestIdBefore = RecommendationHandler.instance.requestId
+        const sessionIdBefore = session.sessionId
+        const validRecsBefore = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestIdBefore.length === 0)
+        assert.ok(sessionIdBefore.length === 0)
+        assert.ok(!validRecsBefore)
+
+        const doc = leftContext + rightContext
+        const filename = 'test.js'
+        const language = 'javascript'
+        const line = 5
+        const character = 39
+        const mockEditor = createMockTextEditor(doc, filename, language, line, character)
+
+        await invokeRecommendation(mockEditor, client, configWithNoRefs)
+
+        const requestId = RecommendationHandler.instance.requestId
+        const sessionId = session.sessionId
+        const validRecs = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestId.length > 0)
+        assert.ok(sessionId.length > 0)
+        // no recs returned because example request returns 1 rec with reference, so no recs returned when references off
+        assert.ok(!validRecs)
+    })
+})

packages/core/src/testE2E/codewhisperer/serviceInvocations.test.ts

@@ -0,0 +1,124 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import assert from 'assert'
+import * as vscode from 'vscode'
+import * as path from 'path'
+import { setValidConnection, skipTestIfNoValidConn } from '../util/connection'
+import { ConfigurationEntry } from '../../codewhisperer/models/model'
+import * as codewhispererClient from '../../codewhisperer/client/codewhisperer'
+import { RecommendationHandler } from '../../codewhisperer/service/recommendationHandler'
+import {
+    createMockTextEditor,
+    createTextDocumentChangeEvent,
+    resetCodeWhispererGlobalVariables,
+} from '../../test/codewhisperer/testUtil'
+import { KeyStrokeHandler } from '../../codewhisperer/service/keyStrokeHandler'
+import { sleep } from '../../shared/utilities/timeoutUtils'
+import { invokeRecommendation } from '../../codewhisperer/commands/invokeRecommendation'
+import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities'
+import { session } from '../../codewhisperer/util/codeWhispererSession'
+
+describe('CodeWhisperer service invocation', async function () {
+    let validConnection: boolean
+    const client = new codewhispererClient.DefaultCodeWhispererClient()
+    const config: ConfigurationEntry = {
+        isShowMethodsEnabled: true,
+        isManualTriggerEnabled: true,
+        isAutomatedTriggerEnabled: true,
+        isSuggestionsWithCodeReferencesEnabled: true,
+    }
+
+    before(async function () {
+        validConnection = await setValidConnection()
+    })
+
+    beforeEach(function () {
+        void resetCodeWhispererGlobalVariables()
+        RecommendationHandler.instance.clearRecommendations()
+        // valid connection required to run tests
+        skipTestIfNoValidConn(validConnection, this)
+    })
+
+    it('manual trigger returns valid recommendation response', async function () {
+        // check that handler is empty before invocation
+        const requestIdBefore = RecommendationHandler.instance.requestId
+        const sessionIdBefore = session.sessionId
+        const validRecsBefore = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestIdBefore.length === 0)
+        assert.ok(sessionIdBefore.length === 0)
+        assert.ok(!validRecsBefore)
+
+        const mockEditor = createMockTextEditor()
+        await invokeRecommendation(mockEditor, client, config)
+
+        const requestId = RecommendationHandler.instance.requestId
+        const sessionId = session.sessionId
+        const validRecs = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestId.length > 0)
+        assert.ok(sessionId.length > 0)
+        assert.ok(validRecs)
+    })
+
+    it('auto trigger returns valid recommendation response', async function () {
+        // check that handler is empty before invocation
+        const requestIdBefore = RecommendationHandler.instance.requestId
+        const sessionIdBefore = session.sessionId
+        const validRecsBefore = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestIdBefore.length === 0)
+        assert.ok(sessionIdBefore.length === 0)
+        assert.ok(!validRecsBefore)
+
+        const mockEditor = createMockTextEditor()
+
+        const mockEvent: vscode.TextDocumentChangeEvent = createTextDocumentChangeEvent(
+            mockEditor.document,
+            new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 1)),
+            '\n'
+        )
+
+        await KeyStrokeHandler.instance.processKeyStroke(mockEvent, mockEditor, client, config)
+        // wait for 5 seconds to allow time for response to be generated
+        await sleep(5000)
+
+        const requestId = RecommendationHandler.instance.requestId
+        const sessionId = session.sessionId
+        const validRecs = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestId.length > 0)
+        assert.ok(sessionId.length > 0)
+        assert.ok(validRecs)
+    })
+
+    it('invocation in unsupported language does not generate a request', async function () {
+        const workspaceFolder = getTestWorkspaceFolder()
+        const appRoot = path.join(workspaceFolder, 'go1-plain-sam-app')
+        const appCodePath = path.join(appRoot, 'hello-world', 'go.mod')
+
+        // check that handler is empty before invocation
+        const requestIdBefore = RecommendationHandler.instance.requestId
+        const sessionIdBefore = session.sessionId
+        const validRecsBefore = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestIdBefore.length === 0)
+        assert.ok(sessionIdBefore.length === 0)
+        assert.ok(!validRecsBefore)
+
+        const doc = await vscode.workspace.openTextDocument(vscode.Uri.file(appCodePath))
+        const editor = await vscode.window.showTextDocument(doc)
+        await invokeRecommendation(editor, client, config)
+
+        const requestId = RecommendationHandler.instance.requestId
+        const sessionId = session.sessionId
+        const validRecs = RecommendationHandler.instance.isValidResponse()
+
+        assert.ok(requestId.length === 0)
+        assert.ok(sessionId.length === 0)
+        assert.ok(!validRecs)
+    })
+})