diff --git a/packages/amazonq/test/e2e/inline/inline.test.ts b/packages/amazonq/test/e2e/inline/inline.test.ts index bcc41851eca..43a9f67ab73 100644 --- a/packages/amazonq/test/e2e/inline/inline.test.ts +++ b/packages/amazonq/test/e2e/inline/inline.test.ts @@ -5,10 +5,18 @@ import * as vscode from 'vscode' import assert from 'assert' -import { closeAllEditors, registerAuthHook, TestFolder, toTextEditor, using } from 'aws-core-vscode/test' +import { + closeAllEditors, + getTestWindow, + registerAuthHook, + resetCodeWhispererGlobalVariables, + TestFolder, + toTextEditor, + using, +} from 'aws-core-vscode/test' +import { RecommendationHandler, RecommendationService, session } from 'aws-core-vscode/codewhisperer' import { Commands, globals, sleep, waitUntil, collectionUtil } from 'aws-core-vscode/shared' import { loginToIdC } from '../amazonq/utils/setup' -import { vsCodeState } from 'aws-core-vscode/codewhisperer' describe('Amazon Q Inline', async function () { const retries = 3 @@ -32,6 +40,7 @@ describe('Amazon Q Inline', async function () { const folder = await TestFolder.create() tempFolder = folder.path await closeAllEditors() + await resetCodeWhispererGlobalVariables() }) afterEach(async function () { @@ -45,6 +54,7 @@ describe('Amazon Q Inline', async function () { const events = getUserTriggerDecision() console.table({ 'telemetry events': JSON.stringify(events), + 'recommendation service status': RecommendationService.instance.isRunning, }) } @@ -61,6 +71,31 @@ describe('Amazon Q Inline', async function () { }) } + async function waitForRecommendations() { + const suggestionShown = await waitUntil(async () => session.getSuggestionState(0) === 'Showed', waitOptions) + if (!suggestionShown) { + throw new Error(`Suggestion did not show. Suggestion States: ${JSON.stringify(session.suggestionStates)}`) + } + const suggestionVisible = await waitUntil( + async () => RecommendationHandler.instance.isSuggestionVisible(), + waitOptions + ) + if (!suggestionVisible) { + throw new Error( + `Suggestions failed to become visible. Suggestion States: ${JSON.stringify(session.suggestionStates)}` + ) + } + console.table({ + 'suggestions states': JSON.stringify(session.suggestionStates), + 'valid recommendation': RecommendationHandler.instance.isValidResponse(), + 'recommendation service status': RecommendationService.instance.isRunning, + recommendations: session.recommendations, + }) + if (!RecommendationHandler.instance.isValidResponse()) { + throw new Error('Did not find a valid response') + } + } + /** * Waits for a specific telemetry event to be emitted with the expected suggestion state. * It looks like there might be a potential race condition in codewhisperer causing telemetry @@ -114,9 +149,8 @@ describe('Amazon Q Inline', async function () { await invokeCompletion() originalEditorContents = vscode.window.activeTextEditor?.document.getText() - // wait until all the recommendations have finished - await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === true), waitOptions) - await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === false), waitOptions) + // wait until the ghost text appears + await waitForRecommendations() } beforeEach(async () => { @@ -129,12 +163,14 @@ describe('Amazon Q Inline', async function () { try { await setup() console.log(`test run ${attempt} succeeded`) + logUserDecisionStatus() break } catch (e) { console.log(`test run ${attempt} failed`) console.log(e) logUserDecisionStatus() attempt++ + await resetCodeWhispererGlobalVariables() } } if (attempt === retries) { @@ -180,6 +216,29 @@ describe('Amazon Q Inline', async function () { assert.deepStrictEqual(vscode.window.activeTextEditor?.document.getText(), originalEditorContents) }) }) + + it(`${name} invoke on unsupported filetype`, async function () { + await setupEditor({ + name: 'test.zig', + contents: `fn doSomething() void { + + }`, + }) + + /** + * Add delay between editor loading and invoking completion + * @see beforeEach in supported filetypes for more information + */ + await sleep(1000) + await invokeCompletion() + + if (name === 'automatic') { + // It should never get triggered since its not a supported file type + assert.deepStrictEqual(RecommendationService.instance.isRunning, false) + } else { + await getTestWindow().waitForMessage('currently not supported by Amazon Q inline suggestions') + } + }) }) } }) diff --git a/packages/core/src/testE2E/codewhisperer/referenceTracker.test.ts b/packages/core/src/testE2E/codewhisperer/referenceTracker.test.ts new file mode 100644 index 00000000000..d173500c608 --- /dev/null +++ b/packages/core/src/testE2E/codewhisperer/referenceTracker.test.ts @@ -0,0 +1,124 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert' +import * as codewhispererClient from '../../codewhisperer/client/codewhisperer' +import { ConfigurationEntry } from '../../codewhisperer/models/model' +import { setValidConnection, skipTestIfNoValidConn } from '../util/connection' +import { RecommendationHandler } from '../../codewhisperer/service/recommendationHandler' +import { createMockTextEditor, resetCodeWhispererGlobalVariables } from '../../test/codewhisperer/testUtil' +import { invokeRecommendation } from '../../codewhisperer/commands/invokeRecommendation' +import { session } from '../../codewhisperer/util/codeWhispererSession' + +/* +New model deployment may impact references returned. +These tests: + 1) are not required for github approval flow + 2) will be auto-skipped until fix for manual runs is posted. +*/ + +const leftContext = `InAuto.GetContent( + InAuto.servers.auto, "vendors.json", + function (data) { + let block = ''; + for(let i = 0; i < data.length; i++) { + block += '' + cars[i].title + ''; + } + $('#cars').html(block); + });` + +describe('CodeWhisperer service invocation', async function () { + let validConnection: boolean + const client = new codewhispererClient.DefaultCodeWhispererClient() + const configWithRefs: ConfigurationEntry = { + isShowMethodsEnabled: true, + isManualTriggerEnabled: true, + isAutomatedTriggerEnabled: true, + isSuggestionsWithCodeReferencesEnabled: true, + } + const configWithNoRefs: ConfigurationEntry = { + isShowMethodsEnabled: true, + isManualTriggerEnabled: true, + isAutomatedTriggerEnabled: true, + isSuggestionsWithCodeReferencesEnabled: false, + } + + before(async function () { + validConnection = await setValidConnection() + }) + + beforeEach(function () { + void resetCodeWhispererGlobalVariables() + RecommendationHandler.instance.clearRecommendations() + // TODO: remove this line (this.skip()) when these tests no longer auto-skipped + this.skip() + // valid connection required to run tests + skipTestIfNoValidConn(validConnection, this) + }) + + it('trigger known to return recs with references returns rec with reference', async function () { + // check that handler is empty before invocation + const requestIdBefore = RecommendationHandler.instance.requestId + const sessionIdBefore = session.sessionId + const validRecsBefore = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestIdBefore.length === 0) + assert.ok(sessionIdBefore.length === 0) + assert.ok(!validRecsBefore) + + const doc = leftContext + rightContext + const filename = 'test.js' + const language = 'javascript' + const line = 5 + const character = 39 + const mockEditor = createMockTextEditor(doc, filename, language, line, character) + + await invokeRecommendation(mockEditor, client, configWithRefs) + + const requestId = RecommendationHandler.instance.requestId + const sessionId = session.sessionId + const validRecs = RecommendationHandler.instance.isValidResponse() + const references = session.recommendations[0].references + + assert.ok(requestId.length > 0) + assert.ok(sessionId.length > 0) + assert.ok(validRecs) + assert.ok(references !== undefined) + // TODO: uncomment this assert when this test is no longer auto-skipped + // assert.ok(references.length > 0) + }) + + // This test will fail if user is logged in with IAM identity center + it('trigger known to return rec with references does not return rec with references when reference tracker setting is off', async function () { + // check that handler is empty before invocation + const requestIdBefore = RecommendationHandler.instance.requestId + const sessionIdBefore = session.sessionId + const validRecsBefore = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestIdBefore.length === 0) + assert.ok(sessionIdBefore.length === 0) + assert.ok(!validRecsBefore) + + const doc = leftContext + rightContext + const filename = 'test.js' + const language = 'javascript' + const line = 5 + const character = 39 + const mockEditor = createMockTextEditor(doc, filename, language, line, character) + + await invokeRecommendation(mockEditor, client, configWithNoRefs) + + const requestId = RecommendationHandler.instance.requestId + const sessionId = session.sessionId + const validRecs = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestId.length > 0) + assert.ok(sessionId.length > 0) + // no recs returned because example request returns 1 rec with reference, so no recs returned when references off + assert.ok(!validRecs) + }) +}) diff --git a/packages/core/src/testE2E/codewhisperer/serviceInvocations.test.ts b/packages/core/src/testE2E/codewhisperer/serviceInvocations.test.ts new file mode 100644 index 00000000000..37f32b130dd --- /dev/null +++ b/packages/core/src/testE2E/codewhisperer/serviceInvocations.test.ts @@ -0,0 +1,124 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert' +import * as vscode from 'vscode' +import * as path from 'path' +import { setValidConnection, skipTestIfNoValidConn } from '../util/connection' +import { ConfigurationEntry } from '../../codewhisperer/models/model' +import * as codewhispererClient from '../../codewhisperer/client/codewhisperer' +import { RecommendationHandler } from '../../codewhisperer/service/recommendationHandler' +import { + createMockTextEditor, + createTextDocumentChangeEvent, + resetCodeWhispererGlobalVariables, +} from '../../test/codewhisperer/testUtil' +import { KeyStrokeHandler } from '../../codewhisperer/service/keyStrokeHandler' +import { sleep } from '../../shared/utilities/timeoutUtils' +import { invokeRecommendation } from '../../codewhisperer/commands/invokeRecommendation' +import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' +import { session } from '../../codewhisperer/util/codeWhispererSession' + +describe('CodeWhisperer service invocation', async function () { + let validConnection: boolean + const client = new codewhispererClient.DefaultCodeWhispererClient() + const config: ConfigurationEntry = { + isShowMethodsEnabled: true, + isManualTriggerEnabled: true, + isAutomatedTriggerEnabled: true, + isSuggestionsWithCodeReferencesEnabled: true, + } + + before(async function () { + validConnection = await setValidConnection() + }) + + beforeEach(function () { + void resetCodeWhispererGlobalVariables() + RecommendationHandler.instance.clearRecommendations() + // valid connection required to run tests + skipTestIfNoValidConn(validConnection, this) + }) + + it('manual trigger returns valid recommendation response', async function () { + // check that handler is empty before invocation + const requestIdBefore = RecommendationHandler.instance.requestId + const sessionIdBefore = session.sessionId + const validRecsBefore = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestIdBefore.length === 0) + assert.ok(sessionIdBefore.length === 0) + assert.ok(!validRecsBefore) + + const mockEditor = createMockTextEditor() + await invokeRecommendation(mockEditor, client, config) + + const requestId = RecommendationHandler.instance.requestId + const sessionId = session.sessionId + const validRecs = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestId.length > 0) + assert.ok(sessionId.length > 0) + assert.ok(validRecs) + }) + + it('auto trigger returns valid recommendation response', async function () { + // check that handler is empty before invocation + const requestIdBefore = RecommendationHandler.instance.requestId + const sessionIdBefore = session.sessionId + const validRecsBefore = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestIdBefore.length === 0) + assert.ok(sessionIdBefore.length === 0) + assert.ok(!validRecsBefore) + + const mockEditor = createMockTextEditor() + + const mockEvent: vscode.TextDocumentChangeEvent = createTextDocumentChangeEvent( + mockEditor.document, + new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 1)), + '\n' + ) + + await KeyStrokeHandler.instance.processKeyStroke(mockEvent, mockEditor, client, config) + // wait for 5 seconds to allow time for response to be generated + await sleep(5000) + + const requestId = RecommendationHandler.instance.requestId + const sessionId = session.sessionId + const validRecs = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestId.length > 0) + assert.ok(sessionId.length > 0) + assert.ok(validRecs) + }) + + it('invocation in unsupported language does not generate a request', async function () { + const workspaceFolder = getTestWorkspaceFolder() + const appRoot = path.join(workspaceFolder, 'go1-plain-sam-app') + const appCodePath = path.join(appRoot, 'hello-world', 'go.mod') + + // check that handler is empty before invocation + const requestIdBefore = RecommendationHandler.instance.requestId + const sessionIdBefore = session.sessionId + const validRecsBefore = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestIdBefore.length === 0) + assert.ok(sessionIdBefore.length === 0) + assert.ok(!validRecsBefore) + + const doc = await vscode.workspace.openTextDocument(vscode.Uri.file(appCodePath)) + const editor = await vscode.window.showTextDocument(doc) + await invokeRecommendation(editor, client, config) + + const requestId = RecommendationHandler.instance.requestId + const sessionId = session.sessionId + const validRecs = RecommendationHandler.instance.isValidResponse() + + assert.ok(requestId.length === 0) + assert.ok(sessionId.length === 0) + assert.ok(!validRecs) + }) +})