Skip to content

Commit a8cdb76

Browse files
atonaamzleigaolaws-toolkit-automationandrewyuqWill-ShaoHua
authored
test(amazonq): adding back inline tests - part 2 (aws#8024)
## Problem ## Solution --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --------- Co-authored-by: Lei Gao <[email protected]> Co-authored-by: Lei Gao <[email protected]> Co-authored-by: aws-toolkit-automation <[email protected]> Co-authored-by: andrewyuq <[email protected]> Co-authored-by: Will Lo <[email protected]>
1 parent 38aed6e commit a8cdb76

File tree

3 files changed

+312
-5
lines changed

3 files changed

+312
-5
lines changed

packages/amazonq/test/e2e/inline/inline.test.ts

Lines changed: 64 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,18 @@
55

66
import * as vscode from 'vscode'
77
import assert from 'assert'
8-
import { closeAllEditors, registerAuthHook, TestFolder, toTextEditor, using } from 'aws-core-vscode/test'
8+
import {
9+
closeAllEditors,
10+
getTestWindow,
11+
registerAuthHook,
12+
resetCodeWhispererGlobalVariables,
13+
TestFolder,
14+
toTextEditor,
15+
using,
16+
} from 'aws-core-vscode/test'
17+
import { RecommendationHandler, RecommendationService, session } from 'aws-core-vscode/codewhisperer'
918
import { Commands, globals, sleep, waitUntil, collectionUtil } from 'aws-core-vscode/shared'
1019
import { loginToIdC } from '../amazonq/utils/setup'
11-
import { vsCodeState } from 'aws-core-vscode/codewhisperer'
1220

1321
describe('Amazon Q Inline', async function () {
1422
const retries = 3
@@ -32,6 +40,7 @@ describe('Amazon Q Inline', async function () {
3240
const folder = await TestFolder.create()
3341
tempFolder = folder.path
3442
await closeAllEditors()
43+
await resetCodeWhispererGlobalVariables()
3544
})
3645

3746
afterEach(async function () {
@@ -45,6 +54,7 @@ describe('Amazon Q Inline', async function () {
4554
const events = getUserTriggerDecision()
4655
console.table({
4756
'telemetry events': JSON.stringify(events),
57+
'recommendation service status': RecommendationService.instance.isRunning,
4858
})
4959
}
5060

@@ -61,6 +71,31 @@ describe('Amazon Q Inline', async function () {
6171
})
6272
}
6373

74+
async function waitForRecommendations() {
75+
const suggestionShown = await waitUntil(async () => session.getSuggestionState(0) === 'Showed', waitOptions)
76+
if (!suggestionShown) {
77+
throw new Error(`Suggestion did not show. Suggestion States: ${JSON.stringify(session.suggestionStates)}`)
78+
}
79+
const suggestionVisible = await waitUntil(
80+
async () => RecommendationHandler.instance.isSuggestionVisible(),
81+
waitOptions
82+
)
83+
if (!suggestionVisible) {
84+
throw new Error(
85+
`Suggestions failed to become visible. Suggestion States: ${JSON.stringify(session.suggestionStates)}`
86+
)
87+
}
88+
console.table({
89+
'suggestions states': JSON.stringify(session.suggestionStates),
90+
'valid recommendation': RecommendationHandler.instance.isValidResponse(),
91+
'recommendation service status': RecommendationService.instance.isRunning,
92+
recommendations: session.recommendations,
93+
})
94+
if (!RecommendationHandler.instance.isValidResponse()) {
95+
throw new Error('Did not find a valid response')
96+
}
97+
}
98+
6499
/**
65100
* Waits for a specific telemetry event to be emitted with the expected suggestion state.
66101
* It looks like there might be a potential race condition in codewhisperer causing telemetry
@@ -114,9 +149,8 @@ describe('Amazon Q Inline', async function () {
114149
await invokeCompletion()
115150
originalEditorContents = vscode.window.activeTextEditor?.document.getText()
116151

117-
// wait until all the recommendations have finished
118-
await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === true), waitOptions)
119-
await waitUntil(() => Promise.resolve(vsCodeState.isRecommendationsActive === false), waitOptions)
152+
// wait until the ghost text appears
153+
await waitForRecommendations()
120154
}
121155

122156
beforeEach(async () => {
@@ -129,12 +163,14 @@ describe('Amazon Q Inline', async function () {
129163
try {
130164
await setup()
131165
console.log(`test run ${attempt} succeeded`)
166+
logUserDecisionStatus()
132167
break
133168
} catch (e) {
134169
console.log(`test run ${attempt} failed`)
135170
console.log(e)
136171
logUserDecisionStatus()
137172
attempt++
173+
await resetCodeWhispererGlobalVariables()
138174
}
139175
}
140176
if (attempt === retries) {
@@ -180,6 +216,29 @@ describe('Amazon Q Inline', async function () {
180216
assert.deepStrictEqual(vscode.window.activeTextEditor?.document.getText(), originalEditorContents)
181217
})
182218
})
219+
220+
it(`${name} invoke on unsupported filetype`, async function () {
221+
await setupEditor({
222+
name: 'test.zig',
223+
contents: `fn doSomething() void {
224+
225+
}`,
226+
})
227+
228+
/**
229+
* Add delay between editor loading and invoking completion
230+
* @see beforeEach in supported filetypes for more information
231+
*/
232+
await sleep(1000)
233+
await invokeCompletion()
234+
235+
if (name === 'automatic') {
236+
// It should never get triggered since its not a supported file type
237+
assert.deepStrictEqual(RecommendationService.instance.isRunning, false)
238+
} else {
239+
await getTestWindow().waitForMessage('currently not supported by Amazon Q inline suggestions')
240+
}
241+
})
183242
})
184243
}
185244
})
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*!
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import assert from 'assert'
7+
import * as codewhispererClient from '../../codewhisperer/client/codewhisperer'
8+
import { ConfigurationEntry } from '../../codewhisperer/models/model'
9+
import { setValidConnection, skipTestIfNoValidConn } from '../util/connection'
10+
import { RecommendationHandler } from '../../codewhisperer/service/recommendationHandler'
11+
import { createMockTextEditor, resetCodeWhispererGlobalVariables } from '../../test/codewhisperer/testUtil'
12+
import { invokeRecommendation } from '../../codewhisperer/commands/invokeRecommendation'
13+
import { session } from '../../codewhisperer/util/codeWhispererSession'
14+
15+
/*
16+
New model deployment may impact references returned.
17+
These tests:
18+
1) are not required for github approval flow
19+
2) will be auto-skipped until fix for manual runs is posted.
20+
*/
21+
22+
const leftContext = `InAuto.GetContent(
23+
InAuto.servers.auto, "vendors.json",
24+
function (data) {
25+
let block = '';
26+
for(let i = 0; i < data.length; i++) {
27+
block += '<a href="' + data`
28+
29+
const rightContext = `[i].title + '">' + cars[i].title + '</a>';
30+
}
31+
$('#cars').html(block);
32+
});`
33+
34+
describe('CodeWhisperer service invocation', async function () {
35+
let validConnection: boolean
36+
const client = new codewhispererClient.DefaultCodeWhispererClient()
37+
const configWithRefs: ConfigurationEntry = {
38+
isShowMethodsEnabled: true,
39+
isManualTriggerEnabled: true,
40+
isAutomatedTriggerEnabled: true,
41+
isSuggestionsWithCodeReferencesEnabled: true,
42+
}
43+
const configWithNoRefs: ConfigurationEntry = {
44+
isShowMethodsEnabled: true,
45+
isManualTriggerEnabled: true,
46+
isAutomatedTriggerEnabled: true,
47+
isSuggestionsWithCodeReferencesEnabled: false,
48+
}
49+
50+
before(async function () {
51+
validConnection = await setValidConnection()
52+
})
53+
54+
beforeEach(function () {
55+
void resetCodeWhispererGlobalVariables()
56+
RecommendationHandler.instance.clearRecommendations()
57+
// TODO: remove this line (this.skip()) when these tests no longer auto-skipped
58+
this.skip()
59+
// valid connection required to run tests
60+
skipTestIfNoValidConn(validConnection, this)
61+
})
62+
63+
it('trigger known to return recs with references returns rec with reference', async function () {
64+
// check that handler is empty before invocation
65+
const requestIdBefore = RecommendationHandler.instance.requestId
66+
const sessionIdBefore = session.sessionId
67+
const validRecsBefore = RecommendationHandler.instance.isValidResponse()
68+
69+
assert.ok(requestIdBefore.length === 0)
70+
assert.ok(sessionIdBefore.length === 0)
71+
assert.ok(!validRecsBefore)
72+
73+
const doc = leftContext + rightContext
74+
const filename = 'test.js'
75+
const language = 'javascript'
76+
const line = 5
77+
const character = 39
78+
const mockEditor = createMockTextEditor(doc, filename, language, line, character)
79+
80+
await invokeRecommendation(mockEditor, client, configWithRefs)
81+
82+
const requestId = RecommendationHandler.instance.requestId
83+
const sessionId = session.sessionId
84+
const validRecs = RecommendationHandler.instance.isValidResponse()
85+
const references = session.recommendations[0].references
86+
87+
assert.ok(requestId.length > 0)
88+
assert.ok(sessionId.length > 0)
89+
assert.ok(validRecs)
90+
assert.ok(references !== undefined)
91+
// TODO: uncomment this assert when this test is no longer auto-skipped
92+
// assert.ok(references.length > 0)
93+
})
94+
95+
// This test will fail if user is logged in with IAM identity center
96+
it('trigger known to return rec with references does not return rec with references when reference tracker setting is off', async function () {
97+
// check that handler is empty before invocation
98+
const requestIdBefore = RecommendationHandler.instance.requestId
99+
const sessionIdBefore = session.sessionId
100+
const validRecsBefore = RecommendationHandler.instance.isValidResponse()
101+
102+
assert.ok(requestIdBefore.length === 0)
103+
assert.ok(sessionIdBefore.length === 0)
104+
assert.ok(!validRecsBefore)
105+
106+
const doc = leftContext + rightContext
107+
const filename = 'test.js'
108+
const language = 'javascript'
109+
const line = 5
110+
const character = 39
111+
const mockEditor = createMockTextEditor(doc, filename, language, line, character)
112+
113+
await invokeRecommendation(mockEditor, client, configWithNoRefs)
114+
115+
const requestId = RecommendationHandler.instance.requestId
116+
const sessionId = session.sessionId
117+
const validRecs = RecommendationHandler.instance.isValidResponse()
118+
119+
assert.ok(requestId.length > 0)
120+
assert.ok(sessionId.length > 0)
121+
// no recs returned because example request returns 1 rec with reference, so no recs returned when references off
122+
assert.ok(!validRecs)
123+
})
124+
})
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*!
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import assert from 'assert'
7+
import * as vscode from 'vscode'
8+
import * as path from 'path'
9+
import { setValidConnection, skipTestIfNoValidConn } from '../util/connection'
10+
import { ConfigurationEntry } from '../../codewhisperer/models/model'
11+
import * as codewhispererClient from '../../codewhisperer/client/codewhisperer'
12+
import { RecommendationHandler } from '../../codewhisperer/service/recommendationHandler'
13+
import {
14+
createMockTextEditor,
15+
createTextDocumentChangeEvent,
16+
resetCodeWhispererGlobalVariables,
17+
} from '../../test/codewhisperer/testUtil'
18+
import { KeyStrokeHandler } from '../../codewhisperer/service/keyStrokeHandler'
19+
import { sleep } from '../../shared/utilities/timeoutUtils'
20+
import { invokeRecommendation } from '../../codewhisperer/commands/invokeRecommendation'
21+
import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities'
22+
import { session } from '../../codewhisperer/util/codeWhispererSession'
23+
24+
describe('CodeWhisperer service invocation', async function () {
25+
let validConnection: boolean
26+
const client = new codewhispererClient.DefaultCodeWhispererClient()
27+
const config: ConfigurationEntry = {
28+
isShowMethodsEnabled: true,
29+
isManualTriggerEnabled: true,
30+
isAutomatedTriggerEnabled: true,
31+
isSuggestionsWithCodeReferencesEnabled: true,
32+
}
33+
34+
before(async function () {
35+
validConnection = await setValidConnection()
36+
})
37+
38+
beforeEach(function () {
39+
void resetCodeWhispererGlobalVariables()
40+
RecommendationHandler.instance.clearRecommendations()
41+
// valid connection required to run tests
42+
skipTestIfNoValidConn(validConnection, this)
43+
})
44+
45+
it('manual trigger returns valid recommendation response', async function () {
46+
// check that handler is empty before invocation
47+
const requestIdBefore = RecommendationHandler.instance.requestId
48+
const sessionIdBefore = session.sessionId
49+
const validRecsBefore = RecommendationHandler.instance.isValidResponse()
50+
51+
assert.ok(requestIdBefore.length === 0)
52+
assert.ok(sessionIdBefore.length === 0)
53+
assert.ok(!validRecsBefore)
54+
55+
const mockEditor = createMockTextEditor()
56+
await invokeRecommendation(mockEditor, client, config)
57+
58+
const requestId = RecommendationHandler.instance.requestId
59+
const sessionId = session.sessionId
60+
const validRecs = RecommendationHandler.instance.isValidResponse()
61+
62+
assert.ok(requestId.length > 0)
63+
assert.ok(sessionId.length > 0)
64+
assert.ok(validRecs)
65+
})
66+
67+
it('auto trigger returns valid recommendation response', async function () {
68+
// check that handler is empty before invocation
69+
const requestIdBefore = RecommendationHandler.instance.requestId
70+
const sessionIdBefore = session.sessionId
71+
const validRecsBefore = RecommendationHandler.instance.isValidResponse()
72+
73+
assert.ok(requestIdBefore.length === 0)
74+
assert.ok(sessionIdBefore.length === 0)
75+
assert.ok(!validRecsBefore)
76+
77+
const mockEditor = createMockTextEditor()
78+
79+
const mockEvent: vscode.TextDocumentChangeEvent = createTextDocumentChangeEvent(
80+
mockEditor.document,
81+
new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 1)),
82+
'\n'
83+
)
84+
85+
await KeyStrokeHandler.instance.processKeyStroke(mockEvent, mockEditor, client, config)
86+
// wait for 5 seconds to allow time for response to be generated
87+
await sleep(5000)
88+
89+
const requestId = RecommendationHandler.instance.requestId
90+
const sessionId = session.sessionId
91+
const validRecs = RecommendationHandler.instance.isValidResponse()
92+
93+
assert.ok(requestId.length > 0)
94+
assert.ok(sessionId.length > 0)
95+
assert.ok(validRecs)
96+
})
97+
98+
it('invocation in unsupported language does not generate a request', async function () {
99+
const workspaceFolder = getTestWorkspaceFolder()
100+
const appRoot = path.join(workspaceFolder, 'go1-plain-sam-app')
101+
const appCodePath = path.join(appRoot, 'hello-world', 'go.mod')
102+
103+
// check that handler is empty before invocation
104+
const requestIdBefore = RecommendationHandler.instance.requestId
105+
const sessionIdBefore = session.sessionId
106+
const validRecsBefore = RecommendationHandler.instance.isValidResponse()
107+
108+
assert.ok(requestIdBefore.length === 0)
109+
assert.ok(sessionIdBefore.length === 0)
110+
assert.ok(!validRecsBefore)
111+
112+
const doc = await vscode.workspace.openTextDocument(vscode.Uri.file(appCodePath))
113+
const editor = await vscode.window.showTextDocument(doc)
114+
await invokeRecommendation(editor, client, config)
115+
116+
const requestId = RecommendationHandler.instance.requestId
117+
const sessionId = session.sessionId
118+
const validRecs = RecommendationHandler.instance.isValidResponse()
119+
120+
assert.ok(requestId.length === 0)
121+
assert.ok(sessionId.length === 0)
122+
assert.ok(!validRecs)
123+
})
124+
})

0 commit comments

Comments
 (0)