telemetry(featureDev): missing "result" field, tests #4167

Problem:
- Missing "result" for some metrics.
- Missing test coverage for telemetry events.

Solution:
- Add tests.
- Set "result" field for amazonq_isReviewedChanges, amazonq_startChat metrics.

Author: jpinkney-aws

src/amazonqFeatureDev/client/featureDev.ts

@@ -63,7 +63,7 @@ const streamResponseErrors: Record<string, number> = {
 }
 
 export class FeatureDevClient {
-    private async getClient() {
+    public async getClient() {
         // Should not be stored for the whole session.
         // Client has to be reinitialized for each request so we always have a fresh bearerToken
         return await createFeatureDevProxyClient()

src/amazonqFeatureDev/controllers/chat/controller.ts

@@ -115,11 +115,13 @@ export class FeatureDevController {
                     telemetry.amazonq_approachThumbsUp.emit({
                         amazonqConversationId: session?.conversationId,
                         value: 1,
+                        result: 'Succeeded',
                     })
                 } else if (vote === 'downvote') {
                     telemetry.amazonq_approachThumbsDown.emit({
                         amazonqConversationId: session?.conversationId,
                         value: 1,
+                        result: 'Succeeded',
                     })
                 }
                 break
@@ -357,7 +359,11 @@ To learn more, visit the _[Amazon Q User Guide](${userGuideURL})_.
 
     private async openDiff(message: OpenDiffMessage) {
         const session = await this.sessionStorage.getSession(message.tabID)
-        telemetry.amazonq_isReviewedChanges.emit({ amazonqConversationId: session.conversationId, enabled: true })
+        telemetry.amazonq_isReviewedChanges.emit({
+            amazonqConversationId: session.conversationId,
+            enabled: true,
+            result: 'Succeeded',
+        })
 
         if (message.deleted) {
             const fileUri = this.getOriginalFileUri(message, session)

src/amazonqFeatureDev/session/session.ts

@@ -27,9 +27,15 @@ export class Session {
     // Used to keep track of whether or not the current session is currently authenticating/needs authenticating
     public isAuthenticating: boolean
 
-    constructor(public readonly config: SessionConfig, private messenger: Messenger, public readonly tabID: string) {
-        this._state = new ConversationNotStartedState('', tabID)
-        this.proxyClient = new FeatureDevClient()
+    constructor(
+        public readonly config: SessionConfig,
+        private messenger: Messenger,
+        public readonly tabID: string,
+        initialState: Omit<SessionState, 'uploadId'> = new ConversationNotStartedState('', tabID),
+        proxyClient: FeatureDevClient = new FeatureDevClient()
+    ) {
+        this._state = initialState
+        this.proxyClient = proxyClient
 
         this.approachRetries = approachRetryLimit
 
@@ -45,7 +51,11 @@ export class Session {
             await this.setupConversation(msg)
             this.preloaderFinished = true
 
-            telemetry.amazonq_startChat.emit({ amazonqConversationId: this.conversationId, value: 1 })
+            telemetry.amazonq_startChat.emit({
+                amazonqConversationId: this.conversationId,
+                value: 1,
+                result: 'Succeeded',
+            })
 
             this.messenger.sendAsyncEventProgress(this.tabID, true, undefined)
         }

src/shared/telemetry/exemptMetrics.ts

@@ -16,7 +16,6 @@
  */
 const validationExemptMetrics: Set<string> = new Set([
     'amazonq_runCommand',
-    'amazonq_isReviewedChanges',
     'apigateway_copyUrl',
     'aws_loadCredentials',
     'aws_validateCredentials',

src/test/amazonqFeatureDev/controllers/chat/controller.test.ts

@@ -8,41 +8,40 @@ import * as assert from 'assert'
 import * as path from 'path'
 import sinon from 'sinon'
 import { waitUntil } from '../../../../shared/utilities/timeoutUtils'
-import { ControllerSetup, createController } from '../../utils'
-import { ChatControllerEventEmitters } from '../../../../amazonqFeatureDev/controllers/chat/controller'
+import { ControllerSetup, createController, createSession } from '../../utils'
 import { FollowUpTypes, createUri } from '../../../../amazonqFeatureDev/types'
 import { Session } from '../../../../amazonqFeatureDev/session/session'
 import { Prompter } from '../../../../shared/ui/prompter'
-import { toFile } from '../../../testUtil'
+import { assertTelemetry, toFile } from '../../../testUtil'
 import { SelectedFolderNotInWorkspaceFolderError } from '../../../../amazonqFeatureDev/errors'
+import { PrepareRefinementState } from '../../../../amazonqFeatureDev/session/sessionState'
+import { FeatureDevClient } from '../../../../amazonqFeatureDev/client/featureDev'
 
 describe('Controller', () => {
     const tabID = '123'
     const conversationID = '456'
     const uploadID = '789'
 
+    let session: Session
     let controllerSetup: ControllerSetup
 
+    before(() => {
+        sinon.stub(performance, 'now').returns(0)
+    })
+
     beforeEach(async () => {
-        controllerSetup = await createController({
-            conversationID,
-            tabID,
-            uploadID,
-        })
+        controllerSetup = await createController()
+        session = await createSession({ messenger: controllerSetup.messenger, conversationID, tabID, uploadID })
     })
 
     afterEach(() => {
         sinon.restore()
     })
 
     describe('openDiff', async () => {
-        async function openDiff(
-            controllerEventEmitter: ChatControllerEventEmitters,
-            filePath: string,
-            deleted = false
-        ) {
+        async function openDiff(filePath: string, deleted = false) {
             const executeDiff = sinon.stub(vscode.commands, 'executeCommand').returns(Promise.resolve(undefined))
-            controllerEventEmitter.openDiff.fire({ tabID, filePath, deleted })
+            controllerSetup.emitters.openDiff.fire({ tabID, filePath, deleted })
 
             // Wait until the controller has time to process the event
             await waitUntil(() => {
@@ -53,7 +52,8 @@ describe('Controller', () => {
         }
 
         it('uses empty file when file is not found locally', async () => {
-            const executedDiff = await openDiff(controllerSetup.emitters, path.join('src', 'mynewfile.js'))
+            sinon.stub(controllerSetup.sessionStorage, 'getSession').resolves(session)
+            const executedDiff = await openDiff(path.join('src', 'mynewfile.js'))
             assert.strictEqual(
                 executedDiff.calledWith(
                     'vscode.diff',
@@ -62,12 +62,15 @@ describe('Controller', () => {
                 ),
                 true
             )
+
+            assertTelemetry('amazonq_isReviewedChanges', { amazonqConversationId: conversationID, enabled: true })
         })
 
         it('uses file location when file is found locally and /src is not available', async () => {
+            sinon.stub(controllerSetup.sessionStorage, 'getSession').resolves(session)
             const newFileLocation = path.join(controllerSetup.workspaceFolder.uri.fsPath, 'mynewfile.js')
             toFile('', newFileLocation)
-            const executedDiff = await openDiff(controllerSetup.emitters, 'mynewfile.js')
+            const executedDiff = await openDiff('mynewfile.js')
             assert.strictEqual(
                 executedDiff.calledWith(
                     'vscode.diff',
@@ -76,12 +79,15 @@ describe('Controller', () => {
                 ),
                 true
             )
+
+            assertTelemetry('amazonq_isReviewedChanges', { amazonqConversationId: conversationID, enabled: true })
         })
 
         it('uses file location when file is found locally and /src is available', async () => {
+            sinon.stub(controllerSetup.sessionStorage, 'getSession').resolves(session)
             const newFileLocation = path.join(controllerSetup.workspaceFolder.uri.fsPath, 'src', 'mynewfile.js')
             toFile('', newFileLocation)
-            const executedDiff = await openDiff(controllerSetup.emitters, path.join('src', 'mynewfile.js'))
+            const executedDiff = await openDiff(path.join('src', 'mynewfile.js'))
             assert.strictEqual(
                 executedDiff.calledWith(
                     'vscode.diff',
@@ -90,18 +96,17 @@ describe('Controller', () => {
                 ),
                 true
             )
+
+            assertTelemetry('amazonq_isReviewedChanges', { amazonqConversationId: conversationID, enabled: true })
         })
 
         it('uses file location when file is found locally and source folder was picked', async () => {
+            sinon.stub(controllerSetup.sessionStorage, 'getSession').resolves(session)
             const newFileLocation = path.join(controllerSetup.workspaceFolder.uri.fsPath, 'foo', 'fi', 'mynewfile.js')
             toFile('', newFileLocation)
             sinon.stub(vscode.workspace, 'getWorkspaceFolder').returns(controllerSetup.workspaceFolder)
-            controllerSetup.session.config.sourceRoot = path.join(
-                controllerSetup.workspaceFolder.uri.fsPath,
-                'foo',
-                'fi'
-            )
-            const executedDiff = await openDiff(controllerSetup.emitters, path.join('foo', 'fi', 'mynewfile.js'))
+            session.config.sourceRoot = path.join(controllerSetup.workspaceFolder.uri.fsPath, 'foo', 'fi')
+            const executedDiff = await openDiff(path.join('foo', 'fi', 'mynewfile.js'))
             assert.strictEqual(
                 executedDiff.calledWith(
                     'vscode.diff',
@@ -110,17 +115,15 @@ describe('Controller', () => {
                 ),
                 true
             )
+
+            assertTelemetry('amazonq_isReviewedChanges', { amazonqConversationId: conversationID, enabled: true })
         })
     })
 
     describe('modifyDefaultSourceFolder', () => {
-        async function modifyDefaultSourceFolder(
-            controllerEventEmitter: ChatControllerEventEmitters,
-            session: Session,
-            sourceRoot: string
-        ) {
+        async function modifyDefaultSourceFolder(sourceRoot: string) {
             const promptStub = sinon.stub(Prompter.prototype, 'prompt').resolves(vscode.Uri.file(sourceRoot))
-            controllerEventEmitter.followUpClicked.fire({
+            controllerSetup.emitters.followUpClicked.fire({
                 tabID,
                 followUp: {
                     type: FollowUpTypes.ModifyDefaultSourceFolder,
@@ -131,14 +134,12 @@ describe('Controller', () => {
             await waitUntil(() => {
                 return Promise.resolve(promptStub.callCount > 0)
             }, {})
-
-            return session
         }
 
         it('fails if selected folder is not under a workspace folder', async () => {
             sinon.stub(vscode.workspace, 'getWorkspaceFolder').returns(undefined)
             const messengerSpy = sinon.spy(controllerSetup.messenger, 'sendAnswer')
-            await modifyDefaultSourceFolder(controllerSetup.emitters, controllerSetup.session, '../../')
+            await modifyDefaultSourceFolder('../../')
             assert.deepStrictEqual(
                 messengerSpy.calledWith({
                     tabID,
@@ -158,23 +159,84 @@ describe('Controller', () => {
         })
 
         it('accepts valid source folders under a workspace root', async () => {
-            const controllerSetup = await createController({
+            sinon.stub(controllerSetup.sessionStorage, 'getSession').resolves(session)
+            sinon.stub(vscode.workspace, 'getWorkspaceFolder').returns(controllerSetup.workspaceFolder)
+            const expectedSourceRoot = path.join(controllerSetup.workspaceFolder.uri.fsPath, 'src')
+            await modifyDefaultSourceFolder(expectedSourceRoot)
+            assert.strictEqual(session.config.sourceRoot, expectedSourceRoot)
+            assert.strictEqual(session.config.workspaceRoot, controllerSetup.workspaceFolder.uri.fsPath)
+        })
+    })
+
+    describe('processChatItemVotedMessage', () => {
+        async function processChatItemVotedMessage(vote: 'upvote' | 'downvote') {
+            const initialState = new PrepareRefinementState(
+                {
+                    conversationId: conversationID,
+                    proxyClient: new FeatureDevClient(),
+                    sourceRoot: '',
+                    workspaceRoot: '',
+                },
+                '',
+                tabID
+            )
+            const newSession = await createSession({
+                messenger: controllerSetup.messenger,
+                sessionState: initialState,
                 conversationID,
                 tabID,
                 uploadID,
             })
-            sinon.stub(vscode.workspace, 'getWorkspaceFolder').returns(controllerSetup.workspaceFolder)
-            const expectedSourceRoot = path.join(controllerSetup.workspaceFolder.uri.fsPath, 'src')
-            const modifiedSourceFolderSession = await modifyDefaultSourceFolder(
-                controllerSetup.emitters,
-                controllerSetup.session,
-                expectedSourceRoot
-            )
-            assert.strictEqual(modifiedSourceFolderSession.config.sourceRoot, expectedSourceRoot)
-            assert.strictEqual(
-                modifiedSourceFolderSession.config.workspaceRoot,
-                controllerSetup.workspaceFolder.uri.fsPath
-            )
+            const getSessionStub = sinon.stub(controllerSetup.sessionStorage, 'getSession').resolves(newSession)
+            controllerSetup.emitters.processChatItemVotedMessage.fire({
+                tabID,
+                messageID: '',
+                vote,
+            })
+
+            // Wait until the controller has time to process the event
+            await waitUntil(() => {
+                return Promise.resolve(getSessionStub.callCount > 0)
+            }, {})
+        }
+
+        it('incoming upvoted message sends telemetry', async () => {
+            await processChatItemVotedMessage('upvote')
+
+            assertTelemetry('amazonq_approachThumbsUp', { amazonqConversationId: conversationID, result: 'Succeeded' })
+        })
+
+        it('incoming downvoted message sends telemetry', async () => {
+            await processChatItemVotedMessage('downvote')
+
+            assertTelemetry('amazonq_approachThumbsDown', {
+                amazonqConversationId: conversationID,
+                result: 'Succeeded',
+            })
+        })
+    })
+
+    describe('newPlan', () => {
+        async function newPlanClicked() {
+            const getSessionStub = sinon.stub(controllerSetup.sessionStorage, 'getSession').resolves(session)
+
+            controllerSetup.emitters.followUpClicked.fire({
+                tabID,
+                followUp: {
+                    type: FollowUpTypes.NewPlan,
+                },
+            })
+
+            // Wait until the controller has time to process the event
+            await waitUntil(() => {
+                return Promise.resolve(getSessionStub.callCount > 0)
+            }, {})
+        }
+
+        it('end chat telemetry is sent', async () => {
+            await newPlanClicked()
+
+            assertTelemetry('amazonq_endChat', { amazonqConversationId: conversationID, result: 'Succeeded' })
         })
     })
 })

src/test/amazonqFeatureDev/session/session.test.ts

@@ -0,0 +1,36 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import sinon from 'sinon'
+import { createMessenger, createSession } from '../utils'
+import { Session } from '../../../amazonqFeatureDev/session/session'
+import { assertTelemetry } from '../../testUtil'
+
+describe('session', () => {
+    const conversationID = '12345'
+
+    let session: Session
+
+    beforeEach(async () => {
+        const messenger = createMessenger()
+        session = await createSession({ messenger, conversationID })
+    })
+
+    afterEach(() => {
+        sinon.restore()
+    })
+
+    describe('preloader', () => {
+        it('emits start chat telemetry', async () => {
+            await session.preloader('implement twosum in typescript')
+
+            assertTelemetry('amazonq_startChat', { amazonqConversationId: conversationID })
+
+            assertTelemetry('amazonq_startConversationInvoke', {
+                amazonqConversationId: conversationID,
+            })
+        })
+    })
+})