feat(transform): improve telemetry (#5403)

* Amazon Q Code Transform: Improve telemetry to capture transform journey

* Update globals.clock.Date usage across transform code path

* Add tech debt unit test

* Use run() to emit telemetry

Author: tincheng

package-lock.json

@@ -68,7 +68,7 @@
         "webpack-merge": "^5.10.0"
     },
     "dependencies": {
-        "@aws-toolkits/telemetry": "^1.0.229",
+        "@aws-toolkits/telemetry": "^1.0.232",
         "vscode-nls": "^5.2.0",
         "vscode-nls-dev": "^4.0.4"
     }

package.json

@@ -27,6 +27,7 @@ import {
     startTransformByQ,
     stopTransformByQ,
     validateCanCompileProject,
+    setMaven,
 } from '../../../codewhisperer/commands/startTransformByQ'
 import { JDKVersion, TransformationCandidateProject, transformByQState } from '../../../codewhisperer/models/model'
 import {
@@ -53,6 +54,8 @@ import { getAuthType } from '../../../codewhisperer/service/transformByQ/transfo
 import DependencyVersions from '../../models/dependencies'
 import { getStringHash } from '../../../shared/utilities/textUtilities'
 import { getTelemetryReasonDesc } from '../../../shared/errors'
+import { getVersionData } from '../../../codewhisperer/service/transformByQ/transformMavenHandler'
+
 // These events can be interactions within the chat,
 // or elsewhere in the IDE
 export interface ChatControllerEventEmitters {
@@ -179,47 +182,64 @@ export class GumbyController {
     }
 
     private async transformInitiated(message: any) {
-        // check that a project is open
-        const workspaceFolders = vscode.workspace.workspaceFolders
-        if (workspaceFolders === undefined || workspaceFolders.length === 0) {
-            this.messenger.sendUnrecoverableErrorResponse('no-project-found', message.tabID)
-            return
-        }
-
-        // check that the session is authenticated
+        // Start /transform chat flow
         const session: Session = this.sessionStorage.getSession()
-        try {
-            const authState = await AuthUtil.instance.getChatAuthState()
-            if (authState.amazonQ !== 'connected') {
-                void this.messenger.sendAuthNeededExceptionMessage(authState, message.tabID)
-                session.isAuthenticating = true
-                return
-            }
+        CodeTransformTelemetryState.instance.setSessionId()
 
-            switch (this.sessionStorage.getSession().conversationState) {
-                case ConversationState.JOB_SUBMITTED:
-                    this.messenger.sendAsyncEventProgress(
-                        message.tabID,
-                        true,
-                        undefined,
-                        GumbyNamedMessages.JOB_SUBMISSION_STATUS_MESSAGE
-                    )
-                    this.messenger.sendJobSubmittedMessage(message.tabID)
+        try {
+            await telemetry.codeTransform_initiateTransform.run(async () => {
+                const authType = await getAuthType()
+                telemetry.record({
+                    codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(),
+                    credentialSourceId: authType,
+                })
+
+                // check that a project is open
+                const workspaceFolders = vscode.workspace.workspaceFolders
+                if (workspaceFolders === undefined || workspaceFolders.length === 0) {
+                    this.messenger.sendUnrecoverableErrorResponse('no-project-found', message.tabID)
+                    telemetry.record({ result: MetadataResult.Fail, reason: 'no-project-found' })
                     return
-                case ConversationState.COMPILING:
-                    this.messenger.sendAsyncEventProgress(
-                        message.tabID,
-                        true,
-                        undefined,
-                        GumbyNamedMessages.COMPILATION_PROGRESS_MESSAGE
-                    )
-                    this.messenger.sendCompilationInProgress(message.tabID)
+                }
+
+                // check that the session is authenticated
+                const authState = await AuthUtil.instance.getChatAuthState()
+                if (authState.amazonQ !== 'connected') {
+                    void this.messenger.sendAuthNeededExceptionMessage(authState, message.tabID)
+                    session.isAuthenticating = true
+                    telemetry.record({ result: MetadataResult.Fail, reason: 'auth-failed' })
                     return
-            }
-            CodeTransformTelemetryState.instance.setSessionId()
-            this.messenger.sendTransformationIntroduction(message.tabID)
+                }
 
-            // start /transform chat flow
+                // If previous transformation was already running
+                switch (this.sessionStorage.getSession().conversationState) {
+                    case ConversationState.JOB_SUBMITTED:
+                        this.messenger.sendAsyncEventProgress(
+                            message.tabID,
+                            true,
+                            undefined,
+                            GumbyNamedMessages.JOB_SUBMISSION_STATUS_MESSAGE
+                        )
+                        this.messenger.sendJobSubmittedMessage(message.tabID)
+                        return
+                    case ConversationState.COMPILING:
+                        this.messenger.sendAsyncEventProgress(
+                            message.tabID,
+                            true,
+                            undefined,
+                            GumbyNamedMessages.COMPILATION_PROGRESS_MESSAGE
+                        )
+                        this.messenger.sendCompilationInProgress(message.tabID)
+                        return
+                }
+                this.messenger.sendTransformationIntroduction(message.tabID)
+            })
+        } catch (e: any) {
+            // if there was an issue getting the list of valid projects, the error message will be shown here
+            this.messenger.sendErrorMessage(e.message, message.tabID)
+        }
+
+        try {
             const validProjects = await this.validateProjectsWithReplyOnError(message)
             if (validProjects.length > 0) {
                 this.sessionStorage.getSession().updateCandidateProjects(validProjects)
@@ -233,14 +253,30 @@ export class GumbyController {
 
     private async validateProjectsWithReplyOnError(message: any): Promise<TransformationCandidateProject[]> {
         let telemetryJavaVersion = JDKToTelemetryValue(JDKVersion.UNSUPPORTED) as CodeTransformJavaSourceVersionsAllowed
+
         let err
         try {
-            const validProjects = await getValidCandidateProjects()
-            if (validProjects.length > 0) {
-                // validProjects[0].JDKVersion will be undefined if javap errors out or no .class files found, so call it UNSUPPORTED
-                const javaVersion = validProjects[0].JDKVersion ?? JDKVersion.UNSUPPORTED
-                telemetryJavaVersion = JDKToTelemetryValue(javaVersion) as CodeTransformJavaSourceVersionsAllowed
-            }
+            const validProjects = await telemetry.codeTransform_validateProject.run(async () => {
+                telemetry.record({
+                    codeTransformBuildSystem: 'Maven', // default for Maven until we add undefined field to CodeTransformBuildSystem
+                    codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(),
+                })
+
+                const validProjects = await getValidCandidateProjects()
+                if (validProjects.length > 0) {
+                    // validProjects[0].JDKVersion will be undefined if javap errors out or no .class files found, so call it UNSUPPORTED
+                    const javaVersion = validProjects[0].JDKVersion ?? JDKVersion.UNSUPPORTED
+                    telemetryJavaVersion = JDKToTelemetryValue(javaVersion) as CodeTransformJavaSourceVersionsAllowed
+                }
+                telemetry.record({ codeTransformLocalJavaVersion: telemetryJavaVersion })
+
+                await setMaven()
+                const versionInfo = await getVersionData()
+                const mavenVersionInfoMessage = `${versionInfo[0]} (${transformByQState.getMavenName()})`
+                telemetry.record({ buildSystemVersion: mavenVersionInfoMessage })
+
+                return validProjects
+            })
             return validProjects
         } catch (e: any) {
             if (e instanceof NoJavaProjectsFoundError) {
@@ -252,7 +288,7 @@ export class GumbyController {
             }
             err = e
         } finally {
-            // New projectDetails metric should always be fired whether the project was valid or invalid
+            // TODO: remove deprecated metric once BI started using new metrics
             telemetry.codeTransform_projectDetails.emit({
                 passive: true,
                 codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(),
@@ -269,9 +305,14 @@ export class GumbyController {
         const typedAction = MessengerUtils.stringToEnumValue(ButtonActions, message.action as any)
         switch (typedAction) {
             case ButtonActions.CONFIRM_TRANSFORMATION_FORM:
-                await this.initiateTransformationOnProject(message)
+                await this.handleUserProjectSelection(message)
                 break
             case ButtonActions.CANCEL_TRANSFORMATION_FORM:
+                telemetry.codeTransform_submitSelection.emit({
+                    codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(),
+                    userChoice: 'Cancel',
+                    result: MetadataResult.Pass,
+                })
                 this.messenger.sendJobFinishedMessage(message.tabID, CodeWhispererConstants.jobCancelledChatMessage)
                 break
             case ButtonActions.VIEW_TRANSFORMATION_HUB:
@@ -306,27 +347,38 @@ export class GumbyController {
     }
 
     // prompt user to pick project and specify source JDK version
-    private async initiateTransformationOnProject(message: any) {
-        const authType = await getAuthType()
-        telemetry.codeTransform_jobStart.emit({
-            codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(),
-            credentialSourceId: authType,
-            result: MetadataResult.Pass,
-        })
-        const pathToProject: string = message.formSelectedValues['GumbyTransformProjectForm']
-        const toJDKVersion: JDKVersion = message.formSelectedValues['GumbyTransformJdkToForm']
-        const fromJDKVersion: JDKVersion = message.formSelectedValues['GumbyTransformJdkFromForm']
+    private async handleUserProjectSelection(message: any) {
+        await telemetry.codeTransform_submitSelection.run(async () => {
+            const pathToProject: string = message.formSelectedValues['GumbyTransformProjectForm']
+            const toJDKVersion: JDKVersion = message.formSelectedValues['GumbyTransformJdkToForm']
+            const fromJDKVersion: JDKVersion = message.formSelectedValues['GumbyTransformJdkFromForm']
+
+            telemetry.record({
+                codeTransformJavaSourceVersionsAllowed: JDKToTelemetryValue(
+                    fromJDKVersion
+                ) as CodeTransformJavaSourceVersionsAllowed,
+                codeTransformJavaTargetVersionsAllowed: JDKToTelemetryValue(
+                    toJDKVersion
+                ) as CodeTransformJavaTargetVersionsAllowed,
+                codeTransformProjectId: pathToProject === undefined ? telemetryUndefined : getStringHash(pathToProject),
+                userChoice: 'Confirm',
+            })
 
-        const projectName = path.basename(pathToProject)
-        this.messenger.sendProjectSelectionMessage(projectName, fromJDKVersion, toJDKVersion, message.tabID)
+            const projectName = path.basename(pathToProject)
+            this.messenger.sendProjectSelectionMessage(projectName, fromJDKVersion, toJDKVersion, message.tabID)
 
-        if (fromJDKVersion === JDKVersion.UNSUPPORTED) {
-            this.messenger.sendUnrecoverableErrorResponse('unsupported-source-jdk-version', message.tabID)
-            return
-        }
+            if (fromJDKVersion === JDKVersion.UNSUPPORTED) {
+                this.messenger.sendUnrecoverableErrorResponse('unsupported-source-jdk-version', message.tabID)
+                telemetry.record({
+                    result: MetadataResult.Fail,
+                    reason: 'unsupported-source-jdk-version',
+                })
+                return
+            }
 
-        await processTransformFormInput(pathToProject, fromJDKVersion, toJDKVersion)
-        await this.validateBuildWithPromptOnError(message)
+            await processTransformFormInput(pathToProject, fromJDKVersion, toJDKVersion)
+            await this.validateBuildWithPromptOnError(message)
+        })
     }
 
     private async prepareProjectForSubmission(message: { pathToJavaHome: string; tabID: string }): Promise<void> {
@@ -338,6 +390,7 @@ export class GumbyController {
         }
 
         const projectPath = transformByQState.getProjectPath()
+        // TODO: remove deprecated metric once BI started using new metrics
         telemetry.codeTransform_jobStartedCompleteFromPopupDialog.emit({
             codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId(),
             codeTransformJavaSourceVersionsAllowed: JDKToTelemetryValue(
@@ -349,6 +402,8 @@ export class GumbyController {
             codeTransformProjectId: projectPath === undefined ? telemetryUndefined : getStringHash(projectPath),
             result: MetadataResult.Pass,
         })
+
+        // Pre-build project locally
         try {
             this.sessionStorage.getSession().conversationState = ConversationState.COMPILING
             this.messenger.sendCompilationInProgress(message.tabID)
@@ -382,6 +437,7 @@ export class GumbyController {
 
     private async validateBuildWithPromptOnError(message: any | undefined = undefined): Promise<void> {
         try {
+            // Check Java Home is set (not yet prebuilding)
             await validateCanCompileProject()
         } catch (err: any) {
             if (err instanceof JavaHomeNotSetError) {

packages/core/src/amazonqGumby/chat/controller/controller.ts

@@ -11,6 +11,7 @@ import {
     CodeTransformJavaTargetVersionsAllowed,
 } from '../../shared/telemetry/telemetry'
 import { JDKVersion } from '../../codewhisperer/models/model'
+import globals from '../../shared/extensionGlobals'
 
 export const telemetryUndefined = 'undefined'
 
@@ -89,4 +90,4 @@ export const javapOutputToTelemetryValue = (javapCommandLineOutput: string) => {
     }
 }
 
-export const calculateTotalLatency = (startTime: number): number => Date.now() - startTime
+export const calculateTotalLatency = (startTime: number): number => globals.clock.Date.now() - startTime

packages/core/src/amazonqGumby/telemetry/codeTransformTelemetry.ts

@@ -5,6 +5,7 @@
 
 import { randomUUID } from '../../shared/crypto'
 import { codeTransformMetaDataToJsonString, ICodeTransformMetaData } from './codeTransformMetadata'
+import globals from '../../shared/extensionGlobals'
 
 interface ICodeTransformTelemetryState {
     sessionId: string
@@ -19,7 +20,7 @@ export class CodeTransformTelemetryState {
     private constructor() {
         this.mainState = {
             sessionId: randomUUID(),
-            sessionStartTime: Date.now(),
+            sessionStartTime: globals.clock.Date.now(),
             resultStatus: '',
             codeTransformMetadata: {},
         }
@@ -36,7 +37,7 @@ export class CodeTransformTelemetryState {
         this.mainState.sessionId = randomUUID()
     }
     public setStartTime = () => {
-        this.mainState.sessionStartTime = Date.now()
+        this.mainState.sessionStartTime = globals.clock.Date.now()
     }
     public setResultStatus = (newValue: string) => {
         this.mainState.resultStatus = newValue